mirror of
https://github.com/sunnypilot/sunnypilot.git
synced 2026-11-07 09:53:43 +08:00
Compare commits
1 Commits
emu
..
boooooost-ui
| Author | SHA1 | Date | |
|---|---|---|---|
| 6c5b7c4c27 |
@@ -34,11 +34,6 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: 'qcom'
|
||||
on_device:
|
||||
description: 'Compile on the tici / chestnut runner instead of on a hosted runner with tinygrad''s qcom / amd mock'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
hf_repo:
|
||||
description: 'Hugging Face dataset repository (e.g. sunnypilot/sunnypilot_models_v1)'
|
||||
required: false
|
||||
@@ -112,11 +107,6 @@ on:
|
||||
options:
|
||||
- qcom
|
||||
- chestnut
|
||||
on_device:
|
||||
description: 'Compile on the tici / chestnut runner instead of on a hosted runner with tinygrad''s qcom / amd mock'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
hf_repo:
|
||||
description: 'Hugging Face dataset repository'
|
||||
required: false
|
||||
@@ -140,45 +130,11 @@ jobs:
|
||||
is_20hz: ${{ inputs.is_20hz }}
|
||||
artifact_suffix: ${{ inputs.artifact_suffix }}
|
||||
target_hardware: ${{ inputs.target_hardware }}
|
||||
on_device: ${{ inputs.on_device || false }}
|
||||
secrets: inherit
|
||||
|
||||
mock_parity:
|
||||
publish_model:
|
||||
if: ${{ !cancelled() && needs.build_model.result == 'success' }}
|
||||
needs: build_model
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
actions: read
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
- uses: ./.github/workflows/setup-amd-mock
|
||||
if: inputs.target_hardware == 'chestnut'
|
||||
- run: python3 -m pip install -q numpy ${{ inputs.target_hardware != 'chestnut' && 'tinymesa==25.2.7.2' || '' }}
|
||||
- name: Download artifact name file
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: artifact-name-${{ inputs.custom_name || inputs.upstream_branch }}
|
||||
path: artifact_name
|
||||
- name: Download the built model
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
|
||||
id=$(gh api -X GET "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts" -f name="$ARTIFACT_NAME" --jq '.artifacts[0].id')
|
||||
url=$(curl -sf -o /dev/null -w '%{redirect_url}' -H "Authorization: Bearer $GH_TOKEN" "https://api.github.com/repos/$GITHUB_REPOSITORY/actions/artifacts/$id/zip")
|
||||
sudo apt-get install -y -qq aria2 > /dev/null
|
||||
aria2c -q -x16 -s16 -k8M --max-tries=5 --file-allocation=none -o model.zip "$url" && unzip -q model.zip -d output && rm model.zip
|
||||
- name: Replay its check frame on the mock and require the hardware's result bit for bit
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}
|
||||
DEV: ${{ inputs.target_hardware == 'chestnut' && 'MOCKUSB+AMD' || 'MOCK+QCOM' }}
|
||||
run: python3 -m openpilot.sunnypilot.modeld_v2.mock_parity check output/${{ inputs.target_hardware == 'chestnut' && 'big_' || '' }}driving_tinygrad.pkl
|
||||
|
||||
publish_model:
|
||||
if: ${{ !cancelled() && needs.build_model.result == 'success' && contains(fromJSON('["success", "skipped"]'), needs.mock_parity.result) }}
|
||||
needs: [build_model, mock_parity]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
name: Chestnut mock build parity
|
||||
description: >-
|
||||
Convert one commaai precompiled chestnut model (LS2) on this runner under tinygrad's amd mock, as a hosted chestnut build does, and require the
|
||||
exact bytes the chestnut runner built (the published artifact). Needs setup-amd-mock, numpy and zstandard; tinygrad_repo checked out.
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Rebuild LS2 on the mock and require the chestnut runner's artifact byte for byte
|
||||
shell: bash
|
||||
env:
|
||||
REF: b92fc4828664f8767bce5e92f1630ed319cbe509 # commaai/openpilot: precompiled big_driving_tinygrad.pkl, pins tinygrad ec91b74b5b
|
||||
PKL_OID: b4dead3ee091d66ab002a6b8e716c26f4f79e3c931e4be5f48e017ba92f6e960
|
||||
PKL_SIZE: "767228907"
|
||||
TG_SHA: ec91b74b5b6b0fc924b72d3f27689679ca2438b9
|
||||
WANT: 680a8d629785f2b1d1a2b0e40d0dfbcd0f7291c21e20c51259f1d768a22ec217 # driving_light_speed_2_november_04_2026_tinygrad.pkl, recompiled46
|
||||
run: |
|
||||
set -euo pipefail
|
||||
work="$RUNNER_TEMP/chestnut_parity" && rm -rf "$work" && mkdir -p "$work"
|
||||
href=$(curl -fsSL --retry 3 -X POST -H "Accept: application/vnd.git-lfs+json" -H "Content-Type: application/vnd.git-lfs+json" \
|
||||
https://huggingface.co/commaai/openpilot-lfs.git/info/lfs/objects/batch \
|
||||
-d "{\"operation\":\"download\",\"transfer\":[\"basic\"],\"objects\":[{\"oid\":\"$PKL_OID\",\"size\":$PKL_SIZE}]}" | jq -r '.objects[0].actions.download.href')
|
||||
curl -fsSL --retry 3 -o "$work/big_driving_tinygrad.pkl" "$href"
|
||||
echo "$PKL_OID $work/big_driving_tinygrad.pkl" | sha256sum -c -
|
||||
tg="$work/tinygrad" && git init -q "$tg" && git -C "$tg" fetch -q --depth 1 https://github.com/tinygrad/tinygrad "$TG_SHA" && git -C "$tg" checkout -q FETCH_HEAD
|
||||
rm -rf "$tg/test/mockgpu" && cp -r "$GITHUB_WORKSPACE/tinygrad_repo/test/mockgpu" "$tg/test/mockgpu"
|
||||
LLVM_PATH="${LLVM_PATH:-$(ls /usr/lib/llvm-20/lib/libLLVM.so* | head -1)}" DEV=MOCKUSB+AMD CHECK_OOB=0 GMMU=0 PYTHONPATH="$tg:$GITHUB_WORKSPACE" \
|
||||
python3 openpilot/sunnypilot/modeld_v2/convert_upstream_pkl.py "$work/big_driving_tinygrad.pkl" "$work/LS2.pkl"
|
||||
got=$(sha256sum "$work/LS2.pkl" | cut -d' ' -f1)
|
||||
if [ "$got" != "$WANT" ]; then
|
||||
echo "::error::the mock's LS2 build ($got) is not the chestnut runner's ($WANT): this runner's LLVM or the mock no longer builds what the chestnut does"
|
||||
exit 1
|
||||
fi
|
||||
echo "LS2 rebuilt on the mock: byte-identical to the chestnut runner's build"
|
||||
@@ -1,18 +0,0 @@
|
||||
name: Set up tinygrad's amd mock
|
||||
description: LLVM 20 and ROCm comgr, as tinygrad_repo/.github/actions/setup-tinygrad installs them for the mock gpu
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install LLVM 20 and ROCm comgr
|
||||
shell: bash
|
||||
run: |
|
||||
. /etc/os-release
|
||||
curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null
|
||||
echo "deb http://apt.llvm.org/$VERSION_CODENAME/ llvm-toolchain-$VERSION_CODENAME-20 main" | sudo tee /etc/apt/sources.list.d/llvm.list
|
||||
sudo mkdir -p /etc/apt/keyrings
|
||||
curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
|
||||
echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/7.1 $VERSION_CODENAME main" | sudo tee /etc/apt/sources.list.d/rocm.list
|
||||
printf 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600\n' | sudo tee /etc/apt/preferences.d/rocm-pin-600
|
||||
sudo apt-get -qq update && sudo apt-get -qq install -y --no-install-recommends libllvm20 clang-20 lld-20 comgr
|
||||
printf '/opt/rocm/lib\n/opt/rocm/lib64\n' | sudo tee /etc/ld.so.conf.d/rocm.conf && sudo ldconfig
|
||||
echo "/usr/lib/llvm-20/bin" >> "$GITHUB_PATH"
|
||||
@@ -35,11 +35,6 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: 'qcom'
|
||||
on_device:
|
||||
description: 'Compile on the tici / chestnut runner instead of on a hosted runner with tinygrad''s qcom / amd mock'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
upstream_branch:
|
||||
@@ -64,11 +59,6 @@ on:
|
||||
- qcom
|
||||
- chestnut
|
||||
default: 'qcom'
|
||||
on_device:
|
||||
description: 'Compile on the tici / chestnut runner instead of on a hosted runner with tinygrad''s qcom / amd mock'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
|
||||
run-name: Build model [${{ inputs.custom_name || inputs.upstream_branch }}] from ref [${{ inputs.upstream_branch }}]
|
||||
@@ -163,48 +153,18 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
build_model:
|
||||
runs-on: ${{ inputs.on_device && fromJSON(inputs.target_hardware == 'chestnut' && '["self-hosted","chestnut"]' || '["self-hosted","tici"]') || 'ubuntu-24.04' }}
|
||||
runs-on: [self-hosted, "${{ inputs.target_hardware == 'chestnut' && 'chestnut' || 'tici' }}"]
|
||||
needs: get_model
|
||||
env:
|
||||
MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})
|
||||
REF: ${{ inputs.upstream_branch }}
|
||||
MOCK_BUILD: ${{ !inputs.on_device }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up the hosted mock build
|
||||
if: env.MOCK_BUILD == 'true'
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install LLVM 20 and ROCm comgr for the amd mock
|
||||
if: env.MOCK_BUILD == 'true' && inputs.target_hardware == 'chestnut'
|
||||
uses: ./.github/workflows/setup-amd-mock
|
||||
|
||||
- name: Install the amd mock build dependencies
|
||||
if: env.MOCK_BUILD == 'true' && inputs.target_hardware == 'chestnut'
|
||||
run: |
|
||||
python3 -m pip install -q numpy zstandard
|
||||
echo "LLVM_PATH=$(ls /usr/lib/llvm-20/lib/libLLVM.so* | head -1)" >> "$GITHUB_ENV" # the chestnut runner's LLVM, never a newer one
|
||||
|
||||
- name: Check this runner builds what the chestnut runner does
|
||||
if: env.MOCK_BUILD == 'true' && inputs.target_hardware == 'chestnut'
|
||||
uses: ./.github/workflows/chestnut-build-parity
|
||||
|
||||
- name: Install the qcom mock build dependencies and check the compiler
|
||||
if: env.MOCK_BUILD == 'true' && inputs.target_hardware != 'chestnut'
|
||||
run: |
|
||||
sudo apt-get -qq update && sudo apt-get -qq install -y --no-install-recommends qemu-user-static aria2
|
||||
python3 -m pip install -q numpy zstandard tinymesa==25.2.7.2
|
||||
# every kernel an a630 has compiled must compile to the same bytes here before a model is built here
|
||||
PYTHONPATH="$TINYGRAD_PATH" python3 "$TINYGRAD_PATH/test/device/qcom/compiler_parity.py" check "$TINYGRAD_PATH/test/mockgpu/qcom/a630_compiled.pkl.xz"
|
||||
|
||||
- name: Set environment variables
|
||||
if: env.MOCK_BUILD != 'true'
|
||||
id: set-env
|
||||
run: |
|
||||
# Set up common environment
|
||||
@@ -217,7 +177,6 @@ jobs:
|
||||
printenv >> $GITHUB_ENV
|
||||
|
||||
- name: Setup build environment
|
||||
if: env.MOCK_BUILD != 'true'
|
||||
run: |
|
||||
mkdir -p "${BUILD_DIR}/"
|
||||
sudo find $BUILD_DIR/ -mindepth 1 -delete
|
||||
@@ -239,28 +198,21 @@ jobs:
|
||||
|
||||
- name: Build Model
|
||||
run: |
|
||||
if [ "$MOCK_BUILD" = "true" ]; then
|
||||
TASKSET=""
|
||||
export CHECK_OOB=0 GMMU=0 # as test/mockgpu runs the amd mock
|
||||
else
|
||||
source /etc/profile
|
||||
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
|
||||
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
TASKSET="taskset -c 7"
|
||||
fi
|
||||
source /etc/profile
|
||||
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
|
||||
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
export PYTHONPATH="${PYTHONPATH}:${{ env.TINYGRAD_PATH }}:${{ github.workspace }}"
|
||||
|
||||
COMPILE_MODELD="${{ github.workspace }}/openpilot/sunnypilot/modeld_v2/compile_modeld.py"
|
||||
MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
|
||||
CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
|
||||
|
||||
MOCK=$([ "$MOCK_BUILD" = "true" ] && echo MOCK || true)
|
||||
TG_FLAGS_QCOM="DEV=${MOCK}QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 OPENPILOT_HACKS=1"
|
||||
TG_FLAGS_QCOM="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 OPENPILOT_HACKS=1"
|
||||
if [ "${{ inputs.target_hardware }}" == "chestnut" ]; then
|
||||
echo "CHESTNUT build"
|
||||
export CHESTNUT=1
|
||||
TG_FLAGS="DEBUG=1 DEV=${MOCK}USB+AMD:LLVM FRAME_DEV=CPU FLOAT16=1 GMMU=0 TC_OPT=2 TC_MIN_GLOBALS=32"
|
||||
TG_FLAGS="DEBUG=1 DEV=USB+AMD:LLVM FRAME_DEV=CPU FLOAT16=1 GMMU=0 TC_OPT=2 TC_MIN_GLOBALS=32"
|
||||
OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl"
|
||||
else
|
||||
echo "QCOM build"
|
||||
@@ -275,17 +227,14 @@ jobs:
|
||||
echo "Converting commaai's precompiled model with tinygrad $TG_SHA"
|
||||
UPSTREAM_TG="${{ runner.temp }}/upstream_tinygrad"
|
||||
rm -rf "$UPSTREAM_TG" && git init -q "$UPSTREAM_TG" && git -C "$UPSTREAM_TG" fetch -q --depth 1 "$TG_URL" "$TG_SHA" && git -C "$UPSTREAM_TG" checkout -q FETCH_HEAD
|
||||
if [ "$MOCK_BUILD" = "true" ]; then # convert under commaai's tinygrad, on our mock: it is the one measured against a chestnut
|
||||
rm -rf "$UPSTREAM_TG/test/mockgpu" && cp -r "$TINYGRAD_PATH/test/mockgpu" "$UPSTREAM_TG/test/mockgpu"
|
||||
fi
|
||||
CONVERTED="${{ runner.temp }}/big_driving_tinygrad_plain.pkl"
|
||||
env DEV=${MOCK}USB+AMD PYTHONPATH="$UPSTREAM_TG:${{ github.workspace }}" $TASKSET python3 openpilot/sunnypilot/modeld_v2/convert_upstream_pkl.py \
|
||||
env DEV=USB+AMD PYTHONPATH="$UPSTREAM_TG:${{ github.workspace }}" taskset -c 7 python3 openpilot/sunnypilot/modeld_v2/convert_upstream_pkl.py \
|
||||
"${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl" "$CONVERTED"
|
||||
mv "$CONVERTED" "$OUTPUT_PKL"
|
||||
elif NEW_ARCH_ONNX=$(find "${{ env.MODELS_DIR }}" -maxdepth 1 -name "${{ inputs.target_hardware == 'chestnut' && 'big_' || '' }}*driving*.onnx" -print -quit) &&
|
||||
[ -n "$NEW_ARCH_ONNX" ] && grep -qa new_img "$NEW_ARCH_ONNX"; then # the new architecture: one ONNX taking the warped frames as new_img
|
||||
echo "Compiling new architecture ONNX: $NEW_ARCH_ONNX"
|
||||
env ${TG_FLAGS} $TASKSET python3 openpilot/sunnypilot/modeld_v2/compile_modeld_v2.py \
|
||||
env ${TG_FLAGS} taskset -c 7 python3 openpilot/sunnypilot/modeld_v2/compile_modeld_v2.py \
|
||||
"$NEW_ARCH_ONNX" "$OUTPUT_PKL" --device-input "*" --benchmark-runs 1
|
||||
else
|
||||
echo "Detected onnx architecture. Routing to native compile_modeld.py."
|
||||
@@ -335,7 +284,7 @@ jobs:
|
||||
|
||||
if [ -n "$MODEL_TYPE" ]; then
|
||||
echo "Detected: $MODEL_TYPE -> $OUTPUT_PKL"
|
||||
env ${TG_FLAGS} $TASKSET python3 "$COMPILE_MODELD" \
|
||||
env ${TG_FLAGS} taskset -c 7 python3 "$COMPILE_MODELD" \
|
||||
--model-type $MODEL_TYPE \
|
||||
--model-size $MODEL_SIZE \
|
||||
--camera-resolutions $CAMERA_RES \
|
||||
@@ -355,7 +304,7 @@ jobs:
|
||||
--include='/${{ inputs.target_hardware == 'chestnut' && 'big_' || '' }}driving_tinygrad.pkl' \
|
||||
--exclude='*' \
|
||||
--delete-excluded \
|
||||
$([ "$MOCK_BUILD" = "true" ] || echo --chown=comma:comma) \
|
||||
--chown=comma:comma \
|
||||
${{ env.MODELS_DIR }}/ ${{ env.OUTPUT_DIR }}/
|
||||
|
||||
python3 "${{ github.workspace }}/release/ci/model_generator.py" \
|
||||
@@ -384,7 +333,7 @@ jobs:
|
||||
path: ${{ env.OUTPUT_DIR }}/artifact_name.txt
|
||||
|
||||
- name: Re-enable powersave
|
||||
if: always() && env.MOCK_BUILD != 'true'
|
||||
if: always()
|
||||
run: |
|
||||
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
||||
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
|
||||
|
||||
@@ -28,30 +28,24 @@ jobs:
|
||||
env:
|
||||
DEV: CPU
|
||||
run: tools/op.sh test -v openpilot/sunnypilot/modeld_v2/tests/test_plain_artifact.py
|
||||
- uses: ./.github/workflows/setup-amd-mock
|
||||
- name: Install LLVM 20 and ROCm comgr for the amd mock, as tinygrad_repo/.github/actions/setup-tinygrad does
|
||||
run: |
|
||||
. /etc/os-release
|
||||
curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null
|
||||
echo "deb http://apt.llvm.org/$VERSION_CODENAME/ llvm-toolchain-$VERSION_CODENAME-20 main" | sudo tee /etc/apt/sources.list.d/llvm.list
|
||||
sudo mkdir -p /etc/apt/keyrings
|
||||
curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
|
||||
echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/7.1 $VERSION_CODENAME main" | sudo tee /etc/apt/sources.list.d/rocm.list
|
||||
printf 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600\n' | sudo tee /etc/apt/preferences.d/rocm-pin-600
|
||||
sudo apt-get -qq update && sudo apt-get -qq install -y --no-install-recommends libllvm20 clang-20 lld-20 comgr
|
||||
printf '/opt/rocm/lib\n/opt/rocm/lib64\n' | sudo tee /etc/ld.so.conf.d/rocm.conf && sudo ldconfig
|
||||
echo "/usr/lib/llvm-20/bin" >> "$GITHUB_PATH"
|
||||
- name: Convert a pickled USB+AMD jit and replay it under tinygrad's amd mock
|
||||
env:
|
||||
DEV: MOCKUSB+AMD
|
||||
CHECK_OOB: 0
|
||||
GMMU: 0
|
||||
run: tools/op.sh test -v openpilot/sunnypilot/modeld_v2/tests/test_convert_upstream_pkl.py
|
||||
- name: Resolve commaai's tinygrad pin, which their precompiled chestnut model is built and converted with
|
||||
id: upstream-tg
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
read -r sha url < <(gh api repos/commaai/openpilot/contents/tinygrad_repo --jq '.sha + " " + .submodule_git_url')
|
||||
echo "commaai/openpilot master pins $url@$sha, ours is $(git rev-parse HEAD:tinygrad_repo)"
|
||||
dir="$RUNNER_TEMP/upstream_tinygrad"
|
||||
git init -q "$dir" && git -C "$dir" fetch -q --depth 1 "$url" "$sha" && git -C "$dir" checkout -q FETCH_HEAD
|
||||
echo "path=$dir" >> "$GITHUB_OUTPUT"
|
||||
- name: Convert a pickled USB+AMD jit under commaai's tinygrad, as the chestnut model build does
|
||||
env:
|
||||
DEV: MOCKUSB+AMD
|
||||
CHECK_OOB: 0
|
||||
GMMU: 0
|
||||
PYTHONPATH: ${{ steps.upstream-tg.outputs.path }}:${{ github.workspace }}
|
||||
run: tools/op.sh test -v openpilot/sunnypilot/modeld_v2/tests/test_convert_upstream_pkl.py
|
||||
- name: Where tinygrad moved, for the helper or failure point named above
|
||||
if: failure() && github.event_name == 'pull_request'
|
||||
run: |
|
||||
@@ -69,8 +63,6 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
models: ${{ steps.set-matrix.outputs.models }}
|
||||
chestnut: ${{ steps.set-matrix.outputs.chestnut }}
|
||||
qcom: ${{ steps.set-matrix.outputs.qcom }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Fetch and Parse json
|
||||
@@ -82,22 +74,16 @@ jobs:
|
||||
with open("openpilot/sunnypilot/models/fetcher.py", "r") as f:
|
||||
urls = re.findall(r"MODEL_URL(?:_CHESTNUT)?\s*=\s*[\"'"'"']([^\"'"'"']+)[\"'"'"']", f.read())
|
||||
|
||||
artifacts, chestnut, qcom = [], [], []
|
||||
artifacts = []
|
||||
for url in urls:
|
||||
data = json.loads(urllib.request.urlopen(url).read())
|
||||
for bundle in data.get("bundles", []):
|
||||
for model in bundle.get("models", []):
|
||||
if "artifact" in model:
|
||||
artifacts.append(model["artifact"])
|
||||
if "chestnut" in url:
|
||||
chestnut.append(model["artifact"])
|
||||
elif "tinygrad" in model["artifact"]["file_name"]:
|
||||
qcom.append(model["artifact"])
|
||||
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
|
||||
f.write(f"models={json.dumps(artifacts)}\n")
|
||||
f.write(f"chestnut={json.dumps(chestnut)}\n")
|
||||
f.write(f"qcom={json.dumps(qcom)}\n")
|
||||
'
|
||||
|
||||
test-model:
|
||||
@@ -134,74 +120,3 @@ jobs:
|
||||
export PYTHONPATH="$GITHUB_WORKSPACE/tinygrad_repo:$GITHUB_WORKSPACE:$PYTHONPATH"
|
||||
python3 -m pip uninstall -y tinygrad || uv pip uninstall -y tinygrad --system || true
|
||||
python3 -m pytest openpilot/sunnypilot/modeld_v2/tests/test_models.py
|
||||
|
||||
chestnut-mock-parity:
|
||||
name: Mock parity ${{ matrix.artifact.file_name }}
|
||||
needs: generate-matrix
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
artifact: ${{ fromJson(needs.generate-matrix.outputs.chestnut) }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
- uses: ./.github/workflows/setup-amd-mock
|
||||
- run: python3 -m pip install -q numpy
|
||||
- name: Download model
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
||||
MODEL_URL: ${{ matrix.artifact.download_uri.url }}
|
||||
MODEL_SHA: ${{ matrix.artifact.download_uri.sha256 }}
|
||||
shell: bash
|
||||
run: |
|
||||
AUTH=()
|
||||
if [ -n "$HF_TOKEN" ]; then AUTH=(-H "Authorization: Bearer $HF_TOKEN"); fi
|
||||
curl --retry 3 --retry-all-errors -s -S -f -L "${AUTH[@]}" -o "$RUNNER_TEMP/model.pkl" "$MODEL_URL"
|
||||
if [ -n "$MODEL_SHA" ]; then echo "$MODEL_SHA $RUNNER_TEMP/model.pkl" | sha256sum -c -; fi
|
||||
- name: Replay its check frame on the amd mock and require the chestnut's result bit for bit
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}
|
||||
run: python3 -m openpilot.sunnypilot.modeld_v2.mock_parity check "$RUNNER_TEMP/model.pkl"
|
||||
|
||||
qcom-mock-parity:
|
||||
name: QCOM mock parity ${{ matrix.artifact.file_name }}
|
||||
needs: generate-matrix
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
artifact: ${{ fromJson(needs.generate-matrix.outputs.qcom) }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
- run: python3 -m pip install -q numpy tinymesa==25.2.7.2
|
||||
- name: Download model
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
||||
MODEL_URL: ${{ matrix.artifact.download_uri.url }}
|
||||
MODEL_SHA: ${{ matrix.artifact.download_uri.sha256 }}
|
||||
shell: bash
|
||||
run: |
|
||||
AUTH=()
|
||||
if [ -n "$HF_TOKEN" ]; then AUTH=(-H "Authorization: Bearer $HF_TOKEN"); fi
|
||||
curl --retry 3 --retry-all-errors -s -S -f -L "${AUTH[@]}" -o "$RUNNER_TEMP/model.pkl" "$MODEL_URL"
|
||||
if [ -n "$MODEL_SHA" ]; then echo "$MODEL_SHA $RUNNER_TEMP/model.pkl" | sha256sum -c -; fi
|
||||
- name: Replay its check frame on the qcom mock and require the comma device's result bit for bit
|
||||
env:
|
||||
DEV: MOCK+QCOM
|
||||
PYTHONPATH: ${{ github.workspace }}/tinygrad_repo:${{ github.workspace }}
|
||||
run: python3 -m openpilot.sunnypilot.modeld_v2.mock_parity check "$RUNNER_TEMP/model.pkl"
|
||||
|
||||
chestnut-mock-build-parity:
|
||||
name: Chestnut build on the amd mock = chestnut runner build
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
- uses: ./.github/workflows/setup-amd-mock
|
||||
- run: python3 -m pip install -q numpy zstandard
|
||||
- uses: ./.github/workflows/chestnut-build-parity
|
||||
|
||||
+1
-1
Submodule opendbc_repo updated: 48611ae9d6...aa9bff6f54
@@ -22,6 +22,7 @@ from openpilot.common.transformations.orientation import rot_from_euler
|
||||
from enum import IntEnum
|
||||
|
||||
if gui_app.sunnypilot_ui():
|
||||
from openpilot.selfdrive.ui.sunnypilot.mici.onroad.augmented_road_view import AugmentedRoadViewSP
|
||||
from openpilot.selfdrive.ui.sunnypilot.mici.onroad.hud_renderer import HudRendererSP as HudRenderer
|
||||
from openpilot.selfdrive.ui.sunnypilot.ui_state import OnroadTimerStatus
|
||||
|
||||
@@ -156,9 +157,10 @@ class BookmarkIcon(Widget):
|
||||
rl.draw_texture_ex(icon, rl.Vector2(icon_x, icon_y), 0.0, 1.0, rl.Color(255, 255, 255, round(255 * alpha)))
|
||||
|
||||
|
||||
class AugmentedRoadView(CameraView):
|
||||
class AugmentedRoadView(CameraView, AugmentedRoadViewSP):
|
||||
def __init__(self, bookmark_callback=None, stream_type: VisionStreamType = VisionStreamType.VISION_STREAM_NARROW_ROAD):
|
||||
super().__init__("camerad", stream_type)
|
||||
AugmentedRoadViewSP.__init__(self)
|
||||
self._bookmark_callback = bookmark_callback
|
||||
self._set_placeholder_color(rl.BLACK)
|
||||
|
||||
@@ -275,7 +277,7 @@ class AugmentedRoadView(CameraView):
|
||||
if show_navigation:
|
||||
self._navigation.render(rl.Rectangle(self._content_rect.x + self._content_rect.width, self.rect.y, PANEL_WIDTH, self.rect.height))
|
||||
else:
|
||||
self._confidence_ball.render(self.rect)
|
||||
AugmentedRoadViewSP.render_side_panel(self, self.rect)
|
||||
|
||||
self._bookmark_icon.render(self.rect)
|
||||
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
"""
|
||||
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
|
||||
|
||||
This file is part of sunnypilot and is licensed under the MIT License.
|
||||
See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
import math
|
||||
import pyray as rl
|
||||
|
||||
from openpilot.cereal import log
|
||||
from openpilot.common.filter_simple import FirstOrderFilter
|
||||
from openpilot.selfdrive.ui.mici.onroad import SIDE_PANEL_WIDTH
|
||||
from openpilot.selfdrive.ui.ui_state import ui_state
|
||||
from openpilot.sunnypilot.selfdrive.controls.lib.accel_boost import get_boost_max
|
||||
from openpilot.system.ui.lib.application import gui_app
|
||||
from openpilot.system.ui.widgets import Widget
|
||||
|
||||
BAR_WIDTH = 26
|
||||
TRACK_TOP_RATIO = 0.08
|
||||
TRACK_BOTTOM_RATIO = 0.92
|
||||
CAP_BLEND_START = 0.45
|
||||
MAX_BOOST_BLEND = 0.55
|
||||
FADE_THRESHOLD = 0.01
|
||||
|
||||
COLOR_TRACK = rl.Color(255, 255, 255, 28)
|
||||
COLOR_FILL = rl.Color(0, 230, 120, 255)
|
||||
|
||||
|
||||
def get_boost_fraction(boost: float, max_boost: float) -> float:
|
||||
if max_boost <= 0.0:
|
||||
return 0.0
|
||||
return min(max(boost / max_boost, 0.0), 1.0)
|
||||
|
||||
|
||||
def lighten(color: rl.Color, amount: float) -> rl.Color:
|
||||
amount = min(max(amount, 0.0), 1.0)
|
||||
return rl.Color(int(color.r + (255 - color.r) * amount),
|
||||
int(color.g + (255 - color.g) * amount),
|
||||
int(color.b + (255 - color.b) * amount), color.a)
|
||||
|
||||
|
||||
def get_boost_color(fraction: float) -> rl.Color:
|
||||
blend = max(0.0, (fraction - CAP_BLEND_START) / (1.0 - CAP_BLEND_START))
|
||||
return lighten(COLOR_FILL, MAX_BOOST_BLEND * blend)
|
||||
|
||||
|
||||
class AccelBoostIndicator(Widget):
|
||||
def __init__(self, panel_width: float = SIDE_PANEL_WIDTH, bar_width: float = BAR_WIDTH):
|
||||
super().__init__()
|
||||
self._panel_width = panel_width
|
||||
self._bar_width = bar_width
|
||||
fps = max(gui_app.target_fps, 1)
|
||||
self._boost_filter = FirstOrderFilter(0.0, 0.15, 1 / fps)
|
||||
self._alpha_filter = FirstOrderFilter(0.0, 0.1, 1 / fps)
|
||||
self._max_boost_filter = FirstOrderFilter(get_boost_max(), 0.25, 1 / fps)
|
||||
|
||||
@property
|
||||
def holding_panel(self) -> bool:
|
||||
return self._alpha_filter.x > FADE_THRESHOLD
|
||||
|
||||
def _update_state(self):
|
||||
sm = ui_state.sm
|
||||
visible = (ui_state.accel_boost and ui_state.has_longitudinal_control and sm.alive['selfdriveState'] and
|
||||
sm.alive['longitudinalPlan'] and sm['selfdriveState'].enabled)
|
||||
plan = sm['longitudinalPlan']
|
||||
applied = visible and plan.longitudinalPlanSource == log.LongitudinalPlan.LongitudinalPlanSource.e2e
|
||||
self._boost_filter.update(float(plan.accelBoost) if applied else 0.0)
|
||||
self._alpha_filter.update(1.0 if visible else 0.0)
|
||||
|
||||
if sm.alive['selfdriveState']:
|
||||
self._max_boost_filter.update(get_boost_max(sm['selfdriveState'].personality))
|
||||
|
||||
def _render(self, _):
|
||||
alpha = self._alpha_filter.x
|
||||
if alpha < FADE_THRESHOLD:
|
||||
return
|
||||
|
||||
panel_x = max(self.rect.x, self.rect.x + self.rect.width - self._panel_width)
|
||||
bar_x = panel_x + (self._panel_width - self._bar_width) / 2
|
||||
track_top = self.rect.y + self.rect.height * TRACK_TOP_RATIO
|
||||
track = rl.Rectangle(bar_x, track_top, self._bar_width,
|
||||
max(1.0, self.rect.height * (TRACK_BOTTOM_RATIO - TRACK_TOP_RATIO)))
|
||||
rl.draw_rectangle_rounded(track, 0.5, 12,
|
||||
rl.Color(COLOR_TRACK.r, COLOR_TRACK.g, COLOR_TRACK.b, int(COLOR_TRACK.a * alpha)))
|
||||
|
||||
fraction = get_boost_fraction(self._boost_filter.x, self._max_boost_filter.x)
|
||||
fill_h = fraction * track.height
|
||||
if fill_h < 1.0:
|
||||
return
|
||||
|
||||
fill_color = get_boost_color(fraction)
|
||||
tip = track.y + track.height - fill_h
|
||||
rl.begin_scissor_mode(int(track.x), int(tip), int(math.ceil(track.width)) + 1, int(math.ceil(fill_h)))
|
||||
rl.draw_rectangle_rounded(track, 0.5, 12, rl.Color(fill_color.r, fill_color.g, fill_color.b, int(235 * alpha)))
|
||||
rl.end_scissor_mode()
|
||||
@@ -0,0 +1,19 @@
|
||||
"""
|
||||
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
|
||||
|
||||
This file is part of sunnypilot and is licensed under the MIT License.
|
||||
See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
import pyray as rl
|
||||
|
||||
from openpilot.selfdrive.ui.sunnypilot.mici.onroad.accel_boost_indicator import AccelBoostIndicator
|
||||
|
||||
|
||||
class AugmentedRoadViewSP:
|
||||
def __init__(self):
|
||||
self._accel_boost_indicator = AccelBoostIndicator()
|
||||
|
||||
def render_side_panel(self, rect: rl.Rectangle) -> None:
|
||||
self._accel_boost_indicator.render(rect)
|
||||
if not self._accel_boost_indicator.holding_panel:
|
||||
self._confidence_ball.render(rect)
|
||||
@@ -0,0 +1,36 @@
|
||||
"""
|
||||
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
|
||||
|
||||
This file is part of sunnypilot and is licensed under the MIT License.
|
||||
See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
from openpilot.cereal import log
|
||||
from openpilot.common.test import OpenpilotTestCase
|
||||
from openpilot.selfdrive.ui.sunnypilot.mici.onroad.accel_boost_indicator import (
|
||||
CAP_BLEND_START, COLOR_FILL, get_boost_color, get_boost_fraction,
|
||||
)
|
||||
from openpilot.sunnypilot.selfdrive.controls.lib.accel_boost import ACCEL_BOOST_MAX, get_boost_max
|
||||
|
||||
|
||||
class TestAccelBoostIndicator(OpenpilotTestCase):
|
||||
def test_bar_fills_at_the_active_profile_cap(self):
|
||||
for personality in (log.LongitudinalPersonality.aggressive, log.LongitudinalPersonality.standard,
|
||||
log.LongitudinalPersonality.relaxed):
|
||||
cap = get_boost_max(personality)
|
||||
assert get_boost_fraction(0.0, cap) == 0.0
|
||||
assert get_boost_fraction(cap * 0.5, cap) == 0.5
|
||||
assert get_boost_fraction(cap, cap) == 1.0
|
||||
|
||||
assert get_boost_fraction(max(ACCEL_BOOST_MAX), min(ACCEL_BOOST_MAX)) == 1.0
|
||||
assert get_boost_fraction(0.0, 0.0) == 0.0
|
||||
assert get_boost_fraction(-1.0, get_boost_max()) == 0.0
|
||||
|
||||
def test_fill_lightens_as_boost_builds(self):
|
||||
base = get_boost_color(0.0)
|
||||
assert (base.r, base.g, base.b) == (COLOR_FILL.r, COLOR_FILL.g, COLOR_FILL.b)
|
||||
assert (get_boost_color(CAP_BLEND_START).r, get_boost_color(CAP_BLEND_START).b) == (COLOR_FILL.r, COLOR_FILL.b)
|
||||
|
||||
full = get_boost_color(1.0)
|
||||
assert full.r > base.r
|
||||
assert full.b > base.b
|
||||
assert full.g > full.r and full.g > full.b
|
||||
@@ -44,6 +44,7 @@ class UIStateSP:
|
||||
self.screensaver = ScreenSaverSP(params=self.params)
|
||||
self.screensaver_enabled: bool = False
|
||||
|
||||
self.accel_boost: bool = False
|
||||
self.active_bundle = None
|
||||
self.model_runner_tinygrad: bool = False
|
||||
self.active_runner_tinygrad: bool = False
|
||||
@@ -171,6 +172,7 @@ class UIStateSP:
|
||||
# stock only counts the default big model's compiled pkl. a downloaded big bundle runs on the
|
||||
# chestnut just the same, so ChestnutState has to see it as available too.
|
||||
self.chestnut_compiled = self.chestnut_compiled or self.model_runner_tinygrad
|
||||
self.accel_boost = self.params.get_bool("AccelBoostSP")
|
||||
self.blindspot = self.params.get_bool("BlindSpot")
|
||||
self.chevron_metrics = self.params.get("ChevronInfo")
|
||||
self.custom_interactive_timeout = self.params.get("InteractivityTimeout", return_default=True)
|
||||
|
||||
@@ -371,7 +371,7 @@ def _compile_warp_resolution_worker(cam_w, cam_h, model_w, model_h, benchmark_ru
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if ('USB' in os.getenv('DEV', '') or os.getenv('CHESTNUT')) and not os.getenv('DEV', '').startswith('MOCK'):
|
||||
if 'USB' in os.getenv('DEV', '') or os.getenv('CHESTNUT'):
|
||||
from openpilot.system.hardware.chestnut.flash import link_up
|
||||
for _ in range(10):
|
||||
if link_up():
|
||||
|
||||
@@ -66,75 +66,47 @@ def _patched(buf, topo) -> tuple[bytearray, dict]:
|
||||
return blob, rows
|
||||
|
||||
|
||||
def _packed_args(program) -> dict[int, int]:
|
||||
from tinygrad.device import TinyELF
|
||||
params = getattr(program.arg, "params", None) or ()
|
||||
return {off: p.arg.slot for (off, _), p in zip(TinyELF.iter_sig(tuple((None, p.addrspace, p.dtype, ()) for p in params)), params, strict=True)}
|
||||
|
||||
|
||||
def _outer_slot(u, packed: dict[int, int]) -> int | None:
|
||||
from tinygrad.uop.ops import Ops
|
||||
if u.op is Ops.PARAM:
|
||||
return None if packed else u.arg.slot
|
||||
if u.op.name == "REFPARAM":
|
||||
return u.arg.slot
|
||||
if packed and u.op is Ops.LOAD and u.src[0].op is Ops.INDEX and _root(u.src[0].src[0]).op is Ops.PARAM:
|
||||
index = _evaluate(u.src[0].src[1], {}, packed)
|
||||
return None if index is None else packed.get(index * u.dtype.itemsize)
|
||||
return None
|
||||
|
||||
|
||||
def _evaluate(u, env, packed: dict[int, int]):
|
||||
def _evaluate(u, env):
|
||||
from tinygrad.uop.ops import GroupOp, Ops, exec_alu
|
||||
if u.op in (Ops.AFTER, Ops.CAST, Ops.BITCAST):
|
||||
return _evaluate(u.src[0], env, packed)
|
||||
return _evaluate(u.src[0], env)
|
||||
if u.op is Ops.CONST:
|
||||
return u.arg
|
||||
if u.op is Ops.RANGE:
|
||||
return env.get(u)
|
||||
if (slot := _outer_slot(u, packed)) is not None:
|
||||
return env.get(slot)
|
||||
if u.op in (Ops.PARAM, Ops.RANGE):
|
||||
return env.get(u.arg.slot if u.op is Ops.PARAM else u)
|
||||
if u.op is Ops.LOAD:
|
||||
table, index = _root(u.src[0].src[0]), _evaluate(u.src[0].src[1], env, packed)
|
||||
table, index = _root(u.src[0].src[0]), _evaluate(u.src[0].src[1], env)
|
||||
return struct.unpack_from(u.dtype.fmt, table.arg, index * u.dtype.itemsize)[0] if table.op is Ops.BINARY and index is not None else None
|
||||
if u.op in GroupOp.ALU and None not in (values := [_evaluate(x, env, packed) for x in u.src]):
|
||||
if u.op in GroupOp.ALU and None not in (values := [_evaluate(x, env) for x in u.src]):
|
||||
return exec_alu(u.op, u.dtype, values, *([False] if "truncate_output" in inspect.signature(exec_alu).parameters else []))
|
||||
return None
|
||||
|
||||
|
||||
def _input_pokes(submit, kernargs) -> dict[int, tuple[int, int]]:
|
||||
from tinygrad.uop.ops import Ops
|
||||
outer, packed = dict(enumerate(submit.src[1:])), _packed_args(submit.src[0])
|
||||
outer = dict(enumerate(submit.src[1:]))
|
||||
kernargs_params = {i for i, a in outer.items() if a.op is Ops.GETADDR and _root(a.src[0]) is kernargs}
|
||||
inputs = {i: _root(a.src[0]).arg.slot for i, a in outer.items() if a.op is Ops.GETADDR and _root(a.src[0]).op is Ops.PARAM}
|
||||
pokes = {}
|
||||
for call in (u for u in submit.src[0].src[0].toposort() if u.op is Ops.CALL and len(u.src) > 3):
|
||||
_, address, value, *_ = call.src[1:]
|
||||
used = list({slot for x in value.toposort() if (slot := _outer_slot(x, packed)) is not None} & inputs.keys())
|
||||
if len(used) != 1 or not kernargs_params & {_outer_slot(x, packed) for x in address.toposort()}:
|
||||
used = list({x.arg.slot for x in value.toposort() if x.op is Ops.PARAM} & inputs.keys())
|
||||
if len(used) != 1 or not kernargs_params & {x.arg.slot for x in address.toposort() if x.op is Ops.PARAM}:
|
||||
continue
|
||||
ranges = list(dict.fromkeys(x for x in (*address.toposort(), *value.toposort()) if x.op is Ops.RANGE))
|
||||
for trip in itertools.product(*[range(int(r.vmax) + 1) for r in ranges]):
|
||||
env = dict.fromkeys(outer, 0) | dict(zip(ranges, trip, strict=True))
|
||||
if (offset := _evaluate(address, env, packed)) % 8 == 0:
|
||||
pokes[offset] = (inputs[used[0]], _evaluate(value, env, packed))
|
||||
if (offset := _evaluate(address, env)) % 8 == 0:
|
||||
pokes[offset] = (inputs[used[0]], _evaluate(value, env))
|
||||
return pokes
|
||||
|
||||
|
||||
def _elf(image: bytes, rodata: int, entry: int) -> bytes:
|
||||
spans = [(".text", 0, rodata), (".rodata", rodata, len(image))] if entry < rodata else \
|
||||
[(".data", 0, rodata), (".rodata", rodata, entry), (".text", entry, len(image))]
|
||||
sections, names = [(0, 0, 0, 0, b"")], b"\0"
|
||||
for name, start, end in (span for span in spans if span[1] < span[2]):
|
||||
sections.append((len(names), 1, 6 if name == ".text" else 2, ALIGNMENT + start, image[start:end]))
|
||||
names += name.encode() + b"\0"
|
||||
sections.append((len(names), 3, 0, 0, names + b".shstrtab\0"))
|
||||
offset, data, headers = 64, b"", []
|
||||
for name, kind, flags, address, content in sections:
|
||||
headers.append(struct.pack("<IIQQQQIIQQ", name, kind, flags, address, offset, len(content), 0, 0, 1, 0))
|
||||
def _elf(image: bytes, rodata: int) -> bytes:
|
||||
names, offset, data, headers = b"\0.text\0.rodata\0.shstrtab\0", 64, b"", []
|
||||
for name, kind, address, content in [(0, 0, 0, b""), (1, 1, ALIGNMENT, image[:rodata]), (7, 1, ALIGNMENT + rodata, image[rodata:]), (15, 3, 0, names)]:
|
||||
headers.append(struct.pack("<IIQQQQIIQQ", name, kind, 2 if kind == 1 else 0, address, offset, len(content), 0, 0, 1, 0))
|
||||
data, offset = data + content, offset + len(content)
|
||||
header = struct.pack("<HHIQQQIHHHHHH", 1, 0xe0, 1, 0, 0, offset, 0, 64, 0, 0, 64, len(sections), len(sections) - 1)
|
||||
return b"\x7fELF\x02\x01\x01" + bytes(9) + header + data + b"".join(headers)
|
||||
return b"\x7fELF\x02\x01\x01" + bytes(9) + struct.pack("<HHIQQQIHHHHHH", 1, 0xe0, 1, 0, 0, offset, 0, 64, 0, 0, 64, 4, 3) + data + b"".join(headers)
|
||||
|
||||
|
||||
def convert(jit, kwargs: dict, frame: dict, target) -> tuple[dict, memoryview]:
|
||||
@@ -183,7 +155,7 @@ def convert(jit, kwargs: dict, frame: dict, target) -> tuple[dict, memoryview]:
|
||||
if image_alloc not in programs:
|
||||
image = bytes(_patched(image_alloc, topo)[0])
|
||||
rodata = next(r for r in range(0, len(image) - 24, 64) if r + struct.unpack_from("<q", image, r + 16)[0] == entry)
|
||||
programs[image_alloc] = blob.add(_elf(image, rodata, entry))
|
||||
programs[image_alloc] = blob.add(_elf(image, rodata))
|
||||
start, size = _view(arguments.src[0])
|
||||
args = []
|
||||
for slot in range(start, start + unwrap(size), 8):
|
||||
@@ -194,8 +166,7 @@ def convert(jit, kwargs: dict, frame: dict, target) -> tuple[dict, memoryview]:
|
||||
base, at, nbytes = _address(argument_words[slot])
|
||||
args.append({"buffer": buffers.setdefault(base, len(buffers)), "offset": at, "nbytes": nbytes or base.max_numel() * base.dtype.itemsize, "dtype": u8})
|
||||
outs, ins = kernel[6]
|
||||
device_target = dataclasses.asdict(dataclasses.replace(target, interface=target.interface.removeprefix("MOCK")))
|
||||
calls.append({"kind": "program", "name": re.sub(r"\x1b\[[0-9;]*m", "", kernel[1]), "lib": programs[image_alloc], "target": device_target,
|
||||
calls.append({"kind": "program", "name": re.sub(r"\x1b\[[0-9;]*m", "", kernel[1]), "lib": programs[image_alloc], "target": dataclasses.asdict(target),
|
||||
"signature": [[f"data{i}", i, u8, [a.get("nbytes", 1)]] for i, a in enumerate(args)], "global_size": grid, "local_size": local,
|
||||
"globals": list(range(len(args))), "outs": list(outs), "ins": list(ins), "args": args})
|
||||
needs_init |= {args[i]["buffer"] for i in ins if "buffer" in args[i] and args[i]["buffer"] not in written}
|
||||
|
||||
@@ -264,8 +264,8 @@ def _program_uop(call: dict, lib: bytes):
|
||||
parameter = UOp.param(call["globals"][compact_slot], _dtype(dtype), math.prod(shape))
|
||||
image = (shape[0], shape[1]) if len(shape) == 3 and shape[2] == 4 else None
|
||||
parameters.append(parameter.replace(arg=_fields(parameter.arg, name=name, image=image)))
|
||||
info = _fields(ProgramInfo, global_size=tuple(call["global_size"]), local_size=tuple(call["local_size"]), params=tuple(parameters),
|
||||
globals=tuple(call["globals"]), outs=tuple(call["outs"]), ins=tuple(call["ins"]), target=_fields(Target, **call["target"]))
|
||||
info = _fields(ProgramInfo, global_size=tuple(call["global_size"]), local_size=tuple(call["local_size"]), globals=tuple(call["globals"]),
|
||||
outs=tuple(call["outs"]), ins=tuple(call["ins"]), target=_fields(Target, **call["target"]))
|
||||
return UOp(Ops.PROGRAM, src=(UOp(Ops.SINK, arg=KernelInfo(name=call["name"])), UOp(Ops.LINEAR, src=tuple(parameters)),
|
||||
UOp(Ops.SOURCE, arg=""), UOp(Ops.BINARY, arg=lib)), arg=info)
|
||||
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
os.environ.setdefault("DEV", "MOCKUSB+AMD")
|
||||
QCOM = "QCOM" in os.environ["DEV"]
|
||||
ON_MOCK = os.environ["DEV"].startswith("MOCK")
|
||||
ARCH, HARDWARE = ("a630", "comma 3x/4 (a630)") if QCOM else ("gfx1200", "chestnut")
|
||||
os.environ.setdefault("CHECK_OOB", "0")
|
||||
os.environ.setdefault("GMMU", "0")
|
||||
if sys.platform == "darwin" and os.environ["DEV"].startswith("MOCKUSB"): # the mock's VRAM is a memfd, which macOS lacks: an unlinked file is the same
|
||||
from tinygrad.runtime.autogen import libc
|
||||
|
||||
def _memfd_create(name, flags):
|
||||
fd, path = tempfile.mkstemp()
|
||||
os.unlink(path)
|
||||
return fd
|
||||
libc.memfd_create = _memfd_create
|
||||
|
||||
import numpy as np
|
||||
|
||||
from openpilot.sunnypilot.modeld_v2.helpers import ALIGNMENT, MAGIC, PlainJit, _device, read_parts
|
||||
|
||||
|
||||
class ParityError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _emulator():
|
||||
try:
|
||||
if QCOM:
|
||||
from test.mockgpu.qcom import emu
|
||||
else:
|
||||
from test.mockgpu.amd import emu
|
||||
except ImportError as e:
|
||||
raise ParityError(f"tinygrad's {'qcom' if QCOM else 'amd'} mock is not importable: put tinygrad_repo on PYTHONPATH") from e
|
||||
if not hasattr(emu, "hw_unverified"):
|
||||
raise ParityError(f"this tinygrad's emulator has no {ARCH} rounding model, its check frames would not match a {HARDWARE}")
|
||||
return emu
|
||||
|
||||
|
||||
def unverified(header: dict, blob: memoryview) -> dict[str, int]:
|
||||
emu, out = _emulator(), {}
|
||||
for call in header["calls"]:
|
||||
if call["kind"] != "program":
|
||||
continue
|
||||
if (arch := call["target"].get("arch", "")).split(",")[0] != ARCH: # IMAGE builds append ",IMAGE_PITCH_ALIGNMENT=64"
|
||||
out[f"arch={arch}"] = out.get(f"arch={arch}", 0) + 1
|
||||
continue
|
||||
for sig, n in emu.hw_unverified(bytes(blob[slice(*call["lib"])])).items():
|
||||
out[sig] = out.get(sig, 0) + n
|
||||
return out
|
||||
|
||||
|
||||
def _array(blob: memoryview, stored: dict) -> np.ndarray:
|
||||
return np.frombuffer(blob[slice(*stored["data"])], dtype=np.dtype(stored["dtype"])).reshape(stored["shape"]).copy()
|
||||
|
||||
|
||||
def replay(header: dict, blob: memoryview, shared: dict) -> list[np.ndarray]:
|
||||
from tinygrad import Tensor
|
||||
check = header["check"]
|
||||
jit = PlainJit(header, blob, None, verify=False, shared=shared)
|
||||
kwargs: dict = {}
|
||||
for path, stored, device in check["inputs"]:
|
||||
tensor = Tensor(_array(blob, stored), device=_device(device, None)).realize()
|
||||
kwargs[path[0]] = tensor if len(path) == 1 else {**kwargs.get(path[0], {}), path[1]: tensor}
|
||||
result = jit(**kwargs)
|
||||
outputs = kwargs[check["output_container"]].values() if check["output_container"] else result
|
||||
return [t.numpy() for t in (outputs if isinstance(outputs, (list, tuple)) or check["output_container"] else [outputs])]
|
||||
|
||||
|
||||
def _parts_with_offsets(path: str) -> tuple[dict, dict]:
|
||||
parts, _ = read_parts(path)
|
||||
with open(path, "rb") as f:
|
||||
head = f.read(len(MAGIC) + 8)
|
||||
start = len(MAGIC) + 8 + struct.unpack_from("<Q", head, len(MAGIC))[0]
|
||||
f.seek(len(MAGIC) + 8)
|
||||
entries = json.loads(f.read(start - len(MAGIC) - 8))["parts"]
|
||||
base = start + -start % ALIGNMENT
|
||||
offsets = {tuple(e["key"]) if isinstance(e["key"], list) else e["key"]: base + e["blob"][0] for e in entries}
|
||||
return parts, offsets
|
||||
|
||||
|
||||
def _gate(parts: dict) -> None:
|
||||
missing: dict[str, int] = {}
|
||||
for header, blob in parts.values():
|
||||
if "check" not in header:
|
||||
raise ParityError("artifact has no check frame")
|
||||
for sig, n in unverified(header, blob).items():
|
||||
missing[sig] = missing.get(sig, 0) + n
|
||||
if missing:
|
||||
raise ParityError(f"kernels use instructions the emulator is not verified against a {HARDWARE} for: " +
|
||||
", ".join(f"{s} ({n}x)" for s, n in sorted(missing.items(), key=lambda kv: -kv[1])))
|
||||
|
||||
|
||||
def check(path: str) -> None:
|
||||
parts, _ = read_parts(path)
|
||||
if ON_MOCK:
|
||||
_gate(parts)
|
||||
shared: dict = {}
|
||||
for key, (header, blob) in parts.items():
|
||||
got = replay(header, blob, shared)
|
||||
for i, (g, stored) in enumerate(zip(got, header["check"]["outputs"], strict=True)):
|
||||
want = _array(blob, stored)
|
||||
if g.shape != want.shape or not np.array_equal(g.view(np.uint8), want.view(np.uint8)):
|
||||
n = int(np.count_nonzero(g != want)) if g.shape == want.shape else -1
|
||||
raise ParityError(f"part {key} output {i}: the {'mock' if ON_MOCK else 'device'}'s check frame differs from the stored one in {n} of {want.size} values")
|
||||
|
||||
|
||||
def record(path: str, out: str) -> None:
|
||||
parts, offsets = _parts_with_offsets(path)
|
||||
_gate(parts)
|
||||
shared, writes = {}, []
|
||||
for key, (header, blob) in parts.items():
|
||||
for g, stored in zip(replay(header, blob, shared), header["check"]["outputs"], strict=True):
|
||||
want = np.dtype(stored["dtype"])
|
||||
if tuple(g.shape) != tuple(stored["shape"]) or g.dtype != want:
|
||||
raise ParityError(f"part {key}: replayed output {g.dtype}{g.shape} does not fit the stored {want}{tuple(stored['shape'])}")
|
||||
writes.append((offsets[key] + stored["data"][0], np.ascontiguousarray(g).tobytes()))
|
||||
shutil.copyfile(path, out)
|
||||
with open(out, "r+b") as f:
|
||||
for offset, data in writes:
|
||||
f.seek(offset)
|
||||
f.write(data)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) < 2 or argv[0] not in ("check", "record") or (argv[0] == "record" and len(argv) != 3):
|
||||
print("usage: mock_parity check ARTIFACT... | record ARTIFACT OUT", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
if argv[0] == "record":
|
||||
st = time.perf_counter()
|
||||
record(argv[1], argv[2])
|
||||
print(f"{argv[2]}: check frame recorded on the mock in {time.perf_counter() - st:.0f}s")
|
||||
return 0
|
||||
for path in argv[1:]:
|
||||
st = time.perf_counter()
|
||||
check(path)
|
||||
print(f"{os.path.basename(path)}: bit-identical to its {HARDWARE} check frame on the {'mock' if ON_MOCK else 'device'} ({time.perf_counter() - st:.0f}s)")
|
||||
return 0
|
||||
except ParityError as e:
|
||||
print(f"PARITY FAIL: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -242,7 +242,7 @@ class ModelState(ModelStateBase):
|
||||
desired_curvature = get_curvature_from_output(model_output, curvature_plan, v_ego, lat_action_t, self.mlsim)
|
||||
else:
|
||||
desired_accel = model_output['action'][0, 1]
|
||||
desired_curvature = model_output['action'][0, 0] / (max(2.5, v_ego))**2
|
||||
desired_curvature = model_output['action'][0, 0] / (max(1.0, v_ego))**2
|
||||
if blendful is not None:
|
||||
desired_curvature += blendful(desired_curvature)
|
||||
|
||||
|
||||
@@ -80,8 +80,7 @@ def serialize_jit(jit, kernel_linear, check: dict | None = None) -> tuple[dict,
|
||||
if ast.op is Ops.PROGRAM:
|
||||
elf, info = ast.to_elf(), ast.arg
|
||||
global_size, local_size = info.launch_dims({})
|
||||
device_target = dataclasses.asdict(dataclasses.replace(elf.target, interface=elf.target.interface.removeprefix("MOCK")))
|
||||
calls.append({"kind": "program", "name": elf.name, "lib": blob.add(bytes(elf.lib)), "target": device_target,
|
||||
calls.append({"kind": "program", "name": elf.name, "lib": blob.add(bytes(elf.lib)), "target": dataclasses.asdict(elf.target),
|
||||
"signature": kernel_signature(ast, elf),
|
||||
"global_size": [int(x) for x in global_size], "local_size": [int(x) for x in local_size],
|
||||
"globals": list(info.globals), "outs": list(info.outs), "ins": list(info.ins), "args": arguments})
|
||||
|
||||
@@ -6,7 +6,6 @@ See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
@@ -26,8 +25,8 @@ if sys.platform == "darwin" and os.environ.get("DEV", "").startswith("MOCKUSB"):
|
||||
|
||||
from openpilot.common.test import OpenpilotTestCase
|
||||
from openpilot.sunnypilot.modeld_v2 import convert_upstream_pkl
|
||||
from openpilot.sunnypilot.modeld_v2.convert_upstream_pkl import _elf, convert
|
||||
from openpilot.sunnypilot.modeld_v2.helpers import ALIGNMENT, PlainJit, load_model
|
||||
from openpilot.sunnypilot.modeld_v2.convert_upstream_pkl import convert
|
||||
from openpilot.sunnypilot.modeld_v2.helpers import PlainJit, load_model
|
||||
from openpilot.sunnypilot.modeld_v2.plain_artifact_compiler import capture_frame
|
||||
|
||||
IMG, DESIRE, STATE, OUT = (6, 8, 8), (8,), (4, 16), (1, 32)
|
||||
@@ -120,18 +119,3 @@ class TestConvertUpstreamPkl(OpenpilotTestCase):
|
||||
bundle = load_model(out, cache=False, verify=True)
|
||||
assert bundle["metadata"] == {"name": "tiny"} and bundle["input_specs"] == self.input_specs and bundle["output_specs"] == self.output_specs
|
||||
self.assert_matches_eager(bundle["run"], 6)
|
||||
|
||||
|
||||
class TestKernelElf(OpenpilotTestCase):
|
||||
def test_sections_follow_the_code(self):
|
||||
from tinygrad.runtime.support.elf import elf_loader
|
||||
code = bytes(range(1, 129))
|
||||
for rodata, entry in ((len(code), 0), (0, 256)):
|
||||
image = bytearray(code + bytes(64) if entry < rodata else bytes(256) + code)
|
||||
struct.pack_into("<q", image, rodata + 16, entry - rodata)
|
||||
loaded, sections, _ = elf_loader(_elf(bytes(image), rodata, entry))
|
||||
headers = {section.name: section.header for section in sections}
|
||||
assert bytes(loaded[ALIGNMENT:]) == bytes(image), "the uploaded program differs from the pickle's"
|
||||
assert headers[".rodata"].sh_addr == ALIGNMENT + rodata, "the loader reads the kernel descriptor at .rodata"
|
||||
text = headers[".text"]
|
||||
assert text.sh_addr <= ALIGNMENT + entry < text.sh_addr + text.sh_size, "the amd mock sizes the code from the entry to the end of .text"
|
||||
|
||||
@@ -13,41 +13,26 @@ from pathlib import Path
|
||||
from datetime import datetime, UTC
|
||||
|
||||
|
||||
_VERSION_REGEX = re.compile(r'^(?:\d+[a-zA-Z]+|\d+[vVbB]\d+|[vVbB]\d+|\d{4})$')
|
||||
_DECIMAL_VERSION_REGEX = re.compile(r'^([vVbB]?)(\d+(?:\.\d+)+)$')
|
||||
|
||||
|
||||
def _version(word: str) -> str | None:
|
||||
match = _DECIMAL_VERSION_REGEX.match(word.strip('()[]{}.,;:'))
|
||||
if match:
|
||||
return match.group(1).upper() + match.group(2)
|
||||
stripped = re.sub(r'[^a-zA-Z0-9]', '', word)
|
||||
return stripped.upper() if _VERSION_REGEX.match(stripped) else None
|
||||
|
||||
|
||||
def _truncate(short_name: str) -> str:
|
||||
return short_name[:8].rstrip('.')
|
||||
|
||||
|
||||
def create_short_name(full_name: str) -> str:
|
||||
# Remove parentheses and extract alphanumeric words, keeping decimals for versioning
|
||||
# Remove parentheses and extract alphanumeric words
|
||||
clean_name = re.sub(r'\([^)]*\)', '', full_name)
|
||||
words = [re.sub(r'[^a-zA-Z0-9.]', '', word) for word in clean_name.split()]
|
||||
words = [word for word in words if word]
|
||||
words = [re.sub(r'[^a-zA-Z0-9]', '', word) for word in clean_name.split() if re.sub(r'[^a-zA-Z0-9]', '', word)]
|
||||
|
||||
if len(words) == 1:
|
||||
return _truncate((_version(words[0]) or words[0]).upper())
|
||||
return words[0][:8].upper()
|
||||
|
||||
# Handle special case: Name + Version (e.g., "Word A1" -> "WordA1", "Model v1.1" -> "ModelV1.1")
|
||||
if len(words) == 2 and re.match(r'^[A-Za-z]?\d+$', words[1]):
|
||||
return _truncate((words[0] + (_version(words[1]) or words[1])).upper())
|
||||
# Handle special case: Name + Version (e.g., "Word A1" -> "WordA1")
|
||||
if len(words) == 2 and re.match(r'^[A-Za-z]\d+$', words[1]):
|
||||
return (words[0] + words[1])[:8].upper()
|
||||
|
||||
result = ""
|
||||
for word in words:
|
||||
# Version or number patterns
|
||||
version = _version(word)
|
||||
if version:
|
||||
result += version
|
||||
if (re.match(r'^\d+[a-zA-Z]+$', word) or
|
||||
re.match(r'^\d+[vVbB]\d+$', word) or
|
||||
re.match(r'^[vVbB]\d+$', word) or
|
||||
re.match(r'^\d{4}$', word)):
|
||||
result += word.upper()
|
||||
# All uppercase abbreviations (2-3 letters)
|
||||
elif re.match(r'^[A-Z]{2,3}$', word):
|
||||
result += word
|
||||
@@ -60,7 +45,7 @@ def create_short_name(full_name: str) -> str:
|
||||
result += word
|
||||
else:
|
||||
result += word[0].upper()
|
||||
return _truncate(result)
|
||||
return result[:8]
|
||||
|
||||
|
||||
def create_pkl_name(full_name: str) -> str:
|
||||
|
||||
+1
-1
Submodule tinygrad_repo updated: cb2fb4ef10...92b742f711
Reference in New Issue
Block a user