mirror of
https://github.com/firestar5683/StarPilot.git
synced 2026-09-15 03:53:58 +08:00
Team Noah
This commit is contained in:
firestar5683
+145
-124
@@ -88,13 +88,13 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
dev: ['METAL', 'AMD', 'NV']
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 30
|
||||
defaults:
|
||||
run:
|
||||
shell: bash -e -o pipefail {0}
|
||||
env:
|
||||
DEV: ${{ matrix.dev }}
|
||||
HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }}
|
||||
HCQ2: '0'
|
||||
if: github.repository_owner == 'tinygrad'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
@@ -102,16 +102,11 @@ jobs:
|
||||
- name: Setup (AMD)
|
||||
if: ${{ matrix.dev == 'AMD' }}
|
||||
run: |
|
||||
./extra/amdpci/setup_python_cap.sh
|
||||
./extra/hcq/hcq_smi.py amd rmmod
|
||||
./extra/hcq/hcq_smi.py amd kill_pids
|
||||
./extra/hcq/hcq_smi.py amd rmmod --expect
|
||||
./extra/hcq/hcq_smi.py amd kill_pids --sudoless
|
||||
- name: Setup (NV)
|
||||
if: ${{ matrix.dev == 'NV' }}
|
||||
run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; }
|
||||
- name: Symlink models and datasets
|
||||
run: |
|
||||
mkdir -p weights
|
||||
ln -s /raid/weights/LLaMA-3 weights/LLaMA-3
|
||||
run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; }
|
||||
- name: setup staging db
|
||||
if: github.ref == 'refs/heads/update_benchmark_staging'
|
||||
run: |
|
||||
@@ -121,18 +116,14 @@ jobs:
|
||||
run: python3 test/external/process_replay/reset.py
|
||||
- name: Run llama3.2
|
||||
run: BENCHMARK_LOG=llama32_3b-f16 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 -m tinygrad.llm -m llama3.2:3b-f16 --benchmark --warmup
|
||||
- name: Run qwen3.6
|
||||
# qwen3.6:35b-a3b doesn't fit on mac
|
||||
- name: Run qwen3.8
|
||||
# qwen3.8:27b doesn't fit on mac
|
||||
if: ${{ matrix.dev != 'METAL' }}
|
||||
run: BENCHMARK_LOG=qwen36_35b-a3b JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 -m tinygrad.llm -m qwen3.6:35b-a3b --benchmark --warmup
|
||||
run: BENCHMARK_LOG=qwen38_27b JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 -m tinygrad.llm -m qwen3.8:27b --benchmark --warmup
|
||||
- name: Run olmoe
|
||||
# just metal for now
|
||||
if: ${{ matrix.dev == 'METAL' }}
|
||||
run: BENCHMARK_LOG=olmoe JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 -m tinygrad.llm -m olmoe --benchmark --warmup
|
||||
- name: Run LLaMA-3 8B on 4 GPUs with BEAM
|
||||
# only run on machines with multiple gpus
|
||||
if: ${{ matrix.dev != 'METAL' }}
|
||||
run: BENCHMARK_LOG=llama3_beam_4gpu JITBEAM=2 IGNORE_BEAM_CACHE=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama3.py --size 8B --shard 4 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0
|
||||
- name: Run process replay tests
|
||||
uses: ./.github/actions/process-replay
|
||||
|
||||
@@ -143,13 +134,13 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
dev: ['METAL', 'AMD', 'NV']
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 10
|
||||
defaults:
|
||||
run:
|
||||
shell: bash -e -o pipefail {0}
|
||||
env:
|
||||
DEV: ${{ matrix.dev }}
|
||||
HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }}
|
||||
HCQ2: '0'
|
||||
if: github.repository_owner == 'tinygrad'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
@@ -157,12 +148,11 @@ jobs:
|
||||
- name: Setup (AMD)
|
||||
if: ${{ matrix.dev == 'AMD' }}
|
||||
run: |
|
||||
./extra/amdpci/setup_python_cap.sh
|
||||
./extra/hcq/hcq_smi.py amd rmmod
|
||||
./extra/hcq/hcq_smi.py amd kill_pids
|
||||
./extra/hcq/hcq_smi.py amd rmmod --expect
|
||||
./extra/hcq/hcq_smi.py amd kill_pids --sudoless
|
||||
- name: Setup (NV)
|
||||
if: ${{ matrix.dev == 'NV' }}
|
||||
run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; }
|
||||
run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; }
|
||||
- name: setup staging db
|
||||
if: github.ref == 'refs/heads/update_benchmark_staging'
|
||||
run: |
|
||||
@@ -182,10 +172,6 @@ jobs:
|
||||
# slow on metal
|
||||
if: ${{ matrix.dev != 'METAL' }}
|
||||
run: time BENCHMARK_LOG=cifar DEFAULT_FLOAT=HALF STEPS=1000 TARGET_EVAL_ACC_PCT=93.0 python3 examples/hlb_cifar10.py
|
||||
- name: Run full CIFAR training steps w 6 GPUS
|
||||
# only run on machines with multiple gpus
|
||||
if: ${{ matrix.dev != 'METAL' }}
|
||||
run: time BENCHMARK_LOG=cifar_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF STEPS=350 BS=1536 GPUS=6 TARGET_EVAL_ACC_PCT=93.0 python3 examples/hlb_cifar10.py
|
||||
- name: Run process replay tests
|
||||
uses: ./.github/actions/process-replay
|
||||
|
||||
@@ -196,13 +182,13 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
dev: ['AMD', 'NV']
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 5
|
||||
defaults:
|
||||
run:
|
||||
shell: bash -e -o pipefail {0}
|
||||
env:
|
||||
DEV: ${{ matrix.dev }}
|
||||
HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }}
|
||||
HCQ2: '0'
|
||||
if: github.repository_owner == 'tinygrad'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
@@ -210,12 +196,11 @@ jobs:
|
||||
- name: Setup (AMD)
|
||||
if: ${{ matrix.dev == 'AMD' }}
|
||||
run: |
|
||||
./extra/amdpci/setup_python_cap.sh
|
||||
./extra/hcq/hcq_smi.py amd rmmod
|
||||
./extra/hcq/hcq_smi.py amd kill_pids
|
||||
./extra/hcq/hcq_smi.py amd rmmod --expect
|
||||
./extra/hcq/hcq_smi.py amd kill_pids --sudoless
|
||||
- name: Setup (NV)
|
||||
if: ${{ matrix.dev == 'NV' }}
|
||||
run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; }
|
||||
run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; }
|
||||
- name: Symlink models and datasets
|
||||
run: |
|
||||
mkdir -p extra/datasets
|
||||
@@ -227,15 +212,8 @@ jobs:
|
||||
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
||||
- name: reset process replay
|
||||
run: test/external/process_replay/reset.py
|
||||
- name: Run MLPerf resnet eval on training data
|
||||
run: time BENCHMARK_LOG=resnet_eval MODEL=resnet python3 examples/mlperf/model_eval.py
|
||||
- name: Run 10 MLPerf ResNet50 training steps (1 gpu)
|
||||
run: BENCHMARK_LOG=resnet_10steps DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py
|
||||
- name: Run 10 MLPerf ResNet50 training steps (6 gpu)
|
||||
run: BENCHMARK_LOG=resnet_10steps_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=1536 GPUS=6 MODEL=resnet python3 examples/mlperf/model_train.py
|
||||
- name: Run 10 MLPerf Bert training steps (6 gpu)
|
||||
# TODO: remove BERT_LAYERS once scheduler is fast
|
||||
run: BENCHMARK_LOG=bert_10steps_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=72 GPUS=6 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py
|
||||
- name: Run process replay tests
|
||||
uses: ./.github/actions/process-replay
|
||||
|
||||
@@ -246,13 +224,13 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
dev: ['METAL', 'AMD', 'NV']
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 15
|
||||
defaults:
|
||||
run:
|
||||
shell: bash -e -o pipefail {0}
|
||||
env:
|
||||
DEV: ${{ matrix.dev }}
|
||||
HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }}
|
||||
HCQ2: '0'
|
||||
if: github.repository_owner == 'tinygrad'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
@@ -260,12 +238,11 @@ jobs:
|
||||
- name: Setup (AMD)
|
||||
if: ${{ matrix.dev == 'AMD' }}
|
||||
run: |
|
||||
./extra/amdpci/setup_python_cap.sh
|
||||
./extra/hcq/hcq_smi.py amd rmmod
|
||||
./extra/hcq/hcq_smi.py amd kill_pids
|
||||
./extra/hcq/hcq_smi.py amd rmmod --expect
|
||||
./extra/hcq/hcq_smi.py amd kill_pids --sudoless
|
||||
- name: Setup (NV)
|
||||
if: ${{ matrix.dev == 'NV' }}
|
||||
run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; }
|
||||
run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; }
|
||||
- name: setup staging db
|
||||
if: github.ref == 'refs/heads/update_benchmark_staging'
|
||||
run: |
|
||||
@@ -285,6 +262,59 @@ jobs:
|
||||
- name: Run process replay tests
|
||||
uses: ./.github/actions/process-replay
|
||||
|
||||
multigpubenchmark:
|
||||
name: Multi-GPU Benchmarks (DEV=${{ matrix.dev }})
|
||||
runs-on: [self-hosted, "${{ matrix.dev == 'AMD' && 'tinybox' || 'tinyboxgreen' }}"]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
dev: ['AMD', 'NV']
|
||||
timeout-minutes: 20
|
||||
defaults:
|
||||
run:
|
||||
shell: bash -e -o pipefail {0}
|
||||
env:
|
||||
DEV: ${{ matrix.dev }}
|
||||
HCQ2: '0'
|
||||
if: github.repository_owner == 'tinygrad'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v6
|
||||
- name: Setup (AMD)
|
||||
if: ${{ matrix.dev == 'AMD' }}
|
||||
run: |
|
||||
./extra/hcq/hcq_smi.py amd rmmod --expect
|
||||
./extra/hcq/hcq_smi.py amd kill_pids --sudoless
|
||||
- name: Setup (NV)
|
||||
if: ${{ matrix.dev == 'NV' }}
|
||||
run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; }
|
||||
- name: Symlink models and datasets
|
||||
run: |
|
||||
mkdir -p weights
|
||||
mkdir -p extra/datasets
|
||||
ln -s /raid/weights/LLaMA-3 weights/LLaMA-3
|
||||
ln -s /raid/datasets/imagenet extra/datasets/imagenet
|
||||
- name: setup staging db
|
||||
if: github.ref == 'refs/heads/update_benchmark_staging'
|
||||
run: |
|
||||
echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
|
||||
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
||||
- name: reset process replay
|
||||
run: python3 test/external/process_replay/reset.py
|
||||
- name: Run LLaMA-3 8B on 4 GPUs with BEAM
|
||||
run: BENCHMARK_LOG=llama3_beam_4gpu JITBEAM=2 IGNORE_BEAM_CACHE=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama3.py --size 8B --shard 4 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0
|
||||
- name: Run full CIFAR training steps w 6 GPUS
|
||||
run: time BENCHMARK_LOG=cifar_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF STEPS=350 BS=1536 GPUS=6 TARGET_EVAL_ACC_PCT=93.0 python3 examples/hlb_cifar10.py
|
||||
- name: Run MLPerf resnet eval on training data
|
||||
run: time BENCHMARK_LOG=resnet_eval MODEL=resnet python3 examples/mlperf/model_eval.py
|
||||
- name: Run 10 MLPerf ResNet50 training steps (6 gpu)
|
||||
run: BENCHMARK_LOG=resnet_10steps_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=1536 GPUS=6 MODEL=resnet python3 examples/mlperf/model_train.py
|
||||
- name: Run 10 MLPerf Bert training steps (6 gpu)
|
||||
# TODO: remove BERT_LAYERS once scheduler is fast
|
||||
run: BENCHMARK_LOG=bert_10steps_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=72 GPUS=6 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py
|
||||
- name: Run process replay tests
|
||||
uses: ./.github/actions/process-replay
|
||||
|
||||
tests:
|
||||
name: Tests (DEV=${{ matrix.dev }})
|
||||
runs-on: [self-hosted, "${{ matrix.dev == 'METAL' && 'macOS' || matrix.dev == 'AMD' && 'tinybox' || 'tinyboxgreen' }}"]
|
||||
@@ -292,7 +322,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
dev: ['METAL', 'AMD', 'NV']
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 10
|
||||
defaults:
|
||||
run:
|
||||
shell: bash -e -o pipefail {0}
|
||||
@@ -305,12 +335,11 @@ jobs:
|
||||
- name: Setup (AMD)
|
||||
if: ${{ matrix.dev == 'AMD' }}
|
||||
run: |
|
||||
./extra/amdpci/setup_python_cap.sh
|
||||
./extra/hcq/hcq_smi.py amd rmmod
|
||||
./extra/hcq/hcq_smi.py amd kill_pids
|
||||
./extra/hcq/hcq_smi.py amd rmmod --expect
|
||||
./extra/hcq/hcq_smi.py amd kill_pids --sudoless
|
||||
- name: Setup (NV)
|
||||
if: ${{ matrix.dev == 'NV' }}
|
||||
run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; }
|
||||
run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; }
|
||||
- name: setup staging db
|
||||
if: github.ref == 'refs/heads/update_benchmark_staging'
|
||||
run: |
|
||||
@@ -374,7 +403,7 @@ jobs:
|
||||
run: python test/external/external_benchmark_multitensor_allreduce.py
|
||||
- name: HEVC Decode Benchmark
|
||||
if: ${{ matrix.dev == 'NV' }}
|
||||
run: VALIDATE=1 MAX_FRAMES=100 ASSERT_FPS=1400 JITBEAM=1 PYTHONPATH=. python3 extra/hevc/decode.py
|
||||
run: IGNORE_BEAM_CACHE=1 VALIDATE=1 MAX_FRAMES=100 ASSERT_FPS=1400 JITBEAM=1 PYTHONPATH=. python3 extra/hevc/decode.py
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: ${{ matrix.dev != 'AMD' }}
|
||||
with:
|
||||
@@ -387,7 +416,7 @@ jobs:
|
||||
testusbgpu:
|
||||
name: UsbGPU Benchmark
|
||||
runs-on: [self-hosted, macOS]
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 3
|
||||
defaults:
|
||||
run:
|
||||
shell: bash -e -o pipefail {0}
|
||||
@@ -402,32 +431,66 @@ jobs:
|
||||
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
||||
- name: Kill stale pids
|
||||
run: |
|
||||
PYTHONPATH=. ./extra/hcq/hcq_smi.py amd kill_pids
|
||||
PYTHONPATH=. ./extra/hcq/hcq_smi.py nv kill_pids
|
||||
# since sudo is required for usbgpu on macos, do not write bytecode, as some of the files are owned by root
|
||||
./extra/hcq/hcq_smi.py amd kill_pids --sudoless
|
||||
./extra/hcq/hcq_smi.py nv kill_pids --sudoless
|
||||
- name: UsbGPU boot time
|
||||
run: sudo -E PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. GMMU=0 DEBUG=2 AM_RESET=1 DEV=USB+AMD time python3.11 test/test_tiny.py TestTiny.test_plus
|
||||
run: GMMU=0 DEBUG=2 AM_RESET=1 DEV=USB+AMD time python3.11 test/test_tiny.py TestTiny.test_plus
|
||||
- name: UsbGPU tiny tests
|
||||
run: sudo -E PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3.11 test/test_tiny.py
|
||||
run: GMMU=0 DEV=USB+AMD python3.11 test/test_tiny.py
|
||||
- name: UsbGPU copy speeds
|
||||
run: sudo -E PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3.11 test/external/external_test_usb_asm24.py TestDevCopySpeeds
|
||||
#- name: UsbGPU openpilot test
|
||||
# run: sudo -E PYTHONPATH=. GMMU=0 DEV=USB+AMD GRAPH_ONE_KERNEL=1 python3.11 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/9118973ed03c1ae1d40cf69a29507ec2cc78efd7/selfdrive/modeld/models/supercombo.onnx
|
||||
run: SIZE=64000000 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3.11 test/external/external_test_usb_asm24.py TestDevCopySpeeds
|
||||
- name: UsbGPU (USB4/TB) install script
|
||||
run: PYTHONPATH=. sh extra/setup_tinygpu_osx.sh
|
||||
run: sh extra/setup_tinygpu_osx.sh
|
||||
- name: UsbGPU (USB4/TB) boot time
|
||||
run: PYTHONPATH=. DEBUG=3 DEV=PCI+NV:NAK time python3.11 test/test_tiny.py TestTiny.test_plus
|
||||
run: DEBUG=3 DEV=PCI+NV:NAK time python3.11 test/test_tiny.py TestTiny.test_plus
|
||||
- name: UsbGPU (USB4/TB) tiny tests
|
||||
run: PYTHONPATH=. DEV=PCI+NV:NAK python3.11 test/test_tiny.py
|
||||
run: DEV=PCI+NV:NAK python3.11 test/test_tiny.py
|
||||
|
||||
testcommalatest:
|
||||
name: comma Benchmark (0.11.2)
|
||||
testcomma:
|
||||
strategy:
|
||||
matrix:
|
||||
dev: ['QCOM', 'QCOM:IR3']
|
||||
version: ['0.11.0', '0.11.2']
|
||||
model: ['vision', 'policy', 'supercombo', 'dmonitoring']
|
||||
# exclude non-existent models
|
||||
exclude: [{ version: '0.11.0', model: supercombo }, { version: '0.11.2', model: vision }, { version: '0.11.2', model: policy }]
|
||||
include:
|
||||
- version: '0.11.0'
|
||||
model: vision
|
||||
url: https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_vision.onnx
|
||||
timing: 18
|
||||
- version: '0.11.0'
|
||||
model: policy
|
||||
url: https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_policy.onnx
|
||||
timing: 3.4
|
||||
- version: '0.11.0'
|
||||
model: dmonitoring
|
||||
url: https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/dmonitoring_model.onnx
|
||||
timing: 12.5
|
||||
- version: '0.11.2'
|
||||
model: supercombo
|
||||
url: https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/433f85f956837606ad1f1cbee4aa7e2158ad23c768dea914b20436c97232741b
|
||||
timing: 28
|
||||
- dev: QCOM:IR3
|
||||
version: '0.11.2'
|
||||
model: supercombo
|
||||
timing: 29
|
||||
- version: '0.11.2'
|
||||
model: dmonitoring
|
||||
url: https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/3e7b31dfbc0a5234f1baf196513b77fc6af12204b8a8ffe8ee0417e48352f316
|
||||
timing: 12.5
|
||||
fail-fast: false
|
||||
name: openpilot ${{ matrix.version }} compile3 ${{ matrix.model }} (DEV=${{ matrix.dev }})
|
||||
runs-on: [self-hosted, Linux, comma]
|
||||
timeout-minutes: 12
|
||||
timeout-minutes: 5
|
||||
defaults:
|
||||
run:
|
||||
shell: bash -e -o pipefail {0}
|
||||
if: github.repository_owner == 'tinygrad'
|
||||
env:
|
||||
DEV: ${{ matrix.dev }}
|
||||
ASSERT_MIN_STEP_TIME: ${{ matrix.timing }}
|
||||
BENCHMARK_LOG: ${{ matrix.dev == 'QCOM:IR3' && 'ir3_' || '' }}openpilot_${{ matrix.version }}_${{ matrix.model }}
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v6
|
||||
@@ -438,45 +501,10 @@ jobs:
|
||||
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
||||
- name: reset process replay
|
||||
run: test/external/process_replay/reset.py
|
||||
- name: openpilot compile3 0.11.2 supercombo
|
||||
run: BENCHMARK_LOG=openpilot_0_11_2_supercombo PYTHONPATH="." ASSERT_MIN_STEP_TIME=26 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/433f85f956837606ad1f1cbee4aa7e2158ad23c768dea914b20436c97232741b
|
||||
- name: openpilot compile3 0.11.2 supercombo (from pickle)
|
||||
run: BENCHMARK_LOG=openpilot_0_11_2_supercombo_run_pickle RUN_PICKLE=1 PYTHONPATH="." ASSERT_MIN_STEP_TIME=26 DEV=QCOM taskset -c 4-7 python3 examples/openpilot/compile3.py
|
||||
- name: IR3 openpilot compile3 0.11.2 supercombo
|
||||
run: BENCHMARK_LOG=ir3_openpilot_0_11_2_supercombo PYTHONPATH="." ASSERT_MIN_STEP_TIME=41 DEV=QCOM:IR3 FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/433f85f956837606ad1f1cbee4aa7e2158ad23c768dea914b20436c97232741b
|
||||
- name: openpilot compile3 0.11.2 dmonitoring
|
||||
run: BENCHMARK_LOG=openpilot_0_11_2_dmonitoring PYTHONPATH="." ASSERT_MIN_STEP_TIME=11 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/3e7b31dfbc0a5234f1baf196513b77fc6af12204b8a8ffe8ee0417e48352f316
|
||||
- name: Run process replay tests
|
||||
uses: ./.github/actions/process-replay
|
||||
|
||||
testcommaold:
|
||||
name: comma Benchmark (0.11.0)
|
||||
runs-on: [self-hosted, Linux, comma]
|
||||
timeout-minutes: 12
|
||||
defaults:
|
||||
run:
|
||||
shell: bash -e -o pipefail {0}
|
||||
if: github.repository_owner == 'tinygrad'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v6
|
||||
- name: setup staging db
|
||||
if: github.ref == 'refs/heads/update_benchmark_staging'
|
||||
run: |
|
||||
echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
|
||||
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
||||
- name: reset process replay
|
||||
run: test/external/process_replay/reset.py
|
||||
- name: openpilot compile3 0.11.0 driving_vision
|
||||
run: BENCHMARK_LOG=openpilot_0_11_0_vision PYTHONPATH="." ASSERT_MIN_STEP_TIME=17 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_vision.onnx
|
||||
- name: openpilot compile3 0.11.0 driving_vision (from pickle)
|
||||
run: BENCHMARK_LOG=openpilot_0_11_0_vision_run_pickle RUN_PICKLE=1 PYTHONPATH="." ASSERT_MIN_STEP_TIME=17 DEV=QCOM taskset -c 4-7 python3 examples/openpilot/compile3.py
|
||||
- name: IR3 openpilot compile3 0.11.0 driving_vision
|
||||
run: BENCHMARK_LOG=ir3_openpilot_0_11_0_vision PYTHONPATH="." ASSERT_MIN_STEP_TIME=18 DEV=QCOM:IR3 FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_vision.onnx
|
||||
- name: openpilot compile3 0.11.0 driving_policy
|
||||
run: BENCHMARK_LOG=openpilot_0_11_0_policy PYTHONPATH="." ASSERT_MIN_STEP_TIME=3.2 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_policy.onnx
|
||||
- name: openpilot compile3 0.11.0 dmonitoring
|
||||
run: BENCHMARK_LOG=openpilot_0_11_0_dmonitoring PYTHONPATH="." ASSERT_MIN_STEP_TIME=11 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/dmonitoring_model.onnx
|
||||
- name: compile
|
||||
run: FLOAT16=1 IMAGE=1 taskset -c 4-7 python3 examples/openpilot/compile3.py ${{ matrix.url }}
|
||||
- name: run pickle
|
||||
run: BENCHMARK_LOG="${BENCHMARK_LOG}_run_pickle" RUN_PICKLE=1 taskset -c 4-7 python3 examples/openpilot/compile3.py
|
||||
- name: Run process replay tests
|
||||
uses: ./.github/actions/process-replay
|
||||
|
||||
@@ -489,15 +517,6 @@ jobs:
|
||||
shell: bash -e -o pipefail {0}
|
||||
if: github.repository_owner == 'tinygrad'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v6
|
||||
- name: setup staging db
|
||||
if: github.ref == 'refs/heads/update_benchmark_staging'
|
||||
run: |
|
||||
echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
|
||||
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
||||
- name: reset process replay
|
||||
run: test/external/process_replay/reset.py
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v6
|
||||
- name: setup staging db
|
||||
@@ -521,7 +540,7 @@ jobs:
|
||||
testcommausbgpubenchmark:
|
||||
name: UsbGPU Benchmark (comma)
|
||||
runs-on: [self-hosted, Linux, comma4]
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 10
|
||||
defaults:
|
||||
run:
|
||||
shell: bash -e -o pipefail {0}
|
||||
@@ -541,7 +560,7 @@ jobs:
|
||||
- name: openpilot run_pickle big_driving_supercombo
|
||||
run: BENCHMARK_LOG=usbgpu_openpilot_big_driving_supercombo_run_pickle RUN_PICKLE=1 PICKLE_OOB=1 PYTHONPATH="." GMMU=0 DEV=USB+AMD ASSERT_MIN_STEP_TIME=50 python3 examples/openpilot/compile3.py - openpilot.pkl
|
||||
- name: Test copy speeds
|
||||
run: SIZE=64e6 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3 test/external/external_test_usb_asm24.py TestDevCopySpeeds
|
||||
run: SIZE=64000000 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3 test/external/external_test_usb_asm24.py TestDevCopySpeeds
|
||||
|
||||
driverbenchmarks:
|
||||
name: PCI Driver Benchmark (DEV=${{ matrix.dev }})
|
||||
@@ -550,7 +569,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
dev: ['AMD', 'NV']
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 5
|
||||
defaults:
|
||||
run:
|
||||
shell: bash -e -o pipefail {0}
|
||||
@@ -562,9 +581,8 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
- name: Setup
|
||||
run: |
|
||||
./extra/amdpci/setup_python_cap.sh
|
||||
./extra/hcq/hcq_smi.py ${{ matrix.dev == 'AMD' && 'amd' || 'nv' }} rmmod
|
||||
./extra/hcq/hcq_smi.py ${{ matrix.dev == 'AMD' && 'amd' || 'nv' }} kill_pids
|
||||
./extra/hcq/hcq_smi.py ${{ matrix.dev }} rmmod --expect
|
||||
./extra/hcq/hcq_smi.py ${{ matrix.dev }} kill_pids --sudoless
|
||||
mkdir -p extra/datasets
|
||||
ln -s /raid/datasets/imagenet extra/datasets/imagenet
|
||||
- name: setup staging db
|
||||
@@ -599,6 +617,9 @@ jobs:
|
||||
run: |
|
||||
GRAPH_ONE_KERNEL=1 NSZ=8192 python3 test/speed/external_test_copy_speed.py TestCopySpeed.testCopyDefaulttoCPUJit
|
||||
GRAPH_ONE_KERNEL=1 NSZ=8192 python3 test/speed/external_test_copy_speed.py TestCopySpeed.testCopyCPUtoDefaultJit
|
||||
- name: HEVC Decode Benchmark
|
||||
if: ${{ matrix.dev == 'NV' }}
|
||||
run: IGNORE_BEAM_CACHE=1 VALIDATE=1 MAX_FRAMES=100 ASSERT_FPS=1400 JITBEAM=1 PYTHONPATH=. python3 extra/hevc/decode.py
|
||||
- name: Run 10 MLPerf ResNet50 training steps (1 gpu)
|
||||
if: ${{ matrix.dev == 'NV' }}
|
||||
run: BENCHMARK_LOG=resnet_10steps MNISTMOCK=1 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py
|
||||
@@ -621,12 +642,12 @@ jobs:
|
||||
llvmspeed:
|
||||
name: LLVM Speed
|
||||
runs-on: [self-hosted, Linux, tinyboxrandom]
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 10
|
||||
if: github.repository_owner == 'tinygrad'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v6
|
||||
- name: Speed Test
|
||||
run: DEV=CPU:LLVM THREADS=0 python3 test/speed/external_test_speed_v_torch.py
|
||||
run: DEV=CPU:LLVM python3 test/speed/external_test_speed_v_torch.py
|
||||
- name: Speed Test (BEAM=2)
|
||||
run: BEAM=2 DEV=CPU:LLVM THREADS=0 python3 test/speed/external_test_speed_v_torch.py
|
||||
run: IGNORE_BEAM_CACHE=1 BEAM=2 DEV=CPU:LLVM python3 test/speed/external_test_speed_v_torch.py
|
||||
|
||||
Reference in New Issue
Block a user