diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5679dbb424..0a291e51bb 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -594,10 +594,10 @@ jobs: - name: Run process replay tests uses: ./.github/actions/process-replay - testqualcommbenchmark: - name: comma Benchmark + testcommalatest: + name: comma Benchmark (0.11.0) runs-on: [self-hosted, Linux, comma] - timeout-minutes: 30 + timeout-minutes: 10 defaults: run: shell: bash -e -o pipefail {0} @@ -622,6 +622,27 @@ jobs: run: BENCHMARK_LOG=openpilot_0_11_0_policy PYTHONPATH="." ASSERT_MIN_STEP_TIME=3.2 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_policy.onnx - name: openpilot compile3 0.11.0 dmonitoring run: BENCHMARK_LOG=openpilot_0_11_0_dmonitoring PYTHONPATH="." ASSERT_MIN_STEP_TIME=11 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/dmonitoring_model.onnx + - name: Run process replay tests + uses: ./.github/actions/process-replay + + testcommaold: + name: comma Benchmark (0.10.1) + runs-on: [self-hosted, Linux, comma] + timeout-minutes: 10 + defaults: + run: + shell: bash -e -o pipefail {0} + if: github.repository_owner == 'tinygrad' + steps: + - name: Checkout Code + uses: actions/checkout@v6 + - name: setup staging db + if: github.ref == 'refs/heads/update_benchmark_staging' + run: | + echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV + rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal + - name: reset process replay + run: test/external/process_replay/reset.py - name: DEBUG=2 openpilot compile3 0.10.1 driving_vision run: PYTHONPATH="." DEBUG=2 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/720392c9a5b986981fdbed1bb8c47a6c5573a50e/selfdrive/modeld/models/driving_vision.onnx - name: openpilot compile3 0.10.1 driving_vision @@ -630,14 +651,44 @@ jobs: run: BENCHMARK_LOG=openpilot_0_10_1_policy PYTHONPATH="." ASSERT_MIN_STEP_TIME=3.2 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/720392c9a5b986981fdbed1bb8c47a6c5573a50e/selfdrive/modeld/models/driving_policy.onnx - name: openpilot compile3 0.10.1 dmonitoring run: BENCHMARK_LOG=openpilot_0_10_1_dmonitoring PYTHONPATH="." ASSERT_MIN_STEP_TIME=11 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/720392c9a5b986981fdbed1bb8c47a6c5573a50e/selfdrive/modeld/models/dmonitoring_model.onnx + - name: Run process replay tests + uses: ./.github/actions/process-replay + + testqualcommdsp: + name: DSP Benchmark + runs-on: [self-hosted, Linux, comma4] + timeout-minutes: 5 + defaults: + run: + shell: bash -e -o pipefail {0} + if: github.repository_owner == 'tinygrad' + steps: + - name: Checkout Code + uses: actions/checkout@v6 + - name: setup staging db + if: github.ref == 'refs/heads/update_benchmark_staging' + run: | + echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV + rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal + - name: reset process replay + run: test/external/process_replay/reset.py + - name: Checkout Code + uses: actions/checkout@v6 + - name: setup staging db + if: github.ref == 'refs/heads/update_benchmark_staging' + run: | + echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV + rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal + - name: reset process replay + run: test/external/process_replay/reset.py - name: benchmark MobileNetV2 on DSP run: | # generate quantized weights ln -s /data/home/tiny/tinygrad/extra/datasets/imagenet extra/datasets/imagenet ln -s /data/home/tiny/tinygrad/testsig-*.so . - PYTHONPATH=. CC=clang-19 DEV=CPU QUANT=1 CNT=0 python3 examples/test_onnx_imagenet.py https://github.com/xamcat/mobcat-samples/raw/refs/heads/master/onnx_runtime/InferencingSample/InferencingSample/mobilenetv2-7.onnx /tmp/model.quant.onnx + PYTHONPATH=. DEV=CPU QUANT=1 CNT=0 python3 examples/test_onnx_imagenet.py https://github.com/xamcat/mobcat-samples/raw/refs/heads/master/onnx_runtime/InferencingSample/InferencingSample/mobilenetv2-7.onnx /tmp/model.quant.onnx # benchmark on DSP with NOOPT=1, the devectorizer has issues - PYTHONPATH=. CC=clang-19 DEV=DSP NOOPT=1 CNT=2 DEBUG=2 python3 examples/test_onnx_imagenet.py /tmp/model.quant.onnx + PYTHONPATH=. DEV=DSP NOOPT=1 CNT=2 DEBUG=2 python3 examples/test_onnx_imagenet.py /tmp/model.quant.onnx - name: Run process replay tests uses: ./.github/actions/process-replay diff --git a/extra/testsig/generate_testsig.py b/extra/testsig/generate_testsig.py index ad7f78d345..285956062c 100644 --- a/extra/testsig/generate_testsig.py +++ b/extra/testsig/generate_testsig.py @@ -255,7 +255,7 @@ def _sign(serial_num, out_dir): final = _build_elf(ehdr, allph, segs) os.makedirs(out_dir, exist_ok=True) - out = os.path.join(out_dir, "testsig-0x%08X.so" % serial_num) + out = os.path.join(out_dir, "testsig-0x%08x.so" % serial_num) with open(out, 'wb') as f: f.write(final) print("Signing complete! Output saved at %s" % out)