diff --git a/.github/workflows/build-all-tinygrad-models.yaml b/.github/workflows/build-all-tinygrad-models.yaml index 0eedb04703..1ba407b3cf 100644 --- a/.github/workflows/build-all-tinygrad-models.yaml +++ b/.github/workflows/build-all-tinygrad-models.yaml @@ -7,6 +7,19 @@ on: description: 'Minimum selector version required for the models (see helpers.py or readme.md)' required: true type: string + target_hardware: + description: 'Hardware target to compile for (qcom or usbgpu)' + required: true + type: choice + default: 'qcom' + options: + - qcom + - usbgpu + hf_repo: + description: 'Hugging Face dataset repository' + required: false + type: string + default: 'sunnypilot/sunnypilot_models_v1' jobs: setup: @@ -46,13 +59,14 @@ jobs: id: get-json run: | cd docs/docs - latest=$(ls driving_models_v*.json | sed -E 's/.*_v([0-9]+)\.json/\1/' | sort -n | tail -1) + PREFIX="driving_models_${{ inputs.target_hardware == 'usbgpu' && 'usbgpu_' || '' }}v" + latest=$(ls ${PREFIX}*.json | sed -E "s/${PREFIX}([0-9]+)\.json/\1/" | sort -n | tail -1) next=$((latest+1)) - json_file="driving_models_v${next}.json" - cp "driving_models_v${latest}.json" "$json_file" + json_file="${PREFIX}${next}.json" + cp "${PREFIX}${latest}.json" "$json_file" echo "json_file=docs/docs/$json_file" >> $GITHUB_OUTPUT echo "json_version=$((next+0))" >> $GITHUB_OUTPUT - echo "SRC_JSON_FILE=docs/docs/driving_models_v${latest}.json" >> $GITHUB_ENV + echo "SRC_JSON_FILE=docs/docs/${PREFIX}${latest}.json" >> $GITHUB_ENV - name: Extract tinygrad models id: set-matrix @@ -61,45 +75,23 @@ jobs: jq -c '[.bundles[] | select(.runner=="tinygrad") | {ref, display_name: (.display_name | gsub(" \\([^)]*\\)"; "")), is_20hz}]' "$(basename "${SRC_JSON_FILE}")" > matrix.json echo "model_matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT - - name: Set up SSH - uses: webfactory/ssh-agent@v0.9.0 - with: - ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }} - - run: | - mkdir -p ~/.ssh - ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts - - - name: Clone GitLab docs repo and create new recompiled dir + - name: Get next recompiled dir number id: create-recompiled-dir env: - GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts' + HF_REPO: ${{ github.event.inputs.hf_repo }} run: | - git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs - cd gitlab_docs - git checkout main - git sparse-checkout set --no-cone models/ - cd models - latest_dir=$(ls -d recompiled* 2>/dev/null | sed -E 's/recompiled([0-9]+)/\1/' | sort -n | tail -1) - if [[ -z "$latest_dir" ]]; then - next_dir=1 - else - next_dir=$((latest_dir+1)) - fi - recompiled_dir="${next_dir}" - mkdir -p "recompiled${recompiled_dir}" - touch "recompiled${recompiled_dir}/.gitkeep" - cd ../.. + pip install huggingface_hub + recompiled_dir=$(python3 -c " + from huggingface_hub import HfApi + import re, sys + api = HfApi() + files = api.list_repo_files(repo_id=sys.argv[1], repo_type='dataset') + dirs = [re.search(r'models/recompiled([0-9]+)', f) for f in files] + nums = [int(m.group(1)) for m in dirs if m] + print(max(nums) + 1) + " "$HF_REPO") echo "recompiled_dir=$recompiled_dir" >> $GITHUB_OUTPUT - - name: Push empty recompiled dir to GitLab - run: | - cd gitlab_docs - git add models/recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }} - git config --global user.name "GitHub Action" - git config --global user.email "action@github.com" - git commit -m "Add recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }} for build-all" || echo "No changes to commit" - git push origin main - - name: Push new JSON to GitHub docs repo run: | cd docs @@ -123,25 +115,30 @@ jobs: is_20hz: ${{ matrix.model.is_20hz }} recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }} json_version: ${{ needs.setup.outputs.json_version }} + target_hardware: ${{ github.event.inputs.target_hardware }} + hf_repo: ${{ github.event.inputs.hf_repo }} + set_min_version: ${{ github.event.inputs.set_min_version }} + tinygrad_ref: ${{ needs.setup.outputs.tinygrad_ref }} secrets: inherit retry_failed_models: needs: [setup, get_and_build] runs-on: ubuntu-latest - if: ${{ needs.setup.result != 'failure' && !cancelled() }} + if: ${{ !cancelled() && needs.setup.result == 'success' && (needs.get_and_build.result == 'success' || needs.get_and_build.result == 'failure') }} outputs: retry_matrix: ${{ steps.set-retry-matrix.outputs.retry_matrix }} steps: - uses: actions/download-artifact@v4 with: - pattern: model-* + pattern: artifact-name-* path: output + continue-on-error: true - id: set-retry-matrix run: | echo '${{ needs.setup.outputs.model_matrix }}' > matrix.json - built=(); while IFS= read -r line; do built+=("$line"); done < <( - find output -maxdepth 1 -name 'model-*' -printf "%f\n" | sed -E 's/^model-//' | sed -E 's/-[0-9]+$//' | sed -E 's/ \([^)]*\)//' | awk '{gsub(/^ +| +$/, ""); print}' + built=(); while IFS= read -r line; do [ -n "$line" ] && built+=("$line"); done < <( + find output -maxdepth 1 -name 'artifact-name-*' -printf "%f\n" 2>/dev/null | sed -E 's/^artifact-name-//' | awk '{gsub(/^ +| +$/, ""); print}' ) jq -c --argjson built "$(printf '%s\n' "${built[@]}" | jq -R . | jq -s .)" \ 'map(select(.display_name as $n | ($built | index($n | gsub("^ +| +$"; "")) | not)))' matrix.json > retry_matrix.json @@ -149,7 +146,7 @@ jobs: retry_get_and_build: needs: [setup, get_and_build, retry_failed_models] - if: ${{ needs.get_and_build.result == 'failure' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '') }} + if: ${{ !cancelled() && needs.retry_failed_models.result == 'success' && needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '' }} strategy: matrix: model: ${{ fromJson(needs.retry_failed_models.outputs.retry_matrix) }} @@ -161,146 +158,9 @@ jobs: is_20hz: ${{ matrix.model.is_20hz }} recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }} json_version: ${{ needs.setup.outputs.json_version }} + target_hardware: ${{ github.event.inputs.target_hardware }} artifact_suffix: -retry + hf_repo: ${{ github.event.inputs.hf_repo }} + set_min_version: ${{ github.event.inputs.set_min_version }} + tinygrad_ref: ${{ needs.setup.outputs.tinygrad_ref }} secrets: inherit - - publish_models: - name: Publish models sequentially - needs: [setup, get_and_build, retry_failed_models, retry_get_and_build] - if: ${{ !cancelled() && (needs.get_and_build.result != 'failure' || needs.retry_get_and_build.result == 'success' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '')) }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - max-parallel: 1 - matrix: - model: ${{ fromJson(needs.setup.outputs.model_matrix) }} - env: - RECOMPILED_DIR: recompiled${{ needs.setup.outputs.recompiled_dir }} - JSON_FILE: ${{ needs.setup.outputs.json_file }} - ARTIFACT_NAME_INPUT: ${{ matrix.model.display_name }} - steps: - - name: Set up SSH - uses: webfactory/ssh-agent@v0.9.0 - with: - ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }} - - - name: Add GitLab.com SSH key to known_hosts - run: | - mkdir -p ~/.ssh - ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts - - - name: Clone GitLab docs repo - env: - GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts' - run: | - echo "Cloning GitLab" - git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs - cd gitlab_docs - echo "checkout models/${RECOMPILED_DIR}" - git sparse-checkout set --no-cone models/${RECOMPILED_DIR} - git checkout main - cd .. - - - name: Checkout docs repo - uses: actions/checkout@v4 - with: - repository: sunnypilot/sunnypilot-models - ref: gh-pages - path: docs - ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }} - - - name: Validate recompiled dir and JSON version - run: | - if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then - echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo" - exit 1 - fi - if [ ! -f "$JSON_FILE" ]; then - echo "JSON file $JSON_FILE does not exist!" - exit 1 - fi - - - name: Download artifact name file - uses: actions/download-artifact@v4 - with: - name: artifact-name-${{ env.ARTIFACT_NAME_INPUT }} - path: artifact_name - - - name: Read artifact name - id: read-artifact-name - run: | - ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt) - echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT - - - name: Download model artifact - uses: actions/download-artifact@v4 - with: - name: ${{ steps.read-artifact-name.outputs.artifact_name }} - path: output - - - name: Remove onnx files bc not needed for recompiled dir since they already exist from single build - run: | - find output -type f -name '*.onnx' -delete - find output -type f -name 'big_*.pkl' -delete - find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete - - - name: Copy model artifacts to gitlab - env: - ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }} - run: | - ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}" - mkdir -p "$ARTIFACT_DIR" - for path in output/*; do - if [ "$(basename "$path")" = "artifact_name.txt" ]; then - continue - fi - name="$(basename "$path")" - if [ -d "$path" ]; then - mkdir -p "$ARTIFACT_DIR/$name" - cp -r "$path"/* "$ARTIFACT_DIR/$name/" - echo "Copied dir $name -> $ARTIFACT_DIR/$name" - else - cp "$path" "$ARTIFACT_DIR/" - echo "Copied file $name -> $ARTIFACT_DIR/" - fi - done - - - name: Push recompiled dir to GitLab - env: - GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }} - run: | - cd gitlab_docs - git checkout main - git pull origin main - for d in models/"$RECOMPILED_DIR"/*/; do - git sparse-checkout add "$d" - done - git add models/"$RECOMPILED_DIR" - git config --global user.name "GitHub Action" - git config --global user.email "action@github.com" - git commit -m "Update $RECOMPILED_DIR with model from build-all-tinygrad-models" || echo "No changes to commit" - git push origin main - - run: | - cd docs - git pull origin gh-pages - - - name: update json - run: | - ARGS="" - [ -n "${{ inputs.set_min_version }}" ] && ARGS="$ARGS --set-min-version \"${{ inputs.set_min_version }}\"" - ARGS="$ARGS --sort-by-date" - ARGS="$ARGS --tinygrad-ref \"${{ needs.setup.outputs.tinygrad_ref }}\"" - eval python3 docs/json_parser.py \ - --json-path "$JSON_FILE" \ - --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \ - $ARGS - - - name: Push updated json to GitHub - run: | - cd docs - git config --global user.name "GitHub Action" - git config --global user.email "action@github.com" - git checkout gh-pages - git add docs/"$(basename $JSON_FILE)" - git commit -m "Update $(basename $JSON_FILE) after recompiling model" || echo "No changes to commit" - git push origin gh-pages diff --git a/.github/workflows/build-single-tinygrad-model.yaml b/.github/workflows/build-single-tinygrad-model.yaml index cf2d870802..1ad06a54e4 100644 --- a/.github/workflows/build-single-tinygrad-model.yaml +++ b/.github/workflows/build-single-tinygrad-model.yaml @@ -29,11 +29,24 @@ on: required: false type: boolean default: true - bypass_push: - description: 'Bypass pushing to GitLab for build-all' + target_hardware: + description: 'Hardware target to compile for (qcom or usbgpu)' required: false - default: true - type: boolean + type: string + default: 'qcom' + hf_repo: + description: 'Hugging Face dataset repository (e.g. sunnypilot/sunnypilot_models_v1)' + required: false + type: string + default: 'sunnypilot/sunnypilot_models_v1' + set_min_version: + description: 'Minimum selector version' + required: false + type: string + tinygrad_ref: + description: 'Tinygrad reference' + required: false + type: string workflow_dispatch: inputs: upstream_branch: @@ -65,8 +78,8 @@ on: - None - Master Models - Release Models - - 2025 World Models - 2026 World Models + - 2026 Deep RL Models - Custom Merge Models - Other custom_model_folder: @@ -81,9 +94,22 @@ on: description: 'Minimum selector version' required: false type: string + target_hardware: + description: 'Hardware target to compile for' + required: false + type: choice + default: 'qcom' + options: + - qcom + - usbgpu + hf_repo: + description: 'Hugging Face dataset repository' + required: false + type: string + default: 'sunnypilot/sunnypilot_models_v1' env: RECOMPILED_DIR: recompiled${{ inputs.recompiled_dir }} - JSON_FILE: docs/docs/driving_models_v${{ inputs.json_version }}.json + JSON_FILE: docs/docs/driving_models_${{ inputs.target_hardware == 'usbgpu' && 'usbgpu_v' || 'v' }}${{ inputs.json_version }}.json jobs: build_model: @@ -93,38 +119,20 @@ jobs: custom_name: ${{ inputs.custom_name || inputs.upstream_branch }} is_20hz: ${{ inputs.is_20hz }} artifact_suffix: ${{ inputs.artifact_suffix }} + target_hardware: ${{ inputs.target_hardware }} secrets: inherit publish_model: - if: ${{ !inputs.bypass_push && !cancelled() }} + if: ${{ !cancelled() && needs.build_model.result == 'success' }} concurrency: - group: gitlab-push-${{ inputs.recompiled_dir }} + group: hf-push-${{ inputs.recompiled_dir }} cancel-in-progress: false needs: build_model runs-on: ubuntu-latest + permissions: + id-token: write + contents: write steps: - - name: Set up SSH - uses: webfactory/ssh-agent@v0.9.0 - with: - ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }} - - - name: Add GitLab.com SSH key to known_hosts - run: | - mkdir -p ~/.ssh - ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts - - - name: Clone GitLab docs repo - env: - GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts' - run: | - echo "Cloning GitLab" - git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs - cd gitlab_docs - echo "checkout models/${RECOMPILED_DIR}" - git sparse-checkout set --no-cone models/${RECOMPILED_DIR} - git checkout main - cd .. - - name: Checkout docs repo uses: actions/checkout@v4 with: @@ -133,16 +141,28 @@ jobs: path: docs ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }} - - name: Validate recompiled dir and JSON version + - name: Install huggingface_hub + run: pip install --upgrade "huggingface_hub>=0.22.0" + + - name: Validate hf_repo and JSON version + env: + HF_OIDC_RESOURCE: datasets/${{ inputs.hf_repo }} run: | - if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then - echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo" - exit 1 - fi if [ ! -f "$JSON_FILE" ]; then echo "JSON file $JSON_FILE does not exist!" exit 1 fi + python3 -c " + import sys + from huggingface_hub import HfApi + try: + api = HfApi() + api.repo_info(repo_id=sys.argv[1], repo_type='dataset') + print(f'Success: Repo {sys.argv[1]} exists.') + except Exception as e: + print('HF validation failed:', e) + sys.exit(1) + " "${{ inputs.hf_repo }}" - name: Download artifact name file uses: actions/download-artifact@v4 @@ -162,49 +182,26 @@ jobs: name: ${{ steps.read-artifact-name.outputs.artifact_name }} path: output - - name: Remove unwanted files - run: | - find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete - find output -type f -name 'dmonitoring_model.onnx' -delete - - - name: Copy model artifact(s) to GitLab recompiled dir + - name: Create models folder env: ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }} run: | - ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}" - mkdir -p "$ARTIFACT_DIR" - for path in output/*; do - if [ "$(basename "$path")" = "artifact_name.txt" ]; then - continue - fi - name="$(basename "$path")" - if [ -d "$path" ]; then - mkdir -p "$ARTIFACT_DIR/$name" - cp -r "$path"/* "$ARTIFACT_DIR/$name/" - echo "Copied dir $name -> $ARTIFACT_DIR/$name" - else - cp "$path" "$ARTIFACT_DIR/" - echo "Copied file $name -> $ARTIFACT_DIR/" - fi - done + mkdir -p "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}" + cp -r output/* "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/" + rm -f "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/artifact_name.txt" - - name: Push recompiled dir to GitLab + - name: Upload to Hugging Face env: - GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }} + HF_OIDC_RESOURCE: datasets/${{ inputs.hf_repo }} + ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }} run: | - cd gitlab_docs - git checkout main - git pull origin main - for d in models/"$RECOMPILED_DIR"/*/; do - git sparse-checkout add "$d" - done - git add models/"$RECOMPILED_DIR" - git config --global user.name "GitHub Action" - git config --global user.email "action@github.com" - git commit -m "Create/Update $RECOMPILED_DIR with new/updated model from build-single-tinygrad-model" || echo "No changes to commit" - git push origin main + hf upload ${{ inputs.hf_repo }} \ + output/ \ + "models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/" \ + --repo-type=dataset - - run: | + - name: Pull gh-pages + run: | cd docs git pull origin gh-pages @@ -220,9 +217,11 @@ jobs: fi [ -n "${{ inputs.generation }}" ] && ARGS="$ARGS --generation \"${{ inputs.generation }}\"" [ -n "${{ inputs.version }}" ] && ARGS="$ARGS --version \"${{ inputs.version }}\"" + [ -n "${{ inputs.set_min_version }}" ] && ARGS="$ARGS --set-min-version \"${{ inputs.set_min_version }}\"" + [ -n "${{ inputs.tinygrad_ref }}" ] && ARGS="$ARGS --tinygrad-ref \"${{ inputs.tinygrad_ref }}\"" eval python3 docs/json_parser.py \ --json-path "$JSON_FILE" \ - --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \ + --recompiled-dir "local_models/$RECOMPILED_DIR" \ --sort-by-date \ $ARGS diff --git a/.github/workflows/sunnypilot-build-model.yaml b/.github/workflows/sunnypilot-build-model.yaml index 2c435e58a0..17981d68e5 100644 --- a/.github/workflows/sunnypilot-build-model.yaml +++ b/.github/workflows/sunnypilot-build-model.yaml @@ -80,6 +80,7 @@ jobs: with: repository: commaai/openpilot ref: ${{ inputs.upstream_branch }} + fetch-depth: 1 submodules: recursive path: openpilot @@ -89,18 +90,25 @@ jobs: with: repository: sunnypilot/sunnypilot ref: ${{ inputs.upstream_branch }} + fetch-depth: 1 submodules: recursive path: openpilot - name: Get commit date id: commit-date run: | - cd ${{ github.workspace }}/openpilot + cd ${{ github.workspace }}/openpilot/openpilot commit_date=$(git log -1 --format=%cd --date=format:'%B %d, %Y') echo "model_date=${commit_date}" >> $GITHUB_OUTPUT cat $GITHUB_OUTPUT - run: | - cd ${{ github.workspace }}/openpilot - git lfs pull + cd ${{ github.workspace }}/openpilot/openpilot + if [ "${{ inputs.target_hardware }}" != "usbgpu" ]; then + git lfs pull -X "selfdrive/modeld/models/big_*.onnx" -X "selfdrive/modeld/models/dmonitoring_*.onnx" + rm -f selfdrive/modeld/models/big_*.onnx selfdrive/modeld/models/dmonitoring_*.onnx + else + git lfs pull -I "selfdrive/modeld/models/big_*.onnx" + find selfdrive/modeld/models -name "*.onnx" ! -name "big_*.onnx" -delete + fi - name: 'Upload Artifact' uses: actions/upload-artifact@v4 with: @@ -116,24 +124,10 @@ jobs: steps: - uses: actions/checkout@v4 with: + fetch-depth: 1 submodules: recursive - run: git lfs pull - - name: Cache SCons - uses: actions/cache@v4 - with: - path: ${{env.SCONS_CACHE_DIR}} - key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model-${{ github.sha }} - # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.) - # for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden. - restore-keys: | - scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model - scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }} - scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }}-model - scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }}-model - scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }} - scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }} - scons-${{ runner.os }}-${{ runner.arch }} - name: Set environment variables id: set-env @@ -144,7 +138,7 @@ jobs: export UV_PYTHON_PREFERENCE=managed export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT - uv sync + uv sync --frozen printenv >> $GITHUB_ENV if [[ "${{ runner.debug }}" == "1" ]]; then cat $GITHUB_OUTPUT @@ -173,8 +167,6 @@ jobs: with: name: models-${{ env.REF }}${{ inputs.artifact_suffix }} path: ${{ env.MODELS_DIR }} - - run: | - rm -f ${{ env.MODELS_DIR }}/{dmonitoring_model,big_driving_policy,big_driving_vision,big_driving_supercombo}.onnx - name: Build Model run: | @@ -191,7 +183,7 @@ jobs: if [ "${{ inputs.target_hardware }}" == "usbgpu" ]; then echo "USBGPU build" export USBGPU=1 - TG_FLAGS="DEV=AMD USBGPU=1 IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1" + TG_FLAGS="DEBUG=2 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2" OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl" else echo "QCOM build" @@ -254,10 +246,8 @@ jobs: # Copy the model files rsync -avm \ --include='*.dlc' \ - --include='*.pkl' \ --include='*.chunk*' \ --include='*.chunkmanifest' \ - --include='*.onnx' \ --exclude='*' \ --delete-excluded \ --chown=comma:comma \ diff --git a/openpilot/common/params_keys.h b/openpilot/common/params_keys.h index bb7989381b..964227e785 100644 --- a/openpilot/common/params_keys.h +++ b/openpilot/common/params_keys.h @@ -195,11 +195,14 @@ inline static std::unordered_map keys = { // Model Manager params {"ModelManager_ActiveBundle", {PERSISTENT, JSON}}, + {"ModelManager_ActiveJson", {CLEAR_ON_MANAGER_START, STRING}}, {"ModelManager_ClearCache", {CLEAR_ON_MANAGER_START, BOOL}}, {"ModelManager_DownloadIndex", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, INT}}, {"ModelManager_Favs", {PERSISTENT | BACKUP, STRING}}, {"ModelManager_LastSyncTime", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}}, + {"ModelManager_LastSyncTime_USBGPU", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}}, {"ModelManager_ModelsCache", {PERSISTENT | BACKUP, JSON}}, + {"ModelManager_ModelsCache_USBGPU", {PERSISTENT | BACKUP, JSON}}, // Neural Network Lateral Control {"NeuralNetworkLateralControl", {PERSISTENT | BACKUP, BOOL, "0"}}, diff --git a/openpilot/sunnypilot/SConscript b/openpilot/sunnypilot/SConscript index 09ad39ab43..587deea5ff 100644 --- a/openpilot/sunnypilot/SConscript +++ b/openpilot/sunnypilot/SConscript @@ -1,3 +1,2 @@ SConscript(['common/transformations/SConscript']) -SConscript(['modeld_v2/SConscript']) SConscript(['selfdrive/locationd/SConscript']) diff --git a/openpilot/sunnypilot/modeld_v2/SConscript b/openpilot/sunnypilot/modeld_v2/SConscript deleted file mode 100644 index daaa199ea9..0000000000 --- a/openpilot/sunnypilot/modeld_v2/SConscript +++ /dev/null @@ -1,84 +0,0 @@ -import os -import glob - -from openpilot.common.transformations.camera import _ar_ox_fisheye, _os_fisheye -from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE -from openpilot.common.hardware import HARDWARE, PC - -Import('env', 'arch', 'release') -lenv = env.Clone() -tinygrad_files = ["#"+x for x in glob.glob(env.Dir("#tinygrad_repo").relpath + "/**", recursive=True, root_dir=env.Dir("#").abspath) if 'pycache' not in x] - - -def get_camera_configs(): - DEVICE_RESOLUTIONS = { - "tici": (_ar_ox_fisheye.width, _ar_ox_fisheye.height), - "tizi": (_ar_ox_fisheye.width, _ar_ox_fisheye.height), - "mici": (_os_fisheye.width, _os_fisheye.height), - } - if release or PC or 'CI' in os.environ: - return set(DEVICE_RESOLUTIONS.values()) - return [DEVICE_RESOLUTIONS[HARDWARE.get_device_type()]] - -CAMERA_CONFIGS = get_camera_configs() - -tg_flags = { - 'larch64': 'DEV=QCOM FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0', - 'Darwin': f'DEV=CPU HOME={os.path.expanduser("~")}', -}.get(arch, 'DEV=CPU:LLVM') - -image_flag = { - 'larch64': 'IMAGE=2', -}.get(arch, 'IMAGE=0') - -model_w, model_h = MEDMODEL_INPUT_SIZE -from openpilot.selfdrive.modeld.constants import ModelConstants -frame_skip = ModelConstants.MODEL_RUN_FREQ // ModelConstants.MODEL_CONTEXT_FREQ -camera_res_args = ' '.join(f'{cw}x{ch}' for cw, ch in CAMERA_CONFIGS) - -pythonpath_string = 'PYTHONPATH="${PYTHONPATH}:' + env.Dir("#tinygrad_repo").abspath + ':' + env.Dir("#").abspath + '"' -compile_modeld_script = File("compile_modeld.py").abspath -upstream_compile_script = File(Dir("#openpilot/selfdrive/modeld").File("compile_modeld.py").abspath) -script_deps = [File("compile_modeld.py"), upstream_compile_script] - -def compile_combined(model_type, onnx_args, output_name): - output_pkl = File(f"models/{output_name}").abspath - cmd = (f'{pythonpath_string} {tg_flags} {image_flag} python3 {compile_modeld_script} ' - f'--model-type {model_type} ' - f'--model-size {model_w}x{model_h} ' - f'--camera-resolutions {camera_res_args} ' - f'{onnx_args} ' - f'--frame-skip {frame_skip} ' - f'--output {output_pkl}') - onnx_files = [f for f in onnx_args.split() if f.endswith('.onnx')] - return lenv.Command(output_pkl, tinygrad_files + script_deps + [File(f) for f in onnx_files if os.path.isfile(f)], cmd) - -# Vision + Policy (stock default model) -vision_onnx = File("models/driving_vision.onnx").abspath -policy_onnx = File("models/driving_policy.onnx").abspath -if os.path.isfile(vision_onnx) and os.path.isfile(policy_onnx): - compile_combined('vision_policy', - f'--vision-onnx {vision_onnx} --policy-onnx {policy_onnx}', - 'driving_combined_tinygrad.pkl') - -# Vision + Off-Policy -off_policy_onnx = File("models/driving_off_policy.onnx").abspath -if os.path.isfile(vision_onnx) and os.path.isfile(off_policy_onnx): - policy_arg = f'--policy-onnx {policy_onnx}' if os.path.isfile(policy_onnx) else '' - compile_combined('vision_multi_policy', - f'--vision-onnx {vision_onnx} {policy_arg} --off-policy-onnx {off_policy_onnx}', - 'driving_combined_multi_tinygrad.pkl') - -# Vision + On-Policy + Off-Policy -on_policy_onnx = File("models/driving_on_policy.onnx").abspath -if os.path.isfile(vision_onnx) and os.path.isfile(on_policy_onnx) and os.path.isfile(off_policy_onnx): - compile_combined('vision_multi_policy', - f'--vision-onnx {vision_onnx} --off-policy-onnx {off_policy_onnx} --on-policy-onnx {on_policy_onnx}', - 'driving_combined_tri_tinygrad.pkl') - -# Supercombo -supercombo_onnx = File("models/supercombo.onnx").abspath -if os.path.isfile(supercombo_onnx): - compile_combined('supercombo', - f'--supercombo-onnx {supercombo_onnx}', - 'driving_combined_supercombo_tinygrad.pkl') diff --git a/openpilot/sunnypilot/modeld_v2/compile_modeld.py b/openpilot/sunnypilot/modeld_v2/compile_modeld.py index def54a4599..4397209005 100755 --- a/openpilot/sunnypilot/modeld_v2/compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/compile_modeld.py @@ -8,10 +8,10 @@ See the LICENSE.md file in the root directory for more details. import argparse import os -import pickle +import tempfile import time -from collections import defaultdict from functools import partial +from openpilot.selfdrive.modeld.helpers import dump_oob, load_oob import numpy as np os.environ['GMMU'] = '0' @@ -38,6 +38,9 @@ from tinygrad.engine.jit import TinyJit from tinygrad.tensor import Tensor MODEL_TYPES = ('vision_policy', 'supercombo', 'vision_multi_policy') +WARP_INPUTS = ['tfm', 'big_tfm'] +POLICY_INPUTS = ['img_q', 'big_img_q', 'feat_q', 'desire_q', 'packed_npy_inputs'] +WARP_DEV = os.getenv('WARP_DEV') def _detect_desire_key(shapes: dict) -> str | None: @@ -76,7 +79,7 @@ def get_policy_npy_shapes(input_shapes: dict, is_supercombo: bool = False) -> tu def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT, - is_supercombo: bool = False, use_packed: bool = True) -> tuple[dict, dict]: + is_supercombo: bool = False) -> tuple[dict, dict]: road_key, _ = _detect_vision_keys(input_shapes) if not road_key: raise ValueError("Vision road key missing from input shapes.") @@ -92,74 +95,75 @@ def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = D desire_shape = input_shapes[desire_key] features_buffer = input_shapes.get('features_buffer') - if use_packed: # remove packed detection block after all models are recompiled - npy_arrays = { - 'tfm': np.zeros((3, 3), dtype=np.float32), - 'big_tfm': np.zeros((3, 3), dtype=np.float32) - } + npy_arrays = { + 'tfm': np.zeros((3, 3), dtype=np.float32), + 'big_tfm': np.zeros((3, 3), dtype=np.float32) + } - shapes, sizes = get_policy_npy_shapes(input_shapes, is_supercombo=is_supercombo) - packed_npy_inputs = np.zeros(sum(sizes), dtype=np.float32) + shapes, sizes = get_policy_npy_shapes(input_shapes, is_supercombo=is_supercombo) + packed_npy_inputs = np.zeros(sum(sizes), dtype=np.float32) - split_indices = np.cumsum(sizes[:-1]) if len(sizes) > 1 else [] - split_views = np.split(packed_npy_inputs, split_indices) if len(sizes) > 0 else [] - for (k, s), v in zip(shapes.items(), split_views, strict=True): - npy_arrays[k] = v.reshape(s) + split_indices = np.cumsum(sizes[:-1]) if len(sizes) > 1 else [] + split_views = np.split(packed_npy_inputs, split_indices) if len(sizes) > 0 else [] + for (k, s), v in zip(shapes.items(), split_views, strict=True): + npy_arrays[k] = v.reshape(s) - queues = { - 'img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), - 'big_img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), - 'desire_q': Tensor(np.zeros((frame_skip * desire_shape[1], desire_shape[0], desire_shape[2]), - dtype=np.float32), device=device).contiguous().realize(), - 'packed_npy_inputs': Tensor(packed_npy_inputs, device='NPY').realize(), - } + queues = { + 'img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), + 'big_img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), + 'desire_q': Tensor(np.zeros((frame_skip * desire_shape[1], desire_shape[0], desire_shape[2]), + dtype=np.float32), device=device).contiguous().realize(), + 'packed_npy_inputs': Tensor(packed_npy_inputs, device='NPY').realize(), + } - if features_buffer: - queues['feat_q'] = Tensor(np.zeros((frame_skip * (features_buffer[1] - 1) + 1, features_buffer[0], features_buffer[2]), - dtype=np.float32), device=device).contiguous().realize() + if features_buffer: + queues['feat_q'] = Tensor(np.zeros((frame_skip * (features_buffer[1] - 1) + 1, features_buffer[0], features_buffer[2]), + dtype=np.float32), device=device).contiguous().realize() - queues.update({key: Tensor(value, device='NPY').realize() for key, value in npy_arrays.items() if key in ('tfm', 'big_tfm')}) - else: - # TODO-SP: Remove legacy queuing fallback else block after all models are recompiled - npy_arrays = { - 'desire': np.zeros(desire_shape[2], dtype=np.float32), - 'tfm': np.zeros((3, 3), dtype=np.float32), - 'big_tfm': np.zeros((3, 3), dtype=np.float32) - } - - for key, shape in input_shapes.items(): - if key not in npy_arrays and 'img' not in key and key not in ('features_buffer', desire_key): - npy_arrays[key] = np.zeros(shape, dtype=np.float32) - - queues = { - 'img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), - 'big_img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), - 'desire_q': Tensor(np.zeros((frame_skip * desire_shape[1], desire_shape[0], desire_shape[2]), - dtype=np.float32), device=device).contiguous().realize() - } - - if features_buffer: - queues['feat_q'] = Tensor(np.zeros((frame_skip * (features_buffer[1] - 1) + 1, features_buffer[0], features_buffer[2]), - dtype=np.float32), device=device).contiguous().realize() - - queues.update({key: Tensor(value, device='NPY').realize() for key, value in npy_arrays.items()}) + queues.update({key: Tensor(value, device='NPY').realize() for key, value in npy_arrays.items() if key in ('tfm', 'big_tfm')}) return queues, npy_arrays def make_split_input_queues(vision_input_shapes: dict, policy_input_shapes: dict, - frame_skip: int, device: str = Device.DEFAULT, use_packed: bool = True) -> tuple[dict, dict]: - return generate_queues_and_npy({**vision_input_shapes, **policy_input_shapes}, frame_skip, device, is_supercombo=False, use_packed=use_packed) + frame_skip: int, device: str = Device.DEFAULT) -> tuple[dict, dict]: + return generate_queues_and_npy({**vision_input_shapes, **policy_input_shapes}, frame_skip, device, is_supercombo=False) def make_supercombo_input_queues(input_shapes: dict, frame_skip: int, - device: str = Device.DEFAULT, use_packed: bool = True) -> tuple[dict, dict]: - return generate_queues_and_npy(input_shapes, frame_skip, device, is_supercombo=True, use_packed=use_packed) + device: str = Device.DEFAULT) -> tuple[dict, dict]: + return generate_queues_and_npy(input_shapes, frame_skip, device, is_supercombo=True) -def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, model_size: tuple[int, int], - features_slice: slice, frame_skip: int, input_shapes: dict, prepare_only: bool): - frame_prepare = make_frame_prepare(nv12, *model_size) +def make_random_images(keys, shape, device): + return {k: Tensor.randint(shape, low=0, high=256, dtype=dtypes.uint8, device=device).realize() for k in keys} + + +def make_warp_queues(device=Device.DEFAULT): + npy = { + 'tfm': np.zeros((3, 3), dtype=np.float32), + 'big_tfm': np.zeros((3, 3), dtype=np.float32), + } + queues = {k: Tensor(v, device='NPY').realize() for k, v in npy.items()} + return queues, npy + + +def make_warp(nv12: NV12Frame, model_w: int, model_h: int): + frame_prepare = make_frame_prepare(nv12, model_w, model_h) + WARP_DEV = os.getenv('WARP_DEV', Device.DEFAULT) + + def warp(tfm, big_tfm, frame, big_frame): + tfm = tfm.to(WARP_DEV) + big_tfm = big_tfm.to(WARP_DEV) + Tensor.realize(tfm, big_tfm) + + warped_frame = frame_prepare(frame, tfm).unsqueeze(0) + warped_big_frame = frame_prepare(big_frame, big_tfm).unsqueeze(0) + return Tensor.cat(warped_frame, warped_big_frame) + return warp + + +def make_run_policy(vision_runner, policy_runners: list, features_slice: slice, frame_skip: int, input_shapes: dict): sample_skip_fn = partial(sample_skip, frame_skip=frame_skip) sample_desire_fn = partial(sample_desire, frame_skip=frame_skip) @@ -172,20 +176,14 @@ def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, mode is_supercombo = vision_runner is None npy_shapes, npy_sizes = get_policy_npy_shapes(input_shapes, is_supercombo=is_supercombo) - def runner(img_q, big_img_q, feat_q, packed_npy_inputs, frame, big_frame, tfm, big_tfm, **kwargs): + def run_policy(warped, img_q, big_img_q, feat_q, packed_npy_inputs, **kwargs): desire_q = kwargs['desire_q'] - packed_npy_inputs_dev = packed_npy_inputs.to(Device.DEFAULT) - tfm_dev = tfm.to(Device.DEFAULT) - big_tfm_dev = big_tfm.to(Device.DEFAULT) + warped_dev = warped.to(Device.DEFAULT) + Tensor.realize(packed_npy_inputs_dev, warped_dev) - Tensor.realize(packed_npy_inputs_dev, tfm_dev, big_tfm_dev) - - img = shift_and_sample(img_q, frame_prepare(frame, tfm_dev).unsqueeze(0), sample_skip_fn).realize() - big_img = shift_and_sample(big_img_q, frame_prepare(big_frame, big_tfm_dev).unsqueeze(0), sample_skip_fn).realize() - - if prepare_only: - return img, big_img + img = shift_and_sample(img_q, warped_dev[0:1], sample_skip_fn).realize() + big_img = shift_and_sample(big_img_q, warped_dev[1:2], sample_skip_fn).realize() unpacked_tensors = [tensor.reshape(shape) for tensor, shape in zip(packed_npy_inputs_dev.split(npy_sizes), npy_shapes.values(), strict=True)] unpacked_dict = dict(zip(npy_shapes.keys(), unpacked_tensors, strict=True)) @@ -220,42 +218,52 @@ def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, mode shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() return policy_out - return runner + return run_policy -def compile_and_warmup(nv12: NV12Frame, model_size: tuple[int, int], prepare_only: bool, frame_skip: int, vision_runner, policy_runners: list, metadata: dict): - print(f"Compiling combined JIT for {nv12.width}x{nv12.height} (prepare_only={prepare_only})...") +def compile_jit(jit, make_random_inputs, input_keys, make_queues): + SEED = 42 + def random_inputs_run(fn, seed, test_val=None, test_buffers=None, expect_match=True): + input_queues, npy = make_queues(Device.DEFAULT) + rng = np.random.default_rng(seed) + Tensor.manual_seed(seed) - all_shapes = {key: value for meta in metadata.values() for key, value in meta['input_shapes'].items()} + testing = test_val is not None or test_buffers is not None + n_runs = 1 if testing else 3 - feat_meta = metadata.get('vision') or metadata.get('model') or metadata.get('policy') - if not feat_meta: - raise ValueError("Could not find vision, model, or policy metadata.") + for i in range(n_runs): + for v in npy.values(): + v[:] = rng.standard_normal(v.shape).astype(v.dtype) + Device.default.synchronize() + random_inputs = make_random_inputs() + st = time.perf_counter() + outs = fn(**{k: input_queues[k] for k in input_keys if k in input_queues}, **random_inputs) + mt = time.perf_counter() + Device.default.synchronize() + et = time.perf_counter() + print(f" [{i+1}/{n_runs}] enqueue {(mt-st)*1e3:6.2f} ms -- total {(et-st)*1e3:6.2f} ms") - features_slice = feat_meta['output_slices']['hidden_state'] - WARP_DEV = 'CPU' if "USBGPU" in os.environ else Device.DEFAULT + if i == 0: + val = [np.copy(v.numpy()) for v in (outs if isinstance(outs, tuple) else [outs])] if outs is not None else [] + buffers = [np.copy(v.numpy().copy()) for v in input_queues.values()] - is_supercombo = vision_runner is None - run_func = create_jit_runner(vision_runner, policy_runners, nv12, model_size, features_slice, frame_skip, all_shapes, prepare_only) - run_jit = TinyJit(run_func, prune=True) - queues, npy_arrays = generate_queues_and_npy(all_shapes, frame_skip, Device.DEFAULT, is_supercombo=is_supercombo) + if test_val is not None: + match = all(np.array_equal(a, b) for a, b in zip(val, test_val, strict=True)) + assert match == expect_match, f"outputs {'differ from' if expect_match else 'match'} baseline (seed={seed})" + if test_buffers is not None: + match = all(np.array_equal(a, b) for a, b in zip(buffers, test_buffers, strict=True)) + assert match == expect_match, f"buffers {'differ from' if expect_match else 'match'} baseline (seed={seed})" + return val, buffers - for i in range(3): - rng = np.random.default_rng(42 + i) - frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8, device=WARP_DEV).realize() - big_frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8, device=WARP_DEV).realize() - for arr in npy_arrays.values(): - arr[:] = rng.standard_normal(arr.shape).astype(arr.dtype) - - Device.default.synchronize() - start_time = time.perf_counter() - run_jit(**queues, frame=frame, big_frame=big_frame) - mid_time = time.perf_counter() - Device.default.synchronize() - print(f" [{i + 1}/3] enqueue {(mid_time - start_time) * 1e3:6.2f} ms -- total {(time.perf_counter() - start_time) * 1e3:6.2f} ms") - - # TODO-SP: switch to dump_oob/load_oob on next full recompile of all models - return pickle.loads(pickle.dumps(run_jit)) if not prepare_only else run_jit + print('capture + replay') + test_val, test_buffers = random_inputs_run(jit, SEED) + print('pickle round trip') + with tempfile.TemporaryFile(dir=".") as f: + dump_oob(jit, f) + f.seek(0) + deserialized_jit = load_oob(f) + random_inputs_run(deserialized_jit, SEED, test_val=test_val, test_buffers=test_buffers) + return deserialized_jit def _parse_size(size_str: str) -> tuple[int, int]: @@ -277,19 +285,6 @@ def read_file_chunked_to_shm(path): return shm_path -def _compile_for_resolutions(camera_resolutions: list, model_size: tuple[int, int], frame_skip: int, - vision_runner, policy_runners: list, metadata: dict) -> dict: - from openpilot.system.camerad.cameras.nv12_info import get_nv12_info - return { - (cam_w, cam_h): { - name: compile_and_warmup(NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h)), model_size, prepare_only, - frame_skip, vision_runner, policy_runners, metadata) - for name, prepare_only in [('warp_enqueue', True), ('run_policy', False)] - } - for cam_w, cam_h in camera_resolutions - } - - def _load_policy_runners(args: argparse.Namespace) -> tuple[list, list]: runners, keys = [], [] for name, onnx_arg in [('policy', args.policy_onnx), ('off_policy', args.off_policy_onnx), ('on_policy', args.on_policy_onnx)]: @@ -300,7 +295,18 @@ def _load_policy_runners(args: argparse.Namespace) -> tuple[list, list]: if __name__ == "__main__": + if 'USB' in os.getenv('DEV', '') or os.getenv('USBGPU'): + from openpilot.system.hardware.chestnut.flash import link_up + for _ in range(10): + if link_up(): + break + time.sleep(1) + else: + raise RuntimeError("Chestnut not ready, skipping big model build") + + from openpilot.common.file_chunker import chunk_file, get_chunk_targets from openpilot.selfdrive.modeld.get_model_metadata import make_metadata_dict + from openpilot.system.camerad.cameras.nv12_info import get_nv12_info from tinygrad.nn.onnx import OnnxRunner parser = argparse.ArgumentParser(description="Compile combined JIT pkl for sunnypilot modeld_v2") @@ -317,7 +323,8 @@ if __name__ == "__main__": parser.add_argument('--supercombo-onnx', help='supercombo ONNX (for supercombo)') args = parser.parse_args() - output_data = defaultdict(dict) + model_w, model_h = args.model_size + output_data = {} args.vision_onnx = read_file_chunked_to_shm(args.vision_onnx) args.policy_onnx = read_file_chunked_to_shm(args.policy_onnx) @@ -348,17 +355,31 @@ if __name__ == "__main__": vision_meta = output_data['metadata'].get('vision', {}) derived_frame_skip = args.frame_skip or derive_frame_skip(vision_meta.get('input_shapes', {}), first_policy_meta.get('input_shapes', {})) - output_data.update(_compile_for_resolutions(args.camera_resolutions, args.model_size, derived_frame_skip, - vision_runner, policy_runners, output_data['metadata'])) + all_shapes = {key: value for meta in output_data['metadata'].values() for key, value in meta['input_shapes'].items()} + feat_meta = output_data['metadata'].get('vision') or output_data['metadata'].get('model') or output_data['metadata'].get('policy') + assert feat_meta is not None + features_slice = feat_meta['output_slices']['hidden_state'] + is_supercombo = vision_runner is None + + print(f"Compiling run_policy JIT (model_size={model_w}x{model_h}, frame_skip={derived_frame_skip})...") + run_policy_func = make_run_policy(vision_runner, policy_runners, features_slice, derived_frame_skip, all_shapes) + run_policy_jit = TinyJit(run_policy_func, prune=True) + make_policy_queues = partial(generate_queues_and_npy, all_shapes, derived_frame_skip, is_supercombo=is_supercombo) + make_random_model_inputs = partial(make_random_images, keys=['warped'], shape=(2, 6, model_h // 2, model_w // 2), device=WARP_DEV) + output_data['run_policy'] = compile_jit(run_policy_jit, make_random_model_inputs, POLICY_INPUTS, make_policy_queues) + + for cam_w, cam_h in args.camera_resolutions: + print(f"Compiling warp JIT for {cam_w}x{cam_h}...") + nv12 = NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h)) + make_random_warp_inputs = partial(make_random_images, keys=['frame', 'big_frame'], shape=nv12.size, device=WARP_DEV) + warp = TinyJit(make_warp(nv12, model_w, model_h), prune=True) + output_data[(cam_w, cam_h)] = compile_jit(warp, make_random_warp_inputs, WARP_INPUTS, make_warp_queues) with open(args.output, "wb") as file: - # TODO-SP: switch to dump_oob from openpilot/selfdrive/helpers on next full recompile of all models - pickle.dump(output_data, file) + dump_oob(output_data, file) pkl_size = os.path.getsize(args.output) print(f"Saved combined JIT to {args.output} ({pkl_size / 1e6:.2f} MB)") - - from openpilot.common.file_chunker import chunk_file, get_chunk_targets chunk_targets = get_chunk_targets(args.output, pkl_size) chunk_file(args.output, chunk_targets) print(f"Chunked into {len(chunk_targets) - 1} file(s)") diff --git a/openpilot/sunnypilot/modeld_v2/modeld.py b/openpilot/sunnypilot/modeld_v2/modeld.py index 66b560802b..9f3d709537 100755 --- a/openpilot/sunnypilot/modeld_v2/modeld.py +++ b/openpilot/sunnypilot/modeld_v2/modeld.py @@ -9,17 +9,13 @@ See the LICENSE.md file in the root directory for more details. import os os.environ['GMMU'] = '0' from openpilot.common.hardware import COMMA_HARDWARE -os.environ['DEV'] = 'QCOM' if COMMA_HARDWARE else 'CPU' -USBGPU = "USBGPU" in os.environ -if USBGPU: - os.environ['DEV'] = 'AMD' - os.environ['AMD_IFACE'] = 'USB' -import pickle +from openpilot.selfdrive.modeld.helpers import usbgpu_present, load_oob import time import numpy as np import openpilot.cereal.messaging as messaging from openpilot.cereal import log from opendbc.car.structs import car +from openpilot.cereal.services import SERVICE_LIST from setproctitle import setproctitle from openpilot.cereal.messaging import PubMaster, SubMaster from openpilot.cereal.visionipc import VisionStreamType @@ -27,7 +23,6 @@ from msgq.visionipc import VisionIpcClient, VisionBuf from opendbc.car.car_helpers import get_demo_car_params from tinygrad.tensor import Tensor -from tinygrad.device import Device from openpilot.common.file_chunker import open_file_chunked from openpilot.common.swaglog import cloudlog @@ -40,12 +35,13 @@ from openpilot.system import sentry from openpilot.system.camerad.cameras.nv12_info import get_nv12_info from openpilot.selfdrive.controls.lib.desire_helper import DesireHelper from openpilot.selfdrive.controls.lib.drive_helpers import get_accel_from_plan, smooth_value +from openpilot.selfdrive.modeld.modeld import ChestnutState from openpilot.sunnypilot.modeld_v2.fill_model_msg import fill_model_msg, fill_pose_msg, PublishState, get_curvature_from_output from openpilot.sunnypilot.modeld_v2.constants import Plan from openpilot.sunnypilot.modeld_v2.meta_helper import load_meta_constants from openpilot.sunnypilot.modeld_v2.camera_offset_helper import CameraOffsetHelper -from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, make_split_input_queues +from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, make_split_input_queues, make_supercombo_input_queues, WARP_INPUTS, POLICY_INPUTS from openpilot.sunnypilot.livedelay.helpers import get_lat_delay from openpilot.sunnypilot.modeld_v2.modeld_base import ModelStateBase @@ -88,7 +84,7 @@ class ModelState(ModelStateBase): inputs: dict[str, np.ndarray] prev_desire: np.ndarray - def __init__(self, cam_w: int, cam_h: int): + def __init__(self, cam_w: int, cam_h: int, usbgpu: bool = False): ModelStateBase.__init__(self) env_pkl = os.environ.get('COMBINED_MODEL_PKL') @@ -103,6 +99,7 @@ class ModelState(ModelStateBase): self.LONG_SMOOTH_SECONDS = float(overrides.get('long', ".0")) self.MIN_LAT_CONTROL_SPEED = 0.3 self.PLANPLUS_CONTROL: float = 1.0 + self.usbgpu = usbgpu pkl_path = _find_driving_pkl(model_bundle) assert pkl_path is not None, "No driving pkl found — all models must be compiled with compile_modeld.py" @@ -110,24 +107,20 @@ class ModelState(ModelStateBase): def _init_combined(self, pkl_path, cam_w, cam_h, bundle): cloudlog.warning(f"loading combined pkl: {pkl_path}") - # TODO-SP: switch to load_oob from openpilot/selfdrive/helpers on next full recompile of all models - jits = pickle.load(open_file_chunked(pkl_path)) + jits = load_oob(open_file_chunked(pkl_path)) - self.DEV = Device.DEFAULT - self.WARP_DEV = 'CPU' if USBGPU else self.DEV + self.WARP_DEV = 'QCOM' if COMMA_HARDWARE else 'CPU' + self.DEV = 'AMD' if self.usbgpu else self.WARP_DEV self.QUEUE_DEV = self.DEV - metadata = jits['metadata'] - self._run_policy = jits[(cam_w, cam_h)]['run_policy'] - self._warp_enqueue = jits[(cam_w, cam_h)]['warp_enqueue'] - - # TODO-SP: Remove legacy use_packed detection block after all models are recompiled - captured = getattr(self._run_policy, 'captured', None) - if captured is not None: - use_packed = 'packed_npy_inputs' in getattr(captured, 'expected_names', []) + self.is_legacy_model = 'run_policy' not in jits # remove after next recompile + if self.is_legacy_model: + self.warp = jits[(cam_w, cam_h)]['warp_enqueue'] + self.run_policy = jits[(cam_w, cam_h)]['run_policy'] else: - use_packed = True + self.run_policy = jits['run_policy'] + self.warp = jits[(cam_w, cam_h)] if 'model' in metadata: model_metadata = metadata['model'] @@ -136,10 +129,9 @@ class ModelState(ModelStateBase): self._policy_slices_list = [] self._combined_model_type = 'supercombo' self._vision_input_names = [key for key in model_metadata['input_shapes'] if 'img' in key] - from openpilot.sunnypilot.modeld_v2.compile_modeld import make_supercombo_input_queues frame_skip = derive_frame_skip({}, model_metadata['input_shapes']) self.input_queues, self.numpy_inputs = make_supercombo_input_queues(model_metadata['input_shapes'], - frame_skip, device=self.QUEUE_DEV, use_packed=use_packed) + frame_skip, device=self.QUEUE_DEV) else: vision_metadata = metadata['vision'] policy_keys = [k for k in metadata if k != 'vision'] @@ -152,13 +144,12 @@ class ModelState(ModelStateBase): self._policy_slices_list = [metadata[k]['output_slices'] for k in policy_keys] self.policy_output_slices = self._policy_slices_list[0] self._has_on_policy = any('on' in k.lower() for k in policy_keys) - first_policy_metadata = metadata[policy_keys[0]] - vision_input_shapes = vision_metadata['input_shapes'] - policy_input_shapes = first_policy_metadata['input_shapes'] - self._vision_input_names = [k for k in vision_input_shapes if 'img' in k] - frame_skip = derive_frame_skip(vision_input_shapes, policy_input_shapes) - self.input_queues, self.numpy_inputs = make_split_input_queues(vision_input_shapes, policy_input_shapes, - frame_skip, device=self.QUEUE_DEV, use_packed=use_packed) + self._vision_input_names = [key for key in vision_metadata['input_shapes'] if 'img' in key] + first_policy_meta = metadata[policy_keys[0]] + frame_skip = derive_frame_skip(vision_metadata['input_shapes'], first_policy_meta['input_shapes']) + self.input_queues, self.numpy_inputs = make_split_input_queues(vision_metadata['input_shapes'], + first_policy_meta['input_shapes'], + frame_skip, device=self.QUEUE_DEV) self._desire_key = next(key for key in self.numpy_inputs if key.startswith('desire')) self._road_key = next(key for key in self._vision_input_names if 'big' not in key) @@ -186,10 +177,33 @@ class ModelState(ModelStateBase): self.frame_buf_params = dict.fromkeys(self._vision_input_names, nv12_info) yuv_size = self.frame_buf_params[self._road_key][3] - self._warp_enqueue( - **self.input_queues, - frame=Tensor(np.zeros(yuv_size, dtype=np.uint8), device=self.WARP_DEV).contiguous().realize(), - big_frame=Tensor(np.zeros(yuv_size, dtype=np.uint8), device=self.WARP_DEV).contiguous().realize()) + frame_tensor = Tensor(np.zeros(yuv_size, dtype=np.uint8), device=self.WARP_DEV).contiguous().realize() + big_frame_tensor = Tensor(np.zeros(yuv_size, dtype=np.uint8), device=self.WARP_DEV).contiguous().realize() + + if self.is_legacy_model: # Remove this conditional hack after recompile + self.warp(**self.input_queues, frame=frame_tensor, big_frame=big_frame_tensor) + else: + self.warp(**{k: self.input_queues[k] for k in WARP_INPUTS}, frame=frame_tensor, big_frame=big_frame_tensor) + + if self.usbgpu: + self.warmup() + + def warmup(self) -> None: + dummy_frames = {k: np.zeros(self.frame_buf_params[k][3], dtype=np.uint8) for k in self._vision_input_names} + transforms = {k: np.eye(3, dtype=np.float32) for k in [self._road_key, self._wide_key] if k} + + dummy_inputs = {} + for k, v in self.numpy_inputs.items(): + if k not in ['tfm', 'big_tfm', 'prev_feat']: + dummy_inputs[k] = np.zeros(v.shape, dtype=v.dtype) + + self.run(dummy_frames, transforms, dummy_inputs, prepare_only=False) + + for v in self.numpy_inputs.values(): + v[:] = 0 + self.prev_desire[:] = 0 + self.full_frames.clear() + self._blob_cache.clear() @property @@ -227,11 +241,17 @@ class ModelState(ModelStateBase): self.numpy_inputs['tfm'][:, :] = transforms[road_key].reshape(3, 3) self.numpy_inputs['big_tfm'][:, :] = transforms[wide_key].reshape(3, 3) - if prepare_only: - self._warp_enqueue(**self.input_queues, frame=self.full_frames[road_key], big_frame=self.full_frames[wide_key]) - return None - - raw_outputs = self._run_policy(**self.input_queues, frame=self.full_frames[road_key], big_frame=self.full_frames[wide_key]) + if self.is_legacy_model: # remove after next recompile + if prepare_only: + self.warp(**self.input_queues, frame=self.full_frames[road_key], big_frame=self.full_frames[wide_key]) + return None + raw_outputs = self.run_policy(**self.input_queues, frame=self.full_frames[road_key], big_frame=self.full_frames[wide_key]) + else: + if prepare_only: + self.warp(**{k: self.input_queues[k] for k in WARP_INPUTS}, frame=self.full_frames[road_key], big_frame=self.full_frames[wide_key]) + return None + warped = self.warp(**{k: self.input_queues[k] for k in WARP_INPUTS}, frame=self.full_frames[road_key], big_frame=self.full_frames[wide_key]) + raw_outputs = self.run_policy(**{k: self.input_queues[k] for k in POLICY_INPUTS if k in self.input_queues}, warped=warped) if self._combined_model_type == 'supercombo': model_output = raw_outputs.numpy().flatten() @@ -267,10 +287,9 @@ class ModelState(ModelStateBase): buf[0, :-1] = buf[0, 1:] buf[0, -1, :] = outputs['desired_curvature'][0, :] if not self.mlsim else 0 - # TODO-SP: This is a hack to prevent GPU corruption by calculating in CPU space, it can be removed on next recompile - if 'prev_feat' not in self.numpy_inputs and 'feat_q' in self.input_queues: - feat_val = self.input_queues['feat_q'].numpy() - self.input_queues['feat_q'].assign(feat_val).realize() + if self.usbgpu and not np.all(np.isfinite(outputs.get('plan', np.array([0.])))): + cloudlog.error("model output not finite, dropping frame") + return None return outputs @@ -278,8 +297,8 @@ class ModelState(ModelStateBase): lat_action_t: float, long_action_t: float, v_ego: float) -> log.ModelDataV2.Action: if 'action' not in model_output: plan = model_output['plan'][0] - desired_accel, should_stop = get_accel_from_plan(plan[:, Plan.VELOCITY][:, 0], plan[:, Plan.ACCELERATION][:, 0], self.constants.T_IDXS, - action_t=long_action_t) + desired_accel = get_accel_from_plan(plan[:, Plan.VELOCITY][:, 0], plan[:, Plan.ACCELERATION][:, 0], self.constants.T_IDXS, + action_t=long_action_t) curvature_plan = (plan + (self.PLANPLUS_CONTROL - 1.0) * model_output['planplus'][0] if 'planplus' in model_output and self.PLANPLUS_CONTROL != 1.0 else plan) @@ -287,8 +306,8 @@ class ModelState(ModelStateBase): else: desired_accel = model_output['action'][0, 1] desired_curvature = model_output['action'][0, 0] / (max(1.0, v_ego))**2 - should_stop = (v_ego < 0.3 and desired_accel < 0.1) + stop = v_ego < 0.3 and desired_accel < 0.1 desired_accel = smooth_value(desired_accel, prev_action.desiredAcceleration, self.LONG_SMOOTH_SECONDS) if self.generation is not None and self.generation >= 10: # smooth curvature for post FOF models @@ -297,7 +316,7 @@ class ModelState(ModelStateBase): else: desired_curvature = prev_action.desiredCurvature - return log.ModelDataV2.Action(desiredCurvature=float(desired_curvature),desiredAcceleration=float(desired_accel), shouldStop=bool(should_stop)) + return log.ModelDataV2.Action(desiredCurvature=float(desired_curvature), desiredAcceleration=float(desired_accel), shouldStop=bool(stop)) def main(demo=False): @@ -308,6 +327,14 @@ def main(demo=False): setproctitle(PROCESS_NAME) config_realtime_process(7, 54) + USBGPU = usbgpu_present() + if USBGPU: + os.environ['HCQDEV_WAIT_TIMEOUT_MS'] = '3000' + + params = Params() + params.put_bool("UsbGpuLoading", USBGPU) + params.remove("UsbGpuActive") + # visionipc clients while True: available_streams = VisionIpcClient.available_streams("camerad", block=False) @@ -332,15 +359,34 @@ def main(demo=False): cloudlog.warning(f"connected extra cam with buffer size: {vipc_client_extra.buffer_len} ({vipc_client_extra.width} x {vipc_client_extra.height})") cloudlog.warning("loading model") - model = ModelState(cam_w=vipc_client_main.width, cam_h=vipc_client_main.height) - cloudlog.warning("models loaded, modeld starting") + st = time.monotonic() + + model = None + if USBGPU: + import threading + def load(): + nonlocal model + model = ModelState(cam_w=vipc_client_main.width, cam_h=vipc_client_main.height, usbgpu=True) + t = threading.Thread(target=load, daemon=True) + t.start() + t.join(60) + if model is None: + params.put_bool("UsbGpuActive", False) + raise RuntimeError("eGPU model load failed or timed out (60s)") + params.put_bool("UsbGpuActive", True) + else: + model = ModelState(cam_w=vipc_client_main.width, cam_h=vipc_client_main.height, usbgpu=False) + + params.put_bool("UsbGpuLoading", False) + cloudlog.warning(f"models loaded in {time.monotonic() - st:.1f}s, modeld starting") # messaging - pm = PubMaster(["modelV2", "drivingModelData", "cameraOdometry", "modelDataV2SP"]) + pub_socks = ["modelV2", "drivingModelData", "cameraOdometry", "modelDataV2SP"] + (["chestnutState"] if USBGPU else []) + pm = PubMaster(pub_socks) sm = SubMaster(["deviceState", "carState", "narrowRoadCameraState", "extrinsicsCalibration", "driverMonitoringState", "carControl", "lateralDelay"]) publish_state = PublishState() - params = Params() + chestnut_state = ChestnutState(pm, USBGPU) if USBGPU else None # setup filter to track dropped frames frame_dropped_filter = FirstOrderFilter(0., 10., 1. / model.constants.MODEL_FREQ) @@ -478,6 +524,7 @@ def main(demo=False): fill_model_msg(drivingdata_send, modelv2_send, model_output, action, publish_state, meta_main.frame_id, meta_extra.frame_id, frame_id, frame_drop_ratio, meta_main.timestamp_eof, model_execution_time, live_calib_seen, meta_constants) + modelv2_send.modelV2.big = model.usbgpu desire_state = modelv2_send.modelV2.meta.desireState l_lane_change_prob = desire_state[log.Desire.laneChangeLeft] @@ -498,6 +545,8 @@ def main(demo=False): pm.send('modelDataV2SP', mdv2sp_send) last_vipc_frame_id = meta_main.frame_id + if chestnut_state is not None and run_count % round(model.constants.MODEL_FREQ / SERVICE_LIST['chestnutState'].frequency) == 0: + chestnut_state.send() if __name__ == "__main__": try: diff --git a/openpilot/sunnypilot/modeld_v2/tests/helpers.py b/openpilot/sunnypilot/modeld_v2/tests/helpers.py index 6925a61f08..ee59e82785 100644 --- a/openpilot/sunnypilot/modeld_v2/tests/helpers.py +++ b/openpilot/sunnypilot/modeld_v2/tests/helpers.py @@ -6,7 +6,6 @@ See the LICENSE.md file in the root directory for more details. """ import pathlib -import pickle import tempfile import openpilot.sunnypilot.models.helpers as helpers @@ -164,14 +163,16 @@ ARCHETYPES = { def make_pkl_data(archetype): return { 'metadata': archetype.metadata_structure, - (CAM_W, CAM_H): {'run_policy': _noop_jit, 'warp_enqueue': _noop_jit}, + 'run_policy': _noop_jit, + (CAM_W, CAM_H): _noop_jit, } def write_pkl(tmp_path, archetype): + from openpilot.selfdrive.modeld.helpers import dump_oob pkl_path = tmp_path / 'driving_test_tinygrad.pkl' with open(pkl_path, 'wb') as f: - pickle.dump(make_pkl_data(archetype), f) + dump_oob(make_pkl_data(archetype), f) return pkl_path diff --git a/openpilot/sunnypilot/modeld_v2/tests/test_recovery_power.py b/openpilot/sunnypilot/modeld_v2/tests/test_recovery_power.py index 85305395ea..fb72022fa5 100644 --- a/openpilot/sunnypilot/modeld_v2/tests/test_recovery_power.py +++ b/openpilot/sunnypilot/modeld_v2/tests/test_recovery_power.py @@ -33,7 +33,7 @@ class TestRecoveryPower(OpenpilotTestCase): def mock_accel(plan_vel, plan_accel, t_idxs, action_t=0.0): recorded_vel.append(plan_vel.copy()) - return 0.0, False + return 0.0 def mock_curvature(output, plan, vego, lat_action_t, mlsim): recorded_curv_plans.append(plan.copy()) diff --git a/openpilot/sunnypilot/models/fetcher.py b/openpilot/sunnypilot/models/fetcher.py index eda1117a2a..e60af2925f 100644 --- a/openpilot/sunnypilot/models/fetcher.py +++ b/openpilot/sunnypilot/models/fetcher.py @@ -13,6 +13,7 @@ from openpilot.common.params import Params from openpilot.common.swaglog import cloudlog from openpilot.common.hardware.hw import Paths from openpilot.sunnypilot.models.helpers import is_bundle_version_compatible +from openpilot.selfdrive.modeld.helpers import usbgpu_present from openpilot.cereal import custom @@ -103,11 +104,11 @@ class ModelParser: class ModelCache: """Handles caching of model data to avoid frequent remote fetches""" - def __init__(self, params: Params, cache_timeout: int = int(3600 * 1e9)): + def __init__(self, params: Params, cache_timeout: int = int(3600 * 1e9), suffix: str = ""): self.params = params self.cache_timeout = cache_timeout - self._LAST_SYNC_KEY = "ModelManager_LastSyncTime" - self._CACHE_KEY = "ModelManager_ModelsCache" + self._LAST_SYNC_KEY = f"ModelManager_LastSyncTime{suffix}" + self._CACHE_KEY = f"ModelManager_ModelsCache{suffix}" def _is_expired(self) -> bool: """Checks if the cache has expired""" @@ -139,24 +140,37 @@ class ModelCache: class ModelFetcher: """Handles fetching and caching of model data from remote source""" - MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v18.json" + MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v19.json" + MODEL_URL_USBGPU = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v19.json" def __init__(self, params: Params): self.params = params - self.model_cache = ModelCache(params) self.model_parser = ModelParser() + self._is_usbgpu: bool | None = None + self.model_cache = ModelCache(params) + self.model_url = self.MODEL_URL + self._update_model_source() + + def _update_model_source(self) -> None: + """Updates what json to use based on usbgpu availability""" + is_usbgpu = usbgpu_present() + if is_usbgpu != self._is_usbgpu: + self._is_usbgpu = is_usbgpu + self.model_cache = ModelCache(self.params, suffix="_USBGPU" if is_usbgpu else "") + self.model_url = self.MODEL_URL_USBGPU if is_usbgpu else self.MODEL_URL + self.params.put("ModelManager_ActiveJson", self.model_url, block=True) def _fetch_and_cache_models(self) -> list[custom.ModelManagerSP.ModelBundle] | None: """Fetches fresh model data from remote and updates cache. Returns None on transport errors. Raises on 404 and other fatal HTTP errors. """ try: - response = requests.get(self.MODEL_URL, timeout=10) + response = requests.get(self.model_url, timeout=10) # Explicitly handle 404 differently if response.status_code == 404: - cloudlog.error(f"Models URL returned 404 Not Found: {self.MODEL_URL}") - raise HTTPError(f"404 Not Found: {self.MODEL_URL}", response=response) + cloudlog.error(f"Models URL returned 404 Not Found: {self.model_url}") + raise HTTPError(f"404 Not Found: {self.model_url}", response=response) # Raise for any other 4xx/5xx response.raise_for_status() @@ -179,6 +193,7 @@ class ModelFetcher: def get_available_bundles(self) -> list[custom.ModelManagerSP.ModelBundle]: """Gets the list of available models, with smart cache handling""" + self._update_model_source() cached_data, is_expired = self.model_cache.get() if cached_data and not is_expired: @@ -202,10 +217,7 @@ if __name__ == "__main__": for bundle in bundles: for model in bundle.models: model_overrides = {override.key: override.value for override in bundle.overrides} - # Print model details print(f"Bundle: {bundle.internalName}, Type: {model.type}, Status: {bundle.status}, Overrides: {model_overrides}") - # Print artifact details print(f"Artifact: {model.artifact.fileName}, Download URI: {model.artifact.downloadUri.uri}") - # Print metadata details if model.artifact.chunks: print(f"Contains {len(model.artifact.chunks)} chunks.") diff --git a/openpilot/sunnypilot/models/helpers.py b/openpilot/sunnypilot/models/helpers.py index 101d8d196e..b5c97467d3 100644 --- a/openpilot/sunnypilot/models/helpers.py +++ b/openpilot/sunnypilot/models/helpers.py @@ -18,7 +18,7 @@ from openpilot.sunnypilot.models.constants import Meta, MetaSimPose, MetaTombRai from openpilot.common.hardware.hw import Paths # SET ME TO THE EXACT JSON VERSION WE SET IN SUNNYPILOT_MODELS REPO -REQUIRED_JSON_VERSION = 16 +REQUIRED_JSON_VERSION = 17 CUSTOM_MODEL_PATH = Paths.model_root() METADATA_PATH = Path(__file__).parent / '../models/supercombo_metadata.pkl' diff --git a/openpilot/sunnypilot/models/tests/test_tinygrad_ref.py b/openpilot/sunnypilot/models/tests/test_tinygrad_ref.py index 1712c60410..fd389f93c0 100644 --- a/openpilot/sunnypilot/models/tests/test_tinygrad_ref.py +++ b/openpilot/sunnypilot/models/tests/test_tinygrad_ref.py @@ -1,12 +1,13 @@ import requests +from openpilot.common.params import Params from openpilot.sunnypilot.models.tinygrad_ref import get_tinygrad_ref from openpilot.sunnypilot.models.fetcher import ModelFetcher from openpilot.common.test import OpenpilotTestCase - def fetch_tinygrad_ref(): - response = requests.get(ModelFetcher.MODEL_URL, timeout=10) + fetcher = ModelFetcher(Params()) + response = requests.get(fetcher.model_url, timeout=10) response.raise_for_status() json_data = response.json() return json_data.get("tinygrad_ref") diff --git a/release/ci/model_generator.py b/release/ci/model_generator.py index 76935f3627..607260f145 100755 --- a/release/ci/model_generator.py +++ b/release/ci/model_generator.py @@ -48,6 +48,11 @@ def create_short_name(full_name: str) -> str: return result[:8] +def create_pkl_name(full_name: str) -> str: + pkl = re.sub(r'[^a-zA-Z0-9]+', '_', full_name).strip('_').lower() + return pkl + + def _read_pkl_bytes(pkl_path: Path) -> bytes: manifest = Path(f"{pkl_path}.chunkmanifest") if manifest.exists(): @@ -154,14 +159,15 @@ if __name__ == "__main__": _output_dir = Path(args.output_dir) _output_dir.mkdir(exist_ok=True, parents=True) _short_name = create_short_name(args.custom_name) if args.custom_name else None + _pkl = create_pkl_name(args.custom_name) if args.custom_name else None _driving_pkl = _find_driving_pkl(_output_dir) if not _driving_pkl: print(f"No driving_tinygrad.pkl found in {_output_dir}", file=sys.stderr) sys.exit(1) - if _short_name: - new_pkl = _output_dir / f"driving_{_short_name.lower()}_tinygrad.pkl" + if _pkl: + new_pkl = _output_dir / f"driving_{_pkl}_tinygrad.pkl" if not new_pkl.exists(): _driving_pkl = _rename_pkl_with_chunks(_driving_pkl, new_pkl) else: diff --git a/tinygrad_repo b/tinygrad_repo index 2fecac4e4a..66ee3cfb4f 160000 --- a/tinygrad_repo +++ b/tinygrad_repo @@ -1 +1 @@ -Subproject commit 2fecac4e4ac32fe369c41f8400b6e7b9adb18683 +Subproject commit 66ee3cfb4f3a3908a6a20ddfbec7774ba7c09b4e