From 0246f03b751fcdb44fa5f6fdc2800c11b29b4069 Mon Sep 17 00:00:00 2001 From: James <91348155+FrogAi@users.noreply.github.com> Date: Mon, 1 Dec 2025 12:00:00 -0700 Subject: [PATCH] FrogPilot workflows --- .github/workflows/compile_frogpilot.yaml | 239 +++++++++++++++++++ .github/workflows/review_pull_request.yaml | 38 +++ .github/workflows/schedule_update.yaml | 71 ++++++ .github/workflows/update_pr_branch.yaml | 97 ++++++++ .github/workflows/update_release_branch.yaml | 107 +++++++++ .github/workflows/update_tinygrad.yaml | 95 ++++++++ frogpilot/tools/compile_models.py | 75 ++++++ 7 files changed, 722 insertions(+) create mode 100644 .github/workflows/compile_frogpilot.yaml create mode 100644 .github/workflows/review_pull_request.yaml create mode 100644 .github/workflows/schedule_update.yaml create mode 100644 .github/workflows/update_pr_branch.yaml create mode 100644 .github/workflows/update_release_branch.yaml create mode 100644 .github/workflows/update_tinygrad.yaml create mode 100644 frogpilot/tools/compile_models.py diff --git a/.github/workflows/compile_frogpilot.yaml b/.github/workflows/compile_frogpilot.yaml new file mode 100644 index 000000000..f81913623 --- /dev/null +++ b/.github/workflows/compile_frogpilot.yaml @@ -0,0 +1,239 @@ +name: Compile FrogPilot + +on: + workflow_dispatch: + inputs: + not_vetted: + description: "This branch is not vetted" + type: boolean + default: false + required: false + publish_custom_branch: + description: "Push to custom branch:" + type: string + default: "" + required: false + publish_frogpilot: + description: "Push to FrogPilot" + type: boolean + default: false + required: false + publish_staging: + description: "Push to FrogPilot-Staging" + type: boolean + default: false + required: false + publish_testing: + description: "Push to FrogPilot-Testing" + type: boolean + default: false + required: false + runner: + description: "Select runner" + type: choice + options: + - c3 + - c3x + default: "c3" + required: true + update_translations: + description: "Update missing/outdated translations" + type: boolean + default: false + required: false + vet_existing_translations: + description: "Vet existing translations" + type: boolean + default: false + required: false + +env: + BASE_DIR: ${{ github.workspace }} + BUILD_DIR: "/data/openpilot" + CUSTOM_BRANCH: ${{ inputs.publish_custom_branch }} + GIT_EMAIL: "91348155+FrogAi@users.noreply.github.com" + GIT_NAME: "James" + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + +jobs: + get_branch: + runs-on: [self-hosted, "${{ inputs.runner }}"] + outputs: + branch: ${{ steps.get_branch.outputs.branch }} + python_version: ${{ steps.get_python_version.outputs.python_version }} + steps: + - name: Get Current Branch + id: get_branch + run: | + cd "$BUILD_DIR" + echo "branch=$(git rev-parse --abbrev-ref HEAD)" >> "$GITHUB_OUTPUT" + + - name: Get Python Version + id: get_python_version + run: | + echo "python_version=$(tr -d '[:space:]' < "$BUILD_DIR/.python-version")" >> $GITHUB_OUTPUT + + translate: + needs: get_branch + if: inputs.update_translations + runs-on: ubuntu-latest + steps: + - name: Configure Git Identity + run: | + git config --global user.name "$GIT_NAME" + git config --global user.email "$GIT_EMAIL" + + - name: Checkout Required Files + uses: actions/checkout@v4 + with: + ref: ${{ needs.get_branch.outputs.branch }} + sparse-checkout: | + frogpilot/ui/ + selfdrive/controls/lib/alerts_offroad.json + selfdrive/ui/ + selfdrive/ui/translations/ + selfdrive/ui/translations/auto_translate.py + selfdrive/ui/update_translations.py + + - name: Set Up Python + uses: actions/setup-python@v4 + with: + cache: "pip" + python-version: ${{ needs.get_branch.outputs.python_version }} + + - name: Install Dependencies + run: | + pip install requests + sudo apt-get update && sudo apt-get install -y --no-install-recommends qttools5-dev-tools + + - name: Update Translations + run: | + python selfdrive/ui/update_translations.py --vanish + + - name: Update Missing Translations + continue-on-error: true + timeout-minutes: 300 + run: | + python selfdrive/ui/translations/auto_translate.py --all-files + + - name: Vet Existing Translations + if: inputs.vet_existing_translations + continue-on-error: true + timeout-minutes: 300 + run: | + python selfdrive/ui/translations/auto_translate.py --all-files --vet-translations + + - name: Commit and Push Translations + run: | + if git diff --quiet selfdrive/ui/translations/*.ts; then + echo "No translation updates detected." + exit 0 + fi + + git fetch --unshallow origin "${{ needs.get_branch.outputs.branch }}" + git checkout "${{ needs.get_branch.outputs.branch }}" + git add selfdrive/ui/translations/*.ts + git commit --amend --no-edit + git push --force origin "${{ needs.get_branch.outputs.branch }}" + + build_and_push: + needs: [get_branch, translate] + if: ${{ !failure() && !cancelled() && needs.get_branch.result == 'success' }} + runs-on: [self-hosted, "${{ inputs.runner }}"] + permissions: + contents: write + defaults: + run: + working-directory: ${{ env.BUILD_DIR }} + steps: + - name: Configure Git + run: | + git config http.postBuffer 104857600 + git config user.name "$GIT_NAME" + git config user.email "$GIT_EMAIL" + git remote set-url origin "https://${{ secrets.PERSONAL_ACCESS_TOKEN }}@github.com/FrogAi/FrogPilot.git" + + - name: Sync Translation Updates + if: inputs.update_translations + run: | + git fetch origin "${{ needs.get_branch.outputs.branch }}" + git reset --hard FETCH_HEAD + + - name: Take Ownership of Build Directory + run: | + sudo chown -R $(whoami):$(whoami) . + + - name: Clean Build Artifacts + run: | + find . -name "matlab.*.md" -delete + + find . -type d \( -iname "debug" -o -iname "test" -o -iname "tests" -o -name '__pycache__' \) -exec rm -rf {} + + + find . -type f \( \ + -name '*.a' -o \ + -name '*.o' -o \ + -name '*.onnx' -o \ + -name '*.os' -o \ + -name '*.pyc' -o \ + -name 'moc_*' \ + \) -delete + + find .github -mindepth 1 -maxdepth 1 ! -name 'workflows' -exec rm -rf {} + + find .github/workflows -mindepth 1 ! \( \ + -type f \( \ + -name 'compile_frogpilot.yaml' -o \ + -name 'review_pull_request.yaml' -o \ + -name 'schedule_update.yaml' -o \ + -name 'update_pr_branch.yaml' -o \ + -name 'update_release_branch.yaml' -o \ + -name 'update_tinygrad.yaml' \ + \) \ + \) -exec rm -rf {} + + + find panda/board -type f \ + ! -name '__init__.py' \ + ! -name 'bootstub.panda.bin' \ + ! -name 'bootstub.panda_h7.bin' \ + ! -name 'panda.bin.signed' \ + ! -name 'panda_h7.bin.signed' \ + -delete + + find third_party/ -name '*Darwin*' -exec rm -rf {} + + find third_party/ -name '*x86*' -exec rm -rf {} + + + rm -f .gitignore .gitmodules .gitattributes .lfsconfig .overlay_init + + rm -rf .sconsign.dblite .vscode/ Jenkinsfile release/ scripts/ site_scons/ teleoprtc_repo/ + + find . -type d -empty ! -path "./.git*" -delete + + touch prebuilt + [ "${{ inputs.not_vetted }}" = "true" ] && touch not_vetted || true + + - name: Add Update Date File + if: inputs.publish_staging + continue-on-error: true + run: | + curl -fLsS https://raw.githubusercontent.com/FrogAi/FrogPilot/FrogPilot-Staging/.github/update_date -o .github/update_date || echo "No update_date found, skipping..." + + - name: Commit and Push Build + run: | + git add -f . + git commit -m "Compile FrogPilot" + git push --force origin HEAD + + if [ "${{ inputs.publish_frogpilot }}" = "true" ]; then + git push --force origin HEAD:FrogPilot + fi + + if [ "${{ inputs.publish_staging }}" = "true" ]; then + git push --force origin HEAD:FrogPilot-Staging + fi + + if [ "${{ inputs.publish_testing }}" = "true" ]; then + git push --force origin HEAD:FrogPilot-Testing + fi + + if [ -n "$CUSTOM_BRANCH" ]; then + git push --force origin HEAD:"$CUSTOM_BRANCH" + fi diff --git a/.github/workflows/review_pull_request.yaml b/.github/workflows/review_pull_request.yaml new file mode 100644 index 000000000..d2e02a3b6 --- /dev/null +++ b/.github/workflows/review_pull_request.yaml @@ -0,0 +1,38 @@ +name: Review Pull Request + +on: + pull_request_target: + types: [opened, reopened] + +env: + EXPECTED_BRANCH: "MAKE-PRS-HERE" + GIT_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +permissions: + pull-requests: write + +jobs: + pr_check: + runs-on: ubuntu-latest + if: ${{ github.actor != 'FrogAi' }} + steps: + - name: Validate Target Branch + env: + GH_TOKEN: ${{ env.GIT_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + TARGET_BRANCH: ${{ github.base_ref }} + run: | + if [ "$TARGET_BRANCH" != "$EXPECTED_BRANCH" ]; then + echo "Invalid target branch: $TARGET_BRANCH" + + gh pr comment "$PR_NUMBER" --repo "$REPO" \ + --body "Please submit your pull request to the \"$EXPECTED_BRANCH\" branch." + + gh pr close "$PR_NUMBER" --repo "$REPO" + else + echo "Valid target branch." + + gh pr comment "$PR_NUMBER" --repo "$REPO" \ + --body "Thank you for your PR! If you're not already in the FrogPilot Discord, [feel free to join](https://discord.FrogPilot.com) and let me know you've opened a PR!" + fi diff --git a/.github/workflows/schedule_update.yaml b/.github/workflows/schedule_update.yaml new file mode 100644 index 000000000..9aa585a3f --- /dev/null +++ b/.github/workflows/schedule_update.yaml @@ -0,0 +1,71 @@ +name: Schedule FrogPilot Update + +on: + workflow_dispatch: + inputs: + scheduled_date: + description: "Enter the date to update the \"FrogPilot\" branch (YYYY-MM-DD)" + required: true + +env: + GIT_EMAIL: "91348155+FrogAi@users.noreply.github.com" + GIT_NAME: "James" + TARGET_BRANCH: "FrogPilot-Staging" + UPDATE_FILE_PATH: ".github/update_date" + +jobs: + schedule_update: + runs-on: ubuntu-latest + steps: + - name: Checkout Target Branch + uses: actions/checkout@v4 + with: + ref: ${{ env.TARGET_BRANCH }} + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + fetch-depth: 3 + + - name: Configure Git Identity + run: | + git config --global user.name "$GIT_NAME" + git config --global user.email "$GIT_EMAIL" + + - name: Write Schedule Date + env: + SCHEDULED_DATE: ${{ github.event.inputs.scheduled_date }} + run: | + echo "$SCHEDULED_DATE" > "$UPDATE_FILE_PATH" + git add "$UPDATE_FILE_PATH" + + - name: Get Target Commit Data + id: get_target + run: | + TARGET_COMMIT=$(git rev-parse HEAD~1) + AUTHOR_DATE=$(git show -s --format=%aD "$TARGET_COMMIT") + COMMITTER_DATE=$(git show -s --format=%cD "$TARGET_COMMIT") + + echo "AUTHOR_DATE=$AUTHOR_DATE" >> "$GITHUB_ENV" + echo "COMMITTER_DATE=$COMMITTER_DATE" >> "$GITHUB_ENV" + echo "TARGET_COMMIT=$TARGET_COMMIT" >> "$GITHUB_ENV" + + - name: Create Fixup Commit + id: fixup_commit + run: | + if git diff --cached --quiet; then + echo "No changes detected." + echo "has_changes=false" >> "$GITHUB_OUTPUT" + else + echo "Changes detected. Creating fixup commit." + git commit --fixup="$TARGET_COMMIT" + echo "has_changes=true" >> "$GITHUB_OUTPUT" + fi + + - name: Autosquash and Restore Timestamps + if: steps.fixup_commit.outputs.has_changes == 'true' + run: | + GIT_SEQUENCE_EDITOR=: git rebase --autosquash -i HEAD~3 + git rebase --exec "GIT_COMMITTER_DATE='$COMMITTER_DATE' git commit --amend --no-edit --date='$AUTHOR_DATE'" HEAD~2 + + - name: Push Changes + if: steps.fixup_commit.outputs.has_changes == 'true' + run: | + git push origin "$TARGET_BRANCH" --force-with-lease diff --git a/.github/workflows/update_pr_branch.yaml b/.github/workflows/update_pr_branch.yaml new file mode 100644 index 000000000..d7c5d231d --- /dev/null +++ b/.github/workflows/update_pr_branch.yaml @@ -0,0 +1,97 @@ +name: Update MAKE-PRS-HERE +run-name: Update MAKE-PRS-HERE + +on: + push: + branches: + - FrogPilot-Testing + +env: + GIT_EMAIL: "91348155+FrogAi@users.noreply.github.com" + GIT_NAME: "James" + GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + SOURCE_BRANCH: FrogPilot-Testing + TARGET_BRANCH: MAKE-PRS-HERE + TZ: America/Phoenix + +permissions: + contents: write + +jobs: + update_branch: + runs-on: ubuntu-latest + steps: + - name: Checkout Source + uses: actions/checkout@v4 + with: + ref: ${{ env.SOURCE_BRANCH }} + fetch-depth: 5 + token: ${{ env.GITHUB_TOKEN }} + + - name: Configure Git Identity + run: | + git config --global user.name "$GIT_NAME" + git config --global user.email "$GIT_EMAIL" + + - name: Revert "Compile FrogPilot" + id: prepare_source + run: | + COMPILE_COMMIT=$(git rev-list HEAD -n 1 --grep="Compile FrogPilot" || true) + + if [ -n "$COMPILE_COMMIT" ]; then + echo "Found 'Compile FrogPilot' at $COMPILE_COMMIT. Reverting..." + git revert --no-edit "$COMPILE_COMMIT" + else + echo "Compile commit not found. Proceeding with current source state." + fi + + echo "source_state=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Checkout Target + run: | + git fetch origin "$TARGET_BRANCH" --depth=2 + git checkout "$TARGET_BRANCH" + + - name: Sync Files from Source to Target + env: + SOURCE_STATE: ${{ steps.prepare_source.outputs.source_state }} + run: | + git rm -r --ignore-unmatch . + git clean -fdx + + git checkout "$SOURCE_STATE" -- . + + rm -f .github/update_date + git add --all + + - name: Commit and Push + env: + REPO_NAME: ${{ github.repository }} + run: | + if git diff --staged --quiet; then + echo "Working tree is clean. No changes to commit." + exit 0 + fi + + DAY=$(TZ="$TZ" date +'%d' | sed 's/^0//') + case "$DAY" in + 1|21|31) SUFFIX="st" ;; + 2|22) SUFFIX="nd" ;; + 3|23) SUFFIX="rd" ;; + *) SUFFIX="th" ;; + esac + + MONTH=$(TZ="$TZ" date +'%B') + YEAR=$(TZ="$TZ" date +'%Y') + COMMIT_MSG="${MONTH} ${DAY}${SUFFIX}, ${YEAR} Update" + + if git log -1 --pretty=%s | grep -q "$COMMIT_MSG"; then + echo "Amending previous commit: $COMMIT_MSG" + git commit --amend --no-edit + else + echo "Creating new commit: $COMMIT_MSG" + git commit -m "$COMMIT_MSG" + fi + + git remote set-url origin "https://${GITHUB_TOKEN}@github.com/${REPO_NAME}" + git push origin "$TARGET_BRANCH" --force diff --git a/.github/workflows/update_release_branch.yaml b/.github/workflows/update_release_branch.yaml new file mode 100644 index 000000000..82f981c3f --- /dev/null +++ b/.github/workflows/update_release_branch.yaml @@ -0,0 +1,107 @@ +name: Update FrogPilot Branch + +on: + schedule: + - cron: "0 18 * * 6" + +env: + BRANCH_FROGPILOT: FrogPilot + BRANCH_PREVIOUS: FrogPilot-Previous + BRANCH_STAGING: FrogPilot-Staging + GIT_EMAIL: "91348155+FrogAi@users.noreply.github.com" + GIT_NAME: "James" + GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + TZ: America/Phoenix + UPDATE_FILE: .github/update_date + +jobs: + check_update: + runs-on: ubuntu-latest + outputs: + update_due: ${{ steps.check_update.outputs.update_due }} + scheduled_date: ${{ steps.check_update.outputs.scheduled_date }} + steps: + - name: Check Update Status + id: check_update + env: + REPO_NAME: ${{ github.repository }} + run: | + URL="https://raw.githubusercontent.com/$REPO_NAME/$BRANCH_STAGING/$UPDATE_FILE" + STATUS=$(curl -o /dev/null -s -w "%{http_code}\n" "$URL") + + if [ "$STATUS" != "200" ]; then + echo "update_due=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + SCHEDULED_DATE=$(curl -s "$URL") + CURRENT_DATE=$(TZ="$TZ" date +%F) + + if [ "$SCHEDULED_DATE" == "$CURRENT_DATE" ]; then + echo "update_due=true" >> "$GITHUB_OUTPUT" + echo "scheduled_date=$SCHEDULED_DATE" >> "$GITHUB_OUTPUT" + else + echo "update_due=false" >> "$GITHUB_OUTPUT" + fi + + update_branch: + needs: check_update + if: ${{ needs.check_update.outputs.update_due == 'true' }} + runs-on: ubuntu-latest + steps: + - name: Checkout Staging + uses: actions/checkout@v4 + with: + ref: ${{ env.BRANCH_STAGING }} + fetch-depth: 0 + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + + - name: Configure Git Identity + run: | + git config --global user.name "$GIT_NAME" + git config --global user.email "$GIT_EMAIL" + + - name: Update README and Cleanup + env: + SCHEDULED_DATE: ${{ needs.check_update.outputs.scheduled_date }} + run: | + DAY=$(TZ="$TZ" date +'%d' | sed 's/^0//') + case "$DAY" in + 1|21|31) SUFFIX="st" ;; + 2|22) SUFFIX="nd" ;; + 3|23) SUFFIX="rd" ;; + *) SUFFIX="th" ;; + esac + + MONTH=$(TZ="$TZ" date +'%B') + YEAR=$(TZ="$TZ" date +'%Y') + DATE_FMT="${MONTH} ${DAY}${SUFFIX}, ${YEAR}" + DATE_ESCAPED=$(printf '%s' "$DATE_FMT" | sed -E 's/ /%20/g; s/,/%2C/g') + + sed -i -E "s|(Last%20Updated-)[^-)]*|\1${DATE_ESCAPED}|g" README.md + + git rm -f "$UPDATE_FILE" + git add README.md + git commit -m "Updated README date to ${DATE_FMT}" + + git reset --soft HEAD~2 + ORIGINAL_MSG=$(git log -1 --pretty=%B HEAD) + + COMMIT_PHX=$(TZ="$TZ" date -d "$SCHEDULED_DATE 12:00" +"%Y-%m-%dT%H:%M:%S %z") + GIT_COMMITTER_DATE="$COMMIT_PHX" GIT_AUTHOR_DATE="$COMMIT_PHX" git commit -m "$ORIGINAL_MSG" + + - name: Wait Until Noon ${{ env.TZ }} + run: | + NOW=$(TZ="$TZ" date +%s) + TARGET=$(TZ="$TZ" date -d "12:00" +%s) + + if [ "$NOW" -lt "$TARGET" ]; then + sleep $((TARGET - NOW)) + fi + + - name: Push and Sync Branches + run: | + git push origin "$BRANCH_STAGING" --force + git fetch origin "$BRANCH_FROGPILOT:$BRANCH_FROGPILOT" + git push origin "$BRANCH_FROGPILOT:$BRANCH_PREVIOUS" --force + git push origin "$BRANCH_STAGING:$BRANCH_FROGPILOT" --force diff --git a/.github/workflows/update_tinygrad.yaml b/.github/workflows/update_tinygrad.yaml new file mode 100644 index 000000000..101fa3edb --- /dev/null +++ b/.github/workflows/update_tinygrad.yaml @@ -0,0 +1,95 @@ +name: Update Tinygrad + +on: + workflow_dispatch: + inputs: + runner: + description: "Select runner" + type: choice + options: + - c3 + - c3x + default: "c3" + required: true + +env: + GIT_EMAIL: "91348155+FrogAi@users.noreply.github.com" + GIT_NAME: "James" + GITLAB_REPO_DIR: "FrogPilot-Resources" + GITLAB_URL: "gitlab.com/FrogAi/FrogPilot-Resources.git" + OPENPILOT_DIR: "/data/openpilot" + +jobs: + update_tinygrad: + runs-on: [self-hosted, "${{ inputs.runner }}"] + steps: + - name: Get Version + id: get_version + run: | + VERSION=$(grep -oP '^VERSION\s*=\s*"\K[^"]+' "$OPENPILOT_DIR/frogpilot/assets/model_manager.py") + echo "VERSION=$VERSION" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + - name: Setup Workspace and Clone GitLab + id: setup + env: + GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} + run: | + WORK_DIR="$RUNNER_TEMP/frogpilot_tinygrad" + rm -rf "$WORK_DIR" && mkdir -p "$WORK_DIR" + echo "work_dir=$WORK_DIR" >> "$GITHUB_OUTPUT" + + cd "$WORK_DIR" + git clone --depth 1 --branch Tinygrad "https://oauth2:${GITLAB_TOKEN}@$GITLAB_URL" + + - name: Create Tinygrad Archive + working-directory: ${{ env.OPENPILOT_DIR }} + env: + WORK_DIR: ${{ steps.setup.outputs.work_dir }} + VERSION: ${{ steps.get_version.outputs.version }} + run: | + set -euo pipefail + + ARCHIVE_DEST="$WORK_DIR/$GITLAB_REPO_DIR" + ARCHIVE_NAME="Tinygrad_$VERSION.tar.gz" + + DUMMY_DIR=$(mktemp -d) + touch "$DUMMY_DIR/SConscript" + + tar -czf "$ARCHIVE_DEST/$ARCHIVE_NAME" \ + --exclude="*.a" \ + --exclude="*.o" \ + --exclude="*.onnx" \ + --exclude="*__pycache__*" \ + --exclude="*tests*" \ + --exclude="selfdrive/modeld/SConscript" \ + selfdrive/modeld tinygrad_repo \ + -C "$DUMMY_DIR" \ + --transform 's|^SConscript$|selfdrive/modeld/SConscript|' \ + SConscript + + rm -rf "$DUMMY_DIR" + + - name: Push Updated Tinygrad + working-directory: ${{ steps.setup.outputs.work_dir }}/${{ env.GITLAB_REPO_DIR }} + env: + VERSION: ${{ steps.get_version.outputs.version }} + run: | + git config user.name "$GIT_NAME" + git config user.email "$GIT_EMAIL" + + git add Tinygrad_*.tar.gz + + if git diff --staged --quiet; then + echo "No changes to commit." + else + git commit -m "Updated Tinygrad: $VERSION" + git push origin Tinygrad + fi + + - name: Cleanup Temporary Files + if: always() + env: + WORK_DIR: ${{ steps.setup.outputs.work_dir }} + run: | + rm -rf "$WORK_DIR" diff --git a/frogpilot/tools/compile_models.py b/frogpilot/tools/compile_models.py new file mode 100644 index 000000000..4cb6e0cfd --- /dev/null +++ b/frogpilot/tools/compile_models.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +import os +import requests +import sys + +from pathlib import Path + +from openpilot.common.basedir import BASEDIR + +from openpilot.frogpilot.common.frogpilot_download_utilities import download_file, verify_download +from openpilot.frogpilot.assets.model_manager import CANCEL_DOWNLOAD_PARAM, DOWNLOAD_PROGRESS_PARAM, MODEL_DOWNLOAD_PARAM +from openpilot.frogpilot.common.frogpilot_utilities import delete_file, run_cmd +from openpilot.frogpilot.common.frogpilot_variables import MODELS_PATH + +METADATA_SCRIPT = Path(BASEDIR) / "frogpilot/tinygrad_modeld/get_model_metadata.py" +TINYGRAD_REPO_PATH = Path(BASEDIR) / "tinygrad_repo" + +MODELS_SOURCE = "https://api.github.com/repos/FrogAi/FrogPilot-Resources/contents/uncompiled?ref=Models" +MODELS_SOURCE_RAW = "https://raw.githubusercontent.com/FrogAi/FrogPilot-Resources/Models/uncompiled" + +COMPILED_DIR = Path(MODELS_PATH) / "compiled" +UNCOMPILED_DIR = Path(MODELS_PATH) / "uncompiled_downloads" + +def compile_model(onnx_path): + onnx_path = Path(onnx_path).resolve() + compiled_path = COMPILED_DIR / f"{onnx_path.stem}_tinygrad.pkl" + + env = os.environ.copy() + env["PYTHONPATH"] = f"{env.get('PYTHONPATH','')}:{TINYGRAD_REPO_PATH}" + + run_cmd([sys.executable, str(TINYGRAD_REPO_PATH / "examples/openpilot/compile3.py"), str(onnx_path), str(compiled_path)], f"{onnx_path.name} compiled successfully!", "Failed to compile the model...", env=env) + run_cmd([sys.executable, str(METADATA_SCRIPT), str(onnx_path)], f"Successfully extracted metadata from {onnx_path.name}!", f"Failed to extract metadata from {onnx_path.name}...") + + delete_file(onnx_path) + +def download_models(): + session = requests.Session() + session.headers.update({"Accept-Language": "en"}) + session.headers.update({"User-Agent": "frogpilot-model-compiler/1.0 (https://github.com/FrogAi/FrogPilot)"}) + + COMPILED_DIR.mkdir(parents=True, exist_ok=True) + UNCOMPILED_DIR.mkdir(parents=True, exist_ok=True) + + local_paths = [] + for name in list_remote_onnx_files(session): + url = f"{MODELS_SOURCE_RAW}/{name}" + destination = UNCOMPILED_DIR / name + + download_file(CANCEL_DOWNLOAD_PARAM, destination, DOWNLOAD_PROGRESS_PARAM, url, MODEL_DOWNLOAD_PARAM, session) + + if not verify_download(destination, url, session): + continue + + print(f"{name} downloaded!") + + local_paths.append(destination) + return local_paths + +def list_remote_onnx_files(session): + response = session.get(MODELS_SOURCE, timeout=10) + response.raise_for_status() + + entries = response.json() + return sorted(entry["name"] for entry in entries if entry["name"].lower().endswith(".onnx")) + +def main(): + downloaded_models = download_models() + for path in downloaded_models: + compile_model(path) + + if UNCOMPILED_DIR.exists(): + delete_file(UNCOMPILED_DIR) + +if __name__ == "__main__": + main()