From a48d43ec2b7413e9d82d6063c283d91f6a3520f9 Mon Sep 17 00:00:00 2001 From: DevTekVE Date: Thu, 19 Dec 2024 15:32:40 +0100 Subject: [PATCH] CI: adding recurring sync to lfs (#489) Allows us to automatically keep our LFS in sync with comma's and also to manually perform a sync if we need to. Even able to sync the LFS from a given commit hash or a given branch. Useful for model stuff. --- .github/workflows/lfs-maintenance.yaml | 72 ++++++++++++++ lfs-comma-pull.sh | 125 ------------------------- 2 files changed, 72 insertions(+), 125 deletions(-) create mode 100644 .github/workflows/lfs-maintenance.yaml delete mode 100755 lfs-comma-pull.sh diff --git a/.github/workflows/lfs-maintenance.yaml b/.github/workflows/lfs-maintenance.yaml new file mode 100644 index 000000000..d2e7bb2e7 --- /dev/null +++ b/.github/workflows/lfs-maintenance.yaml @@ -0,0 +1,72 @@ +name: Sync comma's LFS + +env: + LFS_URL: 'https://gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git/info/lfs' + LFS_PUSH_URL: 'ssh://git@gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git' + +on: + schedule: + - cron: '0 0 * * *' # Runs at 00:00 UTC every day + push: + branches: + - 'master-new' + pull_request: + branches: + - 'master-new' + workflow_dispatch: # enables manual triggering + inputs: + upstream_branch: + default: 'master' + type: string + +jobs: + sync: + runs-on: ubuntu-latest + # Skip if PR is in draft mode + if: github.event_name != 'pull_request' || (github.event_name == 'pull_request' && github.event.pull_request.draft == false) + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + repository: 'commaai/openpilot' + ref: ${{ inputs.upstream_branch }} + + - name: LFS Fetch + run: | + git lfs fetch + + - name: Set up Git + run: | + git config --global user.name 'GitHub Action' + git config --global user.email 'action@github.com' + + - name: Set up SSH + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Add GitLab public keys + run: | + ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts + + - name: Ensure branch + run: | + if git symbolic-ref -q HEAD >/dev/null; then + echo "Already on a branch, proceeding with push" + else + echo "Detached HEAD state detected, creating temporary branch" + git checkout -b temp_branch + fi + + - name: Update LFS Config + run: | + echo '[lfs]' > .lfsconfig + echo ' url = ${{ env.LFS_URL }}' >> .lfsconfig + echo ' pushurl = ${{ env.LFS_PUSH_URL }}' >> .lfsconfig + echo ' locksverify = false' >> .lfsconfig + + - name: Push LFS + id: sync-and-commit + run: | + git lfs ls-files -l + git lfs push --all origin \ No newline at end of file diff --git a/lfs-comma-pull.sh b/lfs-comma-pull.sh deleted file mode 100755 index 5219e757e..000000000 --- a/lfs-comma-pull.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env bash - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -# Parse command line arguments -FETCH_ALL=0 -while [[ "$#" -gt 0 ]]; do - case $1 in - -a|--all) FETCH_ALL=1 ;; - *) echo "Unknown parameter: $1"; exit 1 ;; - esac - shift -done - -# Function to check if a command exists -command_exists() { - command -v "$1" >/dev/null 2>&1 -} - -# Check if git-lfs is installed and initialized -if ! command -v git-lfs >/dev/null 2>&1; then - echo -e "${RED}Error: git-lfs is not installed${NC}" - echo "Please install git-lfs first:" - echo " brew install git-lfs # On macOS" - echo " apt-get install git-lfs # On Ubuntu/Debian" - exit 1 -fi - -# Function to get LFS objects that need syncing -get_missing_lfs_objects() { - git lfs ls-files | awk '$2 == "-" {$1=""; $2=""; sub(/^[ \t]+/, ""); print}' -} - -# Function to get all LFS objects -get_all_lfs_objects() { - git lfs ls-files | awk '{$1=""; $2=""; sub(/^[ \t]+/, ""); print}' -} - -# Clean up any existing LFS config -echo -e "${YELLOW}Cleaning up existing LFS configurations...${NC}" -git config --unset-all lfs.url -git config --unset-all lfs.pushurl -git config --local --unset-all lfs.url -git config --local --unset-all lfs.pushurl - -# Backup current LFS config -if [ -f .lfsconfig ]; then - cp .lfsconfig .lfsconfig.backup -fi - -# Switch to upstream config and pull missing objects -echo -e "${YELLOW}Using upstream LFS configuration...${NC}" -if [ -f .lfsconfig-comma ]; then - cp .lfsconfig-comma .lfsconfig - echo -e "${GREEN}Switched to upstream LFS configuration${NC}" - - if [ $FETCH_ALL -eq 1 ]; then - echo -e "${YELLOW}Fetching all LFS objects (including already present)...${NC}" - - git lfs fetch --all || { - echo -e "${RED}Failed to fetch LFS objects${NC}" - if [ -f .lfsconfig.backup ]; then - mv .lfsconfig.backup .lfsconfig - fi - exit 1 - } - - git lfs checkout || { - echo -e "${RED}Failed to checkout LFS objects${NC}" - if [ -f .lfsconfig.backup ]; then - mv .lfsconfig.backup .lfsconfig - fi - exit 1 - } - - echo -e "${GREEN}Successfully fetched all LFS objects${NC}" - else - # Get list of missing LFS objects - MISSING_FILES=$(get_missing_lfs_objects) - - if [ ! -z "$MISSING_FILES" ]; then - echo -e "${YELLOW}Missing files to fetch:${NC}" - echo "$MISSING_FILES" - - git lfs fetch || { - echo -e "${RED}Failed to fetch LFS objects${NC}" - if [ -f .lfsconfig.backup ]; then - mv .lfsconfig.backup .lfsconfig - fi - exit 1 - } - - git lfs checkout || { - echo -e "${RED}Failed to checkout LFS objects${NC}" - if [ -f .lfsconfig.backup ]; then - mv .lfsconfig.backup .lfsconfig - fi - exit 1 - } - - echo -e "${GREEN}Successfully fetched missing LFS objects${NC}" - else - echo -e "${GREEN}No missing LFS objects to fetch${NC}" - fi - fi -else - echo -e "${RED}Warning: .lfsconfig-comma not found${NC}" -fi - -# Restore original config -if [ -f .lfsconfig.backup ]; then - mv .lfsconfig.backup .lfsconfig - echo -e "${GREEN}Restored original LFS configuration${NC}" -else - rm -f .lfsconfig -fi - -echo -e "${GREEN}LFS sync completed successfully${NC}" - -# Continue with the commit -exit 0