CI: adding recurring sync to lfs (#489)

Allows us to automatically keep our LFS in sync with comma's and also to manually perform a sync if we need to. Even able to sync the LFS from a given commit hash or a given branch. Useful for model stuff.
2026-08-03 08:41:41 +08:00 · 2024-12-19 15:32:40 +01:00
parent c8fe86c552
commit a48d43ec2b
2 changed files with 72 additions and 125 deletions
@@ -0,0 +1,72 @@
+name: Sync comma's LFS
+
+env:
+  LFS_URL: 'https://gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git/info/lfs'
+  LFS_PUSH_URL: 'ssh://git@gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git'
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Runs at 00:00 UTC every day
+  push:
+    branches:
+      - 'master-new'
+  pull_request:
+    branches:
+      - 'master-new'
+  workflow_dispatch:  # enables manual triggering
+    inputs:
+      upstream_branch:
+        default: 'master'
+        type: string
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    # Skip if PR is in draft mode
+    if: github.event_name != 'pull_request' || (github.event_name == 'pull_request' && github.event.pull_request.draft == false)
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          repository: 'commaai/openpilot'
+          ref: ${{ inputs.upstream_branch }}
+      
+      - name: LFS Fetch
+        run: |
+          git lfs fetch
+
+      - name: Set up Git
+        run: |
+          git config --global user.name 'GitHub Action'
+          git config --global user.email 'action@github.com'
+
+      - name: Set up SSH
+        uses: webfactory/ssh-agent@v0.9.0
+        with:
+          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+
+      - name: Add GitLab public keys
+        run: |
+          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
+      
+      - name: Ensure branch
+        run: |
+          if git symbolic-ref -q HEAD >/dev/null; then
+              echo "Already on a branch, proceeding with push"
+            else
+              echo "Detached HEAD state detected, creating temporary branch"
+            git checkout -b temp_branch
+          fi
+
+      - name: Update LFS Config
+        run: |
+          echo '[lfs]' > .lfsconfig
+          echo '    url = ${{ env.LFS_URL }}' >> .lfsconfig
+          echo '    pushurl = ${{ env.LFS_PUSH_URL }}' >> .lfsconfig
+          echo '    locksverify = false' >> .lfsconfig
+
+      - name: Push LFS
+        id: sync-and-commit
+        run: |
+          git lfs ls-files -l
+          git lfs push --all origin
@@ -1,125 +0,0 @@
-#!/usr/bin/env bash
-
-# Colors for output
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-NC='\033[0m' # No Color
-
-# Parse command line arguments
-FETCH_ALL=0
-while [[ "$#" -gt 0 ]]; do
-  case $1 in
-    -a|--all) FETCH_ALL=1 ;;
-    *) echo "Unknown parameter: $1"; exit 1 ;;
-  esac
-  shift
-done
-
-# Function to check if a command exists
-command_exists() {
-  command -v "$1" >/dev/null 2>&1
-}
-
-# Check if git-lfs is installed and initialized
-if ! command -v git-lfs >/dev/null 2>&1; then
-  echo -e "${RED}Error: git-lfs is not installed${NC}"
-  echo "Please install git-lfs first:"
-  echo "  brew install git-lfs    # On macOS"
-  echo "  apt-get install git-lfs # On Ubuntu/Debian"
-  exit 1
-fi
-
-# Function to get LFS objects that need syncing
-get_missing_lfs_objects() {
-  git lfs ls-files | awk '$2 == "-" {$1=""; $2=""; sub(/^[ \t]+/, ""); print}'
-}
-
-# Function to get all LFS objects
-get_all_lfs_objects() {
-  git lfs ls-files | awk '{$1=""; $2=""; sub(/^[ \t]+/, ""); print}'
-}
-
-# Clean up any existing LFS config
-echo -e "${YELLOW}Cleaning up existing LFS configurations...${NC}"
-git config --unset-all lfs.url
-git config --unset-all lfs.pushurl
-git config --local --unset-all lfs.url
-git config --local --unset-all lfs.pushurl
-
-# Backup current LFS config
-if [ -f .lfsconfig ]; then
-  cp .lfsconfig .lfsconfig.backup
-fi
-
-# Switch to upstream config and pull missing objects
-echo -e "${YELLOW}Using upstream LFS configuration...${NC}"
-if [ -f .lfsconfig-comma ]; then
-  cp .lfsconfig-comma .lfsconfig
-  echo -e "${GREEN}Switched to upstream LFS configuration${NC}"
-
-  if [ $FETCH_ALL -eq 1 ]; then
-    echo -e "${YELLOW}Fetching all LFS objects (including already present)...${NC}"
-
-    git lfs fetch --all || {
-      echo -e "${RED}Failed to fetch LFS objects${NC}"
-      if [ -f .lfsconfig.backup ]; then
-        mv .lfsconfig.backup .lfsconfig
-      fi
-      exit 1
-    }
-
-    git lfs checkout || {
-      echo -e "${RED}Failed to checkout LFS objects${NC}"
-      if [ -f .lfsconfig.backup ]; then
-        mv .lfsconfig.backup .lfsconfig
-      fi
-      exit 1
-    }
-
-    echo -e "${GREEN}Successfully fetched all LFS objects${NC}"
-  else
-    # Get list of missing LFS objects
-    MISSING_FILES=$(get_missing_lfs_objects)
-
-    if [ ! -z "$MISSING_FILES" ]; then
-      echo -e "${YELLOW}Missing files to fetch:${NC}"
-      echo "$MISSING_FILES"
-
-      git lfs fetch || {
-        echo -e "${RED}Failed to fetch LFS objects${NC}"
-        if [ -f .lfsconfig.backup ]; then
-          mv .lfsconfig.backup .lfsconfig
-        fi
-        exit 1
-      }
-
-      git lfs checkout || {
-        echo -e "${RED}Failed to checkout LFS objects${NC}"
-        if [ -f .lfsconfig.backup ]; then
-          mv .lfsconfig.backup .lfsconfig
-        fi
-        exit 1
-      }
-
-      echo -e "${GREEN}Successfully fetched missing LFS objects${NC}"
-    else
-      echo -e "${GREEN}No missing LFS objects to fetch${NC}"
-    fi
-  fi
-else
-  echo -e "${RED}Warning: .lfsconfig-comma not found${NC}"
-fi
-
-# Restore original config
-if [ -f .lfsconfig.backup ]; then
-  mv .lfsconfig.backup .lfsconfig
-  echo -e "${GREEN}Restored original LFS configuration${NC}"
-else
-  rm -f .lfsconfig
-fi
-
-echo -e "${GREEN}LFS sync completed successfully${NC}"
-
-# Continue with the commit
-exit 0