Merge branch 'master' into ccnc-port

This reverts commit 15e5d2efb9.

.github/workflows/sunnypilot-build-prebuilt.yaml

@@ -4,8 +4,12 @@ env:
   BUILD_DIR: "/data/openpilot"
   OUTPUT_DIR: ${{ github.workspace }}/output
   CI_DIR: ${{ github.workspace }}/release/ci
+  SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache
   PUBLIC_REPO_URL: "https://github.com/sunnypilot/sunnypilot"
 
+  # Branch configurations
+  STAGING_SOURCE_BRANCH: 'master'
+
   # Runtime configuration
   SOURCE_BRANCH: "${{ github.head_ref || github.ref_name }}"
 
@@ -105,6 +109,11 @@ jobs:
       group: build-${{ github.head_ref || github.ref_name }}
       cancel-in-progress: false
     runs-on: [self-hosted, tici]
+    outputs:
+      new_branch: ${{ needs.prepare_strategy.outputs.new_branch }}
+      version: ${{ needs.prepare_strategy.outputs.version }}
+      extra_version_identifier: ${{ needs.prepare_strategy.outputs.extra_version_identifier }}
+      commit_sha: ${{ github.sha }}
     if: ${{
       (always() && !cancelled() && !failure()) &&
       needs.prepare_strategy.result == 'success' &&
@@ -120,8 +129,26 @@ jobs:
           repository: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.head.repo.full_name || github.repository }}
       - run: git lfs pull
 
+      - name: Cache SCons
+        uses: actions/cache@v4
+        with:
+          path: ${{env.SCONS_CACHE_DIR}}
+          key: scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}-${{ github.sha }}
+          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
+          #   for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
+          restore-keys: |
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.STAGING_SOURCE_BRANCH }}
+            scons-${{ runner.os }}-${{ runner.arch }}
+
       - name: Set environment variables
+        id: set-env
         run: |
+          echo "new_branch=${{ needs.prepare_strategy.outputs.new_branch }}" >> $GITHUB_OUTPUT
+          echo "version=${{ needs.prepare_strategy.outputs.version }}" >> $GITHUB_OUTPUT
+          echo "extra_version_identifier=${{ needs.prepare_strategy.outputs.extra_version_identifier }}" >> $GITHUB_OUTPUT
+          echo "commit_sha=${{ github.sha }}" >> $GITHUB_OUTPUT
+
           # Set up common environment
           source /etc/profile;
           export UV_PROJECT_ENVIRONMENT=${HOME}/venv
@@ -130,6 +157,9 @@ jobs:
           export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
           uv sync
           printenv >> $GITHUB_ENV
+          if [[ "${{ runner.debug }}" == "1" ]]; then
+            cat $GITHUB_OUTPUT
+          fi
 
       - name: Setup build environment
         run: |
@@ -138,7 +168,7 @@ jobs:
           echo "Starting build stage..."
           echo "BUILD_DIR: ${BUILD_DIR}"
           echo "CI_DIR: ${CI_DIR}"
-          echo "VERSION: ${{ needs.prepare_strategy.outputs.version }}"
+          echo "VERSION: ${{ steps.set-env.outputs.version }}"
           echo "UV_PROJECT_ENVIRONMENT: ${UV_PROJECT_ENVIRONMENT}"
           echo "VIRTUAL_ENV: ${VIRTUAL_ENV}"
           echo "-------"
@@ -150,44 +180,61 @@ jobs:
 
       - name: Build Main Project
         run: |
-          export PYTHONPATH="$BUILD_DIR:$BUILD_DIR/msgq_repo:$BUILD_DIR/opendbc_repo:$BUILD_DIR/rednose_repo:$BUILD_DIR/teleoprtc_repo:$BUILD_DIR/tinygrad_repo"
-          ./tools/release/release_files.py | xargs -0 cp -pR --parents -t "$BUILD_DIR" --
-          # outside the checkout, which is wiped each run. /data/scons_cache is the device's, not ours.
-          SCONS_CACHE="$RUNNER_WORKSPACE/scons_cache"
-          mkdir -p "$SCONS_CACHE"
+          export PYTHONPATH="$BUILD_DIR"
+          ./tools/release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/
           cd $BUILD_DIR
-          echo "Building locationd..."
-          # -j1: parallel rednose generators OOM the device
-          scons -j1 cache_dir="$SCONS_CACHE" --minimal \
-            openpilot/selfdrive/locationd openpilot/sunnypilot/selfdrive/locationd
+          ln -sfn msgq_repo/msgq msgq
+          ln -sfn opendbc_repo/opendbc opendbc
+          ln -sfn rednose_repo/rednose rednose
+          ln -sfn teleoprtc_repo/teleoprtc teleoprtc
+          ln -sfn tinygrad_repo/tinygrad tinygrad
+          sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py
+          echo "Building sunnypilot's modeld_v2..."
+          scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal openpilot/sunnypilot/modeld_v2
+          echo "Building sunnypilot's locationd..."
+          scons -j2 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal openpilot/sunnypilot/selfdrive/locationd
+          echo "Building openpilot's locationd..."
+          scons -j1 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal openpilot/selfdrive/locationd
           echo "Building rest of sunnypilot"
-          /usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal
+          scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal
           touch ${BUILD_DIR}/prebuilt
           if [[ "${{ runner.debug }}" == "1" ]]; then
             ls -la ${BUILD_DIR}
           fi
 
-      - name: Strip release tree
+      - name: Prepare Output
         run: |
-          cd $BUILD_DIR
-          find . -name '*.a' -delete
-          find . -name '*.o' -delete
-          find . -name '*.os' -delete
-          find . -name '*.pyc' -delete
-          find . -name 'moc_*' -delete
-          find . -name '__pycache__' -type d -exec rm -rf {} +
-          find . -name 'SConstruct' -delete
-          find . -name 'SConscript' -delete
-          rm -rf .sconsign.dblite Jenkinsfile tools/release/ release/
-          rm -f openpilot/selfdrive/modeld/models/*.onnx*
-          rm -f openpilot/sunnypilot/modeld*/models/*.onnx*
-          find openpilot/third_party/ -name '*x86*' -exec rm -r {} +
-          find openpilot/third_party/ -name '*Darwin*' -exec rm -r {} +
-          cd -
+          sudo rm -rf ${OUTPUT_DIR}
+          mkdir -p ${OUTPUT_DIR}
+          rsync -am${RUNNER_DEBUG:+v} \
+            --exclude='.sconsign.dblite' \
+            --exclude='*.a' \
+            --exclude='*.o' \
+            --exclude='*.os' \
+            --exclude='*.pyc' \
+            --exclude='moc_*' \
+            --exclude='__pycache__' \
+            --exclude='Jenkinsfile' \
+            --exclude='**/release/' \
+            --exclude='**/.github/' \
+            --exclude='**/openpilot/selfdrive/ui/replay/' \
+            --exclude='**/__pycache__/' \
+            --exclude='${{env.SCONS_CACHE_DIR}}' \
+            --exclude='**/.git/' \
+            --exclude='**/SConstruct' \
+            --exclude='**/SConscript' \
+            --exclude='**/.venv/' \
+            --exclude='openpilot/selfdrive/modeld/models/*.onnx*' \
+            --exclude='openpilot/sunnypilot/modeld*/models/*.onnx*' \
+            --exclude='openpilot/third_party/*x86*' \
+            --exclude='openpilot/third_party/*Darwin*' \
+            --delete-excluded \
+            --chown=comma:comma \
+            ${BUILD_DIR}/ ${OUTPUT_DIR}/
 
       - name: 'Tar.gz files'
         run: |
-          tar czf prebuilt.tar.gz -C ${{ env.BUILD_DIR }} .
+          tar czf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} .
           ls -la prebuilt.tar.gz
 
       - name: 'Upload Artifact'
@@ -195,7 +242,6 @@ jobs:
         with:
           name: prebuilt
           path: prebuilt.tar.gz
-          compression-level: 0
 
       - name: Re-enable powersave
         if: always()
@@ -237,22 +283,29 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          echo '${{ toJSON(needs.prepare_strategy.outputs) }}'
+          echo '${{ toJSON(needs.build.outputs) }}'
           ls -la ${{ env.OUTPUT_DIR }}
 
           ${{ env.CI_DIR }}/publish.sh \
             "${{ github.workspace }}" \
             "${{ env.OUTPUT_DIR }}" \
-            "${{ needs.prepare_strategy.outputs.new_branch }}" \
-            "${{ needs.prepare_strategy.outputs.version }}" \
+            "${{ needs.build.outputs.new_branch }}" \
+            "${{ needs.build.outputs.version }}" \
             "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
-            "${{ needs.prepare_strategy.outputs.extra_version_identifier }}"
+            "${{ needs.build.outputs.extra_version_identifier }}"
+
+          echo ""
+          echo "----   ℹ️ To update the list of branches that auto deploy prebuilts  -----"
+          echo ""
+          echo "1. Go to: ${{ github.server_url }}/${{ github.repository }}/settings/variables/actions/AUTO_DEPLOY_PREBUILT_BRANCHES"
+          echo "2. Current value: ${{ vars.AUTO_DEPLOY_PREBUILT_BRANCHES }}"
+          echo "3. Update as needed (JSON array with no spaces)"
 
       - name: Tag ${{ needs.prepare_strategy.outputs.environment }}
         if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }}
         run: |
             TAG="${{ needs.prepare_strategy.outputs.environment }}/${{ needs.prepare_strategy.outputs.version }}/${{ needs.prepare_strategy.outputs.build }}"
-            git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.prepare_strategy.outputs.build }}."
+            git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.build.outputs.build }}."
             git push -f origin ${TAG}
 
   notify:
@@ -271,6 +324,7 @@ jobs:
       - name: Prepare notification message
         id: message
         run: |
+          TEMPLATE='${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}'
           export VERSION="${{ needs.prepare_strategy.outputs.version }}"
           export branch_name="${{ env.SOURCE_BRANCH }}"
           export new_branch="${{ needs.prepare_strategy.outputs.new_branch }}"
@@ -319,7 +373,7 @@ jobs:
               owner: context.repo.owner,
               repo: context.repo.repo,
               issue_number: prNumber,
-              name: process.env.LABEL
+              name: process.env.LABELf
             });
 
             console.log(`Removed '${process.env.LABEL}' label from PR #${prNumber}`);

release/ci/publish.sh

@@ -52,12 +52,6 @@ git fetch origin $DEV_BRANCH || (git checkout -b $DEV_BRANCH && git commit --all
 echo "[-] committing version $VERSION T=$SECONDS"
 git add -f .
 
-# gitlinks break the release tree on device
-if git ls-files -s | awk '$1 == "160000" { found = 1; print } END { exit !found }'; then
-    echo "Error: submodules found in release tree."
-    exit 1
-fi
-
 # include source commit hash and build date in commit
 GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD)
 DATETIME=$(date '+%Y-%m-%dT%H:%M:%S')