ci: prebuilt process improvement & tag of staging prebuilt source (#1190)

* ci: add validate-test-on-staging-c3 branch to deployment triggers and enhance stable branch handling

* fix long overdue mistake lol

* ci: add condition to wait for start on push events in build workflow

* Fix extra version identifier

* no need for this, i validated what I needed

* only care for release tags, not any

* fix: update versioning logic to use build date and run number for tagging

* fix: update tagging logic and enhance commit message format in build scripts

* fix: refine tagging condition to exclude tag pushes for stable branches

* fix: add extra version identifier to output for better version tracking

* trying to keep things clean and simple

* bugfix

.github/workflows/sunnypilot-build-prebuilt.yaml

@@ -9,12 +9,6 @@ env:
   
   # Branch configurations
   STAGING_C3_SOURCE_BRANCH: ${{ vars.STAGING_C3_SOURCE_BRANCH || 'master' }} # vars are set on repo settings.
-  DEV_C3_SOURCE_BRANCH: ${{ vars.DEV_C3_SOURCE_BRANCH || 'master-dev-c3-new' }} # vars are set on repo settings.
-  
-  # Target branch configurations
-  STAGING_TARGET_BRANCH: ${{ vars.STAGING_TARGET_BRANCH || 'staging-c3-new' }} # vars are set on repo settings.
-  DEV_TARGET_BRANCH: ${{ vars.DEV_TARGET_BRANCH || 'dev-c3-new' }} # vars are set on repo settings.
-  RELEASE_TARGET_BRANCH: ${{ vars.RELEASE_TARGET_BRANCH || 'release-c3-new' }} # vars are set on repo settings.
   
   # Runtime configuration
   SOURCE_BRANCH: "${{ github.head_ref || github.ref_name }}"
@@ -22,7 +16,7 @@ env:
 on:
   push:
     branches: [ master, master-dev-c3-new ]
-    tags: [ '*' ]
+    tags: [ 'release/*' ]
   pull_request_target:
     types: [ labeled ]
   workflow_dispatch:
@@ -34,9 +28,72 @@ on:
         default: false
 
 jobs:
+  prepare_strategy:
+    runs-on: ubuntu-24.04
+    if: (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
+    outputs:
+      environment: ${{ steps.strategy.outputs.environment }}
+      new_branch: ${{ steps.strategy.outputs.new_branch }}
+      extra_version_identifier: ${{ steps.strategy.outputs.extra_version_identifier }}
+      version: ${{ steps.strategy.outputs.version }}
+      cancel_publish_in_progress: ${{ steps.strategy.outputs.cancel_publish_in_progress }}
+      publish_concurrency_group: ${{ steps.strategy.outputs.publish_concurrency_group }}
+      is_stable_branch: ${{ steps.strategy.outputs.is_stable_branch }}
+      build: ${{ steps.strategy.outputs.build }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Extract deploy strategy
+        id: strategy
+        run: |
+          echo '::group::Strategy Extraction'
+          BRANCH="${{ github.head_ref || github.ref_name }}"
+          echo "Current branch: $BRANCH"
+
+          STRATEGY_JSON='${{ vars.DEPLOY_STRATEGY }}'
+          CONFIG=$(echo "$STRATEGY_JSON" | jq -r --arg branch "$BRANCH" '
+            .configs[] | select(.branch == $branch)
+          ')
+
+          BUILD="$(date '+%Y.%m.%d')-${{ github.run_number }}"
+          if [[ -z "$CONFIG" || "$CONFIG" == "null" ]]; then
+            echo "No exact strategy match found. Falling back to feature/fork logic."
+            IS_FORK="${{ github.event.pull_request.head.repo.fork && 'true' || 'false' }}"
+            FORK_SUFFIX=$( [[ "$IS_FORK" == "true" ]] && echo "-fork" || echo "" )
+            NEW_BRANCH="${BRANCH}${FORK_SUFFIX}-prebuilt"
+
+            echo "new_branch=$NEW_BRANCH" >> $GITHUB_OUTPUT
+            echo "version=$BUILD" >> $GITHUB_OUTPUT
+            echo "cancel_publish_in_progress=true" >> $GITHUB_OUTPUT
+            echo "publish_concurrency_group=publish-${BRANCH}" >> $GITHUB_OUTPUT
+            echo "environment=feature-branch" >> $GITHUB_OUTPUT
+            echo "extra_version_identifier=feature-branch" >> $GITHUB_OUTPUT
+          else
+          echo "Matched config: $CONFIG"
+            environment=$(echo "$CONFIG" | jq -r '.environment')
+            echo "environment=$environment" >> $GITHUB_OUTPUT
+            echo "new_branch=$(echo "$CONFIG" | jq -r '.target_branch')" >> $GITHUB_OUTPUT
+            cancel="$(echo "$CONFIG" | jq -r '.cancel_publish_in_progress')";
+            echo "cancel_publish_in_progress=$( [ "$cancel" = "null" ] && echo "true" || echo $cancel)" >> $GITHUB_OUTPUT
+            echo "publish_concurrency_group=publish-${BRANCH}$( [ "$cancel" = "null" ] || [ "$cancel" = "true" ] || echo "${{ github.sha }}" )" >> $GITHUB_OUTPUT
+          
+            is_stable_branch="$(echo "$CONFIG" | jq -r '.stable_branch // false')";
+            echo "is_stable_branch=$is_stable_branch" >> $GITHUB_OUTPUT
+
+            stable_version=$(cat common/version.h | grep COMMA_VERSION | sed -e 's/[^0-9|.]//g');
+            echo "version=$([ "$is_stable_branch" = "true" ] && echo "$stable_version" || echo "$BUILD")" >> $GITHUB_OUTPUT
+            echo "extra_version_identifier=${environment}" >> $GITHUB_OUTPUT
+          fi
+          echo "build=$BUILD" >> $GITHUB_OUTPUT
+          cat $GITHUB_OUTPUT
+  
   validate_tests:
     runs-on: ubuntu-24.04
-    if: ((github.event_name == 'workflow_dispatch' && inputs.wait_for_tests) || contains(github.event_name, 'pull_request') && (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
+    needs: [ prepare_strategy ]
+    if: ${{
+      ((github.event_name == 'workflow_dispatch' && inputs.wait_for_tests) ||
+      (github.event_name == 'push' && needs.prepare_strategy.outputs.is_stable_branch == 'true') ||
+      contains(github.event_name, 'pull_request') && (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
+      }}
     steps:
       - uses: actions/checkout@v4
       - name: Wait for Tests
@@ -44,19 +101,26 @@ jobs:
         with:
           workflow: selfdrive_tests.yaml # The workflow file to monitor
           github-token: ${{ secrets.GITHUB_TOKEN }}
+          should-wait-for-start: ${{ github.event_name == 'push' && 'true' || 'false' }}
 
   build:
-    needs: [ validate_tests ]
+    needs: [ validate_tests, prepare_strategy ]
     concurrency:
       group: build-${{ github.head_ref || github.ref_name }}
       cancel-in-progress: false
     runs-on: [self-hosted, tici]
     outputs:
-      new_branch: ${{ steps.set-env.outputs.new_branch }}
-      version: ${{ steps.set-env.outputs.version }}
-      extra_version_identifier: ${{ steps.set-env.outputs.extra_version_identifier }}
-      commit_sha: ${{ steps.set-env.outputs.commit_sha }}
-    if: ${{ (always() && !failure() && !cancelled()) && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
+      new_branch: ${{ needs.prepare_strategy.outputs.new_branch }}
+      version: ${{ needs.prepare_strategy.outputs.version }}
+      extra_version_identifier: ${{ needs.prepare_strategy.outputs.extra_version_identifier }}
+      commit_sha: ${{ github.sha }}
+    if: ${{
+      (always() && !cancelled() && !failure()) &&
+      needs.prepare_strategy.result == 'success' &&
+      (needs.validate_tests.result == 'success' || needs.validate_tests.result == 'skipped') &&
+      (!contains(github.event_name, 'pull_request') ||
+      (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) 
+      }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -77,61 +141,12 @@ jobs:
             scons-${{ runner.os }}-${{ runner.arch }}-${{ env.STAGING_C3_SOURCE_BRANCH }}
             scons-${{ runner.os }}-${{ runner.arch }}
 
-      - name: Set Feature Branch Prebuilt Configuration
-        id: set_feature_configuration
-        if: (
-          !(env.SOURCE_BRANCH == env.DEV_C3_SOURCE_BRANCH) &&
-          !(env.SOURCE_BRANCH == env.STAGING_C3_SOURCE_BRANCH) &&
-          !(startsWith(github.ref, 'refs/tags/'))
-          )
-        run: |
-          echo "NEW_BRANCH=${{ env.SOURCE_BRANCH }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}-prebuilt" >> $GITHUB_ENV
-          echo "VERSION=$(date '+%Y.%m.%d')-${{ github.run_number }}" >> $GITHUB_ENV
-
-      - name: Set dev-c3-new prebuilt Configuration
-        id: set_dev_configuration
-        if: (
-          steps.set_feature_configuration.outcome == 'skipped' &&
-          env.SOURCE_BRANCH == env.DEV_C3_SOURCE_BRANCH
-          )
-        run: |
-          echo "NEW_BRANCH=${{ env.DEV_TARGET_BRANCH }}" >> $GITHUB_ENV
-          echo "VERSION=$(date '+%Y.%m.%d')-${{ github.run_number }}" >> $GITHUB_ENV
-          echo "EXTRA_VERSION_IDENTIFIER=${{ github.run_number }}" >> $GITHUB_ENV
-
-      - name: Set staging-c3-new prebuilt Configuration
-        id: set_staging_configuration
-        if: (
-          steps.set_feature_configuration.outcome == 'skipped' &&
-          !contains(github.event_name, 'pull_request') &&
-          steps.set_dev_configuration.outcome == 'skipped' &&
-          (env.SOURCE_BRANCH == env.STAGING_C3_SOURCE_BRANCH)
-          )
-        run: |
-          echo "NEW_BRANCH=${{ env.STAGING_TARGET_BRANCH }}" >> $GITHUB_ENV
-          echo "EXTRA_VERSION_IDENTIFIER=staging" >> $GITHUB_ENV
-          echo "VERSION=$(cat common/version.h | grep COMMA_VERSION | sed -e 's/[^0-9|.]//g')-staging" >> $GITHUB_ENV
-
-      - name: Set release-c3-new prebuilt Configuration
-        id: set_tag_configuration
-        if: (
-          steps.set_feature_configuration.outcome == 'skipped' &&
-          !contains(github.event_name, 'pull_request') &&
-          steps.set_staging_configuration.outcome == 'skipped' &&
-          startsWith(github.ref, 'refs/tags/')
-          )
-        run: |
-          echo "NEW_BRANCH=${{ env.RELEASE_TARGET_BRANCH }}" >> $GITHUB_ENV
-          echo "EXTRA_VERSION_IDENTIFIER=release" >> $GITHUB_ENV
-          echo "VERSION=$(cat common/version.h | grep COMMA_VERSION | sed -e 's/[^0-9|.]//g')-release" >> $GITHUB_ENV
-
       - name: Set environment variables
         id: set-env
         run: |
-          # Write to GITHUB_OUTPUT from environment variables
-          echo "new_branch=$NEW_BRANCH" >> $GITHUB_OUTPUT
-          [[ ! -z "$EXTRA_VERSION_IDENTIFIER" ]] && echo "extra_version_identifier=$EXTRA_VERSION_IDENTIFIER" >> $GITHUB_OUTPUT
-          [[ ! -z "$VERSION" ]] && echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "new_branch=${{ needs.prepare_strategy.outputs.new_branch }}" >> $GITHUB_OUTPUT
+          echo "version=${{ needs.prepare_strategy.outputs.version }}" >> $GITHUB_OUTPUT
+          echo "extra_version_identifier=${{ needs.prepare_strategy.outputs.extra_version_identifier }}" >> $GITHUB_OUTPUT
           echo "commit_sha=${{ github.sha }}" >> $GITHUB_OUTPUT
           
           # Set up common environment
@@ -226,15 +241,19 @@ jobs:
         if: always()
         run: |
           PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
-
+  
+  
   publish:
     concurrency:
-      group: publish-${{ github.head_ref || github.ref_name }}
-      cancel-in-progress: true
-    if: ${{ (always() && !failure() && !cancelled()) && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
-    needs: [ build ]
+      # We do a bit of a hack here to avoid canceling the publishing job if a new commit comes in while we're publishing by adding the sha to the group name.
+      # This means that if multiple commits come in while we're publishing, they will be queued up and publish one after the other.
+      # Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time.
+      group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}
+      cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }}
+    if: ${{ (always() && !cancelled() && !failure()) && needs.build.result == 'success' && needs.prepare_strategy.result == 'success' && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
+    needs: [ build, prepare_strategy ]
     runs-on: ubuntu-24.04
-    environment: ${{ (contains(fromJSON(vars.AUTO_DEPLOY_PREBUILT_BRANCHES), github.head_ref || github.ref_name) || contains(github.event.pull_request.labels.*.name, 'prebuilt')) && 'auto-deploy' || 'feature-branch' }}
+    environment: ${{ needs.prepare_strategy.outputs.environment }}
     steps:
       - uses: actions/checkout@v4
 
@@ -266,7 +285,7 @@ jobs:
             "${{ needs.build.outputs.new_branch }}" \
             "${{ needs.build.outputs.version }}" \
             "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
-            "-${{ needs.build.outputs.extra_version_identifier }}"
+            "${{ needs.build.outputs.extra_version_identifier }}"
 
           echo ""
           echo "----   ℹ️ To update the list of branches that auto deploy prebuilts  -----"
@@ -274,11 +293,18 @@ jobs:
           echo "1. Go to: ${{ github.server_url }}/${{ github.repository }}/settings/variables/actions/AUTO_DEPLOY_PREBUILT_BRANCHES"
           echo "2. Current value: ${{ vars.AUTO_DEPLOY_PREBUILT_BRANCHES }}"
           echo "3. Update as needed (JSON array with no spaces)"
+      
+      - name: Tag ${{ needs.prepare_strategy.outputs.environment }}
+        if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }}
+        run: |
+            TAG="${{ needs.prepare_strategy.outputs.environment }}/${{ needs.prepare_strategy.outputs.version }}/${{ needs.prepare_strategy.outputs.build }}"
+            git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.build.outputs.build }}."
+            git push -f origin ${TAG}
 
   notify:
     needs: [ build, publish ]
     runs-on: ubuntu-24.04
-    if: ${{ (always() && !failure() && !cancelled()) && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
+    if: ${{ (always() && !cancelled() && !failure()) && needs.publish.result == 'success' && !failure() && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
     steps:
       - uses: actions/checkout@v4
       - name: Setup Alpine Linux environment

release/ci/publish.sh

@@ -51,26 +51,19 @@ git fetch origin $DEV_BRANCH || (git checkout -b $DEV_BRANCH && git commit --all
 
 echo "[-] committing version $VERSION T=$SECONDS"
 git add -f .
-git commit -a -m "sunnypilot v$VERSION release"
-git branch --set-upstream-to=origin/$DEV_BRANCH
 
 # include source commit hash and build date in commit
 GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD)
 DATETIME=$(date '+%Y-%m-%dT%H:%M:%S')
-SP_VERSION=$(cat $SOURCE_DIR/common/version.h | awk -F\" '{print $2}')
+SP_VERSION=$(awk -F\" '{print $2}' $SOURCE_DIR/common/version.h)
 
-# Add built files to git
-git add -f .
-if [ "$EXTRA_VERSION_IDENTIFIER" = "-release" ] || [ "$EXTRA_VERSION_IDENTIFIER" = "-staging" ]; then
-  export VERSION=${VERSION%"$EXTRA_VERSION_IDENTIFIER"}
-  git commit --amend -m "sunnypilot v$VERSION"
-else
-  git commit --amend -m "sunnypilot v$VERSION
-  version: sunnypilot v$SP_VERSION release
-  date: $DATETIME
-  master commit: $GIT_HASH
-  "
-fi
+# Commit with detailed message
+git commit -a -m "sunnypilot v$VERSION
+version: sunnypilot v$SP_VERSION (${EXTRA_VERSION_IDENTIFIER})
+date: $DATETIME
+master commit: $GIT_HASH
+"
+git branch --set-upstream-to=origin/$DEV_BRANCH
 git branch -m $DEV_BRANCH
 
 # Push!

selfdrive/controls/lib/longitudinal_planner.py

@@ -102,8 +102,6 @@ class LongitudinalPlanner(LongitudinalPlannerSP):
       if not self.mlsim:
         self.mpc.mode = dec_mpc_mode
 
-    self.handle_mode_transition(mode)
-
     if len(sm['carControl'].orientationNED) == 3:
       accel_coast = get_coast_accel(sm['carControl'].orientationNED[1])
     else:
@@ -179,7 +177,7 @@ class LongitudinalPlanner(LongitudinalPlannerSP):
       output_a_target = output_a_target_mpc
       self.output_should_stop = output_should_stop_mpc
     else:
-      output_a_target = self.blend_accel_transition(output_a_target_mpc, output_a_target_e2e, v_ego)
+      output_a_target = min(output_a_target_mpc, output_a_target_e2e)
       self.output_should_stop = output_should_stop_e2e or output_should_stop_mpc
 
     for idx in range(2):

sunnypilot/selfdrive/controls/lib/longitudinal_planner.py

@@ -4,11 +4,9 @@ Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
 This file is part of sunnypilot and is licensed under the MIT License.
 See the LICENSE.md file in the root directory for more details.
 """
-import math
 
 from cereal import messaging, custom
 from opendbc.car import structs
-from opendbc.car.interfaces import ACCEL_MIN
 from openpilot.sunnypilot.selfdrive.controls.lib.dec.dec import DynamicExperimentalController
 from openpilot.sunnypilot.models.helpers import get_active_bundle
 
@@ -18,7 +16,6 @@ DecState = custom.LongitudinalPlanSP.DynamicExperimentalControl.DynamicExperimen
 class LongitudinalPlannerSP:
   def __init__(self, CP: structs.CarParams, mpc):
     self.dec = DynamicExperimentalController(CP, mpc)
-    self.transition_init()
     self.generation = int(model_bundle.generation) if (model_bundle := get_active_bundle()) else None
 
   @property
@@ -32,32 +29,6 @@ class LongitudinalPlannerSP:
 
     return self.dec.mode()
 
-  def transition_init(self) -> None:
-    self._transition_counter = 0
-    self._transition_steps = 20
-    self._last_mode = 'acc'
-
-  def handle_mode_transition(self, mode: str) -> None:
-    if self._last_mode != mode:
-      if mode == 'blended':
-        self._transition_counter = 0
-      self._last_mode = mode
-
-  def blend_accel_transition(self, mpc_accel: float, e2e_accel: float, v_ego: float) -> float:
-    if self.dec.enabled():
-      if self._transition_counter < self._transition_steps:
-        self._transition_counter += 1
-        progress = self._transition_counter / self._transition_steps
-        if v_ego > 5.0 and e2e_accel < 0.0:
-          if mpc_accel < 0.0 and e2e_accel > mpc_accel:
-            return mpc_accel
-          # use k3.0 and normalize midpoint at 0.5
-          sigmoid = 1.0 / (1.0 + math.exp(-3.0 * (abs(e2e_accel / ACCEL_MIN) - 0.5)))
-          blend_factor = 1.0 - (1.0 - progress) * (1.0 - sigmoid)
-          blended = mpc_accel + (e2e_accel - mpc_accel) * blend_factor
-          return blended
-    return min(mpc_accel, e2e_accel)
-
   def update(self, sm: messaging.SubMaster) -> None:
     self.dec.update(sm)