From 0977a91d656660c22874bdf6e91267e8d0a4edd6 Mon Sep 17 00:00:00 2001
From: Adeeb Shihadeh <adeebshihadeh@gmail.com>
Date: Thu, 26 Feb 2026 21:17:00 -0800
Subject: [PATCH] CI for the people: no cache (#37437)

* sympathize with our first time cloners

* venv

* rm compile openpilot

* retry for all

* rm setup action
---
 .github/workflows/auto-cache/action.yaml      | 55 ------------------
 .github/workflows/badges.yaml                 |  2 +-
 .../workflows/compile-openpilot/action.yaml   | 21 -------
 .github/workflows/release.yaml                |  2 +-
 .github/workflows/repo-maintenance.yaml       |  4 +-
 .../workflows/setup-with-retry/action.yaml    | 48 ----------------
 .github/workflows/setup/action.yaml           | 56 -------------------
 .github/workflows/tests.yaml                  | 41 ++++++--------
 tools/op.sh                                   | 24 +++++++-
 9 files changed, 42 insertions(+), 211 deletions(-)
 delete mode 100644 .github/workflows/auto-cache/action.yaml
 delete mode 100644 .github/workflows/compile-openpilot/action.yaml
 delete mode 100644 .github/workflows/setup-with-retry/action.yaml
 delete mode 100644 .github/workflows/setup/action.yaml

diff --git a/.github/workflows/auto-cache/action.yaml b/.github/workflows/auto-cache/action.yaml
deleted file mode 100644
index 42c8f8fd2..000000000
--- a/.github/workflows/auto-cache/action.yaml
+++ /dev/null
@@ -1,55 +0,0 @@
-name: 'automatically cache based on current runner'
-
-inputs:
-  path:
-    description: 'path to cache'
-    required: true
-  key:
-    description: 'key'
-    required: true
-  restore-keys:
-    description: 'restore-keys'
-    required: true
-  save:
-    description: 'whether to save the cache'
-    default: 'true'
-    required: false
-outputs:
-  cache-hit:
-    description: 'cache hit occurred'
-    value: ${{ (contains(runner.name, 'nsc') && steps.ns-cache.outputs.cache-hit) ||
-               (!contains(runner.name, 'nsc') && inputs.save != 'false' && steps.gha-cache.outputs.cache-hit) ||
-               (!contains(runner.name, 'nsc') && inputs.save == 'false' && steps.gha-cache-ro.outputs.cache-hit) }}
-
-runs:
-  using: "composite"
-  steps:
-    - name: setup namespace cache
-      id: ns-cache
-      if: ${{ contains(runner.name, 'nsc') }}
-      uses: namespacelabs/nscloud-cache-action@v1
-      with:
-        path: ${{ inputs.path }}
-
-    - name: setup github cache
-      id: gha-cache
-      if: ${{ !contains(runner.name, 'nsc') && inputs.save != 'false' }}
-      uses: 'actions/cache@v4'
-      with:
-        path: ${{ inputs.path }}
-        key: ${{ inputs.key }}
-        restore-keys: ${{ inputs.restore-keys }}
-
-    - name: setup github cache
-      id: gha-cache-ro
-      if: ${{ !contains(runner.name, 'nsc') && inputs.save == 'false' }}
-      uses: 'actions/cache/restore@v4'
-      with:
-        path: ${{ inputs.path }}
-        key: ${{ inputs.key }}
-        restore-keys: ${{ inputs.restore-keys }}
-
-    # make the directory manually in case we didn't get a hit, so it doesn't fail on future steps
-    - id: scons-cache-setup
-      shell: bash
-      run: mkdir -p ${{ inputs.path }}
diff --git a/.github/workflows/badges.yaml b/.github/workflows/badges.yaml
index 23f2c135d..9b99c4f1f 100644
--- a/.github/workflows/badges.yaml
+++ b/.github/workflows/badges.yaml
@@ -18,7 +18,7 @@ jobs:
     - uses: actions/checkout@v6
       with:
         submodules: true
-    - uses: ./.github/workflows/setup-with-retry
+    - run: ./tools/op.sh setup
     - name: Push badges
       run: |
         python3 selfdrive/ui/translations/create_badges.py
diff --git a/.github/workflows/compile-openpilot/action.yaml b/.github/workflows/compile-openpilot/action.yaml
deleted file mode 100644
index 627b4845a..000000000
--- a/.github/workflows/compile-openpilot/action.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-name: 'compile openpilot'
-
-runs:
-  using: "composite"
-  steps:
-    - shell: bash
-      name: Build openpilot with all flags
-      run: |
-        scons -j$(nproc)
-        release/check-dirty.sh
-    - shell: bash
-      name: Cleanup scons cache and rebuild
-      run: |
-        rm -rf /tmp/scons_cache/*
-        scons -j$(nproc) --cache-populate
-    - name: Save scons cache
-      uses: actions/cache/save@v4
-      if: github.ref == 'refs/heads/master'
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 4d731965d..db0e12234 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -26,6 +26,6 @@ jobs:
       with:
         submodules: true
         fetch-depth: 0
-    - uses: ./.github/workflows/setup-with-retry
+    - run: ./tools/op.sh setup
     - name: Push master-ci
       run: BRANCH=__nightly release/build_stripped.sh
diff --git a/.github/workflows/repo-maintenance.yaml b/.github/workflows/repo-maintenance.yaml
index d2c2447d7..2c5d049e4 100644
--- a/.github/workflows/repo-maintenance.yaml
+++ b/.github/workflows/repo-maintenance.yaml
@@ -16,7 +16,7 @@ jobs:
       - uses: actions/checkout@v6
         with:
           submodules: true
-      - uses: ./.github/workflows/setup-with-retry
+      - run: ./tools/op.sh setup
       - name: Update translations
         run: python3 selfdrive/ui/update_translations.py --vanish
       - name: Create Pull Request
@@ -39,7 +39,7 @@ jobs:
     - uses: actions/checkout@v6
       with:
         submodules: true
-    - uses: ./.github/workflows/setup-with-retry
+    - run: ./tools/op.sh setup
     - name: uv lock
       run: uv lock --upgrade
     - name: uv pip tree
diff --git a/.github/workflows/setup-with-retry/action.yaml b/.github/workflows/setup-with-retry/action.yaml
deleted file mode 100644
index 923cc3aad..000000000
--- a/.github/workflows/setup-with-retry/action.yaml
+++ /dev/null
@@ -1,48 +0,0 @@
-name: 'openpilot env setup, with retry on failure'
-
-inputs:
-  sleep_time:
-    description: 'Time to sleep between retries'
-    required: false
-    default: 30
-
-outputs:
-  duration:
-    description: 'Duration of the setup process in seconds'
-    value: ${{ steps.get_duration.outputs.duration }}
-
-runs:
-  using: "composite"
-  steps:
-    - id: start_time
-      shell: bash
-      run: echo "START_TIME=$(date +%s)" >> $GITHUB_ENV
-    - id: setup1
-      uses: ./.github/workflows/setup
-      continue-on-error: true
-      with:
-        is_retried: true
-    - if: steps.setup1.outcome == 'failure'
-      shell: bash
-      run: sleep ${{ inputs.sleep_time }}
-    - id: setup2
-      if: steps.setup1.outcome == 'failure'
-      uses: ./.github/workflows/setup
-      continue-on-error: true
-      with:
-        is_retried: true
-    - if: steps.setup2.outcome == 'failure'
-      shell: bash
-      run: sleep ${{ inputs.sleep_time }}
-    - id: setup3
-      if: steps.setup2.outcome == 'failure'
-      uses: ./.github/workflows/setup
-      with:
-        is_retried: true
-    - id: get_duration
-      shell: bash
-      run: |
-        END_TIME=$(date +%s)
-        DURATION=$((END_TIME - START_TIME))
-        echo "Total duration: $DURATION seconds"
-        echo "duration=$DURATION" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml
deleted file mode 100644
index f3a1a3950..000000000
--- a/.github/workflows/setup/action.yaml
+++ /dev/null
@@ -1,56 +0,0 @@
-name: 'openpilot env setup'
-
-inputs:
-  is_retried:
-    description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly'
-    required: false
-    default: 'false'
-
-runs:
-  using: "composite"
-  steps:
-    # assert that this action is retried using the setup-with-retry
-    - shell: bash
-      if: ${{ inputs.is_retried == 'false' }}
-      run: |
-        echo "You should not run this action directly. Use setup-with-retry instead"
-        exit 1
-
-    - shell: bash
-      name: No retries!
-      run: |
-        if [ "${{ github.run_attempt }}" -gt 1 ]; then
-          echo -e "\033[0;31m##################################################"
-          echo -e "\033[0;31m    Retries not allowed! Fix the flaky test!      "
-          echo -e "\033[0;31m##################################################\033[0m"
-          exit 1
-        fi
-
-    # build cache
-    - id: date
-      shell: bash
-      run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
-    - id: scons-cache
-      uses: ./.github/workflows/auto-cache
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ runner.os }}-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
-        restore-keys: |
-          scons-${{ runner.os }}-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}
-          scons-${{ runner.os }}-${{ runner.arch }}
-    # venv cache
-    - id: venv-cache
-      uses: ./.github/workflows/auto-cache
-      with:
-        path: ${{ github.workspace }}/.venv
-        key: venv-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('uv.lock') }}
-        restore-keys: |
-          venv-${{ runner.os }}-${{ runner.arch }}
-    - shell: bash
-      name: Run setup
-      run: ./tools/op.sh setup
-    - shell: bash
-      name: Setup cache dirs
-      run: |
-        mkdir -p /tmp/comma_download_cache
-        echo "$GITHUB_WORKSPACE/.venv/bin" >> $GITHUB_PATH
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index f95fe7d2e..00fdceda0 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -27,7 +27,7 @@ jobs:
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
+      && fromJSON('["namespace-profile-amd64-8x16"]')
       || fromJSON('["ubuntu-24.04"]') }}
     env:
       STRIPPED_DIR: /tmp/releasepilot
@@ -45,7 +45,7 @@ jobs:
     - name: Build devel
       timeout-minutes: 1
       run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh
-    - uses: ./.github/workflows/setup-with-retry
+    - run: ./tools/op.sh setup
     - name: Build openpilot and run checks
       timeout-minutes: 30
       working-directory: ${{ env.STRIPPED_DIR }}
@@ -70,7 +70,7 @@ jobs:
       run: |
         FILTERED=$(echo "$PATH" | tr ':' '\n' | grep -v '/opt/homebrew' | tr '\n' ':')
         echo "PATH=${FILTERED}/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" >> $GITHUB_ENV
-    - uses: ./.github/workflows/setup-with-retry
+    - run: ./tools/op.sh setup
     - name: Building openpilot
       run: scons
 
@@ -80,13 +80,13 @@ jobs:
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
+      && fromJSON('["namespace-profile-amd64-8x16"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
     - uses: actions/checkout@v6
       with:
         submodules: true
-    - uses: ./.github/workflows/setup-with-retry
+    - run: ./tools/op.sh setup
     - name: Static analysis
       timeout-minutes: 1
       run: scripts/lint/lint.sh
@@ -97,18 +97,17 @@ jobs:
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
+      && fromJSON('["namespace-profile-amd64-8x16"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
     - uses: actions/checkout@v6
       with:
         submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-      id: setup-step
+    - run: ./tools/op.sh setup
     - name: Build openpilot
       run: scons -j$(nproc)
     - name: Run unit tests
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }}
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
       run: |
         source selfdrive/test/setup_xvfb.sh
         # Pre-compile Python bytecode so each pytest worker doesn't need to
@@ -121,24 +120,17 @@ jobs:
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
+      && fromJSON('["namespace-profile-amd64-8x16"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
     - uses: actions/checkout@v6
       with:
         submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-      id: setup-step
-    - name: Cache test routes
-      id: dependency-cache
-      uses: actions/cache@v5
-      with:
-        path: /tmp/comma_download_cache
-        key: proc-replay-${{ hashFiles('selfdrive/test/process_replay/test_processes.py') }}
+    - run: ./tools/op.sh setup
     - name: Build openpilot
       run: scons -j$(nproc)
     - name: Run replay
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }}
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
       continue-on-error: ${{ github.ref == 'refs/heads/master' }}
       run: selfdrive/test/process_replay/test_processes.py -j$(nproc)
     - name: Print diff
@@ -184,19 +176,18 @@ jobs:
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
+      && fromJSON('["namespace-profile-amd64-8x16"]')
       || fromJSON('["ubuntu-24.04"]') }}
     if: false  # FIXME: Started to timeout recently
     steps:
     - uses: actions/checkout@v6
       with:
         submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-      id: setup-step
+    - run: ./tools/op.sh setup
     - name: Build openpilot
       run: scons -j$(nproc)
     - name: Driving test
-      timeout-minutes: ${{ (steps.setup-step.outputs.duration < 18) && 1 || 2 }}
+      timeout-minutes: 2
       run: |
         source selfdrive/test/setup_xvfb.sh
         pytest -s tools/sim/tests/test_metadrive_bridge.py
@@ -207,13 +198,13 @@ jobs:
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
+      && fromJSON('["namespace-profile-amd64-8x16"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
       - uses: actions/checkout@v6
         with:
           submodules: true
-      - uses: ./.github/workflows/setup-with-retry
+      - run: ./tools/op.sh setup
       - name: Build openpilot
         run: scons -j$(nproc)
       - name: Create UI Report
diff --git a/tools/op.sh b/tools/op.sh
index f5c5b6082..9f3d4ee13 100755
--- a/tools/op.sh
+++ b/tools/op.sh
@@ -35,6 +35,21 @@ function loge() {
   fi
 }
 
+function retry() {
+  local attempts=$1
+  shift
+  for i in $(seq 1 "$attempts"); do
+    if "$@"; then
+      return 0
+    fi
+    if [ "$i" -lt "$attempts" ]; then
+      echo "  Attempt $i/$attempts failed, retrying in 5s..."
+      sleep 5
+    fi
+  done
+  return 1
+}
+
 function op_run_command() {
   CMD="$@"
 
@@ -229,7 +244,7 @@ function op_setup() {
 
   echo "Getting git submodules..."
   st="$(date +%s)"
-  if ! git submodule update --jobs 4 --init --recursive; then
+  if ! retry 3 git submodule update --jobs 4 --init --recursive; then
     echo -e " ↳ [${RED}✗${NC}] Getting git submodules failed!"
     loge "ERROR_GIT_SUBMODULES"
     return 1
@@ -239,7 +254,7 @@ function op_setup() {
 
   echo "Pulling git lfs files..."
   st="$(date +%s)"
-  if ! git lfs pull; then
+  if ! retry 3 git lfs pull; then
     echo -e " ↳ [${RED}✗${NC}] Pulling git lfs files failed!"
     loge "ERROR_GIT_LFS"
     return 1
@@ -260,6 +275,11 @@ function op_activate_venv() {
   set +e
   source $OPENPILOT_ROOT/.venv/bin/activate &> /dev/null || true
   set -e
+
+  # persist venv on PATH across GitHub Actions steps
+  if [ -n "$GITHUB_PATH" ]; then
+    echo "$OPENPILOT_ROOT/.venv/bin" >> "$GITHUB_PATH"
+  fi
 }
 
 function op_venv() {