mac: fix setup and re-enable CI (#23365)

* rework mac_setup.sh
* ignore casadi and onnx runtime in pipfile for Darwin
* remove rust stuff
* build and install casadi

* add wget

* explicit python3. dont do onnx just yet

* resource compiler

* replace acados Darwin lib w universal2 libs

* relock Pipfile using linux machine

* Update update_requirements.sh

brackets

* Update update_requirements.sh

oops

* ci: re-enable mac build

* attempt to fix ci build:
* pip stuff?
* move rcc bin path to env script

* oops

* only mac

* k im lazy. does ci like this??

* huh??

* * use curl
* avoid casadi rebuilds
* add comment to remove protobuf
* host detect typo

* python version

* how did the builds not use swig?
* better reinstall check

* sometimes mac clang complain abt error limit

* ci: build OP in venv

* ci: bump to Big Sur
* retry? idk why it failed @ acados link

* * use macos-latest
* move rcc path to scons
* add extra paths for odd homebrew installs
* acados source change

* update macOS README

* uh, maybe?

* k nvm

* ci: ok this is strange. might be scons bug?

* fix conflicts: bump

* just add cppcheck for pre-commit stuff

* agane

* cleanup

* try that

* fix path

* no pyenv update on mac

* source

* fix rpath prefix?

* no examples

* fix exit

* let's get this cached

* add virtualenvs to cache

* why did we cache that

* let's see what's big

* more

* always ruyn

* cache scons cache

* better cache key

* fix for partial hit

* why so long

Co-authored-by: Andrew Tec <andrest@trabus.com>
Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
Co-authored-by: Bruce Wayne <batman@openpilot-mac-mini.local>
old-commit-hash: 6664421710

.github/workflows/selfdrive_tests.yaml

@@ -69,52 +69,62 @@ jobs:
                           rm -rf /tmp/scons_cache/* && \
                           scons -j$(nproc) --cache-populate"
 
-  #build_mac:
-  #  name: build macos
-  #  runs-on: macos-10.15
-  #  timeout-minutes: 60
-  #  steps:
-  #  - uses: actions/checkout@v2
-  #    with:
-  #      submodules: true
-  #  - name: Determine pre-existing Homebrew packages
-  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
-  #    run: |
-  #      echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
-  #      ls -1 /usr/local/Cellar >> $GITHUB_ENV
-  #      echo 'EOF' >> $GITHUB_ENV
-  #  - name: Cache dependencies
-  #    id: dependency-cache
-  #    uses: actions/cache@v2
-  #    with:
-  #      path: |
-  #        ~/.pyenv
-  #        ~/Library/Caches/pip
-  #        ~/Library/Caches/pipenv
-  #        /usr/local/Cellar
-  #        ~/github_brew_cache_entries.txt
-  #      key: macos-cache-${{ hashFiles('tools/mac_setup.sh') }}
-  #  - name: Brew link restored dependencies
-  #    if: steps.dependency-cache.outputs.cache-hit == 'true'
-  #    run: |
-  #      while read pkg; do
-  #        brew link --force "$pkg" # `--force` for keg-only packages
-  #      done < ~/github_brew_cache_entries.txt
-  #  - name: Install dependencies
-  #    run: ./tools/mac_setup.sh
-  #  - name: Build openpilot
-  #    run: eval "$(pyenv init -)" && scons -j$(nproc)
-  #  - name: Remove pre-existing Homebrew packages for caching
-  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
-  #    run: |
-  #      cd /usr/local/Cellar
-  #      new_cellar=$(ls -1)
-  #      comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
-  #        if [[ $pkg != "zstd" ]]; then # caching step needs zstd
-  #          rm -rf "$pkg"
-  #        fi
-  #      done
-  #      comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
+  build_mac:
+    name: build macos
+    runs-on: macos-latest
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - name: Determine pre-existing Homebrew packages
+      if: steps.dependency-cache.outputs.cache-hit != 'true'
+      run: |
+        echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
+        ls -1 /usr/local/Cellar >> $GITHUB_ENV
+        echo 'EOF' >> $GITHUB_ENV
+    - name: Cache dependencies
+      id: dependency-cache
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.pyenv
+          ~/.local/share/virtualenvs/
+          /usr/local/Cellar
+          ~/github_brew_cache_entries.txt
+          /tmp/scons_cache
+        key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'Pipfile*') }}
+        restore-keys: macos-
+    - name: Brew link restored dependencies
+      run: |
+        if [ -f ~/github_brew_cache_entries.txt ]; then
+          while read pkg; do
+            brew link --force "$pkg" # `--force` for keg-only packages
+          done < ~/github_brew_cache_entries.txt
+        else
+          echo "Cache entries not found"
+        fi
+    - name: Install dependencies
+      run: ./tools/mac_setup.sh
+    - name: Build openpilot
+      run: |
+        source tools/openpilot_env.sh
+        pipenv run selfdrive/manager/build.py
+
+        # cleanup scons cache
+        rm -rf /tmp/scons_cache/
+        pipenv run scons -j$(nproc) --cache-populate
+    - name: Remove pre-existing Homebrew packages for caching
+      if: steps.dependency-cache.outputs.cache-hit != 'true'
+      run: |
+        cd /usr/local/Cellar
+        new_cellar=$(ls -1)
+        comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
+          if [[ $pkg != "zstd" ]]; then # caching step needs zstd
+            rm -rf "$pkg"
+          fi
+        done
+        comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
 
   build_webcam:
     name: build webcam