mac: fix setup and re-enable CI (#23365)

* rework mac_setup.sh
* ignore casadi and onnx runtime in pipfile for Darwin
* remove rust stuff
* build and install casadi

* add wget

* explicit python3. dont do onnx just yet

* resource compiler

* replace acados Darwin lib w universal2 libs

* relock Pipfile using linux machine

* Update update_requirements.sh

brackets

* Update update_requirements.sh

oops

* ci: re-enable mac build

* attempt to fix ci build:
* pip stuff?
* move rcc bin path to env script

* oops

* only mac

* k im lazy. does ci like this??

* huh??

* * use curl
* avoid casadi rebuilds
* add comment to remove protobuf
* host detect typo

* python version

* how did the builds not use swig?
* better reinstall check

* sometimes mac clang complain abt error limit

* ci: build OP in venv

* ci: bump to Big Sur
* retry? idk why it failed @ acados link

* * use macos-latest
* move rcc path to scons
* add extra paths for odd homebrew installs
* acados source change

* update macOS README

* uh, maybe?

* k nvm

* ci: ok this is strange. might be scons bug?

* fix conflicts: bump

* just add cppcheck for pre-commit stuff

* agane

* cleanup

* try that

* fix path

* no pyenv update on mac

* source

* fix rpath prefix?

* no examples

* fix exit

* let's get this cached

* add virtualenvs to cache

* why did we cache that

* let's see what's big

* more

* always ruyn

* cache scons cache

* better cache key

* fix for partial hit

* why so long

Co-authored-by: Andrew Tec <andrest@trabus.com>
Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
Co-authored-by: Bruce Wayne <batman@openpilot-mac-mini.local>
old-commit-hash: 6664421710

.github/workflows/selfdrive_tests.yaml

@@ -69,52 +69,62 @@ jobs:
                           rm -rf /tmp/scons_cache/* && \
                           scons -j$(nproc) --cache-populate"
 
-  #build_mac:
-  #  name: build macos
-  #  runs-on: macos-10.15
-  #  timeout-minutes: 60
-  #  steps:
-  #  - uses: actions/checkout@v2
-  #    with:
-  #      submodules: true
-  #  - name: Determine pre-existing Homebrew packages
-  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
-  #    run: |
-  #      echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
-  #      ls -1 /usr/local/Cellar >> $GITHUB_ENV
-  #      echo 'EOF' >> $GITHUB_ENV
-  #  - name: Cache dependencies
-  #    id: dependency-cache
-  #    uses: actions/cache@v2
-  #    with:
-  #      path: |
-  #        ~/.pyenv
-  #        ~/Library/Caches/pip
-  #        ~/Library/Caches/pipenv
-  #        /usr/local/Cellar
-  #        ~/github_brew_cache_entries.txt
-  #      key: macos-cache-${{ hashFiles('tools/mac_setup.sh') }}
-  #  - name: Brew link restored dependencies
-  #    if: steps.dependency-cache.outputs.cache-hit == 'true'
-  #    run: |
-  #      while read pkg; do
-  #        brew link --force "$pkg" # `--force` for keg-only packages
-  #      done < ~/github_brew_cache_entries.txt
-  #  - name: Install dependencies
-  #    run: ./tools/mac_setup.sh
-  #  - name: Build openpilot
-  #    run: eval "$(pyenv init -)" && scons -j$(nproc)
-  #  - name: Remove pre-existing Homebrew packages for caching
-  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
-  #    run: |
-  #      cd /usr/local/Cellar
-  #      new_cellar=$(ls -1)
-  #      comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
-  #        if [[ $pkg != "zstd" ]]; then # caching step needs zstd
-  #          rm -rf "$pkg"
-  #        fi
-  #      done
-  #      comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
+  build_mac:
+    name: build macos
+    runs-on: macos-latest
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - name: Determine pre-existing Homebrew packages
+      if: steps.dependency-cache.outputs.cache-hit != 'true'
+      run: |
+        echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
+        ls -1 /usr/local/Cellar >> $GITHUB_ENV
+        echo 'EOF' >> $GITHUB_ENV
+    - name: Cache dependencies
+      id: dependency-cache
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.pyenv
+          ~/.local/share/virtualenvs/
+          /usr/local/Cellar
+          ~/github_brew_cache_entries.txt
+          /tmp/scons_cache
+        key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'Pipfile*') }}
+        restore-keys: macos-
+    - name: Brew link restored dependencies
+      run: |
+        if [ -f ~/github_brew_cache_entries.txt ]; then
+          while read pkg; do
+            brew link --force "$pkg" # `--force` for keg-only packages
+          done < ~/github_brew_cache_entries.txt
+        else
+          echo "Cache entries not found"
+        fi
+    - name: Install dependencies
+      run: ./tools/mac_setup.sh
+    - name: Build openpilot
+      run: |
+        source tools/openpilot_env.sh
+        pipenv run selfdrive/manager/build.py
+
+        # cleanup scons cache
+        rm -rf /tmp/scons_cache/
+        pipenv run scons -j$(nproc) --cache-populate
+    - name: Remove pre-existing Homebrew packages for caching
+      if: steps.dependency-cache.outputs.cache-hit != 'true'
+      run: |
+        cd /usr/local/Cellar
+        new_cellar=$(ls -1)
+        comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
+          if [[ $pkg != "zstd" ]]; then # caching step needs zstd
+            rm -rf "$pkg"
+          fi
+        done
+        comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
 
   build_webcam:
     name: build webcam

Pipfile

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a3baa202ee29371a3168613072d612a3239f368c0e9122a31b398f7db0eb7b6
-size 1282
+oid sha256:c7d458447a6077d0e58d47e4898fca9a30b4415e829693775e72ec7b2f80ed42
+size 1384

Pipfile.lock

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5aa6aaa2866c3bf2002532ac131e4dce9581adc1e8d3422dc37bba163e5fc874
-size 143236
+oid sha256:deb01290316c7cc8c24bf5a584c06bee95da54f935dd6792bdbe5485029a5e15
+size 143978

SConstruct

@@ -66,7 +66,7 @@ lenv = {
   "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath],
   "PYTHONPATH": Dir("#").abspath + ":" + Dir("#pyextra/").abspath,
 
-  "ACADOS_SOURCE_DIR": Dir("#third_party/acados/acados").abspath,
+  "ACADOS_SOURCE_DIR": Dir("#third_party/acados/include/acados").abspath,
   "ACADOS_PYTHON_INTERFACE_PATH": Dir("#pyextra/acados_template").abspath,
   "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer",
 }
@@ -125,8 +125,10 @@ else:
       f"#third_party/libyuv/{yuv_dir}/lib",
       "/usr/local/lib",
       "/opt/homebrew/lib",
+      "/usr/local/Homebrew/Library",
       "/usr/local/opt/openssl/lib",
       "/opt/homebrew/opt/openssl/lib",
+      "/usr/local/Cellar",
       f"#third_party/acados/{arch}/lib",
       "/System/Library/Frameworks/OpenGL.framework/Libraries",
     ]
@@ -134,6 +136,7 @@ else:
     cxxflags += ["-DGL_SILENCE_DEPRECATION"]
     cpppath += [
       "/opt/homebrew/include",
+      "/usr/local/include",
       "/usr/local/opt/openssl/include",
       "/opt/homebrew/opt/openssl/include"
     ]
@@ -235,6 +238,9 @@ env = Environment(
   tools=["default", "cython", "compilation_db"],
 )
 
+if arch == "Darwin":
+  env['RPATHPREFIX'] = "-rpath "
+
 if GetOption('compile_db'):
   env.CompilationDatabase('compile_commands.json')
 
@@ -299,6 +305,7 @@ if arch == "Darwin":
   qt_dirs += [f"{qt_env['QTDIR']}/include/Qt{m}" for m in qt_modules]
   qt_env["LINKFLAGS"] += ["-F" + os.path.join(qt_env['QTDIR'], "lib")]
   qt_env["FRAMEWORKS"] += [f"Qt{m}" for m in qt_modules] + ["OpenGL"]
+  qt_env.AppendENVPath('PATH', os.path.join(qt_env['QTDIR'], "bin"))
 elif arch == "aarch64":
   qt_env['QTDIR'] = "/system/comma/usr"
   qt_dirs = [

third_party/acados/Darwin/lib/libacados.dylib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:498f02aa0ec9a3f5018e0888b13eeacb64726289f6b9149bd4645252c19e8f2d
-size 357804
+oid sha256:5906c1e63385d550a9d6658f32b280f0a95ac8253ac399365f58f75a40488cfe
+size 816552

third_party/acados/Darwin/lib/libblasfeo.dylib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2422711a5f711325947bb4cfd55368840ce57b33cffacf45fd0a6dcbf71cd8b3
-size 531149
+oid sha256:35174b2221aa673c80277cdf35d0b09663ca47f3a280c8b46dde81e28249f643
+size 2217032

third_party/acados/Darwin/lib/libhpipm.dylib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:40530c77834e7efbf9f7e8b7f76d547d706459e4079a58fb5b93c2d68187a77a
-size 1062411
+oid sha256:deb18d241a950e152d493069aad6c1376f7e790dc7dff12ab8877757dbd1941e
+size 2291256

third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee3876b41af39f6815cb84ecabeada4e9866d34aa2a73874ab0a627a4651952b
-size 218403
+oid sha256:cbc62731fec870121d65930ad8a39c414b895655442738f63a1b39141aef808c
+size 513304

tools/mac_setup.sh

@@ -1,4 +1,6 @@
-#!/bin/bash -e
+#!/bin/bash
+
+set -e
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 ROOT="$(cd $DIR/../ && pwd)"
@@ -7,15 +9,16 @@ ROOT="$(cd $DIR/../ && pwd)"
 if [[ $(command -v brew) == "" ]]; then
   echo "Installing Hombrew"
   /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
+  echo "[ ] installed brew t=$SECONDS"
 fi
 
+# TODO: remove protobuf,protobuf-c,swig when casadi can be pip installed
 brew bundle --file=- <<-EOS
 brew "cmake"
+brew "cppcheck"
 brew "git-lfs"
 brew "zlib"
 brew "bzip2"
-brew "rust"
-brew "rustup-init"
 brew "capnp"
 brew "coreutils"
 brew "eigen"
@@ -29,21 +32,20 @@ brew "openssl"
 brew "pyenv"
 brew "qt@5"
 brew "zeromq"
+brew "protobuf"
+brew "protobuf-c"
+brew "swig"
 cask "gcc-arm-embedded"
 EOS
 
+echo "[ ] finished brew install t=$SECONDS"
+
 if [[ $SHELL == "/bin/zsh" ]]; then
   RC_FILE="$HOME/.zshrc"
 elif [[ $SHELL == "/bin/bash" ]]; then
   RC_FILE="$HOME/.bash_profile"
 fi
 
-# TODO: get rid of this somehow
-# Build requirements for macOS
-# https://github.com/pyenv/pyenv/issues/1740
-# https://github.com/pyca/cryptography/blob/main/docs/installation.rst
-rustup-init -y
-
 export LDFLAGS="$LDFLAGS -L/usr/local/opt/zlib/lib"
 export LDFLAGS="$LDFLAGS -L/usr/local/opt/bzip2/lib"
 export LDFLAGS="$LDFLAGS -L/usr/local/opt/openssl@1.1/lib"
@@ -55,17 +57,40 @@ export PATH="$PATH:/usr/local/bin"
 
 # openpilot environment
 if [ -z "$OPENPILOT_ENV" ] && [ -n "$RC_FILE" ] && [ -z "$CI" ]; then
-  echo "export PATH=\"\$PATH:$HOME/.cargo/bin\"" >> $RC_FILE
   echo "source $ROOT/tools/openpilot_env.sh" >> $RC_FILE
-  export PATH="$PATH:\"\$HOME/.cargo/bin\""
   source "$ROOT/tools/openpilot_env.sh"
   echo "Added openpilot_env to RC file: $RC_FILE"
 fi
 
 # install python dependencies
 $ROOT/update_requirements.sh
+eval "$(pyenv init --path)"
+echo "[ ] installed python dependencies t=$SECONDS"
+
+# install casadi
+VENV=`pipenv --venv`
+PYTHON_VER=3.8
+PYTHON_VERSION=$(cat $ROOT/.python-version)
+if [ ! -f "$VENV/include/casadi/casadi.hpp" ]; then
+  echo "-- casadi manual install"
+  cd /tmp/ && curl -L https://github.com/casadi/casadi/archive/refs/tags/ge6.tar.gz --output casadi.tar.gz
+  tar -xzf casadi.tar.gz
+  cd casadi-ge6/ && mkdir -p build && cd build
+  cmake .. \
+    -DWITH_PYTHON=ON \
+    -DWITH_EXAMPLES=OFF \
+    -DCMAKE_INSTALL_PREFIX:PATH=$VENV \
+    -DPYTHON_PREFIX:PATH=$VENV/lib/python$PYTHON_VER/site-packages \
+    -DPYTHON_LIBRARY:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/lib/libpython$PYTHON_VER.dylib \
+    -DPYTHON_EXECUTABLE:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/bin/python \
+    -DPYTHON_INCLUDE_DIR:PATH=$HOME/.pyenv/versions/$PYTHON_VERSION/include/python$PYTHON_VER \
+    -DCMAKE_CXX_FLAGS="-ferror-limit=0" -DCMAKE_C_FLAGS="-ferror-limit=0"
+  CFLAGS="-ferror-limit=0" make -j$(nproc) && make install
+else
+  echo "----   casadi found in venv. skipping build   ----"
+fi
 
 echo
-echo "----   FINISH OPENPILOT SETUP   ----"
-echo "Configure your active shell env by running:"
+echo "----   OPENPILOT SETUP DONE   ----"
+echo "Open a new shell or configure your active shell env by running:"
 echo "source $RC_FILE"

tools/openpilot_env.sh

@@ -9,13 +9,13 @@ if [ -z "$OPENPILOT_ENV" ]; then
   export PYENV_ROOT="$HOME/.pyenv"
 
   if [[ "$(uname)" == 'Linux' ]]; then
-    eval "$(pyenv init --path)"
     eval "$(pyenv virtualenv-init -)"
   elif [[ "$(uname)" == 'Darwin' ]]; then
     # msgq doesn't work on mac
     export ZMQ=1
     export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
   fi
+  eval "$(pyenv init --path)"
   eval "$(pyenv init -)"
 
   export OPENPILOT_ENV=1

tools/ubuntu_setup.sh

@@ -134,6 +134,6 @@ if [ -z "$OPENPILOT_ENV" ]; then
 fi
 
 echo
-echo "----   FINISH OPENPILOT SETUP   ----"
-echo "Configure your active shell env by running:"
+echo "----   OPENPILOT SETUP DONE   ----"
+echo "Open a new shell or configure your active shell env by running:"
 echo "source ~/.bashrc"

update_requirements.sh

@@ -1,4 +1,6 @@
-#!/bin/bash -e
+#!/bin/bash
+
+set -e
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 cd $DIR
@@ -13,15 +15,18 @@ export MAKEFLAGS="-j$(nproc)"
 
 PYENV_PYTHON_VERSION=$(cat .python-version)
 if ! pyenv prefix ${PYENV_PYTHON_VERSION} &> /dev/null; then
-  echo "pyenv update ..."
-  pyenv update
+  # no pyenv update on mac
+  if [ "$(uname)" == "Linux" ]; then
+    echo "pyenv update ..."
+    pyenv update
+  fi
   echo "python ${PYENV_PYTHON_VERSION} install ..."
   CONFIGURE_OPTS="--enable-shared" pyenv install -f ${PYENV_PYTHON_VERSION}
 fi
+eval "$(pyenv init --path)"
 
-echo "pip install ..."
+echo "update pip"
 pip install pip==21.3.1
-echo "pipenv install ..."
 pip install pipenv==2021.11.23
 
 if [ -d "./xx" ]; then
@@ -36,7 +41,7 @@ else
   RUN=""
 fi
 
-echo "pip packages install ..."
+echo "pip packages install..."
 pipenv install --dev --deploy --clear
 pyenv rehash
 
@@ -45,4 +50,5 @@ if [ -f "$DIR/.pre-commit-config.yaml" ]; then
   $RUN pre-commit install
   [ -d "./xx" ] && (cd xx && $RUN pre-commit install)
   [ -d "./notebooks" ] && (cd notebooks && $RUN pre-commit install)
+  echo "pre-commit hooks installed"
 fi