docker: multiarch images (amd64/aarch64) (#29691)

* arch specific tags

* merge multiarch script

* arm64 build and merge jobs

* update tag script

* casual ci test

* change image to 2204

* docker -v test

* arch specific naming

* Remove test_buildjet

* build -arch images only when CURRENT_ARCH_BUILD is set

* support new tags in tag_multiarch

* Toggle CURRENT_ARCH_BUILD on in ci

* Docker common

* Fix condition

* Fix path to docker_common

* Add more stuff to common

* Add build_arm job

* composite job for building

* Run checkout before build composite

* add shell arg

* move timeout to selfdrive_tests

* TARGET ARCHITECTURE var

* Support TARGET_ARCHITECTURE in workflow

* Rewrite to Single build job with matrix

* Remove shebang from docker_common

* Attempt for fix build matrix

* Remove setup arch

* build matrix for docker push

* Use 2vcpu for arm build

* 2vcpu for docker_push too

* temporarly unlock docker_push

* Remove requirement for target arch when pushing

* Unset target architecture in docker_push cl

* fix sha tags

* Rename action to compile-openpilot

* move push_image line to tag_multiarch step

* arch suffix for scons cache

* cache_key_prefix for setup-with-retry

* Re-disable docker_push on non-master

* Add newlines

.github/workflows/compile-openpilot/action.yaml

@@ -0,0 +1,27 @@
+name: 'compile openpilot'
+
+inputs:
+  cache_key_prefix:
+    description: 'Prefix for caching key'
+    required: false
+    default: 'scons'
+
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      name: Build openpilot with all flags
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+        ${{ env.RUN }} "release/check-dirty.sh"
+    - shell: bash
+      name: Cleanup scons cache and rebuild
+      run: |
+          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
+                          scons -j$(nproc) --cache-populate"
+    - name: Save scons cache
+      uses: actions/cache/save@v3
+      if: github.ref == 'refs/heads/master'
+      with:
+        path: .ci_cache/scons_cache
+        key: ${{ inputs.cache_key_prefix }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}

.github/workflows/selfdrive_tests.yaml

@@ -67,30 +67,20 @@ jobs:
         cd $STRIPPED_DIR
         ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all"
 
-  build_all:
-    name: build all
-    runs-on: ubuntu-20.04
+  build:
+    strategy:
+      matrix:
+        arch: ${{ fromJson( (github.repository == 'commaai/openpilot') && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
+    runs-on: ${{ (matrix.arch == 'aarch64') && 'buildjet-2vcpu-ubuntu-2204-arm' || 'ubuntu-20.04' }}
     steps:
     - uses: actions/checkout@v3
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry
-    - name: Build openpilot with all flags
-      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 12 || 30) }} # allow more time when we missed the scons cache
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-        ${{ env.RUN }} "release/check-dirty.sh"
-    - name: Cleanup scons cache and rebuild
-      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 2 || 30) }} # allow more time when we missed the scons cache
-      run: |
-          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
-                          scons -j$(nproc) --cache-populate"
-    - name: Save scons cache
-      uses: actions/cache/save@v3
-      if: github.ref == 'refs/heads/master'
+    - uses: ./.github/workflows/compile-openpilot
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache
       with:
-        path: .ci_cache/scons_cache
-        key: scons-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        cache_key_prefix: scons_${{ matrix.arch }}
 
   build_mac:
     name: build macos
@@ -185,8 +175,10 @@ jobs:
         sudo rm -rf /Applications/ArmGNUToolchain/*/*/.fseventsd
 
   docker_push:
-    name: docker push
-    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        arch: ${{ fromJson( (github.repository == 'commaai/openpilot') && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
+    runs-on: ${{ (matrix.arch == 'aarch64') && 'buildjet-2vcpu-ubuntu-2204-arm' || 'ubuntu-20.04' }}
     if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
     steps:
     - uses: actions/checkout@v3
@@ -195,14 +187,34 @@ jobs:
     - name: Setup to push to repo
       run: |
         echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
+        echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV"
         $DOCKER_LOGIN
     - uses: ./.github/workflows/setup-with-retry
       with:
         git-lfs: false
     - name: Build and push CL Docker image
+      if: matrix.arch == 'x86_64'
       run: |
+        echo "TARGET_ARCHITECTURE=''" >> "$GITHUB_ENV"
         eval "$BUILD_CL"
 
+  docker_push_multiarch:
+    name: docker push multiarch tag
+    runs-on: ubuntu-20.04
+    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
+    needs: [docker_push]
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: false
+    - name: Setup docker
+      run: |
+        $DOCKER_LOGIN
+    - name: Merge x64 and arm64 tags
+      run: |
+        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
+        selfdrive/test/docker_tag_multiarch.sh base x86_64 aarch64
+
   static_analysis:
     name: static analysis
     runs-on: ubuntu-20.04

.github/workflows/setup-with-retry/action.yaml

@@ -5,6 +5,10 @@ inputs:
     description: 'Whether or not to pull the git lfs'
     required: false
     default: 'true'
+  cache_key_prefix:
+    description: 'Prefix for caching key'
+    required: false
+    default: 'scons_x86_64'
 
 env:
   SLEEP_TIME: 30 # Time to sleep between retries
@@ -17,6 +21,7 @@ runs:
       continue-on-error: true
       with:
         git_lfs: ${{ inputs.git_lfs }}
+        cache_key_prefix: ${{ inputs.cache_key_prefix }}
         is_retried: true
     - if: steps.setup1.outcome == 'failure'
       shell: bash
@@ -27,6 +32,7 @@ runs:
       continue-on-error: true
       with:
         git_lfs: ${{ inputs.git_lfs }}
+        cache_key_prefix: ${{ inputs.cache_key_prefix }}
         is_retried: true
     - if: steps.setup2.outcome == 'failure'
       shell: bash
@@ -36,4 +42,5 @@ runs:
       uses: ./.github/workflows/setup
       with:
         git_lfs: ${{ inputs.git_lfs }}
-        is_retried: true
+        cache_key_prefix: ${{ inputs.cache_key_prefix }}
+        is_retried: true

.github/workflows/setup/action.yaml

@@ -5,6 +5,10 @@ inputs:
     description: 'Whether or not to pull the git lfs'
     required: false
     default: 'true'
+  cache_key_prefix:
+    description: 'Prefix for caching key'
+    required: false
+    default: 'scons_x86_64'
   is_retried:
     description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly'
     required: false
@@ -35,10 +39,10 @@ runs:
       uses: actions/cache/restore@v3
       with:
         path: .ci_cache/scons_cache
-        key: scons-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        key: ${{ inputs.cache_key_prefix }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
         restore-keys: |
-          scons-${{ env.CACHE_COMMIT_DATE }}-
-          scons-
+          ${{ inputs.cache_key_prefix }}-${{ env.CACHE_COMMIT_DATE }}-
+          ${{ inputs.cache_key_prefix }}-
     # if we didn't get a cache hit, make the directory manually so it doesn't fail on future steps
     - id: scons-cache-setup
       shell: bash