Distribute custom pytest workloads without dropping cases

.github/workflows/tests.yaml

@@ -203,13 +203,6 @@ jobs:
         RAYLIB_BACKEND: headless
         STARPILOT_TEST_DIAGNOSTICS: ci-results/host-test-diagnostics/unittest
       run: tools/op.sh test --json-output unit-test-results.json
-    - name: Test custom module-level pytest cases
-      if: ${{ !cancelled() && steps.build.outcome == 'success' }}
-      timeout-minutes: 15
-      env:
-        RAYLIB_BACKEND: headless
-        STARPILOT_TEST_DIAGNOSTICS: ci-results/host-test-diagnostics/pytest
-      run: python3 tools/ci/run_host_tests.py --pytest
     - name: Test GM vehicle configurations and CAN behavior
       if: ${{ !cancelled() && steps.build.outcome == 'success' }}
       timeout-minutes: 5
@@ -244,6 +237,80 @@ jobs:
           ci-results/host-test-diagnostics/
         if-no-files-found: error
 
+  custom_pytest:
+    name: custom pytest (${{ matrix.shard }})
+    runs-on: ubuntu-24.04
+    timeout-minutes: 35
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]
+    steps:
+    - uses: actions/checkout@v7
+    - name: Provide JavaScript runtime for Galaxy contract tests
+      uses: actions/setup-node@v7
+      with:
+        node-version: '24'
+        package-manager-cache: false
+    - name: Provide pinned map provider toolchain
+      uses: actions/setup-go@v7
+      with:
+        go-version: '1.25.1'
+        cache-dependency-path: mapd_repo/go.sum
+    - name: Fetch pinned map provider dependencies
+      working-directory: mapd_repo
+      env:
+        GOTOOLCHAIN: local
+      run: go mod download
+    - run: ./tools/op.sh setup
+      id: setup
+    - name: Build openpilot
+      id: build
+      run: scons
+    - name: Provide H.264 encoder for Android Auto integration tests
+      if: ${{ !cancelled() && steps.build.outcome == 'success' }}
+      run: |
+        uv pip install --python .venv/bin/python --only-binary av av==19.0.1
+        .venv/bin/python -c "import av; assert 'libx264' in av.codecs_available"
+    - name: Test every assigned custom pytest node serially
+      if: ${{ !cancelled() && steps.build.outcome == 'success' }}
+      timeout-minutes: 15
+      env:
+        RAYLIB_BACKEND: headless
+      run: .venv/bin/python -m tools.ci.custom_pytest_shards run --index "${{ matrix.shard }}" --count 24 --output "ci-results/custom-pytest-${{ matrix.shard }}"
+    - name: Save collection, every outcome, JUnit and durable worker diagnostics
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: custom-pytest-${{ matrix.shard }}
+        path: ci-results/custom-pytest-${{ matrix.shard }}/
+        if-no-files-found: error
+
+  custom_pytest_complete:
+    name: complete custom pytest catalog
+    needs: custom_pytest
+    if: always()
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+    steps:
+    - uses: actions/checkout@v7
+    - uses: actions/download-artifact@v4
+      with:
+        pattern: custom-pytest-*
+        path: ci-results/custom-pytest-shards
+    - name: Require every case passed or audited capability-unavailable
+      run: python3 -m tools.ci.custom_pytest_shards aggregate --directory ci-results/custom-pytest-shards --count 24 --output ci-results/custom-pytest-complete.json
+    - name: Save complete custom catalog verdict
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: custom-pytest-complete
+        path: ci-results/custom-pytest-complete.json
+        if-no-files-found: error
+
   aol_lifecycles:
     name: AOL lifecycles (${{ matrix.shard }})
     runs-on: ubuntu-24.04