Process replay: add diff report (#37048)

* rm upload

* use ci-artifacts

* sanitize

* rm ref_commit

* add ci

* handle exept

* bootstrap

* always

* fix

* replay

* keep ref_commit fork compatibility

* remove upload-only

* apply comments

* safe diffs in master

* Revert "safe diffs in master"

This reverts commit 369fccac786a67799193e9152488813c6df20414.

* continue on master diff

* imports

* copy formatting from car_diff

* main

* setup refs and cur

* copy diff

* copy formatting

* comment

* rm token

* rm hash

* continue on master diff

* use ci-artifacts refs

* add run card diff

* checkout

* shebang

* card_diff.yml

* rm ci-artifacts

* apply ci-artifacts

* call differ

* rename

* uv lock

* tests

* readme

* checkout

* add all configs

* import base_url

* rename yaml

* integrate in test_processes

* fix diff report

* var names

* extract to module

* print report

* add msg count to diff

* traceback

* diff format

* typing

* name step

* allow NaN

* replace join

.github/workflows/diff_report.yaml

@@ -0,0 +1,45 @@
+name: diff report
+
+on:
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  comment:
+    name: comment
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      pull-requests: write
+      actions: read
+    steps:
+    - name: Wait for process replay
+      id: wait
+      continue-on-error: true
+      uses: lewagon/wait-on-check-action@v1.3.4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        check-name: process replay
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        allowed-conclusions: success,failure
+        wait-interval: 20
+    - name: Download diff
+      if: steps.wait.outcome == 'success'
+      uses: dawidd6/action-download-artifact@v6
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        workflow: tests.yaml
+        workflow_conclusion: ''
+        pr: ${{ github.event.number }}
+        name: diff_report_${{ github.event.number }}
+        path: .
+        allow_forks: true
+    - name: Comment on PR
+      if: steps.wait.outcome == 'success'
+      uses: thollander/actions-comment-pull-request@v2
+      with:
+        filePath: diff_report.txt
+        comment_tag: diff_report
+        pr_number: ${{ github.event.number }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/tests.yaml

@@ -139,12 +139,22 @@ jobs:
       id: print-diff
       if: always()
       run: cat selfdrive/test/process_replay/diff.txt
+    - name: Print diff report
+      if: always()
+      run: cat selfdrive/test/process_replay/diff_report.txt
     - uses: actions/upload-artifact@v6
       if: always()
       continue-on-error: true
       with:
         name: process_replay_diff.txt
         path: selfdrive/test/process_replay/diff.txt
+    - name: Upload diff report
+      uses: actions/upload-artifact@v6
+      if: always() && github.event_name == 'pull_request'
+      continue-on-error: true
+      with:
+        name: diff_report_${{ github.event.number }}
+        path: selfdrive/test/process_replay/diff_report.txt
     - name: Checkout ci-artifacts
       if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
       uses: actions/checkout@v4