diff --git a/.github/workflows/model_review.yaml b/.github/workflows/model_review.yaml deleted file mode 100644 index bdcc2e8977..0000000000 --- a/.github/workflows/model_review.yaml +++ /dev/null @@ -1,47 +0,0 @@ -name: "model review" - -on: - pull_request: - types: [opened, reopened, synchronize] - paths: - - 'openpilot/selfdrive/modeld/models/*.onnx' - workflow_dispatch: - -env: - GIT_CONFIG_COUNT: 1 - GIT_CONFIG_KEY_0: lfs.fetchexclude - GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx - -jobs: - comment: - permissions: - contents: read - pull-requests: write - runs-on: ubuntu-latest - if: github.repository == 'commaai/openpilot' - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - submodules: true - - name: Checkout master - uses: actions/checkout@v7 - with: - ref: master - path: base - - run: git lfs pull - - run: cd base && git lfs pull - - - name: scripts/reporter.py - id: report - run: | - echo "content<> $GITHUB_OUTPUT - echo "## Model Review" >> $GITHUB_OUTPUT - PYTHONPATH=${{ github.workspace }}:${{ github.workspace }}/tinygrad_repo MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - - name: Post model report comment - uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 - with: - header: model-review - message: ${{ steps.report.outputs.content }} diff --git a/scripts/reporter.py b/scripts/reporter.py deleted file mode 100755 index 175a06fa67..0000000000 --- a/scripts/reporter.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 -import os -import glob - -from tinygrad.nn.onnx import OnnxPBParser - -BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../")) - -MASTER_PATH = os.getenv("MASTER_PATH", BASEDIR) -MODEL_PATH = "/openpilot/selfdrive/modeld/models/" - - -class MetadataOnnxPBParser(OnnxPBParser): - def _parse_ModelProto(self) -> dict: - obj = {"metadata_props": []} - for fid, wire_type in self._parse_message(self.reader.len): - match fid: - case 14: - obj["metadata_props"].append(self._parse_StringStringEntryProto()) - case _: - self.reader.skip_field(wire_type) - return obj - - -def get_checkpoint(f): - model = MetadataOnnxPBParser(f).parse() - metadata = {prop["key"]: prop["value"] for prop in model["metadata_props"]} - # "" or "...//"; combined models list vision then policy - parts = metadata['model_checkpoint'].split('/') - return parts[-2] if len(parts) > 1 else parts[0] - - -if __name__ == "__main__": - print("| | master | PR branch |") - print("|-| ----- | --------- |") - - for f in glob.glob(BASEDIR + MODEL_PATH + "/*.onnx"): - fn = os.path.basename(f) - if fn == "big_driving_supercombo.onnx": - continue - master_path = MASTER_PATH + MODEL_PATH + fn - if os.path.exists(master_path): - master = get_checkpoint(master_path) - master_col = f"[{master}](https://reporter.comma.life/{master})" - else: - master_col = "N/A (new model)" - pr = get_checkpoint(BASEDIR + MODEL_PATH + fn) - print("|", fn, "|", master_col, "|", f"[{pr}](https://reporter.comma.life/{pr})", "|")