From b1a6223b14f4ea14b748588da7667525ad259548 Mon Sep 17 00:00:00 2001 From: Jason Wen Date: Sun, 10 May 2026 01:04:45 -0400 Subject: [PATCH] ci: simplify cereal validation to sparse-checkout + pycapnp, drop scons (#1836) * ci: simplify cereal validation to sparse-checkout + pycapnp, drop scons build Co-Authored-By: Claude Sonnet 4.6 (1M context) * more * fix: resolve cereal_dir to absolute path before passing to capnp.load Co-Authored-By: Claude Sonnet 4.6 (1M context) * ci: init opendbc submodule after sparse checkout to resolve car.capnp symlink Co-Authored-By: Claude Sonnet 4.6 (1M context) * try to break it * Revert "try to break it" This reverts commit 79ce135c5f59b093aeee7cc564c29565498f74d9. * try to break it * Revert "try to break it" This reverts commit 1eaa9e79e6824b9236e69ee2e34f359499fabf38. --------- Co-authored-by: Claude Sonnet 4.6 (1M context) --- .github/workflows/cereal_validation.yaml | 73 ++++++++----------- .../tests/validate_sp_cereal_upstream.py | 29 +++++--- 2 files changed, 47 insertions(+), 55 deletions(-) diff --git a/.github/workflows/cereal_validation.yaml b/.github/workflows/cereal_validation.yaml index e9c0e5d03..d71354dd5 100644 --- a/.github/workflows/cereal_validation.yaml +++ b/.github/workflows/cereal_validation.yaml @@ -23,56 +23,43 @@ env: CI: 1 jobs: - generate_cereal_artifact: - name: Generate cereal validation artifacts - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v6 - with: - submodules: true - - run: ./tools/op.sh setup - - name: Build openpilot - run: scons -j$(nproc) cereal - - name: Dump sunnypilot schema - run: | - export PYTHONPATH=${{ github.workspace }} - python3 cereal/messaging/tests/validate_sp_cereal_upstream.py -g -f schema.json - - name: 'Prepare artifact' - run: | - mkdir -p "cereal/messaging/tests/cereal_validations" - cp cereal/messaging/tests/validate_sp_cereal_upstream.py "cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py" - cp schema.json "cereal/messaging/tests/cereal_validations/schema.json" - - name: 'Upload Artifact' - uses: actions/upload-artifact@v4 - with: - name: cereal_validations - path: cereal/messaging/tests/cereal_validations - validate_cereal_with_upstream: name: Validate cereal with Upstream runs-on: ubuntu-24.04 - needs: generate_cereal_artifact steps: - - name: Checkout sunnypilot + - name: Checkout sunnypilot cereal uses: actions/checkout@v6 - - name: Checkout upstream openpilot + with: + sparse-checkout: cereal + + - name: Init sunnypilot opendbc submodule + run: git submodule update --init --depth 1 opendbc_repo + + - name: Checkout upstream openpilot cereal uses: actions/checkout@v6 with: repository: 'commaai/openpilot' - path: openpilot - submodules: true + path: upstream_openpilot + sparse-checkout: cereal ref: "refs/heads/master" - - run: ./tools/op.sh setup - - name: Build openpilot - working-directory: openpilot - run: scons -j$(nproc) cereal - - name: Download build artifacts - uses: actions/download-artifact@v4 - with: - name: cereal_validations - path: openpilot/cereal/messaging/tests/cereal_validations - - name: 'Validate sunnypilot schema against upstream' + + - name: Init upstream opendbc submodule + working-directory: upstream_openpilot + run: git submodule update --init --depth 1 opendbc_repo + + - name: Install uv + run: pip install uv + + - name: Generate sunnypilot schema run: | - export PYTHONPATH=${{ github.workspace }}/openpilot - chmod +x openpilot/cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py - python3 openpilot/cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py -r -f openpilot/cereal/messaging/tests/cereal_validations/schema.json + PYCAPNP_VER=$(python3 -c "import re; m=re.search(r'name = \"pycapnp\"\nversion = \"([^\"]+)\"', open('uv.lock').read()); print(m.group(1))") + uv run --isolated --with "pycapnp==${PYCAPNP_VER}" \ + python3 cereal/messaging/tests/validate_sp_cereal_upstream.py \ + -g -f /tmp/sp_schema.json --cereal-dir cereal + + - name: Validate against upstream + run: | + PYCAPNP_VER=$(python3 -c "import re; m=re.search(r'name = \"pycapnp\"\nversion = \"([^\"]+)\"', open('uv.lock').read()); print(m.group(1))") + uv run --isolated --with "pycapnp==${PYCAPNP_VER}" \ + python3 cereal/messaging/tests/validate_sp_cereal_upstream.py \ + -r -f /tmp/sp_schema.json --cereal-dir upstream_openpilot/cereal diff --git a/cereal/messaging/tests/validate_sp_cereal_upstream.py b/cereal/messaging/tests/validate_sp_cereal_upstream.py index d673f7db0..11e39cd6c 100755 --- a/cereal/messaging/tests/validate_sp_cereal_upstream.py +++ b/cereal/messaging/tests/validate_sp_cereal_upstream.py @@ -13,6 +13,7 @@ from __future__ import annotations import argparse import json +import os import sys from typing import Any @@ -104,8 +105,15 @@ def collect_schema(root: Any) -> dict[str, dict]: return structs -def dump_schema(path: str) -> None: - from cereal import log +def load_log(cereal_dir: str) -> Any: + import capnp + cereal_dir = os.path.abspath(cereal_dir) + capnp.remove_import_hook() + return capnp.load(os.path.join(cereal_dir, "log.capnp"), imports=[cereal_dir]) + + +def dump_schema(cereal_dir: str, path: str) -> None: + log = load_log(cereal_dir) payload = { "root": hex_id(log.Event.schema.node.id), "structs": collect_schema(log.Event.schema), @@ -206,8 +214,8 @@ def load_peer(path: str) -> dict: return json.load(handle) -def run_read(peer_path: str) -> int: - from cereal import log +def run_read(cereal_dir: str, peer_path: str) -> int: + log = load_log(cereal_dir) peer_dump = load_peer(peer_path) local_dump = { "root": hex_id(log.Event.schema.node.id), @@ -235,16 +243,13 @@ def main() -> int: mode.add_argument("-g", "--generate", action="store_true", help="dump local schema to JSON") mode.add_argument("-r", "--read", action="store_true", help="load peer JSON and diff against local") parser.add_argument("-f", "--file", default="schema.json", help="JSON file path (default: schema.json)") + parser.add_argument("--cereal-dir", required=True, help="path to cereal directory containing log.capnp") args = parser.parse_args() - try: - if args.generate: - dump_schema(args.file) - return 0 - return run_read(args.file) - except ImportError as exc: - print(f"error: cannot import cereal ({exc}). did scons build cereal?") - return 2 + if args.generate: + dump_schema(args.cereal_dir, args.file) + return 0 + return run_read(args.cereal_dir, args.file) if __name__ == "__main__":