diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 5b2d462..0000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -.sconsign.dblite diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cab495a..3fb4523 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,61 +1,26 @@ name: tests -on: [push, pull_request] +on: + push: + branches: + - master + pull_request: -env: - DOCKER_REGISTRY: ghcr.io/commaai - RUN: docker run -e PYTHONWARNINGS=error --shm-size 1G --name msgq msgq /bin/sh -c - RUN_NAMED: docker run -e PYTHONWARNINGS=error --shm-size 1G --rm msgq /bin/sh -c - CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm msgqci /bin/bash -c - BUILD: docker buildx build --pull --load --cache-to type=inline --cache-from $DOCKER_REGISTRY/msgq:latest -t msgq -f Dockerfile . - PYTHONWARNINGS: error +concurrency: + group: ${{ github.workflow }}-${{ github.ref != 'refs/heads/master' && github.ref || github.run_id }}-${{ github.event_name }} + cancel-in-progress: true jobs: - build: - name: build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Build docker image - run: eval "$BUILD" - - name: Push to dockerhub - if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/msgq' - run: | - docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} - docker tag msgq $DOCKER_REGISTRY/msgq:latest - docker push $DOCKER_REGISTRY/msgq:latest - - unit_tests: - name: unit tests - runs-on: ubuntu-latest + test: + name: ./test.sh + runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: - flags: ['', '--asan', '--ubsan'] - backend: ['MSGQ', 'ZMQ'] + # TODO: fixup macOS + #os: ['macos-latest', 'ubuntu-latest'] + os: ['ubuntu-latest'] + timeout-minutes: 10 steps: - - uses: actions/checkout@v3 - - name: Build docker image - run: eval "$BUILD" - - name: C++ tests - run: | - $RUN "export ${{ matrix.backend }}=1 && \ - scons ${{ matrix.flags }} -j$(nproc) && \ - msgq/test_runner && \ - msgq/visionipc/test_runner" - - name: python tests - run: $RUN_NAMED "${{ matrix.backend }}=1 coverage run -m pytest" - - name: Upload coverage - run: | - docker commit msgq msgqci - $CI_RUN "cd /project/msgq && bash <(curl -s https://codecov.io/bash) -v -F unit_tests_${{ matrix.backend }}" - - static_analysis: - name: static analysis - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Build docker image - run: eval "$BUILD" - - name: Static analysis - # TODO: a package pre-commit installs has a warning, remove the unset once that's fixed - run: $RUN "git init && git add -A && unset PYTHONWARNINGS && pre-commit run --all" + - uses: actions/checkout@v4 + - run: ./test.sh diff --git a/.gitignore b/.gitignore index 0113da4..324effb 100644 --- a/.gitignore +++ b/.gitignore @@ -8,11 +8,12 @@ __pycache__ *.so *.o *.a +uv.lock +catch2/ test_runner libmessaging.* libmessaging_shared.* -services.h .sconsign.dblite -.mypy_cache/ +.mypy_cache/ \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b32abe2..c24e0b2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,6 +30,7 @@ repos: - --force - --quiet - -j4 + - --check-level=exhaustive - repo: https://github.com/cpplint/cpplint rev: 1.6.1 hooks: diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index ddc3a83..0000000 --- a/Dockerfile +++ /dev/null @@ -1,52 +0,0 @@ -FROM ubuntu:24.04 - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - autoconf \ - build-essential \ - ca-certificates \ - capnproto \ - clang \ - cppcheck \ - curl \ - git \ - libbz2-dev \ - libcapnp-dev \ - libclang-rt-dev \ - libffi-dev \ - liblzma-dev \ - libncurses5-dev \ - libncursesw5-dev \ - libreadline-dev \ - libsqlite3-dev \ - libssl-dev \ - libtool \ - libzmq3-dev \ - llvm \ - make \ - cmake \ - ocl-icd-opencl-dev \ - opencl-headers \ - python3-dev \ - python3-pip \ - tk-dev \ - wget \ - xz-utils \ - zlib1g-dev \ - && rm -rf /var/lib/apt/lists/* - -RUN pip3 install --break-system-packages --no-cache-dir pyyaml Cython scons pycapnp pre-commit ruff parameterized coverage numpy pytest - -WORKDIR /project/msgq -RUN cd /tmp/ && \ - git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git && \ - cd Catch2 && \ - mv single_include/* /project/msgq/ && \ - cd .. \ - rm -rf Catch2 - -ENV PYTHONPATH=/project/msgq - -COPY . . -RUN ls && rm -rf .git && \ - scons -c && scons -j$(nproc) diff --git a/pyproject.toml b/pyproject.toml index 87fb21b..d7c4e33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,29 @@ +[project] +name = "msgq" +version = "0.0.1" +description = "Code powering the comma.ai panda" +readme = "README.md" +requires-python = ">=3.11,<3.13" +license = {text = "MIT"} +authors = [{name = "comma.ai"}] +classifiers = [ + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Topic :: System :: Hardware", +] +dependencies = [ + "setuptools", # for distutils + "Cython", + "scons", + "pre-commit", + "ruff", + "parameterized", + "coverage", + "numpy", + "pytest", + "cppcheck", +] + # https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml [tool.ruff] lint.select = ["E", "F", "W", "PIE", "C4", "ISC", "RUF100", "A"] diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000..abba95d --- /dev/null +++ b/setup.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -e + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +cd $DIR + +PLATFORM=$(uname -s) + +echo "installing dependencies" +if [[ $PLATFORM == "Darwin" ]]; then + export HOMEBREW_NO_AUTO_UPDATE=1 + brew install python3 zeromq +elif [[ $PLATFORM == "Linux" ]]; then + # for AGNOS since we clear the apt lists + if [[ ! -d /"var/lib/apt/" ]]; then + sudo apt update + fi + + sudo apt-get install -y --no-install-recommends \ + curl ca-certificates \ + libzmq3-dev \ + ocl-icd-opencl-dev opencl-headers \ + python3-dev python3-pip python3-venv +else + echo "WARNING: unsupported platform. skipping apt/brew install." +fi + +# catch2 +if [ ! -d $DIR/msgq/catch2/ ]; then + rm -rf /tmp/catch2/ $DIR/msgq/catch2/ + git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git /tmp/catch2 + pushd /tmp/catch2 + mv single_include/* $DIR/msgq/ + popd +fi + +if ! command -v uv &>/dev/null; then + echo "'uv' is not installed. Installing 'uv'..." + curl -LsSf https://astral.sh/uv/install.sh | sh + + # doesn't require sourcing on all platforms + set +e + source $HOME/.local/bin/env + set -e +fi + +export UV_PROJECT_ENVIRONMENT="$DIR/.venv" +uv sync --all-extras +source "$DIR/.venv/bin/activate" diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..e0cfe7c --- /dev/null +++ b/test.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -e + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +cd $DIR + +# *** env setup *** +source ./setup.sh + +# *** build *** +scons -j8 + +# *** lint *** +#ruff check . +#mypy python/ +pre-commit run --all-files + +# *** test *** +pytest