better

.github/workflows/tests.yml

@@ -10,7 +10,7 @@ jobs:
       matrix:
         flags: ['', '--asan', '--ubsan']
         backend: ['MSGQ', 'ZMQ']
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.11", "3.12"]
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5

.github/workflows/wheels.yml

@@ -0,0 +1,51 @@
+name: wheels
+
+on: [push, pull_request]
+
+jobs:
+  wheels:
+    runs-on: ${{ matrix.platform.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        platform: [ {os: "ubuntu-latest", target: "manylinux_x86_64",  arch: "x86_64"},
+                    {os: "ubuntu-latest", target: "manylinux_aarch64", arch: "aarch64"},
+                    {os: "macos-14",      target: "macosx_arm64",      arch: "arm64"} ]
+        python:   [ {cp: "cp311", py: "3.11"}, {cp: "cp312", py: "3.12"} ]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up QEMU
+      if: runner.os == 'Linux'
+      uses: docker/setup-qemu-action@v3
+      with:
+        platforms: all
+
+    - name: Building wheel
+      uses: pypa/cibuildwheel@v2.19.2
+      env:
+        CIBW_BUILD: "${{ matrix.python.cp }}-${{ matrix.platform.target }}"
+        CIBW_ARCHS: "${{ matrix.platform.arch }}"
+        CIBW_BEFORE_ALL_LINUX: "bash {project}/scripts/manylinux_dependencies.sh"
+        CIBW_BEFORE_BUILD_MACOS: "bash {project}/scripts/macos_dependencies.sh"
+        CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
+        CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28"
+        CIBW_TEST_COMMAND: pip install -r {project}/requirements.txt && pytest {package}
+
+
+    - uses: actions/setup-python@v5
+      if: ${{ matrix.platform.arch }} != 'aarch64'
+      with:
+        python-version: ${{ matrix.python.py }}
+
+    - name: Installing the wheel
+      if: ${{ matrix.platform.arch }} != 'aarch64'
+      run: |
+        pip install --break-system-packages ./wheelhouse/*.whl
+
+    - name: Saving wheel
+      uses: actions/upload-artifact@v4
+      with:
+        name: cibw-wheels-${{ matrix.platform.target }}-${{ matrix.python.cp }}
+        path: ./wheelhouse/*.whl

pyproject.toml

@@ -1,17 +1,17 @@
 [project]
 name = "msgq"
-requires-python = ">= 3.8"
+requires-python = ">= 3.11"
 version = "0.1.0"
 
 dependencies = [
   "numpy",
   "Cython",
   "pycapnp",
+  "setuptools",
 ]
 
 [project.optional-dependencies]
 dev = [
-  "pyyaml",
   "scons",
   "pre-commit",
   "ruff",

requirements.txt

@@ -0,0 +1,50 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile --all-extras -o requirements.txt pyproject.toml
+cfgv==3.4.0
+    # via pre-commit
+coverage==7.6.0
+    # via
+    #   msgq (pyproject.toml)
+    #   pytest-cov
+cython==3.0.10
+    # via msgq (pyproject.toml)
+distlib==0.3.8
+    # via virtualenv
+filelock==3.15.4
+    # via virtualenv
+identify==2.6.0
+    # via pre-commit
+iniconfig==2.0.0
+    # via pytest
+nodeenv==1.9.1
+    # via pre-commit
+numpy==2.0.0
+    # via msgq (pyproject.toml)
+packaging==24.1
+    # via pytest
+parameterized==0.9.0
+    # via msgq (pyproject.toml)
+platformdirs==4.2.2
+    # via virtualenv
+pluggy==1.5.0
+    # via pytest
+pre-commit==3.7.1
+    # via msgq (pyproject.toml)
+pycapnp==2.0.0
+    # via msgq (pyproject.toml)
+pytest==8.2.2
+    # via
+    #   msgq (pyproject.toml)
+    #   pytest-cov
+pytest-cov==5.0.0
+    # via msgq (pyproject.toml)
+pyyaml==6.0.1
+    # via pre-commit
+ruff==0.5.3
+    # via msgq (pyproject.toml)
+scons==4.8.0
+    # via msgq (pyproject.toml)
+setuptools==71.0.4
+    # via msgq (pyproject.toml)
+virtualenv==20.26.3
+    # via pre-commit

scripts/build_wheel.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-set -e
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-cd $DIR
-
-./install_dependencies.sh
-cd ..
-python3 -m build

scripts/install_dependencies.sh

@@ -1,58 +0,0 @@
-#!/bin/bash
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-
-cd $DIR/../
-
-SUDO=""
-
-# Use sudo if not root
-if [[ ! $(id -u) -eq 0 ]]; then
-  if [[ -z $(which sudo) ]]; then
-    echo "Please install sudo or run as root"
-    exit 1
-  fi
-  SUDO="sudo"
-fi
-
-$SUDO apt-get update
-$SUDO apt-get install -y --no-install-recommends \
-  autoconf \
-  build-essential \
-  ca-certificates \
-  capnproto \
-  clang \
-  cppcheck \
-  curl \
-  git \
-  libbz2-dev \
-  libcapnp-dev \
-  libclang-rt-dev \
-  libffi-dev \
-  liblzma-dev \
-  libncurses5-dev \
-  libncursesw5-dev \
-  libreadline-dev \
-  libsqlite3-dev \
-  libssl-dev \
-  libtool \
-  libzmq3-dev \
-  llvm \
-  make \
-  cmake \
-  ocl-icd-opencl-dev \
-  opencl-headers  \
-  python3-dev \
-  python3-pip \
-  tk-dev \
-  wget \
-  xz-utils \
-  zlib1g-dev
-
-if [[ -n "$BUILD_TESTS" ]]; then
-  git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git
-  cd Catch2
-  mv single_include/* ../
-  cd ..
-  rm -rf Catch2
-fi

scripts/macos_dependencies.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+brew bundle --file=- <<-EOS
+brew "zeromq"
+cask "gcc-arm-embedded"
+brew "gcc@13"
+EOS

scripts/manylinux_dependencies.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+dnf install -y clang opencl-headers ocl-icd-devel cppcheck wget
+
+wget https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz
+tar -xvf zeromq-4.3.5.tar.gz
+cd zeromq-4.3.5
+./configure
+make -j$(nproc)
+make install

scripts/ubuntu_dependencies.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [[ ! $(id -u) -eq 0 ]]; then
+  if [[ -z $(which sudo) ]]; then
+    echo "Please install sudo or run as root"
+    exit 1
+  fi
+  SUDO="sudo"
+fi
+
+$SUDO apt-get update
+$SUDO apt-get install -y --no-install-recommends clang opencl-headers libzmq3-dev ocl-icd-opencl-dev cppcheck

setup.py

@@ -1,8 +1,13 @@
-from setuptools import Command, setup
+from setuptools import Command, setup, Distribution
 from setuptools.command.build import build
 import subprocess
 import os
 
+class BinaryDistribution(Distribution):
+  def has_ext_modules(self):
+    return True
+
+
 class SconsBuild(Command):
   def initialize_options(self) -> None:
     pass
@@ -12,14 +17,16 @@ class SconsBuild(Command):
 
   def run(self) -> None:
     scons_flags = '' if 'BUILD_TESTS' in os.environ else '--minimal'
-    subprocess.run([f"scons {scons_flags} -j$(nproc)"], shell=True).check_returncode()
+    subprocess.run([f"scons {scons_flags} -j$(nproc || sysctl -n hw.logicalcpu)"], shell=True).check_returncode()
+
 
 class CustomBuild(build):
   sub_commands = [('scons_build', None)] + build.sub_commands
 
 setup(
     packages = ["msgq", "msgq.visionipc"],
-    package_data={'msgq': ['**/*.cc', '**/*.h', '**/*.pxd', '**/*.pyx', '**/*.so']},
+    package_data={'': ['**/*.cc', '**/*.c', '**/*.h', '**/*.pxd', '**/*.pyx', '**/*.py', '**/*.so', '**/*.npy']},
     include_package_data=True,
-    cmdclass={'build': CustomBuild, 'scons_build': SconsBuild}
+    cmdclass={'build': CustomBuild, 'scons_build': SconsBuild},
+    distclass=BinaryDistribution,
     )