openpilot v0.9.8 toyota-test-build

date: 2024-11-27T07:44:05
master commit: 2b6b8414700cbb048f8a76329ff52d0112f86bc2

.devcontainer/.gitignore

@@ -1,3 +0,0 @@
-.Xauthority
-.env
-.host/

.devcontainer/Dockerfile

@@ -1,18 +0,0 @@
-FROM ghcr.io/commaai/openpilot-base:latest
-
-RUN apt update && apt install -y vim net-tools usbutils htop ripgrep tmux wget mesa-utils xvfb libxtst6 libxv1 libglu1-mesa gdb bash-completion
-RUN python3 -m ensurepip --upgrade
-RUN pip3 install ipython jupyter jupyterlab
-
-RUN cd /tmp && \
-    ARCH=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && \
-    curl -L -o virtualgl.deb "https://github.com/VirtualGL/virtualgl/releases/download/3.1.1/virtualgl_3.1.1_$ARCH.deb" && \
-    dpkg -i virtualgl.deb
-
-RUN usermod -aG video batman
-
-USER batman
-
-RUN cd $HOME && \
-    curl -O https://raw.githubusercontent.com/commaai/agnos-builder/master/userspace/home/.tmux.conf && \
-    curl -O https://raw.githubusercontent.com/commaai/agnos-builder/master/userspace/home/.vimrc

.devcontainer/container_post_create.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-
-TARGET_USER=batman
-source .devcontainer/.host/.env
-
-# override display flag for mac hosts
-if [[ $HOST_OS == darwin ]]; then
-  echo "Setting up DISPLAY override for macOS..."
-  cat <<EOF >> /home/$TARGET_USER/.bashrc
-source .devcontainer/.host/.env
-if [ -n "\$HOST_DISPLAY" ]; then
-  DISPLAY_NUM=\$(echo "\$HOST_DISPLAY" | awk -F: '{print \$NF}')
-  export DISPLAY=host.docker.internal:\$DISPLAY_NUM
-fi
-EOF
-fi
-
-# setup virtualgl for mac hosts
-if [[ $HOST_OS == darwin ]]; then
-  echo "Setting up virtualgl for macOS..."
-  cat <<EOF >> /home/$TARGET_USER/.bashrc
-if [ -n "\$HOST_DISPLAY" ]; then
-  export VGL_PORT=10000
-  export VGL_CLIENT=host.docker.internal
-  export VGL_COMPRESS=rgb
-  export VGL_DISPLAY=:99
-  export VGL_FPS=60
-  # prevent vglrun from running exec
-  alias exec=:; source vglrun :; unalias exec
-fi
-EOF
-fi
-
-# These lines are temporary, to remain backwards compatible with old devcontainers
-# that were running as root and therefore had their caches written as root
-sudo chown -R $TARGET_USER: /tmp/scons_cache
-sudo chown -R $TARGET_USER: /tmp/comma_download_cache
-sudo chown -R $TARGET_USER: /home/batman/.comma

.devcontainer/container_post_start.sh

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-source .devcontainer/.host/.env
-
-# setup safe directories for submodules
-SUBMODULE_DIRS=$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }')
-for DIR in $SUBMODULE_DIRS; do 
-  git config --global --add safe.directory "$PWD/$DIR"
-done
-
-# virtual display for virtualgl
-if [[ "$HOST_OS" == "darwin" ]] && [[ -n "$HOST_DISPLAY" ]]; then
-  echo "Starting virtual display at :99 ..."
-  tmux new-session -d -s fakedisplay Xvfb :99 -screen 0 1920x1080x24
-fi

.devcontainer/devcontainer.json

@@ -1,53 +0,0 @@
-{
-  "name": "openpilot devcontainer",
-  "build": {
-    "dockerfile": "Dockerfile"
-  },
-  "postCreateCommand": ".devcontainer/container_post_create.sh",
-  "postStartCommand": ".devcontainer/container_post_start.sh",
-  "initializeCommand": [".devcontainer/host_setup"],
-  "privileged": true,
-  "containerEnv": {
-    "DISPLAY": "${localEnv:DISPLAY}",
-    "PYTHONPATH": "${containerWorkspaceFolder}",
-    "TERM": "xterm-256color",
-    "force_color_prompt": "1"
-  },
-  "runArgs": [
-    "--volume=/dev:/dev",
-    "--volume=/tmp/.X11-unix:/tmp/.X11-unix",
-    "--volume=${localWorkspaceFolder}/.devcontainer/.host/.Xauthority:/home/batman/.Xauthority",
-    "--volume=${localEnv:HOME}/.comma:/home/batman/.comma",
-    "--volume=${localEnv:HOME}/.azure:/home/batman/.azure",
-    "--volume=/tmp/comma_download_cache:/tmp/comma_download_cache",
-    "--shm-size=1G",
-    "--add-host=host.docker.internal:host-gateway", // required to use host.docker.internal on linux
-    "--publish=0.0.0.0:8070-8079:8070-8079"  // body ZMQ services
-  ],
-  "features": {
-    "ghcr.io/devcontainers/features/common-utils:2": {
-      "installZsh": false,
-      "installOhMyZsh": false,
-      "upgradePackages": false,
-      "username": "batman"
-    },
-    "ghcr.io/devcontainers-contrib/features/gh-cli:1": {},
-    "ghcr.io/devcontainers/features/azure-cli:1": {}
-  },
-  "containerUser": "batman",
-  "remoteUser": "batman",
-  "customizations": {
-    "vscode": {
-      "extensions": [
-        "ms-python.python",
-        "ms-vscode.cpptools",
-        "ms-toolsai.jupyter",
-        "guyskk.language-cython",
-        "lharri73.dbc"
-      ]
-    }
-  },
-  "mounts": [
-    "type=volume,source=scons_cache,target=/tmp/scons_cache"
-  ]
-}

.devcontainer/host_setup

@@ -1,47 +0,0 @@
-#!/usr/bin/env bash
-
-# pull base image
-if [[ -z $USE_LOCAL_IMAGE ]]; then
-  echo "Updating openpilot_base image if needed..."
-  docker pull ghcr.io/commaai/openpilot-base:latest
-fi
-
-# setup .host dir
-mkdir -p .devcontainer/.host
-
-# setup links to Xauthority
-XAUTHORITY_LINK=".devcontainer/.host/.Xauthority"
-rm -f $XAUTHORITY_LINK
-if [[ -z $XAUTHORITY ]]; then
-  echo "XAUTHORITY not set. Fallback to ~/.Xauthority ..."
-  if ! [[ -f $HOME/.Xauthority ]]; then
-    echo "~/.XAuthority file does not exist. GUI tools may not work properly."
-    touch $XAUTHORITY_LINK # dummy file to satisfy container volume mount
-  else
-    ln -sf $HOME/.Xauthority $XAUTHORITY_LINK
-  fi
-else
-    ln -sf $XAUTHORITY $XAUTHORITY_LINK
-fi
-
-# setup host env file
-HOST_INFO_FILE=".devcontainer/.host/.env"
-SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]')
-echo "HOST_OS=\"$SYSTEM\"" > $HOST_INFO_FILE
-echo "HOST_DISPLAY=\"$DISPLAY\"" >> $HOST_INFO_FILE
-
-# run virtualgl if macos
-if [[ $SYSTEM == "darwin" ]]; then
-  echo
-  if [[ -f /opt/VirtualGL/bin/vglclient ]]; then
-    echo "Starting VirtualGL client at port 10000..."
-    VGL_LOG_FILE=".devcontainer/.host/.vgl/vglclient.log"
-    mkdir -p "$(dirname $VGL_LOG_FILE)"
-    /opt/VirtualGL/bin/vglclient -l "$VGL_LOG_FILE" -display "$DISPLAY" -port 10000 -detach
-  else
-    echo "VirtualGL not found. GUI tools may not work properly. Some GUI tools require OpenGL to work properly. To use them with XQuartz on mac, VirtualGL needs to be installed. To install it run:"
-    echo
-    echo "  brew install --cask virtualgl"
-    echo
-  fi
-fi

.devcontainer/host_setup.cmd

@@ -1,10 +0,0 @@
-:: pull base image
-IF NOT DEFINED USE_LOCAL_IMAGE ^
-echo "Updating openpilot_base image if needed..." && ^
-docker pull ghcr.io/commaai/openpilot-base:latest
-
-:: setup .host dir
-mkdir .devcontainer\.host
-
-:: setup host env file
-echo "" > .devcontainer\.host\.env

.gitattributes

@@ -1,23 +0,0 @@
-* text=auto
-
-# to move existing files into LFS:
-# git add --renormalize .
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.svg filter=lfs diff=lfs merge=lfs -text
-*.png filter=lfs diff=lfs merge=lfs -text
-*.gif filter=lfs diff=lfs merge=lfs -text
-*.ttf filter=lfs diff=lfs merge=lfs -text
-*.wav filter=lfs diff=lfs merge=lfs -text
-
-selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
-system/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text
-selfdrive/ui/qt/spinner_larch64 filter=lfs diff=lfs merge=lfs -text
-selfdrive/ui/qt/text_larch64 filter=lfs diff=lfs merge=lfs -text
-third_party/**/*.a filter=lfs diff=lfs merge=lfs -text
-third_party/**/*.so filter=lfs diff=lfs merge=lfs -text
-third_party/**/*.so.* filter=lfs diff=lfs merge=lfs -text
-third_party/**/*.dylib filter=lfs diff=lfs merge=lfs -text
-third_party/acados/*/t_renderer filter=lfs diff=lfs merge=lfs -text
-third_party/qt5/larch64/bin/lrelease filter=lfs diff=lfs merge=lfs -text
-third_party/qt5/larch64/bin/lupdate filter=lfs diff=lfs merge=lfs -text
-third_party/catch2/include/catch2/catch.hpp filter=lfs diff=lfs merge=lfs -text

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,47 +0,0 @@
-name: Bug report
-description: For issues with running openpilot on your comma device
-labels: ["bug"]
-body:
-
-  - type: markdown
-    attributes:
-      value: >
-        Before creating a **bug report**, please check the following:
-          * If the issue likely only affects your car model or make, go back and open a **car bug report** instead.
-          * If the issue is related to the driving or driver monitoring models, you should open a [discussion](https://github.com/commaai/openpilot/discussions/categories/model-feedback) instead.
-          * Ensure you're running the latest openpilot release.
-          * Ensure you're using officially supported hardware. Issues running on PCs have a different issue template.
-          * Ensure there isn't an existing issue for your bug. If there is, leave a comment on the existing issue.
-          * Ensure you're running stock openpilot. We cannot look into bug reports from forks.
-
-        If you're unsure whether you've hit a bug, check out the #installation-help channel in the [community Discord server](https://discord.comma.ai).
-
-  - type: textarea
-    attributes:
-      label: Describe the bug
-      description: Also include a description of how to reproduce the bug
-    validations:
-      required: true
-
-  - type: input
-    id: route
-    attributes:
-      label: Provide a route where the issue occurs
-      description: Ensure the route is fully uploaded at https://useradmin.comma.ai. We cannot look into issues without routes, or at least a Dongle ID.
-      placeholder: 77611a1fac303767|2020-05-11--16-37-07
-    validations:
-      required: true
-
-  - type: input
-    id: version
-    attributes:
-      label: openpilot version
-      description: If you're not on release, provide the commit hash
-      placeholder: 0.8.10
-    validations:
-      required: true
-
-  - type: textarea
-    attributes:
-      label: Additional info
-

.github/ISSUE_TEMPLATE/car_bug_report.yml

@@ -1,53 +0,0 @@
-name: Car bug report
-description: For issues with a particular car make or model
-labels: ["car", "bug"]
-body:
-
-  - type: markdown
-    attributes:
-      value: >
-        Before creating a **bug report**, please check the following:
-          * Ensure you're running the latest openpilot release.
-          * Ensure you're using officially supported hardware. Issues running on PCs have a different issue template.
-          * Ensure there isn't an existing issue for your bug. If there is, leave a comment on the existing issue.
-          * Ensure you're running stock openpilot. We cannot look into bug reports from forks.
-
-        If you're unsure whether you've hit a bug, check out the #installation-help channel in the [community Discord server](https://discord.comma.ai).
-
-  - type: textarea
-    attributes:
-      label: Describe the bug
-      description: Also include a description of how to reproduce the bug
-    validations:
-      required: true
-
-  - type: input
-    id: car
-    attributes:
-      label: Which car does this affect?
-      placeholder: Toyota Prius 2017
-    validations:
-      required: true
-
-  - type: input
-    id: route
-    attributes:
-      label: Provide a route where the issue occurs
-      description: Ensure the route is fully uploaded at https://useradmin.comma.ai
-      placeholder: 77611a1fac303767|2020-05-11--16-37-07
-    validations:
-      required: true
-
-  - type: input
-    id: version
-    attributes:
-      label: openpilot version
-      description: If you're not on release, provide the commit hash
-      placeholder: 0.8.10
-    validations:
-      required: true
-
-  - type: textarea
-    attributes:
-      label: Additional info
-

.github/ISSUE_TEMPLATE/config.yml

@@ -1,11 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: Join the Discord
-    url: https://discord.comma.ai
-    about: The community Discord is for both openpilot development and experience discussion
-  - name: Report model bugs
-    url: https://discord.com/channels/469524606043160576/1254834193066623017
-    about: Feedback for the driving and driver monitoring models goes in the #driving-feedback in Discord
-  - name: Community Wiki
-    url: https://github.com/commaai/openpilot/wiki
-    about: Check out our community wiki

.github/ISSUE_TEMPLATE/enhancement.md

@@ -1,8 +0,0 @@
----
-name: Enhancement
-about: For openpilot enhancement suggestions
-title: ''
-labels: 'enhancement'
-assignees: ''
----
-

.github/ISSUE_TEMPLATE/pc_bug_report.yml

@@ -1,42 +0,0 @@
-name: PC bug report
-description: For issues with running openpilot on PC
-labels: ["PC"]
-body:
-
-  - type: markdown
-    attributes:
-      value: >
-        Before creating a **bug report**, please check the following:
-          * Ensure you're running the latest openpilot release.
-          * Ensure there isn't an existing issue for your bug. If there is, leave a comment on the existing issue.
-          * Ensure you're running stock openpilot. We cannot look into bug reports from forks.
-
-        If you're unsure whether you've hit a bug, check out the #installation-help channel in the [community Discord server](https://discord.comma.ai).
-
-  - type: textarea
-    attributes:
-      label: Describe the bug
-      description: Also include a description of how to reproduce the bug
-    validations:
-      required: true
-
-  - type: input
-    id: os-version
-    attributes:
-      label: OS Version
-      placeholder: Ubuntu 24.04
-    validations:
-      required: true
-
-  - type: input
-    id: version
-    attributes:
-      label: openpilot version or commit
-      placeholder: bd36f2ec8d3559909678eff2690c10a520938367
-    validations:
-      required: false
-
-  - type: textarea
-    attributes:
-      label: Additional info
-

.github/PULL_REQUEST_TEMPLATE/bugfix.md

@@ -1,15 +0,0 @@
----
-name: Bug fix
-about: For openpilot bug fixes
-title: ''
-labels: 'bugfix'
-assignees: ''
----
-
-**Description**
-
-<!-- A description of the bug and the fix. Also link the issue if it exists. -->
-
-**Verification**
-
-<!-- Explain how you tested this bug fix. -->

.github/PULL_REQUEST_TEMPLATE/car_bugfix.md

@@ -1,19 +0,0 @@
----
-name: Car Bug fix
-about: For vehicle/brand specific bug fixes
-title: ''
-labels: 'car bug fix'
-assignees: ''
----
-
-**Description**
-
-<!-- A description of the bug and the fix. Also link the issue if it exists. -->
-
-**Verification**
-
-<!-- Explain how you tested this bug fix. -->
-
-**Route**
-
-Route: [a route with the bug fix]

.github/PULL_REQUEST_TEMPLATE/car_port.md

@@ -1,15 +0,0 @@
----
-name: Car port
-about: For new car ports
-title: ''
-labels: 'car port'
-assignees: ''
----
-
-**Checklist**
-
-- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/opcar/docs.py` to generate new docs
-- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
-- [ ] route with openpilot:
-- [ ] route with stock system:
-- [ ] car harness used (if comma doesn't sell it, put N/A):

.github/PULL_REQUEST_TEMPLATE/fingerprint.md

@@ -1,13 +0,0 @@
----
-name: Fingerprint
-about: For adding fingerprints to existing cars
-title: ''
-labels: 'fingerprint'
-assignees: ''
----
-
-**Car**
-Which car (make, model, year) this fingerprint is for
-
-**Route**
-A route with the fingerprint

.github/PULL_REQUEST_TEMPLATE/refactor.md

@@ -1,15 +0,0 @@
----
-name: Refactor
-about: For code refactors
-title: ''
-labels: 'refactor'
-assignees: ''
----
-
-**Description**
-
-<!-- A description of the refactor, including the goals it accomplishes. -->
-
-**Verification**
-
-<!-- Explain how you tested the refactor for regressions. -->

.github/PULL_REQUEST_TEMPLATE/tuning.md

@@ -1,31 +0,0 @@
----
-name: Tuning
-about: For openpilot tuning changes
-title: ''
-labels: 'tuning'
-assignees: ''
----
-
-**Description**
-
-<!-- A description of what is wrong with the current tuning and how the PR addresses this. -->
-
-**Verification**
-
-<!-- To verify tuning, capture the following scenarios (broadly, not exactly), with current tune and this tune.
-Use the PlotJuggler tuning layout to compare planned versus actual behavior.
-
-Run ./juggle.py <route> --layout layouts/tuning.xml , screenshot the full tab of interest, and paste into this PR.
-
-Longitudinal:
-* Maintaining speed at 25, 40, 65mph
-* Driving up and down hills
-* Accelerating from a stop
-* Decelerating to a stop
-* Following large changes in set speed
-* Coming to a stop behind a lead car
-
-Lateral:
-* Straight driving at ~25, ~45 and ~65mph
-* Turns driving at ~25, ~45 and ~65mph
--->

.github/build.py

@@ -1,30 +0,0 @@
-import pathlib
-
-GITHUB_FOLDER = pathlib.Path(__file__).parent
-
-PULL_REQUEST_TEMPLATES = (GITHUB_FOLDER / "PULL_REQUEST_TEMPLATE")
-
-order = ["fingerprint", "car_bugfix", "bugfix", "car_port", "refactor"]
-
-def create_pull_request_template():
-  with open(GITHUB_FOLDER / "pull_request_template.md", "w") as f:
-    f.write("<!-- Please copy and paste the relevant template -->\n\n")
-
-    for t in order:
-      template = PULL_REQUEST_TEMPLATES / f"{t}.md"
-      text = template.read_text()
-
-      # Remove metadata for GitHub
-      start = text.find("---")
-      end = text.find("---", start+1)
-      text = text[end + 4:]
-
-      # Remove comments
-      text = text.replace("<!-- ", "").replace("-->", "")
-
-      f.write(f"<!--- ***** Template: {template.stem.replace('_', ' ').title()} *****\n")
-      f.write(text)
-      f.write("\n\n")
-      f.write("-->\n\n")
-
-create_pull_request_template()

.github/labeler.yaml

@@ -1,27 +0,0 @@
-CI / testing:
-  - changed-files:
-    - any-glob-to-all-files: "{.github/**,**/test_*,Jenkinsfile}"
-
-car:
-  - changed-files:
-    - any-glob-to-all-files: '{selfdrive/car/**,opendbc_repo}'
-
-simulation:
-  - changed-files:
-    - any-glob-to-all-files: 'tools/sim/**'
-
-ui:
-  - changed-files:
-    - any-glob-to-all-files: 'selfdrive/ui/**'
-
-tools:
-  - changed-files:
-    - any-glob-to-all-files: 'tools/**'
-
-multilanguage:
-  - changed-files:
-    - any-glob-to-all-files: 'selfdrive/ui/translations/**'
-
-autonomy:
-  - changed-files:
-    - any-glob-to-all-files: "{selfdrive/modeld/models/**,selfdrive/test/process_replay/model_replay_ref_commit}"

.github/pull_request_template.md

@@ -1,68 +0,0 @@
-<!-- Please copy and paste the relevant template -->
-
-<!--- ***** Template: Fingerprint *****
-
-**Car**
-Which car (make, model, year) this fingerprint is for
-
-**Route**
-A route with the fingerprint
-
--->
-
-<!--- ***** Template: Car Bugfix *****
-
-**Description**
-
-A description of the bug and the fix. Also link the issue if it exists. 
-
-**Verification**
-
-Explain how you tested this bug fix. 
-
-**Route**
-
-Route: [a route with the bug fix]
-
-
--->
-
-<!--- ***** Template: Bugfix *****
-
-**Description**
-
-A description of the bug and the fix. Also link the issue if it exists. 
-
-**Verification**
-
-Explain how you tested this bug fix. 
-
-
--->
-
-<!--- ***** Template: Car Port *****
-
-**Checklist**
-
-- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/opcar/docs.py` to generate new docs
-- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
-- [ ] route with openpilot:
-- [ ] route with stock system:
-- [ ] car harness used (if comma doesn't sell it, put N/A):
-
-
--->
-
-<!--- ***** Template: Refactor *****
-
-**Description**
-
-A description of the refactor, including the goals it accomplishes. 
-
-**Verification**
-
-Explain how you tested the refactor for regressions. 
-
-
--->
-

.github/workflows/auto-cache/action.yaml

@@ -1,58 +0,0 @@
-name: 'automatically cache based on current runner'
-
-inputs:
-  path:
-    description: 'path to cache'
-    required: true
-  key:
-    description: 'key'
-    required: true
-  restore-keys:
-    description: 'restore-keys'
-    required: true
-  save:
-    description: 'whether to save the cache'
-    default: 'false'
-    required: false
-outputs:
-  cache-hit:
-    description: 'cache hit occurred'
-    value: ${{ (contains(runner.name, 'nsc') && steps.ns-cache.outputs.cache-hit) ||
-               (!contains(runner.name, 'nsc') && inputs.save != 'false' && steps.gha-cache.outputs.cache-hit) ||
-               (!contains(runner.name, 'nsc') && inputs.save == 'false' && steps.gha-cache-ro.outputs.cache-hit) }}
-
-runs:
-  using: "composite"
-  steps:
-    - name: setup namespace cache
-      id: ns-cache
-      if: ${{ contains(runner.name, 'nsc') }}
-      uses: namespacelabs/nscloud-cache-action@v1
-      with:
-        path: ${{ inputs.path }}
-
-    - name: setup github cache
-      id: gha-cache
-      if: ${{ !contains(runner.name, 'nsc') && inputs.save != 'false' }}
-      uses: 'actions/cache@v4'
-      with:
-        path: ${{ inputs.path }}
-        key: ${{ inputs.key }}
-        restore-keys: ${{ inputs.restore-keys }}
-
-    - name: setup github cache
-      id: gha-cache-ro
-      if: ${{ !contains(runner.name, 'nsc') && inputs.save == 'false' }}
-      uses: 'actions/cache/restore@v4'
-      with:
-        path: ${{ inputs.path }}
-        key: ${{ inputs.key }}
-        restore-keys: ${{ inputs.restore-keys }}
-
-    # make the directory manually in case we didn't get a hit, so it doesn't fail on future steps
-    - id: scons-cache-setup
-      shell: bash
-      run: |
-        mkdir -p ${{ inputs.path }}
-        sudo chmod -R 777 ${{ inputs.path }}
-        sudo chown -R $USER ${{ inputs.path }}

.github/workflows/auto_pr_review.yaml

@@ -1,53 +0,0 @@
-name: "PR review"
-on:
-  pull_request_target:
-    types: [opened, reopened, synchronize, edited, edited]
-
-jobs:
-  labeler:
-    name: review
-    permissions:
-      contents: read
-      pull-requests: write
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: false
-
-    # Label PRs
-    - uses: actions/labeler@v5.0.0
-      with:
-        dot: true
-        configuration-path: .github/labeler.yaml
-
-    # Check PR target branch
-    - name: check branch
-      uses: Vankka/pr-target-branch-action@def32ec9d93514138d6ac0132ee62e120a72aed5
-      if: github.repository == 'sunnypilot/sunnypilot'
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        target: /^(?!master$).*/
-        exclude: /commaai:.*/
-        change-to: ${{ github.base_ref }}
-        already-exists-action: close_this
-        already-exists-comment: "Your PR should be made against the `master` branch"
-
-    # Welcome comment
-    - name: "First timers PR"
-      uses: actions/first-interaction@v1
-      if: github.event.pull_request.head.repo.full_name != 'sunnypilot/sunnypilot'
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        pr-message: |
-            <!-- _(run_id **${{ github.run_id }}**)_ -->
-            Thanks for contributing to openpilot! In order for us to review your PR as quickly as possible, check the following:
-            * Convert your PR to a draft unless it's ready to review
-            * Read the [contributing docs](https://github.com/sunnypilot/sunnypilot/blob/master/docs/CONTRIBUTING.md)
-            * Before marking as "ready for review", ensure:
-              * the goal is clearly stated in the description
-              * all the tests are passing
-              * the change is [something we merge](https://github.com/sunnypilot/sunnypilot/blob/master/docs/CONTRIBUTING.md#what-gets-merged)
-              * include a route or your device' dongle ID if relevant
-

.github/workflows/badges.yaml

@@ -1,37 +0,0 @@
-name: badges
-on:
-  schedule:
-    - cron: '0 * * * *'
-  workflow_dispatch:
-
-env:
-  BASE_IMAGE: openpilot-base
-  DOCKER_REGISTRY: ghcr.io/commaai
-  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c
-
-jobs:
-  badges:
-    name: create badges
-    runs-on: ubuntu-latest
-    if: github.repository == 'sunnypilot/sunnypilot'
-    permissions:
-      contents: write
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Push badges
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc) && python3 selfdrive/ui/translations/create_badges.py"
-
-        rm .gitattributes
-
-        git checkout --orphan badges
-        git rm -rf --cached .
-        git config user.email "badge-researcher@sunnypilot.ai"
-        git config user.name "Badge Researcher"
-
-        git add translation_badge.svg
-        git commit -m "Add/Update badges"
-        git push -f origin HEAD

.github/workflows/ci_weekly_report.yaml

@@ -1,101 +0,0 @@
-name: weekly CI test report
-on:
-  schedule:
-    - cron: '37 9 * * 1' # 9:37AM UTC -> 2:37AM PST every monday
-  workflow_dispatch:
-    inputs:
-      ci_runs:
-        description: 'The amount of runs to trigger in CI test report'
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-env:
-  CI_RUNS: ${{ github.event.inputs.ci_runs || '50' }}
-
-jobs:
-  setup:
-    if: github.repository == 'sunnypilot/sunnypilot'
-    runs-on: ubuntu-latest
-    outputs:
-      ci_runs: ${{ steps.ci_runs_setup.outputs.matrix }}
-    steps:
-      - id: ci_runs_setup
-        name: CI_RUNS=${{ env.CI_RUNS }}
-        run: |
-          matrix=$(python3 -c "import json; print(json.dumps({ 'run_number' : list(range(${{ env.CI_RUNS }})) }))")
-          echo "matrix=$matrix" >> $GITHUB_OUTPUT
-
-  ci_matrix_run:
-    needs: [ setup ]
-    strategy:
-      fail-fast: false
-      matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}}
-    uses: sunnypilot/sunnypilot/.github/workflows/ci_weekly_run.yaml@master
-    with:
-      run_number: ${{ matrix.run_number }}
-
-  report:
-    needs: [ci_matrix_run]
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Get job results
-        uses: actions/github-script@v7
-        id: get-job-results
-        with:
-          script: |
-            const jobs = await github
-              .paginate("GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", {
-                owner: "commaai",
-                repo: "${{ github.event.repository.name }}",
-                run_id: "${{ github.run_id }}",
-                attempt: "${{ github.run_attempt }}",
-              })
-            var report = {}
-            jobs.slice(1, jobs.length-1).forEach(job => {
-              if (job.conclusion === "skipped") return;
-              const jobName = job.name.split(" / ")[2];
-              const runRegex = /\((.*?)\)/;
-              const run = job.name.match(runRegex)[1];
-              report[jobName] = report[jobName] || { successes: [], failures: [], canceled: [] };
-              switch (job.conclusion) {
-                case "success":
-                  report[jobName].successes.push({ "run_number": run, "link": job.html_url}); break;
-                case "failure":
-                  report[jobName].failures.push({ "run_number": run, "link": job.html_url }); break;
-                case "canceled":
-                  report[jobName].canceled.push({ "run_number": run, "link": job.html_url }); break;
-              }
-            });
-            return JSON.stringify({"jobs": report});
-
-      - name: Add job results to summary
-        env:
-          JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }}
-        run: |
-          cat <<EOF >> template.html
-          <table>
-            <thead>
-              <tr>
-                <th></th>
-                <th>Job</th>
-                <th>✅ Passing</th>
-                <th>❌ Failure Details</th>
-              </tr>
-            </thead>
-            <tbody>
-              {% for key in jobs.keys() %}<tr>
-                <td>{% for i in range(5) %}{% if i+1 <= (5 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }}) %}🟩{% else %}🟥{% endif %}{% endfor%}</td>
-                <td>{{ key }}</td>
-                <td>{{ 100 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }} }}%</td>
-                <td>{% if jobs[key]["failures"]|length > 0 %}<details>{% for failure in jobs[key]["failures"] %}<a href="{{ failure['link'] }}">Log for run #{{ failure['run_number'] }}</a><br>{% endfor %}</details>{% else %}{% endif %}</td>
-                </td>
-              </tr>{% endfor %}
-          </table>
-          EOF
-
-          pip install jinja2-cli
-          echo $JOB_RESULTS | jinja2 template.html > report.html
-          echo "# CI Test Report - ${{ env.CI_RUNS }} Runs" >> $GITHUB_STEP_SUMMARY
-          cat report.html >> $GITHUB_STEP_SUMMARY

.github/workflows/ci_weekly_run.yaml

@@ -1,21 +0,0 @@
-name: weekly CI test run
-on:
-  workflow_call:
-    inputs:
-      run_number:
-        required: true
-        type: string
-
-concurrency:
-  group: ci-run-${{ inputs.run_number }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  selfdrive_tests:
-    uses: sunnypilot/sunnypilot/.github/workflows/selfdrive_tests.yaml@master
-    with:
-      run_number: ${{ inputs.run_number }}
-  tools_tests:
-    uses: sunnypilot/sunnypilot/.github/workflows/tools_tests.yaml@master
-    with:
-      run_number: ${{ inputs.run_number }}

.github/workflows/compile-openpilot/action.yaml

@@ -1,21 +0,0 @@
-name: 'compile openpilot'
-
-runs:
-  using: "composite"
-  steps:
-    - shell: bash
-      name: Build openpilot with all flags
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-        ${{ env.RUN }} "release/check-dirty.sh"
-    - shell: bash
-      name: Cleanup scons cache and rebuild
-      run: |
-          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
-                          scons -j$(nproc) --cache-populate"
-    - name: Save scons cache
-      uses: actions/cache/save@v4
-      if: github.ref == 'refs/heads/master'
-      with:
-        path: .ci_cache/scons_cache
-        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}

.github/workflows/docs.yaml

@@ -1,65 +0,0 @@
-name: docs
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-  workflow_call:
-    inputs:
-      run_number:
-        default: '1'
-        required: true
-        type: string
-concurrency:
-  group: docs-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
-  cancel-in-progress: true
-
-jobs:
-  docs:
-    name: build docs
-    runs-on: ubuntu-latest
-    steps:
-    - uses: commaai/timeout@v1
-
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-
-    # Build
-    - name: Build docs
-      run: |
-        # TODO: can we install just the "docs" dependency group without the normal deps?
-        pip install mkdocs
-        mkdocs build
-
-    # Push to docs.comma.ai
-    - uses: actions/checkout@v4
-      if: github.ref == 'refs/heads/master' && github.repository == 'sunnypilot/sunnypilot'
-      with:
-        path: openpilot-docs
-        ssh-key: ${{ secrets.OPENPILOT_DOCS_KEY }}
-        repository: sunnypilot/sunnypilot-docs
-    - name: Push
-      if: github.ref == 'refs/heads/master' && github.repository == 'sunnypilot/sunnypilot'
-      run: |
-        set -x
-
-        source release/identity.sh
-
-        cd openpilot-docs
-        git checkout --orphan tmp
-        git rm -rf .
-
-        # copy over docs
-        cp -r ../docs_site/ docs/
-
-        # GitHub pages config
-        touch docs/.nojekyll
-        echo -n docs.comma.ai > docs/CNAME
-
-        git add -f .
-        git commit -m "build docs"
-
-        # docs live in different repo to not bloat openpilot's full clone size
-        git push -f origin tmp:gh-pages

.github/workflows/jenkins-pr-trigger.yaml

@@ -1,45 +0,0 @@
-name: jenkins scan
-
-on:
-  issue_comment:
-    types: [created, edited]
-
-jobs:
-  # TODO: gc old branches in a separate job in this workflow
-  scan-comments:
-    runs-on: ubuntu-latest
-    if: ${{ github.event.issue.pull_request }}
-    steps:
-    - name: Check for trigger phrase
-      id: check_comment
-      uses: actions/github-script@v7
-      with:
-        script: |
-          const triggerPhrase = "trigger-jenkins";
-          const comment = context.payload.comment.body;
-          const commenter = context.payload.comment.user.login;
-
-          const { data: permissions } = await github.rest.repos.getCollaboratorPermissionLevel({
-            owner: context.repo.owner,
-            repo: context.repo.repo,
-            username: commenter
-          });
-
-          const hasWriteAccess = permissions.permission === 'write' || permissions.permission === 'admin';
-
-          return (hasWriteAccess && comment.includes(triggerPhrase));
-        result-encoding: json
-
-    - name: Checkout repository
-      if: steps.check_comment.outputs.result == 'true'
-      uses: actions/checkout@v4
-      with:
-        ref: refs/pull/${{ github.event.issue.number }}/head
-
-    - name: Push to tmp-jenkins branch
-      if: steps.check_comment.outputs.result == 'true'
-      run: |
-        git config --global user.name "github-actions[bot]"
-        git config --global user.email "github-actions[bot]@users.noreply.github.com"
-        git checkout -b tmp-jenkins-${{ github.event.issue.number }}
-        GIT_LFS_SKIP_PUSH=1 git push -f origin tmp-jenkins-${{ github.event.issue.number }}

.github/workflows/prebuilt.yaml

@@ -1,39 +0,0 @@
-name: prebuilt
-on:
-  schedule:
-    - cron: '0 * * * *'
-  workflow_dispatch:
-
-env:
-  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: selfdrive/test/docker_build.sh prebuilt
-
-jobs:
-  build_prebuilt:
-    name: build prebuilt
-    runs-on: ubuntu-latest
-    if: github.repository == 'sunnypilot/sunnypilot'
-    env:
-      PUSH_IMAGE: true
-    permissions:
-      checks: read
-      contents: read
-      packages: write
-    steps:
-    - name: Wait for green check mark
-      if: ${{ github.event_name != 'workflow_dispatch' }}
-      uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
-      with:
-        ref: master
-        wait-interval: 30
-        running-workflow-name: 'build prebuilt'
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        check-regexp: ^((?!.*(build master-ci).*).)*$
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - run: git lfs pull
-    - name: Build and Push docker image
-      run: |
-        $DOCKER_LOGIN
-        eval "$BUILD"

.github/workflows/release.yaml

@@ -1,54 +0,0 @@
-name: release
-on:
-  schedule:
-    - cron: '0 9 * * *'
-  workflow_dispatch:
-
-jobs:
-  build_masterci:
-    name: build master-ci
-    env:
-      TARGET_DIR: /tmp/openpilot
-      ImageOS: ubuntu20
-    container:
-      image: ghcr.io/commaai/openpilot-base:latest
-    runs-on: ubuntu-latest
-    if: github.repository == 'sunnypilot/sunnypilot'
-    permissions:
-      checks: read
-      contents: write
-    steps:
-    - name: Install wait-on-check-action dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y libyaml-dev
-    - name: Wait for green check mark
-      if: ${{ github.event_name != 'workflow_dispatch' }}
-      uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
-      with:
-        ref: master
-        wait-interval: 30
-        running-workflow-name: 'build master-ci'
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        check-regexp: ^((?!.*(build prebuilt).*).)*$
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-        fetch-depth: 0
-    - name: Pull LFS
-      run: |
-        git config --global --add safe.directory '*'
-        git lfs pull
-    - name: Build master-ci
-      run: |
-        release/build_devel.sh
-    - name: Run tests
-      run: |
-        export PYTHONPATH=$TARGET_DIR
-        cd $TARGET_DIR
-        scons -j$(nproc)
-        pytest -n logical selfdrive/car/tests/test_car_interfaces.py
-    - name: Push master-ci
-      run: |
-        unset TARGET_DIR
-        BRANCH=master-ci release/build_devel.sh

.github/workflows/repo-maintenance.yaml

@@ -1,45 +0,0 @@
-name: repo maintenance
-
-on:
-  schedule:
-    - cron: "0 14 * * 1" # every Monday at 2am UTC (6am PST)
-  workflow_dispatch:
-
-jobs:
-  package_updates:
-    name: package_updates
-    runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/commaai/openpilot-base:latest
-    if: github.repository == 'sunnypilot/sunnypilot'
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - name: uv lock
-      run: |
-        python3 -m ensurepip --upgrade
-        pip3 install uv
-        uv lock --upgrade
-    - name: bump submodules
-      run: |
-        git config --global --add safe.directory '*'
-        git -c submodule."tinygrad".update=none submodule update --remote
-        git add .
-    - name: update car docs
-      run: |
-        scons -j$(nproc) --minimal opendbc
-        PYTHONPATH=. python selfdrive/car/docs.py
-        git add docs/CARS.md
-    - name: Create Pull Request
-      uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
-      with:
-        author: Vehicle Researcher <user@comma.ai>
-        token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
-        commit-message: Update Python packages
-        title: '[bot] Update Python packages'
-        branch: auto-package-updates
-        base: master
-        delete-branch: true
-        body: 'Automatic PR from repo-maintenance -> package_updates'
-        labels: bot

.github/workflows/selfdrive_tests.yaml

@@ -1,389 +0,0 @@
-name: selfdrive
-
-on:
-  push:
-    branches:
-      - master
-      - master-new
-  pull_request:
-  workflow_dispatch:
-  workflow_call:
-    inputs:
-      run_number:
-        default: '1'
-        required: true
-        type: string
-
-concurrency:
-  group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
-  cancel-in-progress: true
-
-env:
-  REPORT_NAME: report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && 'master' || github.event.number }}
-  PYTHONWARNINGS: error
-  BASE_IMAGE: openpilot-base
-  AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
-
-  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: selfdrive/test/docker_build.sh base
-
-  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
-
-  PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 -n logical
-
-jobs:
-  build_release:
-    if: github.repository == 'commaai/openpilot' # build_release blocked for the time being to only comma as we may have a different process.
-    name: build release
-    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
-                   ((github.event_name != 'pull_request') ||
-                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
-    env:
-      STRIPPED_DIR: /tmp/releasepilot
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - name: Getting LFS files
-      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
-      with:
-        timeout_minutes: 2
-        max_attempts: 3
-        command: git lfs pull
-    - name: Build devel
-      timeout-minutes: 1
-      run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Check submodules
-      if: github.repository == 'sunnypilot/sunnypilot'
-      timeout-minutes: 3
-      run: release/check-submodules.sh
-    - name: Build openpilot and run checks
-      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
-      run: |
-        cd $STRIPPED_DIR
-        ${{ env.RUN }} "python3 system/manager/build.py"
-    - name: Run tests
-      timeout-minutes: 1
-      run: |
-        cd $STRIPPED_DIR
-        ${{ env.RUN }} "release/check-dirty.sh && \
-                        MAX_EXAMPLES=5 $PYTEST -m 'not slow' selfdrive/car system/manager"
-    - name: Static analysis
-      timeout-minutes: 1
-      run: |
-        cd $GITHUB_WORKSPACE
-        cp pyproject.toml $STRIPPED_DIR
-        cd $STRIPPED_DIR
-        ${{ env.RUN }} "scripts/lint/lint.sh --skip check_added_large_files"
-
-  build:
-    strategy:
-      matrix:
-        arch: ${{ fromJson(
-           ((github.repository == 'commaai/openpilot') &&
-              ((github.event_name != 'pull_request') ||
-               (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
-    runs-on: ${{ ((matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8') ||
-                 ((matrix.arch == 'x86_64') && ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16') ||
-                   'ubuntu-latest'}}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - name: Setup docker push
-      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
-      run: |
-        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
-        echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV"
-        $DOCKER_LOGIN
-    - uses: ./.github/workflows/setup-with-retry
-    - uses: ./.github/workflows/compile-openpilot
-      timeout-minutes: 30
-
-  build_mac:
-    if: github.repository == 'commaai/openpilot' # Blocking macos builds as well since they have a 10x miltiplier for GH action minutes, waaaay too much!
-    name: build macOS
-    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
-                   ((github.event_name != 'pull_request') ||
-                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - name: Homebrew cache
-      uses: ./.github/workflows/auto-cache
-      with:
-        path: ~/Library/Caches/Homebrew
-        key: build_macos_${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}
-    - name: Install dependencies
-      run: ./tools/mac_setup.sh
-      env:
-        # package install has DeprecationWarnings
-        PYTHONWARNINGS: default
-    - run: git lfs pull
-    - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
-    - name: Getting scons cache
-      uses: ./.github/workflows/auto-cache
-      with:
-        path: /tmp/scons_cache
-        key: build_macos_${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}
-    - name: Building openpilot
-      run: . .venv/bin/activate && scons -j$(nproc)
-
-  docker_push_multiarch:
-    name: docker push multiarch tag
-    runs-on: ubuntu-latest
-    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
-    needs: [build]
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: false
-    - name: Setup docker
-      run: |
-        $DOCKER_LOGIN
-    - name: Merge x64 and arm64 tags
-      run: |
-        export PUSH_IMAGE=true
-        scripts/retry.sh selfdrive/test/docker_tag_multiarch.sh base x86_64 aarch64
-
-  static_analysis:
-    name: static analysis
-    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
-                   ((github.event_name != 'pull_request') ||
-                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
-    env:
-      PYTHONWARNINGS: default
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Static analysis
-      timeout-minutes: 1
-      run: ${{ env.RUN }} "scripts/lint/lint.sh"
-
-  unit_tests:
-    name: unit tests
-    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
-                   ((github.event_name != 'pull_request') ||
-                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Build openpilot
-      run: ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Run unit tests
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && 1 || 20 }}
-      run: |
-        ${{ env.RUN }} "$PYTEST --collect-only -m 'not slow' &> /dev/null && \
-                        MAX_EXAMPLES=1 $PYTEST -m 'not slow' && \
-                        ./selfdrive/ui/tests/create_test_translations.sh && \
-                        QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
-                        chmod -R 777 /tmp/comma_download_cache"
-    - name: "Upload coverage to Codecov"
-      uses: codecov/codecov-action@v4
-      with:
-        name: ${{ github.job }}
-      env:
-        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
-  process_replay:
-    name: process replay
-    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
-                   ((github.event_name != 'pull_request') ||
-                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Cache test routes
-      id: dependency-cache
-      uses: actions/cache@v4
-      with:
-        path: .ci_cache/comma_download_cache
-        key: proc-replay-${{ hashFiles('selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_processes.py') }}
-    - name: Build openpilot
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Run replay
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && 1 || 20 }}
-      run: |
-        ${{ env.RUN }} "coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
-                        chmod -R 777 /tmp/comma_download_cache && \
-                        coverage combine && \
-                        coverage xml"
-    - name: Print diff
-      id: print-diff
-      if: always()
-      run: cat selfdrive/test/process_replay/diff.txt
-    - uses: actions/upload-artifact@v4
-      if: always()
-      continue-on-error: true
-      with:
-        name: process_replay_diff.txt
-        path: selfdrive/test/process_replay/diff.txt
-    - name: Upload reference logs
-      if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
-      run: |
-        ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python3 selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
-    - name: Run regen
-      if: false
-      timeout-minutes: 4
-      run: |
-        ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
-                        chmod -R 777 /tmp/comma_download_cache"
-    - name: "Upload coverage to Codecov"
-      uses: codecov/codecov-action@v4
-      with:
-        name: ${{ github.job }}
-      env:
-        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
-  test_cars:
-    name: cars
-    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
-                   ((github.event_name != 'pull_request') ||
-                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
-    strategy:
-      fail-fast: false
-      matrix:
-        job: [0, 1, 2, 3]
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Cache test routes
-      id: routes-cache
-      uses: actions/cache@v4
-      with:
-        path: .ci_cache/comma_download_cache
-        key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
-    - name: Build openpilot
-      run: ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Test car models
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.routes-cache.outputs.cache-hit == 'true') && 1 || 20 }}
-      run: |
-        ${{ env.RUN }} "MAX_EXAMPLES=1 $PYTEST selfdrive/car/tests/test_models.py && \
-                        chmod -R 777 /tmp/comma_download_cache"
-      env:
-        NUM_JOBS: 4
-        JOB_ID: ${{ matrix.job }}
-    - name: "Upload coverage to Codecov"
-      uses: codecov/codecov-action@v4
-      with:
-        name: ${{ github.job }}-${{ matrix.job }}
-      env:
-        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
-  car_docs_diff:
-    name: PR comments
-    runs-on: ubuntu-latest
-    #if: github.event_name == 'pull_request'
-    if: false  # TODO: run this in opendbc?
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          ref: ${{ github.event.pull_request.base.ref }}
-      - run: git lfs pull
-      - uses: ./.github/workflows/setup-with-retry
-      - name: Get base car info
-        run: |
-          ${{ env.RUN }} "scons -j$(nproc) && python3 selfdrive/debug/dump_car_docs.py --path /tmp/openpilot_cache/base_car_docs"
-          sudo chown -R $USER:$USER ${{ github.workspace }}
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          path: current
-      - run: cd current && git lfs pull
-      - name: Save car docs diff
-        id: save_diff
-        run: |
-          cd current
-          ${{ env.RUN }} "scons -j$(nproc)"
-          output=$(${{ env.RUN }} "python3 selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_docs")
-          output="${output//$'\n'/'%0A'}"
-          echo "::set-output name=diff::$output"
-      - name: Find comment
-        if: ${{ env.AZURE_TOKEN != '' }}
-        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e
-        id: fc
-        with:
-          issue-number: ${{ github.event.pull_request.number }}
-          body-includes: This PR makes changes to
-      - name: Update comment
-        if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }}
-        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043
-        with:
-          comment-id: ${{ steps.fc.outputs.comment-id }}
-          issue-number: ${{ github.event.pull_request.number }}
-          body: "${{ steps.save_diff.outputs.diff }}"
-          edit-mode: replace
-      - name: Delete comment
-        if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }}
-        uses: actions/github-script@v7
-        with:
-          script: |
-            github.rest.issues.deleteComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              comment_id: ${{ steps.fc.outputs.comment-id }}
-            })
-
-  simulator_driving:
-    name: simulator driving
-    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
-                   ((github.event_name != 'pull_request') ||
-                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Build openpilot
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Driving test
-      timeout-minutes: 1
-      run: |
-        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
-                        source selfdrive/test/setup_vsound.sh && \
-                        CI=1 pytest tools/sim/tests/test_metadrive_bridge.py"
-
-  create_ui_report:
-    # This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml
-    name: Create UI Report
-    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
-                   ((github.event_name != 'pull_request') ||
-                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-      - uses: ./.github/workflows/setup-with-retry
-      - name: caching frames
-        id: frames-cache
-        uses: actions/cache@v4
-        with:
-          path: .ci_cache/comma_download_cache
-          key: ui_screenshots_test_${{ hashFiles('selfdrive/ui/tests/test_ui/run.py') }}
-      - name: Build openpilot
-        run: ${{ env.RUN }} "scons -j$(nproc)"
-      - name: Create Test Report
-        timeout-minutes: ${{ ((steps.frames-cache.outputs.cache-hit == 'true') && 1 || 3) }}
-        run: >
-            ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
-                            source selfdrive/test/setup_xvfb.sh &&
-                            CACHE_ROOT=/tmp/comma_download_cache python3 selfdrive/ui/tests/test_ui/run.py &&
-                            chmod -R 777 /tmp/comma_download_cache"
-      - name: Upload Test Report
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ env.REPORT_NAME }}
-          path: selfdrive/ui/tests/test_ui/report_1/screenshots

.github/workflows/setup-with-retry/action.yaml

@@ -1,37 +0,0 @@
-name: 'openpilot env setup, with retry on failure'
-
-inputs:
-  docker_hub_pat:
-    description: 'Auth token for Docker Hub, required for BuildJet jobs'
-    required: false
-    default: ''
-  sleep_time:
-    description: 'Time to sleep between retries'
-    required: false
-    default: 30
-
-runs:
-  using: "composite"
-  steps:
-    - id: setup1
-      uses: ./.github/workflows/setup
-      continue-on-error: true
-      with:
-        is_retried: true
-    - if: steps.setup1.outcome == 'failure'
-      shell: bash
-      run: sleep ${{ inputs.sleep_time }}
-    - id: setup2
-      if: steps.setup1.outcome == 'failure'
-      uses: ./.github/workflows/setup
-      continue-on-error: true
-      with:
-        is_retried: true
-    - if: steps.setup2.outcome == 'failure'
-      shell: bash
-      run: sleep ${{ inputs.sleep_time }}
-    - id: setup3
-      if: steps.setup2.outcome == 'failure'
-      uses: ./.github/workflows/setup
-      with:
-        is_retried: true

.github/workflows/setup/action.yaml

@@ -1,56 +0,0 @@
-name: 'openpilot env setup'
-
-inputs:
-  is_retried:
-    description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly'
-    required: false
-    default: 'false'
-
-runs:
-  using: "composite"
-  steps:
-    # assert that this action is retried using the setup-with-retry
-    - shell: bash
-      if: ${{ inputs.is_retried == 'false' }}
-      run: |
-        echo "You should not run this action directly. Use setup-with-retry instead"
-        exit 1
-
-    - shell: bash
-      name: No retries!
-      run: |
-        if [ "${{ github.run_attempt }}" -gt 1 ]; then
-          echo -e "\033[0;31m##################################################"
-          echo -e "\033[0;31m    Retries not allowed! Fix the flaky test!      "
-          echo -e "\033[0;31m##################################################\033[0m"
-          exit 1
-        fi
-
-    # do this after checkout to ensure our custom LFS config is used to pull from GitLab
-    - shell: bash
-      run: git lfs pull
-
-    # build cache
-    - id: date
-      shell: bash
-      run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
-    - shell: bash
-      run: echo "$CACHE_COMMIT_DATE"
-    - id: scons-cache
-      uses: ./.github/workflows/auto-cache
-      with:
-        path: .ci_cache/scons_cache
-        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
-        restore-keys: |
-          scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}
-          scons-${{ runner.arch }}
-    # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001
-    - id: normalize-file-permissions
-      shell: bash
-      name: Normalize file permissions to ensure a consistent docker build cache
-      run: |
-        find . -type f -executable -not -perm 755 -exec chmod 755 {} \;
-        find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \;
-    # build our docker image
-    - shell: bash
-      run: eval ${{ env.BUILD }}

.github/workflows/stale.yaml

@@ -1,29 +0,0 @@
-name: stale
-on:
-  schedule:
-    - cron: '30 1 * * *'
-  workflow_dispatch:
-
-env:
-  DAYS_BEFORE_PR_CLOSE: 2
-  DAYS_BEFORE_PR_STALE: 9
-
-jobs:
-  stale:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/stale@v9
-        with:
-          exempt-all-milestones: true
-
-          # pull request config
-          stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.'
-          close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.'
-          stale-pr-label: stale
-          delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'sunnypilot/sunnypilot' }} # only delete branches on the main repo
-          exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale
-          days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }}
-          days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}
-
-          # issue config
-          days-before-issue-stale: -1 # ignore issues for now

.github/workflows/tools_tests.yaml

@@ -1,55 +0,0 @@
-name: tools
-
-on:
-  push:
-    branches:
-      - master
-      - master-new
-  pull_request:
-  workflow_call:
-    inputs:
-      run_number:
-        default: '1'
-        required: true
-        type: string
-concurrency:
-  group: tools-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
-  cancel-in-progress: true
-
-env:
-  BASE_IMAGE: openpilot-base
-  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-
-  BUILD: selfdrive/test/docker_build.sh base
-
-  RUN: docker run --shm-size 2G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
-
-
-jobs:
-  devcontainer:
-    name: devcontainer
-    runs-on: ubuntu-latest
-    if: false  # we can re-enable once this is faster
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Use local image for testing devcontainer with latest base image
-      run: |
-        echo "USE_LOCAL_IMAGE=true" >> "$GITHUB_ENV"
-    - name: Setup Dev Container CLI
-      run: npm install -g @devcontainers/cli
-    - name: Build dev container image
-      run: ./scripts/retry.sh devcontainer build --workspace-folder .
-    - name: Run dev container
-      run: |
-        mkdir -p /tmp/devcontainer_scons_cache/
-        cp -r $GITHUB_WORKSPACE/.ci_cache/scons_cache/. /tmp/devcontainer_scons_cache/
-        devcontainer up --workspace-folder .
-    - name: Test environment
-      run: |
-        devcontainer exec --workspace-folder . scons -j$(nproc) cereal/ common/
-        devcontainer exec --workspace-folder . pip3 install pip-install-test
-        devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json
-        devcontainer exec --workspace-folder . sudo touch /root/test.txt

.github/workflows/ui_preview.yaml

@@ -1,163 +0,0 @@
-name: "ui preview"
-on:
-  push:
-    branches:
-      - master
-      - master-new
-  pull_request_target:
-    types: [assigned, opened, synchronize, reopened, edited]
-    branches:
-      - 'master'
-      - 'master-new'
-    paths:
-      - 'selfdrive/ui/**'
-  workflow_dispatch:
-
-env:
-  UI_JOB_NAME: "Create UI Report"
-  REPORT_NAME: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && 'master' || github.event.number }}
-  SHA: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.sha || github.event.pull_request.head.sha }}
-  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}"
-
-jobs:
-  preview:
-    if: github.repository == 'sunnypilot/sunnypilot'
-    name: preview
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    permissions:
-      contents: read
-      pull-requests: write
-      actions: read
-    steps:
-      - name: Waiting for ui generation to start
-        run: sleep 30
-
-      - name: Waiting for ui generation to end
-        uses: lewagon/wait-on-check-action@v1.3.4
-        with:
-          ref: ${{ env.SHA }}
-          check-name: ${{ env.UI_JOB_NAME }}
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          allowed-conclusions: success
-          wait-interval: 20
-
-      - name: Getting workflow run ID
-        id: get_run_id
-        run: |
-          echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
-
-      - name: Getting proposed ui
-        id: download-artifact
-        uses: dawidd6/action-download-artifact@v6
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          run_id: ${{ steps.get_run_id.outputs.run_id }}
-          search_artifacts: true
-          name: report-1-${{ env.REPORT_NAME }}
-          path: ${{ github.workspace }}/pr_ui
-
-      - name: Getting master ui
-        uses: actions/checkout@v4
-        with:
-          repository: sunnypilot/ci-artifacts
-          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/master_ui
-          ref: openpilot_master_ui
-
-      - name: Saving new master ui
-        if: (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.event_name == 'push'
-        working-directory: ${{ github.workspace }}/master_ui
-        run: |
-          git checkout --orphan=new_master_ui
-          git rm -rf *
-          git branch -D openpilot_master_ui
-          git branch -m openpilot_master_ui
-          git config user.name "GitHub Actions Bot"
-          git config user.email "<>"
-          mv ${{ github.workspace }}/pr_ui/*.png .
-          git add .
-          git commit -m "screenshots for commit ${{ env.SHA }}"
-          git push origin openpilot_master_ui --force
-
-      - name: Finding diff
-        if: github.event_name == 'pull_request_target'
-        id: find_diff
-        run: >-
-          sudo apt-get install -y imagemagick
-
-          scenes="homescreen settings_device settings_software settings_toggles settings_developer offroad_alert update_available prime onroad onroad_disengaged onroad_override onroad_sidebar onroad_wide onroad_wide_sidebar onroad_alert_small onroad_alert_mid onroad_alert_full driver_camera body keyboard"
-          A=($scenes)
-
-          DIFF=""
-          TABLE="<details><summary>All Screenshots</summary>"
-          TABLE="${TABLE}<table>"
-
-          for ((i=0; i<${#A[*]}; i=i+1));
-          do
-
-            if ! compare -fuzz 2% -highlight-color DeepSkyBlue1 -lowlight-color Black -compose Src ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png; then
-              convert ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png -transparent black mask.png
-              composite mask.png ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png
-              convert -delay 100 ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png -loop 0 ${{ github.workspace }}/pr_ui/${A[$i]}_diff.gif
-
-              mv ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_master_ref.png
-
-              DIFF="${DIFF}<details open>"
-              DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{red}\\text{DIFFERENT}}\$\$</summary>"
-              DIFF="${DIFF}<table>"
-
-              DIFF="${DIFF}<tr>"
-              DIFF="${DIFF}  <td> master <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_master_ref.png\"> </td>"
-              DIFF="${DIFF}  <td> proposed <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
-              DIFF="${DIFF}</tr>"
-
-              DIFF="${DIFF}<tr>"
-              DIFF="${DIFF}  <td> diff <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.png\"> </td>"
-              DIFF="${DIFF}  <td> composite diff <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.gif\"> </td>"
-              DIFF="${DIFF}</tr>"
-
-              DIFF="${DIFF}</table>"
-              DIFF="${DIFF}</details>"
-            else
-              rm -f ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png
-            fi
-
-            INDEX=$(($i % 2))
-            if [[ $INDEX -eq 0 ]]; then
-              TABLE="${TABLE}<tr>"
-            fi
-            TABLE="${TABLE}  <td> <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
-            if [[ $INDEX -eq 1 || $(($i + 1)) -eq ${#A[*]} ]]; then
-              TABLE="${TABLE}</tr>"
-            fi
-          done
-
-          TABLE="${TABLE}</table></details>"
-
-          echo "DIFF=$DIFF$TABLE" >> "$GITHUB_OUTPUT"
-
-      - name: Saving proposed ui
-        if: github.event_name == 'pull_request_target'
-        working-directory: ${{ github.workspace }}/master_ui
-        run: |
-          git config user.name "GitHub Actions Bot"
-          git config user.email "<>"
-          git checkout --orphan=${{ env.BRANCH_NAME }}
-          git rm -rf *
-          mv ${{ github.workspace }}/pr_ui/* .
-          git add .
-          git commit -m "screenshots for PR #${{ github.event.number }}"
-          git push origin ${{ env.BRANCH_NAME }} --force
-
-      - name: Comment Screenshots on PR
-        if: github.event_name == 'pull_request_target'
-        uses: thollander/actions-comment-pull-request@v2
-        with:
-          message: |
-            <!-- _(run_id_screenshots **${{ github.run_id }}**)_ -->
-            ## UI Preview
-            ${{ steps.find_diff.outputs.DIFF }}
-          comment_tag: run_id_screenshots
-          pr_number: ${{ github.event.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitmodules

@@ -1,18 +0,0 @@
-[submodule "panda"]
-  path = panda
-  url = https://github.com/sunnyhaibin/panda.git
-[submodule "opendbc"]
-  path = opendbc_repo
-  url = https://github.com/sunnypilot/opendbc.git
-[submodule "msgq"]
-  path = msgq_repo
-  url = https://github.com/sunnypilot/msgq.git
-[submodule "rednose_repo"]
-  path = rednose_repo
-  url = https://github.com/commaai/rednose.git
-[submodule "teleoprtc_repo"]
-  path = teleoprtc_repo
-  url = https://github.com/commaai/teleoprtc
-[submodule "tinygrad"]
-  path = tinygrad_repo
-  url = https://github.com/tinygrad/tinygrad.git

.lfsconfig

@@ -1,4 +0,0 @@
-[lfs]
-	url = https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
-	pushurl = ssh://git@gitlab.com/commaai/openpilot-lfs.git
-	locksverify = false

.vscode/extensions.json

@@ -1,8 +0,0 @@
-{
-  "recommendations": [
-    "ms-python.python",
-    "ms-vscode.cpptools",
-    "elagil.pre-commit-helper",
-    "charliermarsh.ruff",
-  ]
-}

.vscode/launch.json

@@ -1,46 +0,0 @@
-{
-  "version": "0.2.0",
-  "inputs": [
-    {
-      "id": "python_process",
-      "type": "pickString",
-      "description": "Select the process to debug",
-      "options": [
-        "selfdrive/controls/controlsd.py",
-        "system/timed/timed.py",
-        "tools/sim/run_bridge.py"
-      ]
-    },
-    {
-      "id": "cpp_process",
-      "type": "pickString",
-      "description": "Select the process to debug",
-      "options": [
-        "selfdrive/ui/ui"
-      ]
-    },
-    {
-      "id": "args",
-      "description": "Arguments to pass to the process",
-      "type": "promptString"
-    }
-  ],
-  "configurations": [
-    {
-      "name": "Python: openpilot Process",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "${input:python_process}",
-      "console": "integratedTerminal",
-      "justMyCode": true,
-      "args": "${input:args}"
-    },
-    {
-      "name": "C++: openpilot Process",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "${workspaceFolder}/${input:cpp_process}",
-      "cwd": "${workspaceFolder}",
-    }
-  ]
-}

.vscode/settings.json

@@ -1,27 +0,0 @@
-{
-  "editor.tabSize": 2,
-  "editor.insertSpaces": true,
-  "editor.renderWhitespace": "trailing",
-  "files.trimTrailingWhitespace": true,
-  "search.exclude": {
-    "**/.git": true,
-    "**/.venv": true,
-    "**/__pycache__": true
-  },
-  "files.exclude": {
-    "**/.git": true,
-    "**/.venv": true,
-    "**/__pycache__": true
-  },
-  "python.analysis.exclude": [
-    "**/.git",
-    "**/.venv",
-    "**/__pycache__",
-    // exclude directories that should be using the symlinked version
-    "common/**",
-    "selfdrive/**",
-    "system/**",
-    "third_party/**",
-    "tools/**",
-  ]
-}

Dockerfile.openpilot

@@ -1,13 +0,0 @@
-FROM ghcr.io/commaai/openpilot-base:latest
-
-ENV PYTHONUNBUFFERED 1
-
-ENV OPENPILOT_PATH /home/batman/openpilot
-ENV PYTHONPATH ${OPENPILOT_PATH}:${PYTHONPATH}
-
-RUN mkdir -p ${OPENPILOT_PATH}
-WORKDIR ${OPENPILOT_PATH}
-
-COPY . ${OPENPILOT_PATH}/
-
-RUN scons --cache-readonly -j$(nproc)

Dockerfile.openpilot_base

@@ -1,84 +0,0 @@
-FROM ubuntu:24.04
-
-ENV PYTHONUNBUFFERED 1
-
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
-
-COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
-RUN /tmp/tools/install_ubuntu_dependencies.sh && \
-    rm -rf /var/lib/apt/lists/* /tmp/* && \
-    cd /usr/lib/gcc/arm-none-eabi/* && \
-    rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
-
-# Add OpenCL
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    apt-utils \
-    alien \
-    unzip \
-    tar \
-    curl \
-    xz-utils \
-    dbus \
-    gcc-arm-none-eabi \
-    tmux \
-    vim \
-    libx11-6 \
-    wget \
-  && rm -rf /var/lib/apt/lists/*
-
-RUN mkdir -p /tmp/opencl-driver-intel && \
-    cd /tmp/opencl-driver-intel && \
-    wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
-    wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz && \
-    mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
-    cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
-    tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
-    mkdir -p /etc/OpenCL/vendors && \
-    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd && \
-    cd /opt/intel && \
-    tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    mkdir -p /etc/ld.so.conf.d && \
-    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf && \
-    ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf && \
-    cd / && \
-    rm -rf /tmp/opencl-driver-intel
-
-ENV NVIDIA_VISIBLE_DEVICES all
-ENV NVIDIA_DRIVER_CAPABILITIES graphics,utility,compute
-ENV QTWEBENGINE_DISABLE_SANDBOX 1
-
-RUN dbus-uuidgen > /etc/machine-id
-
-ARG USER=batman
-ARG USER_UID=1001
-RUN useradd -m -s /bin/bash -u $USER_UID $USER
-RUN usermod -aG sudo $USER
-RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-USER $USER
-
-COPY --chown=$USER pyproject.toml uv.lock /tmp/
-COPY --chown=$USER tools/install_python_dependencies.sh /tmp/tools/
-
-ENV VIRTUAL_ENV=/home/$USER/.venv
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-RUN cd /tmp && \
-    tools/install_python_dependencies.sh && \
-    mkdir -p $VIRTUAL_ENV && \
-    cp -r /tmp/.venv/* $VIRTUAL_ENV && \
-    rm -rf /tmp/* && \
-    rm -rf /home/$USER/.cache
-
-USER root
-RUN sudo git config --global --add safe.directory /tmp/openpilot

LICENSE

@@ -1,7 +0,0 @@
-Copyright (c) 2018, Comma.ai, Inc.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

LICENSE.md

@@ -1,21 +0,0 @@
-# Custom MIT License
-
-Copyright (c) 2024, Haibin Wen, SUNNYPILOT LLC
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to view and modify the Software, subject to the following conditions:
-
-1. **Permission Required**: Permission Required for Commercial, For-Profit, or Closed Source Use: Use of the Software, in whole or in part, for any commercial purposes, for-profit projects, or in closed source projects requires explicit written permission from the original author(s).
-
-2. **Redistribution**: Any redistribution of the Software, modified or unmodified, must retain this license notice and the following acknowledgment:
-   "This software is licensed under a custom license requiring permission for use."
-
-3. **Visibility**: Any project that uses the Software must visibly mention the following acknowledgment:
-   "This project uses software from Haibin Wen and SUNNYPILOT LLC and is licensed under a custom license requiring permission for use."
-
-4. **No Warranty**: THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-Contact sunnypilot Support <support@sunnypilot.ai> for permission requests.
-
----
-
-Haibin Wen, SUNNYPILOT LLC

cereal/custom.capnp

@@ -8,7 +8,23 @@ $Cxx.namespace("cereal");
 # cereal, so use these if you want custom events in your fork.
 
 # you can rename the struct, but don't change the identifier
-struct CustomReserved0 @0x81c2f05a394cf4af {
+struct SelfdriveStateSP @0x81c2f05a394cf4af {
+  mads @0 :ModularAssistiveDrivingSystem;
+
+  struct ModularAssistiveDrivingSystem {
+    state @0 :ModularAssistiveDrivingSystemState;
+    enabled @1 :Bool;
+    active @2 :Bool;
+    available @3 :Bool;
+
+    enum ModularAssistiveDrivingSystemState {
+      disabled @0;
+      paused @1;
+      enabled @2;
+      softDisabling @3;
+      overriding @4;
+    }
+  }
 }
 
 struct CustomReserved1 @0xaedffd8f31e7b55d {

cereal/log.capnp

@@ -125,6 +125,79 @@ struct OnroadEvent @0xc4fa6047f024e718 {
     espActive @90;
     personalityChanged @91;
     aeb @92;
+    eventReserved93 @93;
+    eventReserved94 @94;
+    eventReserved95 @95;
+    eventReserved96 @96;
+    eventReserved97 @97;
+    eventReserved98 @98;
+    eventReserved99 @99;
+    eventReserved100 @100;
+    eventReserved101 @101;
+    eventReserved102 @102;
+    eventReserved103 @103;
+    eventReserved104 @104;
+    eventReserved105 @105;
+    eventReserved106 @106;
+    eventReserved107 @107;
+    eventReserved108 @108;
+    eventReserved109 @109;
+    eventReserved110 @110;
+    eventReserved111 @111;
+    eventReserved112 @112;
+    eventReserved113 @113;
+    eventReserved114 @114;
+    eventReserved115 @115;
+    eventReserved116 @116;
+    eventReserved117 @117;
+    eventReserved118 @118;
+    eventReserved119 @119;
+    eventReserved120 @120;
+    eventReserved121 @121;
+    eventReserved122 @122;
+    eventReserved123 @123;
+    eventReserved124 @124;
+    eventReserved125 @125;
+    eventReserved126 @126;
+    eventReserved127 @127;
+    eventReserved128 @128;
+    eventReserved129 @129;
+    eventReserved130 @130;
+    eventReserved131 @131;
+    eventReserved132 @132;
+    eventReserved133 @133;
+    eventReserved134 @134;
+    eventReserved135 @135;
+    eventReserved136 @136;
+    eventReserved137 @137;
+    eventReserved138 @138;
+    eventReserved139 @139;
+    eventReserved140 @140;
+    eventReserved141 @141;
+    eventReserved142 @142;
+    eventReserved143 @143;
+    eventReserved144 @144;
+    eventReserved145 @145;
+    eventReserved146 @146;
+    eventReserved147 @147;
+    eventReserved148 @148;
+    eventReserved149 @149;
+    eventReserved150 @150;
+
+    # sunnypilot
+    lkasEnable @151;
+    lkasDisable @152;
+    manualSteeringRequired @153;
+    manualLongitudinalRequired @154;
+    silentPedalPressed @155;
+    silentLkasEnable @156;
+    silentLkasDisable @157;
+    silentBrakeHold @158;
+    silentWrongGear @159;
+    silentReverseGear @160;
+    silentDoorOpen @161;
+    silentSeatbeltNotLatched @162;
+    silentParkBrake @163;
 
     soundsUnavailableDEPRECATED @47;
   }
@@ -586,6 +659,7 @@ struct PandaState @0xa7649e2575e4591e {
 
   # safety stuff
   controlsAllowed @3 :Bool;
+  controlsAllowedLat @5 :Bool;
   safetyRxInvalid @19 :UInt32;
   safetyTxBlocked @24 :UInt32;
   safetyModel @14 :Car.CarParams.SafetyModel;
@@ -693,7 +767,6 @@ struct PandaState @0xa7649e2575e4591e {
   }
 
   gasInterceptorDetectedDEPRECATED @4 :Bool;
-  startedSignalDetectedDEPRECATED @5 :Bool;
   hasGpsDEPRECATED @6 :Bool;
   gmlanSendErrsDEPRECATED @9 :UInt32;
   fanSpeedRpmDEPRECATED @11 :UInt16;
@@ -2544,7 +2617,7 @@ struct Event {
     customReservedRawData2 @126 :Data;
 
     # *********** Custom: reserved for forks ***********
-    customReserved0 @107 :Custom.CustomReserved0;
+    selfdriveStateSP @107 :Custom.SelfdriveStateSP;
     customReserved1 @108 :Custom.CustomReserved1;
     customReserved2 @109 :Custom.CustomReserved2;
     customReserved3 @110 :Custom.CustomReserved3;

cereal/messaging/tests/test_messaging.py

@@ -1,185 +0,0 @@
-import os
-import capnp
-import multiprocessing
-import numbers
-import random
-import threading
-import time
-from parameterized import parameterized
-import pytest
-
-from cereal import log, car
-import cereal.messaging as messaging
-from cereal.services import SERVICE_LIST
-
-events = [evt for evt in log.Event.schema.union_fields if evt in SERVICE_LIST.keys()]
-
-def random_sock():
-  return random.choice(events)
-
-def random_socks(num_socks=10):
-  return list({random_sock() for _ in range(num_socks)})
-
-def random_bytes(length=1000):
-  return bytes([random.randrange(0xFF) for _ in range(length)])
-
-def zmq_sleep(t=1):
-  if "ZMQ" in os.environ:
-    time.sleep(t)
-
-
-# TODO: this should take any capnp struct and returrn a msg with random populated data
-def random_carstate():
-  fields = ["vEgo", "aEgo", "gas", "steeringAngleDeg"]
-  msg = messaging.new_message("carState")
-  cs = msg.carState
-  for f in fields:
-    setattr(cs, f, random.random() * 10)
-  return msg
-
-# TODO: this should compare any capnp structs
-def assert_carstate(cs1, cs2):
-  for f in car.CarState.schema.non_union_fields:
-    # TODO: check all types
-    val1, val2 = getattr(cs1, f), getattr(cs2, f)
-    if isinstance(val1, numbers.Number):
-      assert val1 == val2, f"{f}: sent '{val1}' vs recvd '{val2}'"
-
-def delayed_send(delay, sock, dat):
-  def send_func():
-    sock.send(dat)
-  threading.Timer(delay, send_func).start()
-
-
-class TestMessaging:
-  def setUp(self):
-    # TODO: ZMQ tests are too slow; all sleeps will need to be
-    # replaced with logic to block on the necessary condition
-    if "ZMQ" in os.environ:
-      pytest.skip()
-
-    # ZMQ pub socket takes too long to die
-    # sleep to prevent multiple publishers error between tests
-    zmq_sleep()
-
-  @parameterized.expand(events)
-  def test_new_message(self, evt):
-    try:
-      msg = messaging.new_message(evt)
-    except capnp.lib.capnp.KjException:
-      msg = messaging.new_message(evt, random.randrange(200))
-    assert (time.monotonic() - msg.logMonoTime) < 0.1
-    assert not msg.valid
-    assert evt == msg.which()
-
-  @parameterized.expand(events)
-  def test_pub_sock(self, evt):
-    messaging.pub_sock(evt)
-
-  @parameterized.expand(events)
-  def test_sub_sock(self, evt):
-    messaging.sub_sock(evt)
-
-  @parameterized.expand([
-    (messaging.drain_sock, capnp._DynamicStructReader),
-    (messaging.drain_sock_raw, bytes),
-  ])
-  def test_drain_sock(self, func, expected_type):
-    sock = "carState"
-    pub_sock = messaging.pub_sock(sock)
-    sub_sock = messaging.sub_sock(sock, timeout=1000)
-    zmq_sleep()
-
-    # no wait and no msgs in queue
-    msgs = func(sub_sock)
-    assert isinstance(msgs, list)
-    assert len(msgs) == 0
-
-    # no wait but msgs are queued up
-    num_msgs = random.randrange(3, 10)
-    for _ in range(num_msgs):
-      pub_sock.send(messaging.new_message(sock).to_bytes())
-    time.sleep(0.1)
-    msgs = func(sub_sock)
-    assert isinstance(msgs, list)
-    assert all(isinstance(msg, expected_type) for msg in msgs)
-    assert len(msgs) == num_msgs
-
-  def test_recv_sock(self):
-    sock = "carState"
-    pub_sock = messaging.pub_sock(sock)
-    sub_sock = messaging.sub_sock(sock, timeout=100)
-    zmq_sleep()
-
-    # no wait and no msg in queue, socket should timeout
-    recvd = messaging.recv_sock(sub_sock)
-    assert recvd is None
-
-    # no wait and one msg in queue
-    msg = random_carstate()
-    pub_sock.send(msg.to_bytes())
-    time.sleep(0.01)
-    recvd = messaging.recv_sock(sub_sock)
-    assert isinstance(recvd, capnp._DynamicStructReader)
-    # https://github.com/python/mypy/issues/13038
-    assert_carstate(msg.carState, recvd.carState)
-
-  def test_recv_one(self):
-    sock = "carState"
-    pub_sock = messaging.pub_sock(sock)
-    sub_sock = messaging.sub_sock(sock, timeout=1000)
-    zmq_sleep()
-
-    # no msg in queue, socket should timeout
-    recvd = messaging.recv_one(sub_sock)
-    assert recvd is None
-
-    # one msg in queue
-    msg = random_carstate()
-    pub_sock.send(msg.to_bytes())
-    recvd = messaging.recv_one(sub_sock)
-    assert isinstance(recvd, capnp._DynamicStructReader)
-    assert_carstate(msg.carState, recvd.carState)
-
-  @pytest.mark.xfail(condition="ZMQ" in os.environ, reason='ZMQ detected')
-  def test_recv_one_or_none(self):
-    sock = "carState"
-    pub_sock = messaging.pub_sock(sock)
-    sub_sock = messaging.sub_sock(sock)
-    zmq_sleep()
-
-    # no msg in queue, socket shouldn't block
-    recvd = messaging.recv_one_or_none(sub_sock)
-    assert recvd is None
-
-    # one msg in queue
-    msg = random_carstate()
-    pub_sock.send(msg.to_bytes())
-    recvd = messaging.recv_one_or_none(sub_sock)
-    assert isinstance(recvd, capnp._DynamicStructReader)
-    assert_carstate(msg.carState, recvd.carState)
-
-  def test_recv_one_retry(self):
-    sock = "carState"
-    sock_timeout = 0.1
-    pub_sock = messaging.pub_sock(sock)
-    sub_sock = messaging.sub_sock(sock, timeout=round(sock_timeout*1000))
-    zmq_sleep()
-
-    # this test doesn't work with ZMQ since multiprocessing interrupts it
-    if "ZMQ" not in os.environ:
-      # wait 5 socket timeouts and make sure it's still retrying
-      p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,))
-      p.start()
-      time.sleep(sock_timeout*5)
-      assert p.is_alive()
-      p.terminate()
-
-    # wait 5 socket timeouts before sending
-    msg = random_carstate()
-    delayed_send(sock_timeout*5, pub_sock, msg.to_bytes())
-    start_time = time.monotonic()
-    recvd = messaging.recv_one_retry(sub_sock)
-    assert (time.monotonic() - start_time) >= sock_timeout*5
-    assert isinstance(recvd, capnp._DynamicStructReader)
-    assert_carstate(msg.carState, recvd.carState)

cereal/messaging/tests/test_pub_sub_master.py

@@ -1,156 +0,0 @@
-import random
-import time
-from typing import Sized, cast
-
-import cereal.messaging as messaging
-from cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
-                                                  random_bytes, random_carstate, assert_carstate, \
-                                                  zmq_sleep
-
-
-class TestSubMaster:
-
-  def setup_method(self):
-    # ZMQ pub socket takes too long to die
-    # sleep to prevent multiple publishers error between tests
-    zmq_sleep(3)
-
-  def test_init(self):
-    sm = messaging.SubMaster(events)
-    for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
-              sm.sock, sm.data, sm.logMonoTime, sm.valid]:
-      assert len(cast(Sized, p)) == len(events)
-
-  def test_init_state(self):
-    socks = random_socks()
-    sm = messaging.SubMaster(socks)
-    assert sm.frame == -1
-    assert not any(sm.updated.values())
-    assert not any(sm.alive.values())
-    assert all(t == 0. for t in sm.recv_time.values())
-    assert all(f == 0 for f in sm.recv_frame.values())
-    assert all(t == 0 for t in sm.logMonoTime.values())
-
-    for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
-              sm.sock, sm.data, sm.logMonoTime, sm.valid]:
-      assert len(cast(Sized, p)) == len(socks)
-
-  def test_getitem(self):
-    sock = "carState"
-    pub_sock = messaging.pub_sock(sock)
-    sm = messaging.SubMaster([sock,])
-    zmq_sleep()
-
-    msg = random_carstate()
-    pub_sock.send(msg.to_bytes())
-    sm.update(1000)
-    assert_carstate(msg.carState, sm[sock])
-
-  # TODO: break this test up to individually test SubMaster.update and SubMaster.update_msgs
-  def test_update(self):
-    sock = "carState"
-    pub_sock = messaging.pub_sock(sock)
-    sm = messaging.SubMaster([sock,])
-    zmq_sleep()
-
-    for i in range(10):
-      msg = messaging.new_message(sock)
-      pub_sock.send(msg.to_bytes())
-      sm.update(1000)
-      assert sm.frame == i
-      assert all(sm.updated.values())
-
-  def test_update_timeout(self):
-    sock = random_sock()
-    sm = messaging.SubMaster([sock,])
-    timeout = random.randrange(1000, 3000)
-    start_time = time.monotonic()
-    sm.update(timeout)
-    t = time.monotonic() - start_time
-    assert t >= timeout/1000.
-    assert t < 3
-    assert not any(sm.updated.values())
-
-  def test_avg_frequency_checks(self):
-    for poll in (True, False):
-      sm = messaging.SubMaster(["modelV2", "carParams", "carState", "cameraOdometry", "liveCalibration"],
-                               poll=("modelV2" if poll else None),
-                               frequency=(20. if not poll else None))
-
-      checks = {
-        "carState": (20, 20),
-        "modelV2": (20, 20 if poll else 10),
-        "cameraOdometry": (20, 10),
-        "liveCalibration": (4, 4),
-        "carParams": (None, None),
-      }
-
-      for service, (max_freq, min_freq) in checks.items():
-        if max_freq is not None:
-          assert sm._check_avg_freq(service)
-          assert sm.freq_tracker[service].max_freq == max_freq*1.2
-          assert sm.freq_tracker[service].min_freq == min_freq*0.8
-        else:
-          assert not sm._check_avg_freq(service)
-
-  def test_alive(self):
-    pass
-
-  def test_ignore_alive(self):
-    pass
-
-  def test_valid(self):
-    pass
-
-  # SubMaster should always conflate
-  def test_conflate(self):
-    sock = "carState"
-    pub_sock = messaging.pub_sock(sock)
-    sm = messaging.SubMaster([sock,])
-
-    n = 10
-    for i in range(n+1):
-      msg = messaging.new_message(sock)
-      msg.carState.vEgo = i
-      pub_sock.send(msg.to_bytes())
-      time.sleep(0.01)
-    sm.update(1000)
-    assert sm[sock].vEgo == n
-
-
-class TestPubMaster:
-
-  def setup_method(self):
-    # ZMQ pub socket takes too long to die
-    # sleep to prevent multiple publishers error between tests
-    zmq_sleep(3)
-
-  def test_init(self):
-    messaging.PubMaster(events)
-
-  def test_send(self):
-    socks = random_socks()
-    pm = messaging.PubMaster(socks)
-    sub_socks = {s: messaging.sub_sock(s, conflate=True, timeout=1000) for s in socks}
-    zmq_sleep()
-
-    # PubMaster accepts either a capnp msg builder or bytes
-    for capnp in [True, False]:
-      for i in range(100):
-        sock = socks[i % len(socks)]
-
-        if capnp:
-          try:
-            msg = messaging.new_message(sock)
-          except Exception:
-            msg = messaging.new_message(sock, random.randrange(50))
-        else:
-          msg = random_bytes()
-
-        pm.send(sock, msg)
-        recvd = sub_socks[sock].receive()
-
-        if capnp:
-          msg.clear_write_flag()
-          msg = msg.to_bytes()
-        assert msg == recvd, i

cereal/messaging/tests/test_services.py

@@ -1,21 +0,0 @@
-import os
-import tempfile
-from typing import Dict
-from parameterized import parameterized
-
-import cereal.services as services
-from cereal.services import SERVICE_LIST
-
-
-class TestServices:
-
-  @parameterized.expand(SERVICE_LIST.keys())
-  def test_services(self, s):
-    service = SERVICE_LIST[s]
-    assert service.frequency <= 104
-    assert service.decimation != 0
-
-  def test_generated_header(self):
-    with tempfile.NamedTemporaryFile(suffix=".h") as f:
-      ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name}")
-      assert ret == 0, "generated services header is not valid C"

cereal/services.py

@@ -73,6 +73,9 @@ _services: dict[str, tuple] = {
   "userFlag": (True, 0., 1),
   "microphone": (True, 10., 10),
 
+  # sunnypilot
+  "selfdriveStateSP": (True, 100., 10),
+
   # debug
   "uiDebug": (True, 0., 1),
   "testJoystick": (True, 0.),

common/params.cc

@@ -201,6 +201,16 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"UpdaterTargetBranch", CLEAR_ON_MANAGER_START},
     {"UpdaterLastFetchTime", PERSISTENT},
     {"Version", PERSISTENT},
+
+    // sunnypilot params
+    {"Mads", PERSISTENT},
+    {"MadsCruiseMain", PERSISTENT},
+    {"MadsDisengageLateralOnBrake", PERSISTENT},
+    {"MadsUnifiedEngagementMode", PERSISTENT},
+
+    {"ToyotaAutoHold", PERSISTENT},
+    {"ToyotaEnhancedBsm", PERSISTENT},
+    {"ToyotaTSS2Long", PERSISTENT},
 };
 
 } // namespace

common/tests/.gitignore

@@ -1 +0,0 @@
-test_common

common/tests/test_file_helpers.py

@@ -1,19 +0,0 @@
-import os
-from uuid import uuid4
-
-from openpilot.common.file_helpers import atomic_write_in_dir
-
-
-class TestFileHelpers:
-  def run_atomic_write_func(self, atomic_write_func):
-    path = f"/tmp/tmp{uuid4()}"
-    with atomic_write_func(path) as f:
-      f.write("test")
-      assert not os.path.exists(path)
-
-    with open(path) as f:
-      assert f.read() == "test"
-    os.remove(path)
-
-  def test_atomic_write_in_dir(self):
-    self.run_atomic_write_func(atomic_write_in_dir)

common/tests/test_markdown.py

@@ -1,15 +0,0 @@
-import os
-
-from openpilot.common.basedir import BASEDIR
-from openpilot.common.markdown import parse_markdown
-
-
-class TestMarkdown:
-  def test_all_release_notes(self):
-    with open(os.path.join(BASEDIR, "RELEASES.md")) as f:
-      release_notes = f.read().split("\n\n")
-      assert len(release_notes) > 10
-
-      for rn in release_notes:
-        md = parse_markdown(rn)
-        assert len(md) > 0

common/tests/test_numpy_fast.py

@@ -1,21 +0,0 @@
-import numpy as np
-
-from openpilot.common.numpy_fast import interp
-
-
-class TestInterp:
-  def test_correctness_controls(self):
-    _A_CRUISE_MIN_BP = np.asarray([0., 5., 10., 20., 40.])
-    _A_CRUISE_MIN_V = np.asarray([-1.0, -.8, -.67, -.5, -.30])
-    v_ego_arr = [-1, -1e-12, 0, 4, 5, 6, 7, 10, 11, 15.2, 20, 21, 39,
-                 39.999999, 40, 41]
-
-    expected = np.interp(v_ego_arr, _A_CRUISE_MIN_BP, _A_CRUISE_MIN_V)
-    actual = interp(v_ego_arr, _A_CRUISE_MIN_BP, _A_CRUISE_MIN_V)
-
-    np.testing.assert_equal(actual, expected)
-
-    for v_ego in v_ego_arr:
-      expected = np.interp(v_ego, _A_CRUISE_MIN_BP, _A_CRUISE_MIN_V)
-      actual = interp(v_ego, _A_CRUISE_MIN_BP, _A_CRUISE_MIN_V)
-      np.testing.assert_equal(actual, expected)

common/tests/test_params.cc

@@ -1,27 +0,0 @@
-#include "catch2/catch.hpp"
-#define private public
-#include "common/params.h"
-#include "common/util.h"
-
-TEST_CASE("params_nonblocking_put") {
-  char tmp_path[] = "/tmp/asyncWriter_XXXXXX";
-  const std::string param_path = mkdtemp(tmp_path);
-  auto param_names = {"CarParams", "IsMetric"};
-  {
-    Params params(param_path);
-    for (const auto &name : param_names) {
-      params.putNonBlocking(name, "1");
-      // param is empty
-      REQUIRE(params.get(name).empty());
-    }
-
-    // check if thread is running
-    REQUIRE(params.future.valid());
-    REQUIRE(params.future.wait_for(std::chrono::milliseconds(0)) == std::future_status::timeout);
-  }
-  // check results
-  Params p(param_path);
-  for (const auto &name : param_names) {
-    REQUIRE(p.get(name) == "1");
-  }
-}

common/tests/test_params.py

@@ -1,109 +0,0 @@
-import pytest
-import os
-import threading
-import time
-import uuid
-
-from openpilot.common.params import Params, ParamKeyType, UnknownKeyName
-
-class TestParams:
-  def setup_method(self):
-    self.params = Params()
-
-  def test_params_put_and_get(self):
-    self.params.put("DongleId", "cb38263377b873ee")
-    assert self.params.get("DongleId") == b"cb38263377b873ee"
-
-  def test_params_non_ascii(self):
-    st = b"\xe1\x90\xff"
-    self.params.put("CarParams", st)
-    assert self.params.get("CarParams") == st
-
-  def test_params_get_cleared_manager_start(self):
-    self.params.put("CarParams", "test")
-    self.params.put("DongleId", "cb38263377b873ee")
-    assert self.params.get("CarParams") == b"test"
-
-    undefined_param = self.params.get_param_path(uuid.uuid4().hex)
-    with open(undefined_param, "w") as f:
-      f.write("test")
-    assert os.path.isfile(undefined_param)
-
-    self.params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START)
-    assert self.params.get("CarParams") is None
-    assert self.params.get("DongleId") is not None
-    assert not os.path.isfile(undefined_param)
-
-  def test_params_two_things(self):
-    self.params.put("DongleId", "bob")
-    self.params.put("AthenadPid", "123")
-    assert self.params.get("DongleId") == b"bob"
-    assert self.params.get("AthenadPid") == b"123"
-
-  def test_params_get_block(self):
-    def _delayed_writer():
-      time.sleep(0.1)
-      self.params.put("CarParams", "test")
-    threading.Thread(target=_delayed_writer).start()
-    assert self.params.get("CarParams") is None
-    assert self.params.get("CarParams", True) == b"test"
-
-  def test_params_unknown_key_fails(self):
-    with pytest.raises(UnknownKeyName):
-      self.params.get("swag")
-
-    with pytest.raises(UnknownKeyName):
-      self.params.get_bool("swag")
-
-    with pytest.raises(UnknownKeyName):
-      self.params.put("swag", "abc")
-
-    with pytest.raises(UnknownKeyName):
-      self.params.put_bool("swag", True)
-
-  def test_remove_not_there(self):
-    assert self.params.get("CarParams") is None
-    self.params.remove("CarParams")
-    assert self.params.get("CarParams") is None
-
-  def test_get_bool(self):
-    self.params.remove("IsMetric")
-    assert not self.params.get_bool("IsMetric")
-
-    self.params.put_bool("IsMetric", True)
-    assert self.params.get_bool("IsMetric")
-
-    self.params.put_bool("IsMetric", False)
-    assert not self.params.get_bool("IsMetric")
-
-    self.params.put("IsMetric", "1")
-    assert self.params.get_bool("IsMetric")
-
-    self.params.put("IsMetric", "0")
-    assert not self.params.get_bool("IsMetric")
-
-  def test_put_non_blocking_with_get_block(self):
-    q = Params()
-    def _delayed_writer():
-      time.sleep(0.1)
-      Params().put_nonblocking("CarParams", "test")
-    threading.Thread(target=_delayed_writer).start()
-    assert q.get("CarParams") is None
-    assert q.get("CarParams", True) == b"test"
-
-  def test_put_bool_non_blocking_with_get_block(self):
-    q = Params()
-    def _delayed_writer():
-      time.sleep(0.1)
-      Params().put_bool_nonblocking("CarParams", True)
-    threading.Thread(target=_delayed_writer).start()
-    assert q.get("CarParams") is None
-    assert q.get("CarParams", True) == b"1"
-
-  def test_params_all_keys(self):
-    keys = Params().all_keys()
-
-    # sanity checks
-    assert len(keys) > 20
-    assert len(keys) == len(set(keys))
-    assert b"CarParams" in keys

common/tests/test_simple_kalman.py

@@ -1,29 +0,0 @@
-from openpilot.common.simple_kalman import KF1D
-
-
-class TestSimpleKalman:
-  def setup_method(self):
-    dt = 0.01
-    x0_0 = 0.0
-    x1_0 = 0.0
-    A0_0 = 1.0
-    A0_1 = dt
-    A1_0 = 0.0
-    A1_1 = 1.0
-    C0_0 = 1.0
-    C0_1 = 0.0
-    K0_0 = 0.12287673
-    K1_0 = 0.29666309
-
-    self.kf = KF1D(x0=[[x0_0], [x1_0]],
-                   A=[[A0_0, A0_1], [A1_0, A1_1]],
-                   C=[C0_0, C0_1],
-                   K=[[K0_0], [K1_0]])
-
-  def test_getter_setter(self):
-    self.kf.set_x([[1.0], [1.0]])
-    assert self.kf.x == [[1.0], [1.0]]
-
-  def update_returns_state(self):
-    x = self.kf.update(100)
-    assert x == self.kf.x

common/tests/test_swaglog.cc

@@ -1,88 +0,0 @@
-#include <zmq.h>
-
-#include <iostream>
-
-#include "catch2/catch.hpp"
-#include "common/swaglog.h"
-#include "common/util.h"
-#include "common/version.h"
-#include "system/hardware/hw.h"
-#include "third_party/json11/json11.hpp"
-
-std::string daemon_name = "testy";
-std::string dongle_id = "test_dongle_id";
-int LINE_NO = 0;
-
-void log_thread(int thread_id, int msg_cnt) {
-  for (int i = 0; i < msg_cnt; ++i) {
-    LOGD("%d", thread_id);
-    LINE_NO = __LINE__ - 1;
-    usleep(1);
-  }
-}
-
-void recv_log(int thread_cnt, int thread_msg_cnt) {
-  void *zctx = zmq_ctx_new();
-  void *sock = zmq_socket(zctx, ZMQ_PULL);
-  zmq_bind(sock, Path::swaglog_ipc().c_str());
-  std::vector<int> thread_msgs(thread_cnt);
-  int total_count = 0;
-
-  for (auto start = std::chrono::steady_clock::now(), now = start;
-       now < start + std::chrono::seconds{1} && total_count < (thread_cnt * thread_msg_cnt);
-       now = std::chrono::steady_clock::now()) {
-    char buf[4096] = {};
-    if (zmq_recv(sock, buf, sizeof(buf), ZMQ_DONTWAIT) <= 0) {
-      if (errno == EAGAIN || errno == EINTR || errno == EFSM) continue;
-      break;
-    }
-
-    REQUIRE(buf[0] == CLOUDLOG_DEBUG);
-    std::string err;
-    auto msg = json11::Json::parse(buf + 1, err);
-    REQUIRE(!msg.is_null());
-
-    REQUIRE(msg["levelnum"].int_value() == CLOUDLOG_DEBUG);
-    REQUIRE_THAT(msg["filename"].string_value(), Catch::Contains("test_swaglog.cc"));
-    REQUIRE(msg["funcname"].string_value() == "log_thread");
-    REQUIRE(msg["lineno"].int_value() == LINE_NO);
-
-    auto ctx = msg["ctx"];
-
-    REQUIRE(ctx["daemon"].string_value() == daemon_name);
-    REQUIRE(ctx["dongle_id"].string_value() == dongle_id);
-    REQUIRE(ctx["dirty"].bool_value() == true);
-
-    REQUIRE(ctx["version"].string_value() == COMMA_VERSION);
-
-    std::string device = Hardware::get_name();
-    REQUIRE(ctx["device"].string_value() == device);
-
-    int thread_id = atoi(msg["msg"].string_value().c_str());
-    REQUIRE((thread_id >= 0 && thread_id < thread_cnt));
-    thread_msgs[thread_id]++;
-    total_count++;
-  }
-  for (int i = 0; i < thread_cnt; ++i) {
-    INFO("thread :" << i);
-    REQUIRE(thread_msgs[i] == thread_msg_cnt);
-  }
-  zmq_close(sock);
-  zmq_ctx_destroy(zctx);
-}
-
-TEST_CASE("swaglog") {
-  setenv("MANAGER_DAEMON", daemon_name.c_str(), 1);
-  setenv("DONGLE_ID", dongle_id.c_str(), 1);
-  setenv("dirty", "1", 1);
-  const int thread_cnt = 5;
-  const int thread_msg_cnt = 100;
-
-  std::vector<std::thread> log_threads;
-  for (int i = 0; i < thread_cnt; ++i) {
-    log_threads.push_back(std::thread(log_thread, i, thread_msg_cnt));
-  }
-  for (auto &t : log_threads) t.join();
-
-  recv_log(thread_cnt, thread_msg_cnt);
-}

common/tests/test_util.cc

@@ -1,147 +0,0 @@
-
-#include <dirent.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-
-#include <algorithm>
-#include <climits>
-#include <fstream>
-#include <random>
-#include <string>
-
-#include "catch2/catch.hpp"
-#include "common/util.h"
-
-std::string random_bytes(int size) {
-  std::random_device rd;
-  std::independent_bits_engine<std::default_random_engine, CHAR_BIT, unsigned char> rbe(rd());
-  std::string bytes(size + 1, '\0');
-  std::generate(bytes.begin(), bytes.end(), std::ref(rbe));
-  return bytes;
-}
-
-TEST_CASE("util::read_file") {
-  SECTION("read /proc/version") {
-    std::string ret = util::read_file("/proc/version");
-    REQUIRE(ret.find("Linux version") != std::string::npos);
-  }
-  SECTION("read from sysfs") {
-    std::string ret = util::read_file("/sys/power/wakeup_count");
-    REQUIRE(!ret.empty());
-  }
-  SECTION("read file") {
-    char filename[] = "/tmp/test_read_XXXXXX";
-    int fd = mkstemp(filename);
-
-    REQUIRE(util::read_file(filename).empty());
-
-    std::string content = random_bytes(64 * 1024);
-    write(fd, content.c_str(), content.size());
-    std::string ret = util::read_file(filename);
-    bool equal = (ret == content);
-    REQUIRE(equal);
-    close(fd);
-  }
-  SECTION("read directory") {
-    REQUIRE(util::read_file(".").empty());
-  }
-  SECTION("read non-existent file") {
-    std::string ret = util::read_file("does_not_exist");
-    REQUIRE(ret.empty());
-  }
-  SECTION("read non-permission") {
-    REQUIRE(util::read_file("/proc/kmsg").empty());
-  }
-}
-
-TEST_CASE("util::file_exists") {
-  char filename[] = "/tmp/test_file_exists_XXXXXX";
-  int fd = mkstemp(filename);
-  REQUIRE(fd != -1);
-  close(fd);
-
-  SECTION("existent file") {
-    REQUIRE(util::file_exists(filename));
-    REQUIRE(util::file_exists("/tmp"));
-  }
-  SECTION("nonexistent file") {
-    std::string fn = filename;
-    REQUIRE(!util::file_exists(fn + "/nonexistent"));
-  }
-  SECTION("file has no access permissions") {
-    std::string fn = "/proc/kmsg";
-    std::ifstream f(fn);
-    REQUIRE(f.good() == false);
-    REQUIRE(util::file_exists(fn));
-  }
-  ::remove(filename);
-}
-
-TEST_CASE("util::read_files_in_dir") {
-  char tmp_path[] = "/tmp/test_XXXXXX";
-  const std::string test_path = mkdtemp(tmp_path);
-  const std::string files[] = {".test1", "'test2'", "test3"};
-  for (auto fn : files) {
-    std::ofstream{test_path + "/" + fn} << fn;
-  }
-  mkdir((test_path + "/dir").c_str(), 0777);
-
-  std::map<std::string, std::string> result = util::read_files_in_dir(test_path);
-  REQUIRE(result.find("dir") == result.end());
-  REQUIRE(result.size() == std::size(files));
-  for (auto& [k, v] : result) {
-    REQUIRE(k == v);
-  }
-}
-
-
-TEST_CASE("util::safe_fwrite") {
-  char filename[] = "/tmp/XXXXXX";
-  int fd = mkstemp(filename);
-  close(fd);
-  std::string dat = random_bytes(1024 * 1024);
-
-  FILE *f = util::safe_fopen(filename, "wb");
-  REQUIRE(f != nullptr);
-  size_t size = util::safe_fwrite(dat.data(), 1, dat.size(), f);
-  REQUIRE(size == dat.size());
-  int ret = util::safe_fflush(f);
-  REQUIRE(ret == 0);
-  ret = fclose(f);
-  REQUIRE(ret == 0);
-  bool equal = (dat == util::read_file(filename));
-  REQUIRE(equal);
-}
-
-TEST_CASE("util::create_directories") {
-  system("rm /tmp/test_create_directories -rf");
-  std::string dir = "/tmp/test_create_directories/a/b/c/d/e/f";
-
-  auto check_dir_permissions = [](const std::string &dir, mode_t mode) -> bool {
-    struct stat st = {};
-    return stat(dir.c_str(), &st) == 0 && (st.st_mode & S_IFMT) == S_IFDIR && (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) == mode;
-  };
-
-  SECTION("create_directories") {
-    REQUIRE(util::create_directories(dir, 0755));
-    REQUIRE(check_dir_permissions(dir, 0755));
-  }
-  SECTION("dir already exists") {
-    REQUIRE(util::create_directories(dir, 0755));
-    REQUIRE(util::create_directories(dir, 0755));
-  }
-  SECTION("a file exists with the same name") {
-    REQUIRE(util::create_directories(dir, 0755));
-    int f = open((dir + "/file").c_str(), O_RDWR | O_CREAT);
-    REQUIRE(f != -1);
-    close(f);
-    REQUIRE(util::create_directories(dir + "/file", 0755) == false);
-    REQUIRE(util::create_directories(dir + "/file/1/2/3", 0755) == false);
-  }
-  SECTION("end with slashes") {
-    REQUIRE(util::create_directories(dir + "/", 0755));
-  }
-  SECTION("empty") {
-    REQUIRE(util::create_directories("", 0755) == false);
-  }
-}

docs/assets/icon-star-empty.svg

@@ -1,3 +1,56 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3142658fa2782d266028dcab491e02ab329a2cadcf9efb1846a5f9631d64cac1
-size 1947
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="19.999983"
+   height="19"
+   viewBox="0 0 19.999983 19"
+   version="1.1"
+   id="svg1425"
+   sodipodi:docname="icon-star-empty.svg"
+   style="fill:none"
+   inkscape:version="0.92.5 (2060ec1f9f, 2020-04-08)">
+  <metadata
+     id="metadata1431">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <defs
+     id="defs1429" />
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1198"
+     inkscape:window-height="631"
+     id="namedview1427"
+     showgrid="false"
+     inkscape:zoom="9.8333333"
+     inkscape:cx="-8.3559301"
+     inkscape:cy="8.9999925"
+     inkscape:window-x="0"
+     inkscape:window-y="1267"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg1425" />
+  <path
+     d="m 10.258402,14.841908 -0.2584,-0.1559 -0.2583999,0.1559 -5.16547,3.1177 1.3708,-5.876 0.06856,-0.2939 -0.2281,-0.1976 -4.56431,-3.9540698 6.00919,-0.50982 0.30043,-0.02549 0.11766,-0.27761 2.3496399,-5.54381 2.3496,5.54381 0.1177,0.27761 0.3004,0.02549 6.0092,0.50982 -4.5643,3.9540698 -0.2281,0.1976 0.0686,0.2939 1.3708,5.876 z"
+     id="path1423"
+     inkscape:connector-curvature="0"
+     style="stroke:#b7b7b7" />
+</svg>

docs/assets/icon-star-full.svg

@@ -1,3 +1,56 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5fdcc85f3d8c9d0c2fb76b98ddd9f118c0e4a12c859a4281a04a0870d5de12c8
-size 1950
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="19.999983"
+   height="19"
+   viewBox="0 0 19.999983 19"
+   version="1.1"
+   id="svg817"
+   sodipodi:docname="icon-star-full.svg"
+   style="fill:none"
+   inkscape:version="0.92.5 (2060ec1f9f, 2020-04-08)">
+  <metadata
+     id="metadata823">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <defs
+     id="defs821" />
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1198"
+     inkscape:window-height="1898"
+     id="namedview819"
+     showgrid="false"
+     inkscape:zoom="20.005229"
+     inkscape:cx="8.4325646"
+     inkscape:cy="3.9969093"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg817" />
+  <path
+     d="m 10.258402,14.841908 -0.2584,-0.1559 -0.2583999,0.1559 -5.16547,3.1177 1.3708,-5.876 0.06856,-0.2939 -0.2281,-0.1976 -4.56431,-3.9540698 6.00919,-0.50982 0.30043,-0.02549 0.11766,-0.27761 2.3496399,-5.54381 2.3496,5.54381 0.1177,0.27761 0.3004,0.02549 6.0092,0.50982 -4.5643,3.9540698 -0.2281,0.1976 0.0686,0.2939 1.3708,5.876 z"
+     id="path815"
+     inkscape:connector-curvature="0"
+     style="fill:#f5c543;stroke:#f0a43b" />
+</svg>

docs/assets/icon-star-half.svg

@@ -1,3 +1,66 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cdf5e7ae57626f4cda35973873a4696252babbbe473913fe0aafdd8ba39d7fc3
-size 2508
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="19.999983"
+   height="19.000008"
+   viewBox="0 0 19.999983 19.000008"
+   version="1.1"
+   id="svg831"
+   sodipodi:docname="icon-star-half.svg"
+   style="fill:none"
+   inkscape:version="0.92.5 (2060ec1f9f, 2020-04-08)">
+  <metadata
+     id="metadata837">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <defs
+     id="defs835" />
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1198"
+     inkscape:window-height="948"
+     id="namedview833"
+     showgrid="false"
+     inkscape:zoom="9.8333333"
+     inkscape:cx="-8.3559301"
+     inkscape:cy="9"
+     inkscape:window-x="0"
+     inkscape:window-y="950"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg831" />
+  <path
+     d="M 10.000002,8.1671631e-6 V 15.270008 l -6.1799999,3.73 1.64,-7.03 L 2.0565151e-6,7.2400082 7.1900021,6.6300082 Z"
+     id="path825"
+     inkscape:connector-curvature="0"
+     style="fill:#f5c543" />
+  <path
+     d="m 10.258402,14.841908 -0.2584,-0.1559 -0.2583999,0.1559 -5.16547,3.1177 1.3708,-5.876 0.06856,-0.2939 -0.2281,-0.1976 -4.56431,-3.9540698 6.00919,-0.50982 0.30043,-0.02549 0.11766,-0.27761 2.3496399,-5.54381 2.3496,5.54381 0.1177,0.27761 0.3004,0.02549 6.0092,0.50982 -4.5643,3.9540698 -0.2281,0.1976 0.0686,0.2939 1.3708,5.876 z"
+     id="path827"
+     inkscape:connector-curvature="0"
+     style="stroke:#b7b7b7" />
+  <path
+     d="m 5.3322621,16.919208 3.66774,-2.2137 0.9999999,-0.6055 v 1.17 l -6.1799999,3.73 1.64,-7.03 L 2.0565151e-6,7.2400082 7.1900021,6.6300082 10.000002,8.1671631e-6 V 2.6000082 l -0.9999999,2.32203 -1.1246,2.65341 -5.42923,0.46066 4.12481,3.5732998 z"
+     id="path829"
+     inkscape:connector-curvature="0"
+     style="fill:#f0a43b" />
+</svg>

docs/assets/icon-youtube.svg

@@ -1,3 +1,12 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6cb64f9da10b818c56763a7c48347f6043da20a2a77fb14f6d60d9457c575b6b
-size 1278
+<?xml version="1.0" encoding="utf-8"?>
+<svg viewBox="500.972 178.81 20 14.375" width="20" height="14.375" xmlns="http://www.w3.org/2000/svg" xmlns:bx="https://boxy-svg.com">
+  <defs>
+    <clipPath id="clip0_1674_1768">
+      <rect width="20" height="20" fill="white"/>
+    </clipPath>
+  </defs>
+  <g clip-path="url(#clip0_1674_1768)" transform="matrix(1, 0, 0, 1, 500.971771, 175.81041)">
+    <path fill-rule="evenodd" clip-rule="evenodd" d="M19.582 5.2447C19.352 4.3611 18.6743 3.66531 17.8138 3.42916C16.2542 3 10 3 10 3C10 3 3.7458 3 2.18614 3.42916C1.32557 3.66531 0.647841 4.3611 0.417841 5.2447C0 6.84612 0 10.1875 0 10.1875C0 10.1875 0 13.5288 0.417841 15.1303C0.647841 16.0139 1.32557 16.7097 2.18614 16.946C3.7458 17.375 10 17.375 10 17.375C10 17.375 16.2542 17.375 17.8138 16.946C18.6743 16.7097 19.352 16.0139 19.582 15.1303C20 13.5288 20 10.1875 20 10.1875C20 10.1875 20 6.84612 19.582 5.2447ZM8.12509 13.6255V7.37549L13.1251 10.5006L8.12509 13.6255Z" fill="#FF0000"/>
+  </g>
+  <path d="M 425.503 122.113 L 428.63 127.117 L 422.376 127.117 L 425.503 122.113 Z" style="stroke: rgb(0, 0, 0); stroke-width: 0px; fill: rgb(255, 255, 255);" transform="matrix(0, 1, -1, 0, 636.212463, -239.192383)" bx:shape="triangle 422.376 122.113 6.254 5.004 0.5 0 1@278e08c4"/>
+</svg>

msgq_repo/.dockerignore

@@ -0,0 +1 @@
+.sconsign.dblite

msgq_repo/.gitignore

@@ -0,0 +1,18 @@
+/gen/
+*.tmp
+*.pyc
+__pycache__
+.*.swp
+.*.swo
+*.os
+*.so
+*.o
+*.a
+
+test_runner
+
+libmessaging.*
+libmessaging_shared.*
+services.h
+.sconsign.dblite
+.mypy_cache/

msgq_repo/README.md

@@ -0,0 +1,54 @@
+# MSGQ: A lock free single producer multi consumer message queue
+
+## What is this library?
+MSGQ is a generic high performance IPC pub sub system with a single publisher and multiple subscribers. MSGQ is designed to be a high performance replacement for ZMQ-like SUB/PUB patterns. It uses a ring buffer in shared memory to efficiently read and write data. Each read requires a copy. Writing can be done without a copy, as long as the size of the data is known in advance. While MSGQ is the core of this library, this library also allows replacing the MSGQ backend with ZMQ or a spoofed implementation that can be used for deterministic testing. This library also contains visionipc, an IPC system specifically for large contiguous buffers (like images/video).
+
+## Storage
+The storage for the queue consists of an area of metadata, and the actual buffer. The metadata contains:
+
+1. A counter to the number of readers that are active
+2. A pointer to the head of the queue for writing. From now on referred to as *write pointer*
+3. A cycle counter for the writer. This counter is incremented when the writer wraps around
+4. N pointers, pointing to the current read position for all the readers. From now on referred to as *read pointer*
+5. N counters,  counting the number of cycles for all the readers
+6. N booleans, indicating validity for all the readers. From now on referred to as *validity flag*
+
+The counter and the pointer are both 32 bit values, packed into 64 bit so they can be read and written atomically.
+
+The data buffer is a ring buffer. All messages are prefixed by an 8 byte size field, followed by the data. A size of -1 indicates a wrap-around, and means the next message is stored at the beginning of the buffer.
+
+
+## Writing
+Writing involves the following steps:
+
+1. Check if the area that is to be written overlaps with any of the read pointers, mark those readers as invalid by clearing the validity flag.
+2. Write the message
+3. Increase the write pointer by the size of the message
+
+In case there is not enough space at the end of the buffer, a special empty message with a prefix of -1 is written. The cycle counter is incremented by one. In this case step 1 will check there are no read pointers pointing to the remainder of the buffer. Then another write cycle will start with the actual message.
+
+There always needs to be 8 bytes of empty space at the end of the buffer. By doing this there is always space to write the -1.
+
+## Reset reader
+When the reader is lagging too much behind the read pointer becomes invalid and no longer points to the beginning of a valid message. To reset a reader to the current write pointer, the following steps are performed:
+
+1. Set valid flag
+2. Set read cycle counter to that of the writer
+3. Set read pointer to write pointer
+
+## Reading
+Reading involves the following steps:
+
+1. Read the size field at the current read pointer
+2. Read the validity flag
+3. Copy the data out of the buffer
+4. Increase the read pointer by the size of the message
+5. Check the validity flag again
+
+Before starting the copy, the valid flag is checked. This is to prevent a race condition where the size prefix was invalid, and the read could read outside of the buffer. Make sure that step 1 and 2 are not reordered by your compiler or CPU.
+
+If a writer overwrites the data while it's being copied out, the data will be invalid. Therefore the validity flag is also checked after reading it. The order of step 4 and 5 does not matter.
+
+If at steps 2 or 5 the validity flag is not set, the reader is reset. Any data that was already read is discarded. After the reader is reset, the reading starts from the beginning.
+
+If a message with size -1 is encountered, step 3 and 4 are replaced by increasing the cycle counter and setting the read pointer to the beginning of the buffer. After that another read is performed.

msgq_repo/SConscript

@@ -0,0 +1,48 @@
+Import('env', 'envCython', 'arch', 'common')
+
+
+visionipc_dir = Dir('msgq/visionipc')
+gen_dir = Dir('gen')
+
+
+# Build msgq
+msgq_objects = env.SharedObject([
+  'msgq/ipc.cc',
+  'msgq/event.cc',
+  'msgq/impl_zmq.cc',
+  'msgq/impl_msgq.cc',
+  'msgq/impl_fake.cc',
+  'msgq/msgq.cc',
+])
+msgq = env.Library('msgq', msgq_objects)
+msgq_python = envCython.Program('msgq/ipc_pyx.so', 'msgq/ipc_pyx.pyx', LIBS=envCython["LIBS"]+[msgq, "zmq", common])
+
+# Build Vision IPC
+vipc_files = ['visionipc.cc', 'visionipc_server.cc', 'visionipc_client.cc', 'visionbuf.cc']
+vipc_sources = [f'{visionipc_dir.abspath}/{f}' for f in vipc_files]
+
+if arch == "larch64":
+  vipc_sources += [f'{visionipc_dir.abspath}/visionbuf_ion.cc']
+else:
+  vipc_sources += [f'{visionipc_dir.abspath}/visionbuf_cl.cc']
+
+vipc_objects = env.SharedObject(vipc_sources)
+visionipc = env.Library('visionipc', vipc_objects)
+
+
+vipc_frameworks = []
+vipc_libs = envCython["LIBS"] + [visionipc, msgq, common, "zmq"]
+if arch == "Darwin":
+  vipc_frameworks.append('OpenCL')
+else:
+  vipc_libs.append('OpenCL')
+envCython.Program(f'{visionipc_dir.abspath}/visionipc_pyx.so', f'{visionipc_dir.abspath}/visionipc_pyx.pyx',
+                  LIBS=vipc_libs, FRAMEWORKS=vipc_frameworks)
+
+if GetOption('extras'):
+  env.Program('msgq/test_runner', ['msgq/test_runner.cc', 'msgq/msgq_tests.cc'], LIBS=[msgq, common])
+  env.Program(f'{visionipc_dir.abspath}/test_runner',
+             [f'{visionipc_dir.abspath}/test_runner.cc', f'{visionipc_dir.abspath}/visionipc_tests.cc'],
+              LIBS=['pthread'] + vipc_libs, FRAMEWORKS=vipc_frameworks)
+
+Export('visionipc', 'msgq', 'msgq_python')

msgq_repo/SConstruct

@@ -0,0 +1,91 @@
+import os
+import platform
+import subprocess
+import sysconfig
+import numpy as np
+
+arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+if platform.system() == "Darwin":
+  arch = "Darwin"
+
+common = ''
+
+cpppath = [
+  f"#/",
+  '#msgq/',
+  '/usr/lib/include',
+  '/opt/homebrew/include',
+  sysconfig.get_paths()['include'],
+]
+
+libpath = [
+  '/opt/homebrew/lib',
+]
+
+AddOption('--minimal',
+          action='store_false',
+          dest='extras',
+          default=True,
+          help='the minimum build. no tests, tools, etc.')
+
+AddOption('--asan',
+          action='store_true',
+          help='turn on ASAN')
+
+AddOption('--ubsan',
+          action='store_true',
+          help='turn on UBSan')
+
+ccflags = []
+ldflags = []
+if GetOption('ubsan'):
+  flags = [
+    "-fsanitize=undefined",
+    "-fno-sanitize-recover=undefined",
+  ]
+  ccflags += flags
+  ldflags += flags
+elif GetOption('asan'):
+  ccflags += ["-fsanitize=address", "-fno-omit-frame-pointer"]
+  ldflags += ["-fsanitize=address"]
+
+env = Environment(
+  ENV=os.environ,
+  CC='clang',
+  CXX='clang++',
+  CCFLAGS=[
+    "-g",
+    "-fPIC",
+    "-O2",
+    "-Wunused",
+    "-Werror",
+    "-Wshadow",
+    "-Wno-vla-cxx-extension",
+    "-Wno-unknown-warning-option",
+  ] + ccflags,
+  LDFLAGS=ldflags,
+  LINKFLAGS=ldflags,
+
+  CFLAGS="-std=gnu11",
+  CXXFLAGS="-std=c++1z",
+  CPPPATH=cpppath,
+  LIBPATH=libpath,
+  CYTHONCFILESUFFIX=".cpp",
+  tools=["default", "cython"]
+)
+
+Export('env', 'arch', 'common')
+
+envCython = env.Clone(LIBS=[])
+envCython["CPPPATH"] += [np.get_include()]
+envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
+envCython["CCFLAGS"].remove('-Werror')
+if arch == "Darwin":
+  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"]
+else:
+  envCython["LINKFLAGS"] = ["-pthread", "-shared"]
+
+Export('envCython')
+
+
+SConscript(['SConscript'])

msgq_repo/codecov.yml

@@ -0,0 +1,8 @@
+comment: false
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+    patch: off
+

msgq_repo/msgq/.gitignore

@@ -0,0 +1 @@
+ipc_pyx.cpp

msgq_repo/msgq/__init__.py

@@ -0,0 +1,61 @@
+# must be built with scons
+from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
+                                set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
+from msgq.ipc_pyx import MultiplePublishersError, IpcError
+
+from typing import Optional, List
+
+assert MultiplePublishersError
+assert IpcError
+assert toggle_fake_events
+assert set_fake_prefix
+assert get_fake_prefix
+assert delete_fake_prefix
+assert wait_for_one_event
+
+NO_TRAVERSAL_LIMIT = 2**64-1
+
+context = Context()
+
+
+def fake_event_handle(endpoint: str, identifier: Optional[str] = None, override: bool = True, enable: bool = False) -> SocketEventHandle:
+  identifier = identifier or get_fake_prefix()
+  handle = SocketEventHandle(endpoint, identifier, override)
+  if override:
+    handle.enabled = enable
+
+  return handle
+
+def pub_sock(endpoint: str) -> PubSocket:
+  sock = PubSocket()
+  sock.connect(context, endpoint)
+  return sock
+
+
+def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1",
+             conflate: bool = False, timeout: Optional[int] = None) -> SubSocket:
+  sock = SubSocket()
+  sock.connect(context, endpoint, addr.encode('utf8'), conflate)
+
+  if timeout is not None:
+    sock.setTimeout(timeout)
+
+  if poller is not None:
+    poller.registerSocket(sock)
+  return sock
+
+def drain_sock_raw(sock: SubSocket, wait_for_one: bool = False) -> List[bytes]:
+  """Receive all message currently available on the queue"""
+  ret: List[bytes] = []
+  while 1:
+    if wait_for_one and len(ret) == 0:
+      dat = sock.receive()
+    else:
+      dat = sock.receive(non_blocking=True)
+
+    if dat is None:
+      break
+
+    ret.append(dat)
+
+  return ret

msgq_repo/msgq/event.cc

@@ -0,0 +1,236 @@
+#include <cassert>
+#include <cstring>
+#include <cstdlib>
+#include <iostream>
+#include <string>
+#include <exception>
+#include <filesystem>
+
+#include <unistd.h>
+#include <poll.h>
+#include <signal.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+
+#include "msgq/event.h"
+
+#ifndef __APPLE__
+#include <sys/eventfd.h>
+
+void event_state_shm_mmap(std::string endpoint, std::string identifier, char **shm_mem, std::string *shm_path) {
+  const char* op_prefix = std::getenv("OPENPILOT_PREFIX");
+
+  std::string full_path = "/dev/shm/";
+  if (op_prefix) {
+    full_path += std::string(op_prefix) + "/";
+  }
+  full_path += CEREAL_EVENTS_PREFIX + "/";
+  if (identifier.size() > 0) {
+    full_path += identifier + "/";
+  }
+  std::filesystem::create_directories(full_path);
+  full_path += endpoint;
+
+  int shm_fd = open(full_path.c_str(), O_RDWR | O_CREAT, 0664);
+  if (shm_fd < 0) {
+    throw std::runtime_error("Could not open shared memory file.");
+  }
+
+  int rc = ftruncate(shm_fd, sizeof(EventState));
+  if (rc < 0){
+    close(shm_fd);
+    throw std::runtime_error("Could not truncate shared memory file.");
+  }
+
+  char * mem = (char*)mmap(NULL, sizeof(EventState), PROT_READ | PROT_WRITE, MAP_SHARED, shm_fd, 0);
+  close(shm_fd);
+  if (mem == nullptr) {
+    throw std::runtime_error("Could not map shared memory file.");
+  }
+
+  if (shm_mem != nullptr)
+    *shm_mem = mem;
+  if (shm_path != nullptr)
+    *shm_path = full_path;
+}
+
+SocketEventHandle::SocketEventHandle(std::string endpoint, std::string identifier, bool override) {
+  char *mem;
+  event_state_shm_mmap(endpoint, identifier, &mem, &this->shm_path);
+
+  this->state = (EventState*)mem;
+  if (override) {
+    this->state->fds[0] = eventfd(0, EFD_NONBLOCK);
+    this->state->fds[1] = eventfd(0, EFD_NONBLOCK);
+  }
+}
+
+SocketEventHandle::~SocketEventHandle() {
+  close(this->state->fds[0]);
+  close(this->state->fds[1]);
+  munmap(this->state, sizeof(EventState));
+  unlink(this->shm_path.c_str());
+}
+
+bool SocketEventHandle::is_enabled() {
+  return this->state->enabled;
+}
+
+void SocketEventHandle::set_enabled(bool enabled) {
+  this->state->enabled = enabled;
+}
+
+Event SocketEventHandle::recv_called() {
+  return Event(this->state->fds[0]);
+}
+
+Event SocketEventHandle::recv_ready() {
+  return Event(this->state->fds[1]);
+}
+
+void SocketEventHandle::toggle_fake_events(bool enabled) {
+  if (enabled)
+    setenv("CEREAL_FAKE", "1", true);
+  else
+    unsetenv("CEREAL_FAKE");
+}
+
+void SocketEventHandle::set_fake_prefix(std::string prefix) {
+  if (prefix.size() == 0) {
+    unsetenv("CEREAL_FAKE_PREFIX");
+  } else {
+    setenv("CEREAL_FAKE_PREFIX", prefix.c_str(), true);
+  }
+}
+
+std::string SocketEventHandle::fake_prefix() {
+  const char* prefix = std::getenv("CEREAL_FAKE_PREFIX");
+  if (prefix == nullptr) {
+    return "";
+  } else {
+    return std::string(prefix);
+  }
+}
+
+Event::Event(int fd): event_fd(fd) {}
+
+void Event::set() const {
+  throw_if_invalid();
+
+  uint64_t val = 1;
+  size_t count = write(this->event_fd, &val, sizeof(uint64_t));
+  assert(count == sizeof(uint64_t));
+}
+
+int Event::clear() const {
+  throw_if_invalid();
+
+  uint64_t val = 0;
+  // read the eventfd to clear it
+  read(this->event_fd, &val, sizeof(uint64_t));
+
+  return val;
+}
+
+void Event::wait(int timeout_sec) const {
+  throw_if_invalid();
+
+  int event_count;
+  struct pollfd fds = { this->event_fd, POLLIN, 0 };
+  struct timespec timeout = { timeout_sec, 0 };;
+
+  sigset_t signals;
+  sigfillset(&signals);
+  sigdelset(&signals, SIGALRM);
+  sigdelset(&signals, SIGINT);
+  sigdelset(&signals, SIGTERM);
+  sigdelset(&signals, SIGQUIT);
+
+  event_count = ppoll(&fds, 1, timeout_sec < 0 ? nullptr : &timeout, &signals);
+
+  if (event_count == 0) {
+    throw std::runtime_error("Event timed out pid: " + std::to_string(getpid()));
+  } else if (event_count < 0) {
+    throw std::runtime_error("Event poll failed, errno: " + std::to_string(errno) + " pid: " + std::to_string(getpid()));
+  }
+}
+
+bool Event::peek() const {
+  throw_if_invalid();
+
+  int event_count;
+
+  struct pollfd fds = { this->event_fd, POLLIN, 0 };
+
+  // poll with timeout zero to return status immediately
+  event_count = poll(&fds, 1, 0);
+
+  return event_count != 0;
+}
+
+bool Event::is_valid() const {
+  return event_fd != -1;
+}
+
+int Event::fd() const {
+  return event_fd;
+}
+
+int Event::wait_for_one(const std::vector<Event>& events, int timeout_sec) {
+  struct pollfd fds[events.size()];
+  for (size_t i = 0; i < events.size(); i++) {
+    fds[i] = { events[i].fd(), POLLIN, 0 };
+  }
+
+  struct timespec timeout = { timeout_sec, 0 };
+
+  sigset_t signals;
+  sigfillset(&signals);
+  sigdelset(&signals, SIGALRM);
+  sigdelset(&signals, SIGINT);
+  sigdelset(&signals, SIGTERM);
+  sigdelset(&signals, SIGQUIT);
+
+  int event_count = ppoll(fds, events.size(), timeout_sec < 0 ? nullptr : &timeout, &signals);
+
+  if (event_count == 0) {
+    throw std::runtime_error("Event timed out pid: " + std::to_string(getpid()));
+  } else if (event_count < 0) {
+    throw std::runtime_error("Event poll failed, errno: " + std::to_string(errno) + " pid: " + std::to_string(getpid()));
+  }
+
+  for (size_t i = 0; i < events.size(); i++) {
+    if (fds[i].revents & POLLIN) {
+      return i;
+    }
+  }
+
+  throw std::runtime_error("Event poll failed, no events ready");
+}
+#else
+// Stub implementation for Darwin, which does not support eventfd
+void event_state_shm_mmap(std::string endpoint, std::string identifier, char **shm_mem, std::string *shm_path) {}
+
+SocketEventHandle::SocketEventHandle(std::string endpoint, std::string identifier, bool override) {
+  std::cerr << "SocketEventHandle not supported on macOS" << std::endl;
+  assert(false);
+}
+SocketEventHandle::~SocketEventHandle() {}
+bool SocketEventHandle::is_enabled() { return this->state->enabled; }
+void SocketEventHandle::set_enabled(bool enabled) {}
+Event SocketEventHandle::recv_called() { return Event(); }
+Event SocketEventHandle::recv_ready() { return Event(); }
+void SocketEventHandle::toggle_fake_events(bool enabled) {}
+void SocketEventHandle::set_fake_prefix(std::string prefix) {}
+std::string SocketEventHandle::fake_prefix() { return ""; }
+
+Event::Event(int fd): event_fd(fd) {}
+void Event::set() const {}
+int Event::clear() const { return 0; }
+void Event::wait(int timeout_sec) const {}
+bool Event::peek() const { return false; }
+bool Event::is_valid() const { return false; }
+int Event::fd() const { return this->event_fd; }
+int Event::wait_for_one(const std::vector<Event>& events, int timeout_sec) { return -1; }
+#endif

msgq_repo/msgq/event.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+#define CEREAL_EVENTS_PREFIX std::string("cereal_events")
+
+void event_state_shm_mmap(std::string endpoint, std::string identifier, char **shm_mem, std::string *shm_path);
+
+enum EventPurpose {
+  RECV_CALLED,
+  RECV_READY
+};
+
+struct EventState {
+  int fds[2];
+  bool enabled;
+};
+
+class Event {
+private:
+  int event_fd = -1;
+
+  inline void throw_if_invalid() const {
+    if (!this->is_valid()) {
+      throw std::runtime_error("Event does not have valid file descriptor.");
+    }
+  }
+public:
+  Event(int fd = -1);
+
+  void set() const;
+  int clear() const;
+  void wait(int timeout_sec = -1) const;
+  bool peek() const;
+  bool is_valid() const;
+  int fd() const;
+
+  static int wait_for_one(const std::vector<Event>& events, int timeout_sec = -1);
+};
+
+class SocketEventHandle {
+private:
+  std::string shm_path;
+  EventState* state;
+public:
+  SocketEventHandle(std::string endpoint, std::string identifier = "", bool override = true);
+  ~SocketEventHandle();
+
+  bool is_enabled();
+  void set_enabled(bool enabled);
+  Event recv_called();
+  Event recv_ready();
+
+  static void toggle_fake_events(bool enabled);
+  static void set_fake_prefix(std::string prefix);
+  static std::string fake_prefix();
+};

msgq_repo/msgq/impl_fake.cc

@@ -0,0 +1,9 @@
+#include "msgq/impl_fake.h"
+
+void FakePoller::registerSocket(SubSocket *socket) {
+  this->sockets.push_back(socket);
+}
+
+std::vector<SubSocket*> FakePoller::poll(int timeout) {
+  return this->sockets;
+}

msgq_repo/msgq/impl_fake.h

@@ -0,0 +1,67 @@
+#pragma once
+
+#include <cassert>
+#include <iostream>
+#include <string>
+#include <vector>
+#include <filesystem>
+
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "msgq/ipc.h"
+#include "msgq/event.h"
+
+template<typename TSubSocket>
+class FakeSubSocket: public TSubSocket {
+private:
+  Event *recv_called = nullptr;
+  Event *recv_ready = nullptr;
+  EventState *state = nullptr;
+
+public:
+  FakeSubSocket(): TSubSocket() {}
+  ~FakeSubSocket() {
+    delete recv_called;
+    delete recv_ready;
+    if (state != nullptr) {
+      munmap(state, sizeof(EventState));
+    }
+  }
+
+  int connect(Context *context, std::string endpoint, std::string address, bool conflate=false, bool check_endpoint=true) override {
+    const char* cereal_prefix = std::getenv("CEREAL_FAKE_PREFIX");
+
+    char* mem;
+    std::string identifier = cereal_prefix != nullptr ?  std::string(cereal_prefix) : "";
+    event_state_shm_mmap(endpoint, identifier, &mem, nullptr);
+
+    this->state = (EventState*)mem;
+    this->recv_called = new Event(state->fds[EventPurpose::RECV_CALLED]);
+    this->recv_ready = new Event(state->fds[EventPurpose::RECV_READY]);
+
+    return TSubSocket::connect(context, endpoint, address, conflate, check_endpoint);
+  }
+
+  Message *receive(bool non_blocking=false) override {
+    if (this->state->enabled) {
+      this->recv_called->set();
+      this->recv_ready->wait();
+      this->recv_ready->clear();
+    }
+
+    return TSubSocket::receive(non_blocking);
+  }
+};
+
+class FakePoller: public Poller {
+private:
+  std::vector<SubSocket*> sockets;
+
+public:
+  void registerSocket(SubSocket *socket) override;
+  std::vector<SubSocket*> poll(int timeout) override;
+  ~FakePoller() {}
+};

msgq_repo/msgq/impl_msgq.cc

@@ -0,0 +1,207 @@
+#include <cassert>
+#include <cstring>
+#include <iostream>
+#include <cstdlib>
+#include <csignal>
+#include <cerrno>
+
+#include "msgq/impl_msgq.h"
+
+
+volatile sig_atomic_t msgq_do_exit = 0;
+
+void sig_handler(int signal) {
+  assert(signal == SIGINT || signal == SIGTERM);
+  msgq_do_exit = 1;
+}
+
+
+MSGQContext::MSGQContext() {
+}
+
+MSGQContext::~MSGQContext() {
+}
+
+void MSGQMessage::init(size_t sz) {
+  size = sz;
+  data = new char[size];
+}
+
+void MSGQMessage::init(char * d, size_t sz) {
+  size = sz;
+  data = new char[size];
+  memcpy(data, d, size);
+}
+
+void MSGQMessage::takeOwnership(char * d, size_t sz) {
+  size = sz;
+  data = d;
+}
+
+void MSGQMessage::close() {
+  if (size > 0){
+    delete[] data;
+  }
+  size = 0;
+}
+
+MSGQMessage::~MSGQMessage() {
+  this->close();
+}
+
+int MSGQSubSocket::connect(Context *context, std::string endpoint, std::string address, bool conflate, bool check_endpoint){
+  assert(context);
+  assert(address == "127.0.0.1");
+
+  q = new msgq_queue_t;
+  int r = msgq_new_queue(q, endpoint.c_str(), DEFAULT_SEGMENT_SIZE);
+  if (r != 0){
+    return r;
+  }
+
+  msgq_init_subscriber(q);
+
+  if (conflate){
+    q->read_conflate = true;
+  }
+
+  timeout = -1;
+
+  return 0;
+}
+
+
+Message * MSGQSubSocket::receive(bool non_blocking){
+  msgq_do_exit = 0;
+
+  void (*prev_handler_sigint)(int);
+  void (*prev_handler_sigterm)(int);
+  if (!non_blocking){
+    prev_handler_sigint = std::signal(SIGINT, sig_handler);
+    prev_handler_sigterm = std::signal(SIGTERM, sig_handler);
+  }
+
+  msgq_msg_t msg;
+
+  MSGQMessage *r = NULL;
+
+  int rc = msgq_msg_recv(&msg, q);
+
+  // Hack to implement blocking read with a poller. Don't use this
+  while (!non_blocking && rc == 0 && msgq_do_exit == 0){
+    msgq_pollitem_t items[1];
+    items[0].q = q;
+
+    int t = (timeout != -1) ? timeout : 100;
+
+    int n = msgq_poll(items, 1, t);
+    rc = msgq_msg_recv(&msg, q);
+
+    // The poll indicated a message was ready, but the receive failed. Try again
+    if (n == 1 && rc == 0){
+      continue;
+    }
+
+    if (timeout != -1){
+      break;
+    }
+  }
+
+
+  if (!non_blocking){
+    std::signal(SIGINT, prev_handler_sigint);
+    std::signal(SIGTERM, prev_handler_sigterm);
+  }
+
+  errno = msgq_do_exit ? EINTR : 0;
+
+  if (rc > 0){
+    if (msgq_do_exit){
+      msgq_msg_close(&msg); // Free unused message on exit
+    } else {
+      r = new MSGQMessage;
+      r->takeOwnership(msg.data, msg.size);
+    }
+  }
+
+  return (Message*)r;
+}
+
+void MSGQSubSocket::setTimeout(int t){
+  timeout = t;
+}
+
+MSGQSubSocket::~MSGQSubSocket(){
+  if (q != NULL){
+    msgq_close_queue(q);
+    delete q;
+  }
+}
+
+int MSGQPubSocket::connect(Context *context, std::string endpoint, bool check_endpoint){
+  assert(context);
+
+  // TODO
+  //if (check_endpoint && !service_exists(std::string(endpoint))){
+  //  std::cout << "Warning, " << std::string(endpoint) << " is not in service list." << std::endl;
+  //}
+
+  q = new msgq_queue_t;
+  int r = msgq_new_queue(q, endpoint.c_str(), DEFAULT_SEGMENT_SIZE);
+  if (r != 0){
+    return r;
+  }
+
+  msgq_init_publisher(q);
+
+  return 0;
+}
+
+int MSGQPubSocket::sendMessage(Message *message){
+  msgq_msg_t msg;
+  msg.data = message->getData();
+  msg.size = message->getSize();
+
+  return msgq_msg_send(&msg, q);
+}
+
+int MSGQPubSocket::send(char *data, size_t size){
+  msgq_msg_t msg;
+  msg.data = data;
+  msg.size = size;
+
+  return msgq_msg_send(&msg, q);
+}
+
+bool MSGQPubSocket::all_readers_updated() {
+  return msgq_all_readers_updated(q);
+}
+
+MSGQPubSocket::~MSGQPubSocket(){
+  if (q != NULL){
+    msgq_close_queue(q);
+    delete q;
+  }
+}
+
+
+void MSGQPoller::registerSocket(SubSocket * socket){
+  assert(num_polls + 1 < MAX_POLLERS);
+  polls[num_polls].q = (msgq_queue_t*)socket->getRawSocket();
+
+  sockets.push_back(socket);
+  num_polls++;
+}
+
+std::vector<SubSocket*> MSGQPoller::poll(int timeout){
+  std::vector<SubSocket*> r;
+
+  msgq_poll(polls, num_polls, timeout);
+  for (size_t i = 0; i < num_polls; i++){
+    if (polls[i].revents){
+      r.push_back(sockets[i]);
+    }
+  }
+
+  return r;
+}

msgq_repo/msgq/impl_msgq.h

@@ -0,0 +1,67 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "msgq/ipc.h"
+#include "msgq/msgq.h"
+
+#define MAX_POLLERS 128
+
+class MSGQContext : public Context {
+private:
+  void * context = NULL;
+public:
+  MSGQContext();
+  void * getRawContext() {return context;}
+  ~MSGQContext();
+};
+
+class MSGQMessage : public Message {
+private:
+  char * data;
+  size_t size;
+public:
+  void init(size_t size);
+  void init(char *data, size_t size);
+  void takeOwnership(char *data, size_t size);
+  size_t getSize(){return size;}
+  char * getData(){return data;}
+  void close();
+  ~MSGQMessage();
+};
+
+class MSGQSubSocket : public SubSocket {
+private:
+  msgq_queue_t * q = NULL;
+  int timeout;
+public:
+  int connect(Context *context, std::string endpoint, std::string address, bool conflate=false, bool check_endpoint=true);
+  void setTimeout(int timeout);
+  void * getRawSocket() {return (void*)q;}
+  Message *receive(bool non_blocking=false);
+  ~MSGQSubSocket();
+};
+
+class MSGQPubSocket : public PubSocket {
+private:
+  msgq_queue_t * q = NULL;
+public:
+  int connect(Context *context, std::string endpoint, bool check_endpoint=true);
+  int sendMessage(Message *message);
+  int send(char *data, size_t size);
+  bool all_readers_updated();
+  ~MSGQPubSocket();
+};
+
+class MSGQPoller : public Poller {
+private:
+  std::vector<SubSocket*> sockets;
+  msgq_pollitem_t polls[MAX_POLLERS];
+  size_t num_polls = 0;
+
+public:
+  void registerSocket(SubSocket *socket);
+  std::vector<SubSocket*> poll(int timeout);
+  ~MSGQPoller(){}
+};

msgq_repo/msgq/impl_zmq.cc

@@ -0,0 +1,177 @@
+#include <cassert>
+#include <cstring>
+#include <iostream>
+#include <cstdlib>
+#include <cerrno>
+#include <unistd.h>
+
+#include "msgq/impl_zmq.h"
+
+static size_t fnv1a_hash(const std::string &str) {
+    const size_t fnv_prime = 0x100000001b3;
+    size_t hash_value = 0xcbf29ce484222325;
+    for (char c : str) {
+        hash_value ^= (unsigned char)c;
+        hash_value *= fnv_prime;
+    }
+    return hash_value;
+}
+
+//FIXME: This is a hack to get the port number from the socket name, might have collisions
+static int get_port(std::string endpoint) {
+    size_t hash_value = fnv1a_hash(endpoint);
+    int start_port = 8023;
+    int max_port = 65535;
+    int port = start_port + (hash_value % (max_port - start_port));
+    return port;
+}
+
+ZMQContext::ZMQContext() {
+  context = zmq_ctx_new();
+}
+
+ZMQContext::~ZMQContext() {
+  zmq_ctx_term(context);
+}
+
+void ZMQMessage::init(size_t sz) {
+  size = sz;
+  data = new char[size];
+}
+
+void ZMQMessage::init(char * d, size_t sz) {
+  size = sz;
+  data = new char[size];
+  memcpy(data, d, size);
+}
+
+void ZMQMessage::close() {
+  if (size > 0){
+    delete[] data;
+  }
+  size = 0;
+}
+
+ZMQMessage::~ZMQMessage() {
+  this->close();
+}
+
+
+int ZMQSubSocket::connect(Context *context, std::string endpoint, std::string address, bool conflate, bool check_endpoint){
+  sock = zmq_socket(context->getRawContext(), ZMQ_SUB);
+  if (sock == NULL){
+    return -1;
+  }
+
+  zmq_setsockopt(sock, ZMQ_SUBSCRIBE, "", 0);
+
+  if (conflate){
+    int arg = 1;
+    zmq_setsockopt(sock, ZMQ_CONFLATE, &arg, sizeof(int));
+  }
+
+  int reconnect_ivl = 500;
+  zmq_setsockopt(sock, ZMQ_RECONNECT_IVL_MAX, &reconnect_ivl, sizeof(reconnect_ivl));
+
+
+  full_endpoint = "tcp://" + address + ":";
+  if (check_endpoint){
+    full_endpoint += std::to_string(get_port(endpoint));
+  } else {
+    full_endpoint += endpoint;
+  }
+
+  return zmq_connect(sock, full_endpoint.c_str());
+}
+
+
+Message * ZMQSubSocket::receive(bool non_blocking){
+  zmq_msg_t msg;
+  assert(zmq_msg_init(&msg) == 0);
+
+  int flags = non_blocking ? ZMQ_DONTWAIT : 0;
+  int rc = zmq_msg_recv(&msg, sock, flags);
+  Message *r = NULL;
+
+  if (rc >= 0){
+    // Make a copy to ensure the data is aligned
+    r = new ZMQMessage;
+    r->init((char*)zmq_msg_data(&msg), zmq_msg_size(&msg));
+  }
+
+  zmq_msg_close(&msg);
+  return r;
+}
+
+void ZMQSubSocket::setTimeout(int timeout){
+  zmq_setsockopt(sock, ZMQ_RCVTIMEO, &timeout, sizeof(int));
+}
+
+ZMQSubSocket::~ZMQSubSocket(){
+  zmq_close(sock);
+}
+
+int ZMQPubSocket::connect(Context *context, std::string endpoint, bool check_endpoint){
+  sock = zmq_socket(context->getRawContext(), ZMQ_PUB);
+  if (sock == NULL){
+    return -1;
+  }
+
+  full_endpoint = "tcp://*:";
+  if (check_endpoint){
+    full_endpoint += std::to_string(get_port(endpoint));
+  } else {
+    full_endpoint += endpoint;
+  }
+
+  // ZMQ pub sockets cannot be shared between processes, so we need to ensure pid stays the same
+  pid = getpid();
+
+  return zmq_bind(sock, full_endpoint.c_str());
+}
+
+int ZMQPubSocket::sendMessage(Message *message) {
+  assert(pid == getpid());
+  return zmq_send(sock, message->getData(), message->getSize(), ZMQ_DONTWAIT);
+}
+
+int ZMQPubSocket::send(char *data, size_t size) {
+  assert(pid == getpid());
+  return zmq_send(sock, data, size, ZMQ_DONTWAIT);
+}
+
+bool ZMQPubSocket::all_readers_updated() {
+  assert(false); // TODO not implemented
+  return false;
+}
+
+ZMQPubSocket::~ZMQPubSocket(){
+  zmq_close(sock);
+}
+
+
+void ZMQPoller::registerSocket(SubSocket * socket){
+  assert(num_polls + 1 < MAX_POLLERS);
+  polls[num_polls].socket = socket->getRawSocket();
+  polls[num_polls].events = ZMQ_POLLIN;
+
+  sockets.push_back(socket);
+  num_polls++;
+}
+
+std::vector<SubSocket*> ZMQPoller::poll(int timeout){
+  std::vector<SubSocket*> r;
+
+  int rc = zmq_poll(polls, num_polls, timeout);
+  if (rc < 0){
+    return r;
+  }
+
+  for (size_t i = 0; i < num_polls; i++){
+    if (polls[i].revents){
+      r.push_back(sockets[i]);
+    }
+  }
+
+  return r;
+}

msgq_repo/msgq/impl_zmq.h

@@ -0,0 +1,68 @@
+#pragma once
+
+#include <zmq.h>
+#include <string>
+#include <vector>
+
+#include "msgq/ipc.h"
+
+#define MAX_POLLERS 128
+
+class ZMQContext : public Context {
+private:
+  void * context = NULL;
+public:
+  ZMQContext();
+  void * getRawContext() {return context;}
+  ~ZMQContext();
+};
+
+class ZMQMessage : public Message {
+private:
+  char * data;
+  size_t size;
+public:
+  void init(size_t size);
+  void init(char *data, size_t size);
+  size_t getSize(){return size;}
+  char * getData(){return data;}
+  void close();
+  ~ZMQMessage();
+};
+
+class ZMQSubSocket : public SubSocket {
+private:
+  void * sock;
+  std::string full_endpoint;
+public:
+  int connect(Context *context, std::string endpoint, std::string address, bool conflate=false, bool check_endpoint=true);
+  void setTimeout(int timeout);
+  void * getRawSocket() {return sock;}
+  Message *receive(bool non_blocking=false);
+  ~ZMQSubSocket();
+};
+
+class ZMQPubSocket : public PubSocket {
+private:
+  void * sock;
+  std::string full_endpoint;
+  int pid = -1;
+public:
+  int connect(Context *context, std::string endpoint, bool check_endpoint=true);
+  int sendMessage(Message *message);
+  int send(char *data, size_t size);
+  bool all_readers_updated();
+  ~ZMQPubSocket();
+};
+
+class ZMQPoller : public Poller {
+private:
+  std::vector<SubSocket*> sockets;
+  zmq_pollitem_t polls[MAX_POLLERS];
+  size_t num_polls = 0;
+
+public:
+  void registerSocket(SubSocket *socket);
+  std::vector<SubSocket*> poll(int timeout);
+  ~ZMQPoller(){}
+};

msgq_repo/msgq/ipc.cc

@@ -0,0 +1,120 @@
+#include <cassert>
+#include <iostream>
+
+#include "msgq/ipc.h"
+#include "msgq/impl_zmq.h"
+#include "msgq/impl_msgq.h"
+#include "msgq/impl_fake.h"
+
+#ifdef __APPLE__
+const bool MUST_USE_ZMQ = true;
+#else
+const bool MUST_USE_ZMQ = false;
+#endif
+
+bool messaging_use_zmq(){
+  if (std::getenv("ZMQ") || MUST_USE_ZMQ) {
+    if (std::getenv("OPENPILOT_PREFIX")) {
+      std::cerr << "OPENPILOT_PREFIX not supported with ZMQ backend\n";
+      assert(false);
+    }
+    return true;
+  }
+  return false;
+}
+
+bool messaging_use_fake(){
+  char* fake_enabled = std::getenv("CEREAL_FAKE");
+  return fake_enabled != NULL;
+}
+
+Context * Context::create(){
+  Context * c;
+  if (messaging_use_zmq()){
+    c = new ZMQContext();
+  } else {
+    c = new MSGQContext();
+  }
+  return c;
+}
+
+SubSocket * SubSocket::create(){
+  SubSocket * s;
+  if (messaging_use_fake()) {
+    if (messaging_use_zmq()) {
+      s = new FakeSubSocket<ZMQSubSocket>();
+    } else {
+      s = new FakeSubSocket<MSGQSubSocket>();
+    }
+  } else {
+    if (messaging_use_zmq()){
+      s = new ZMQSubSocket();
+    } else {
+      s = new MSGQSubSocket();
+    }
+  }
+
+  return s;
+}
+
+SubSocket * SubSocket::create(Context * context, std::string endpoint, std::string address, bool conflate, bool check_endpoint){
+  SubSocket *s = SubSocket::create();
+  int r = s->connect(context, endpoint, address, conflate, check_endpoint);
+
+  if (r == 0) {
+    return s;
+  } else {
+    std::cerr << "Error, failed to connect SubSocket to " << endpoint << ": " << strerror(errno) << std::endl;
+
+    delete s;
+    return nullptr;
+  }
+}
+
+PubSocket * PubSocket::create(){
+  PubSocket * s;
+  if (messaging_use_zmq()){
+    s = new ZMQPubSocket();
+  } else {
+    s = new MSGQPubSocket();
+  }
+
+  return s;
+}
+
+PubSocket * PubSocket::create(Context * context, std::string endpoint, bool check_endpoint){
+  PubSocket *s = PubSocket::create();
+  int r = s->connect(context, endpoint, check_endpoint);
+
+  if (r == 0) {
+    return s;
+  } else {
+    std::cerr << "Error, failed to bind PubSocket to " << endpoint << ": " << strerror(errno) << std::endl;
+
+    delete s;
+    return nullptr;
+  }
+}
+
+Poller * Poller::create(){
+  Poller * p;
+  if (messaging_use_fake()) {
+    p = new FakePoller();
+  } else {
+    if (messaging_use_zmq()){
+      p = new ZMQPoller();
+    } else {
+      p = new MSGQPoller();
+    }
+  }
+  return p;
+}
+
+Poller * Poller::create(std::vector<SubSocket*> sockets){
+  Poller * p = Poller::create();
+
+  for (auto s : sockets){
+    p->registerSocket(s);
+  }
+  return p;
+}

msgq_repo/msgq/ipc.h

@@ -0,0 +1,68 @@
+#pragma once
+
+#include <cstddef>
+#include <map>
+#include <string>
+#include <vector>
+#include <utility>
+#include <time.h>
+
+
+
+#ifdef __APPLE__
+#define CLOCK_BOOTTIME CLOCK_MONOTONIC
+#endif
+
+#define MSG_MULTIPLE_PUBLISHERS 100
+
+bool messaging_use_zmq();
+
+class Context {
+public:
+  virtual void * getRawContext() = 0;
+  static Context * create();
+  virtual ~Context(){}
+};
+
+class Message {
+public:
+  virtual void init(size_t size) = 0;
+  virtual void init(char * data, size_t size) = 0;
+  virtual void close() = 0;
+  virtual size_t getSize() = 0;
+  virtual char * getData() = 0;
+  virtual ~Message(){}
+};
+
+
+class SubSocket {
+public:
+  virtual int connect(Context *context, std::string endpoint, std::string address, bool conflate=false, bool check_endpoint=true) = 0;
+  virtual void setTimeout(int timeout) = 0;
+  virtual Message *receive(bool non_blocking=false) = 0;
+  virtual void * getRawSocket() = 0;
+  static SubSocket * create();
+  static SubSocket * create(Context * context, std::string endpoint, std::string address="127.0.0.1", bool conflate=false, bool check_endpoint=true);
+  virtual ~SubSocket(){}
+};
+
+class PubSocket {
+public:
+  virtual int connect(Context *context, std::string endpoint, bool check_endpoint=true) = 0;
+  virtual int sendMessage(Message *message) = 0;
+  virtual int send(char *data, size_t size) = 0;
+  virtual bool all_readers_updated() = 0;
+  static PubSocket * create();
+  static PubSocket * create(Context * context, std::string endpoint, bool check_endpoint=true);
+  static PubSocket * create(Context * context, std::string endpoint, int port, bool check_endpoint=true);
+  virtual ~PubSocket(){}
+};
+
+class Poller {
+public:
+  virtual void registerSocket(SubSocket *socket) = 0;
+  virtual std::vector<SubSocket*> poll(int timeout) = 0;
+  static Poller * create();
+  static Poller * create(std::vector<SubSocket*> sockets);
+  virtual ~Poller(){}
+};

msgq_repo/msgq/ipc.pxd

@@ -0,0 +1,68 @@
+# distutils: language = c++
+#cython: language_level=3
+
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+from libcpp cimport bool
+
+
+cdef extern from "msgq/impl_fake.h":
+  cdef cppclass Event:
+    @staticmethod
+    int wait_for_one(vector[Event], int) except +
+
+    Event()
+    Event(int)
+    void set()
+    int clear()
+    void wait(int) except +
+    bool peek()
+    int fd()
+
+  cdef cppclass SocketEventHandle:
+    @staticmethod
+    void toggle_fake_events(bool)
+    @staticmethod
+    void set_fake_prefix(string)
+    @staticmethod
+    string fake_prefix()
+
+    SocketEventHandle(string, string, bool)
+    bool is_enabled()
+    void set_enabled(bool)
+    Event recv_called()
+    Event recv_ready()
+
+
+cdef extern from "msgq/ipc.h":
+  cdef cppclass Context:
+    @staticmethod
+    Context * create()
+
+  cdef cppclass Message:
+    void init(size_t)
+    void init(char *, size_t)
+    void close()
+    size_t getSize()
+    char *getData()
+
+  cdef cppclass SubSocket:
+    @staticmethod
+    SubSocket * create()
+    int connect(Context *, string, string, bool)
+    Message * receive(bool)
+    void setTimeout(int)
+
+  cdef cppclass PubSocket:
+    @staticmethod
+    PubSocket * create()
+    int connect(Context *, string)
+    int sendMessage(Message *)
+    int send(char *, size_t)
+    bool all_readers_updated()
+
+  cdef cppclass Poller:
+    @staticmethod
+    Poller * create()
+    void registerSocket(SubSocket *)
+    vector[SubSocket*] poll(int) nogil

msgq_repo/msgq/ipc_pyx.pyx

@@ -0,0 +1,247 @@
+# distutils: language = c++
+# cython: c_string_encoding=ascii, language_level=3
+
+import sys
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+from libcpp cimport bool
+from libc cimport errno
+from libc.string cimport strerror
+from cython.operator import dereference
+
+
+from .ipc cimport Context as cppContext
+from .ipc cimport SubSocket as cppSubSocket
+from .ipc cimport PubSocket as cppPubSocket
+from .ipc cimport Poller as cppPoller
+from .ipc cimport Message as cppMessage
+from .ipc cimport Event as cppEvent, SocketEventHandle as cppSocketEventHandle
+
+
+class IpcError(Exception):
+  def __init__(self, endpoint=None):
+    suffix = f"with {endpoint.decode('utf-8')}" if endpoint else ""
+    message = f"Messaging failure {suffix}: {strerror(errno.errno).decode('utf-8')}"
+    super().__init__(message)
+
+
+class MultiplePublishersError(IpcError):
+  pass
+
+
+def toggle_fake_events(bool enabled):
+  cppSocketEventHandle.toggle_fake_events(enabled)
+
+
+def set_fake_prefix(string prefix):
+  cppSocketEventHandle.set_fake_prefix(prefix)
+
+
+def get_fake_prefix():
+  return cppSocketEventHandle.fake_prefix()
+
+
+def delete_fake_prefix():
+  cppSocketEventHandle.set_fake_prefix(b"")
+
+
+def wait_for_one_event(list events, int timeout=-1):
+  cdef vector[cppEvent] items
+  for event in events:
+    items.push_back(dereference(<cppEvent*><size_t>event.ptr))
+  return cppEvent.wait_for_one(items, timeout)
+
+
+cdef class Event:
+  cdef cppEvent event;
+
+  def __cinit__(self):
+    pass
+
+  cdef setEvent(self, cppEvent event):
+    self.event = event
+
+  def set(self):
+    self.event.set()
+
+  def clear(self):
+    return self.event.clear()
+
+  def wait(self, int timeout=-1):
+    self.event.wait(timeout)
+
+  def peek(self):
+    return self.event.peek()
+
+  @property
+  def fd(self):
+    return self.event.fd()
+
+  @property
+  def ptr(self):
+    return <size_t><void*>&self.event
+
+
+cdef class SocketEventHandle:
+  cdef cppSocketEventHandle * handle;
+
+  def __cinit__(self, string endpoint, string identifier, bool override):
+    self.handle = new cppSocketEventHandle(endpoint, identifier, override)
+
+  def __dealloc__(self):
+    del self.handle
+
+  @property
+  def enabled(self):
+    return self.handle.is_enabled()
+
+  @enabled.setter
+  def enabled(self, bool value):
+    self.handle.set_enabled(value)
+
+  @property
+  def recv_called_event(self):
+    e = Event()
+    e.setEvent(self.handle.recv_called())
+
+    return e
+
+  @property
+  def recv_ready_event(self):
+    e = Event()
+    e.setEvent(self.handle.recv_ready())
+
+    return e
+
+
+cdef class Context:
+  cdef cppContext * context
+
+  def __cinit__(self):
+    self.context = cppContext.create()
+
+  def term(self):
+    del self.context
+    self.context = NULL
+
+  def __dealloc__(self):
+    pass
+    # Deleting the context will hang if sockets are still active
+    # TODO: Figure out a way to make sure the context is closed last
+    # del self.context
+
+
+cdef class Poller:
+  cdef cppPoller * poller
+  cdef list sub_sockets
+
+  def __cinit__(self):
+    self.sub_sockets = []
+    self.poller = cppPoller.create()
+
+  def __dealloc__(self):
+    del self.poller
+
+  def registerSocket(self, SubSocket socket):
+    self.sub_sockets.append(socket)
+    self.poller.registerSocket(socket.socket)
+
+  def poll(self, timeout):
+    sockets = []
+    cdef int t = timeout
+
+    with nogil:
+      result = self.poller.poll(t)
+
+    for s in result:
+      socket = SubSocket()
+      socket.setPtr(s)
+      sockets.append(socket)
+
+    return sockets
+
+
+cdef class SubSocket:
+  cdef cppSubSocket * socket
+  cdef bool is_owner
+
+  def __cinit__(self):
+    self.socket = cppSubSocket.create()
+    self.is_owner = True
+
+    if self.socket == NULL:
+      raise IpcError
+
+  def __dealloc__(self):
+    if self.is_owner:
+      del self.socket
+
+  cdef setPtr(self, cppSubSocket * ptr):
+    if self.is_owner:
+      del self.socket
+
+    self.is_owner = False
+    self.socket = ptr
+
+  def connect(self, Context context, string endpoint, string address=b"127.0.0.1", bool conflate=False):
+    r = self.socket.connect(context.context, endpoint, address, conflate)
+
+    if r != 0:
+      if errno.errno == errno.EADDRINUSE:
+        raise MultiplePublishersError(endpoint)
+      else:
+        raise IpcError(endpoint)
+
+  def setTimeout(self, int timeout):
+    self.socket.setTimeout(timeout)
+
+  def receive(self, bool non_blocking=False):
+    msg = self.socket.receive(non_blocking)
+
+    if msg == NULL:
+      # If a blocking read returns no message check errno if SIGINT was caught in the C++ code
+      if errno.errno == errno.EINTR:
+        print("SIGINT received, exiting")
+        sys.exit(1)
+
+      return None
+    else:
+      sz = msg.getSize()
+      m = msg.getData()[:sz]
+      del msg
+
+      return m
+
+
+cdef class PubSocket:
+  cdef cppPubSocket * socket
+
+  def __cinit__(self):
+    self.socket = cppPubSocket.create()
+    if self.socket == NULL:
+      raise IpcError
+
+  def __dealloc__(self):
+    del self.socket
+
+  def connect(self, Context context, string endpoint):
+    r = self.socket.connect(context.context, endpoint)
+
+    if r != 0:
+      if errno.errno == errno.EADDRINUSE:
+        raise MultiplePublishersError(endpoint)
+      else:
+        raise IpcError(endpoint)
+
+  def send(self, bytes data):
+    length = len(data)
+    r = self.socket.send(<char*>data, length)
+
+    if r != length:
+      if errno.errno == errno.EADDRINUSE:
+        raise MultiplePublishersError
+      else:
+        raise IpcError
+
+  def all_readers_updated(self):
+    return self.socket.all_readers_updated()