Vendor dependencies and resources for standalone builds

Keep official openpilot ancestry and dependency licenses. Store resources directly in Git, remove LFS setup requirements, and leave the downloadable big model outside the source tree.
This commit is contained in:
firestar5683
2026-10-02 22:49:52 -05:00
parent ec95db3f1f
commit 22a8575199
2635 changed files with 817345 additions and 414 deletions
+18 -10
View File
@@ -2,14 +2,22 @@
# to move existing files into LFS:
# git add --renormalize .
*.onnx filter=lfs diff=lfs merge=lfs -text
openpilot/selfdrive/modeld/models/big_*_tinygrad.pkl filter=lfs diff=lfs merge=lfs -text
*.svg filter=lfs diff=lfs merge=lfs -text
*.png filter=lfs diff=lfs merge=lfs -text
*.gif filter=lfs diff=lfs merge=lfs -text
*.ttf filter=lfs diff=lfs merge=lfs -text
*.otf filter=lfs diff=lfs merge=lfs -text
*.wav filter=lfs diff=lfs merge=lfs -text
*.onnx -text
openpilot/selfdrive/modeld/models/big_*_tinygrad.pkl -text
*.svg -text
*.png -text
*.gif -text
*.ttf -text
*.otf -text
*.wav -text
openpilot/selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
openpilot/common/hardware/comma/updater filter=lfs diff=lfs merge=lfs -text
openpilot/selfdrive/car/tests/test_models_segs.txt -text
openpilot/common/hardware/comma/updater -text
# Dependency snapshots keep their own storage attributes.
msgq_repo/** !filter !diff !merge -text
opendbc_repo/** !filter !diff !merge -text
panda/** !filter !diff !merge -text
rednose_repo/** !filter !diff !merge -text
teleoprtc_repo/** !filter !diff !merge -text
tinygrad_repo/** !filter !diff !merge -text
-2
View File
@@ -12,8 +12,6 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
with:
submodules: false
# Label PRs
- uses: actions/labeler@v6
-5
View File
@@ -15,11 +15,6 @@ concurrency:
group: docs-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
cancel-in-progress: true
env:
GIT_CONFIG_COUNT: 1
GIT_CONFIG_KEY_0: lfs.fetchexclude
GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl
jobs:
docs:
name: build docs
+1 -1
View File
@@ -82,7 +82,7 @@ jobs:
git config --global user.name "github-actions[bot]"
git config --global user.email "github-actions[bot]@users.noreply.github.com"
git checkout -b tmp-jenkins-${{ github.event.issue.number }}
GIT_LFS_SKIP_PUSH=1 git push -f origin tmp-jenkins-${{ github.event.issue.number }}
git push -f origin tmp-jenkins-${{ github.event.issue.number }}
- name: Delete trigger comment
if: steps.check_comment.outputs.result == 'true' && always()
-7
View File
@@ -4,11 +4,6 @@ on:
- cron: '0 9 * * *'
workflow_dispatch:
env:
GIT_CONFIG_COUNT: 1
GIT_CONFIG_KEY_0: lfs.fetchexclude
GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl
jobs:
build_masterci:
name: build master-ci
@@ -29,12 +24,10 @@ jobs:
check-regexp: ^((?!.*(build prebuilt|create badges).*).)*$
- uses: actions/checkout@v4 # checkout version > v4 breaks nightly release script
with:
submodules: true
fetch-depth: 0
- run: ./tools/op.sh setup
- name: Push master-ci
run: BRANCH=__nightly tools/release/build_stripped.sh
- name: Push chestnut nightly
run: |
git lfs pull --exclude=''
INCLUDE_BIG_MODEL=1 BRANCH=__nightly-chestnut tools/release/build_stripped.sh
+2 -9
View File
@@ -9,9 +9,6 @@ on:
env:
PYTHONPATH: ${{ github.workspace }}
GIT_CONFIG_COUNT: 1
GIT_CONFIG_KEY_0: lfs.fetchexclude
GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl
jobs:
package_updates:
@@ -22,8 +19,6 @@ jobs:
(github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
steps:
- uses: actions/checkout@v7
with:
submodules: true
- run: ./tools/op.sh setup
- name: uv lock
run: uv lock --upgrade
@@ -47,10 +42,8 @@ jobs:
| head -10 \
| while IFS=$'\t' read size path; do echo "$size ${path##*/}"; done >> $GITHUB_OUTPUT
echo 'EOF' >> $GITHUB_OUTPUT
- name: bump submodules
run: |
git submodule update --remote
git add .
- name: Check tracked dependencies
run: python3 tools/vendor/check.py
- name: update car docs
run: |
python openpilot/selfdrive/car/docs.py
+23 -15
View File
@@ -4,6 +4,7 @@ on:
push:
branches:
- master
- Domathon
pull_request:
workflow_dispatch:
workflow_call:
@@ -20,11 +21,28 @@ concurrency:
env:
CI: 1
PYTHONPATH: ${{ github.workspace }}
GIT_CONFIG_COUNT: 1
GIT_CONFIG_KEY_0: lfs.fetchexclude
GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl
jobs:
source_integrity:
name: source integrity
runs-on: ubuntu-24.04
permissions:
contents: read
steps:
- uses: actions/checkout@v7
- name: Validate tracked dependencies
run: python3 tools/vendor/check.py --revision HEAD
- name: Test dependency and packaging tools
run: |
python3 -m unittest discover -s tools/vendor/tests -v
python3 -m unittest discover -s tools/release/tests -v
python3 -m unittest discover -s openpilot/system/updated/tests -v
- name: Check setup and release shell syntax
run: |
for script in tools/op.sh tools/release/*.sh openpilot/selfdrive/test/setup_device_ci.sh; do
bash -n "$script"
done
build_release:
name: build release
runs-on: ${{
@@ -38,14 +56,6 @@ jobs:
PYTHONPATH: /tmp/releasepilot
steps:
- uses: actions/checkout@v7
with:
submodules: true
- name: Getting LFS files
uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60
with:
timeout_minutes: 2
max_attempts: 3
command: git lfs pull
- name: Build devel
timeout-minutes: 1
run: TARGET_DIR=$STRIPPED_DIR tools/release/build_stripped.sh
@@ -58,10 +68,9 @@ jobs:
timeout-minutes: 1
working-directory: ${{ env.STRIPPED_DIR }}
run: tools/release/check-dirty.sh
- name: Check submodules
if: github.repository == 'commaai/openpilot'
- name: Check tracked dependencies
timeout-minutes: 3
run: tools/release/check-submodules.sh
run: python3 tools/vendor/check.py --revision HEAD
build_mac:
name: build macOS
@@ -125,7 +134,6 @@ jobs:
run: scons
- name: Run replay
timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
continue-on-error: ${{ github.ref == 'refs/heads/master' }}
run: openpilot/selfdrive/test/process_replay/test_processes.py -j$(nproc)
- name: Print diff
id: print-diff
-2
View File
@@ -36,8 +36,6 @@ jobs:
actions: read
steps:
- uses: actions/checkout@v7
with:
submodules: true
- name: Waiting for ui generation to end
uses: lewagon/wait-on-check-action@1d57e2c51a58d812d2765e036a028b6bdb5a6154
-19
View File
@@ -1,19 +0,0 @@
[submodule "panda"]
path = panda
url = ../../commaai/panda.git
[submodule "opendbc"]
path = opendbc_repo
url = ../../commaai/opendbc.git
[submodule "msgq"]
path = msgq_repo
url = ../../commaai/msgq.git
[submodule "rednose_repo"]
path = rednose_repo
url = ../../commaai/rednose.git
[submodule "teleoprtc_repo"]
path = teleoprtc_repo
url = ../../commaai/teleoprtc
[submodule "tinygrad"]
path = tinygrad_repo
url = https://github.com/tinygrad/tinygrad.git
branch = master
-4
View File
@@ -1,4 +0,0 @@
[lfs]
url = https://huggingface.co/commaai/openpilot-lfs.git/info/lfs
pushurl = https://huggingface.co/commaai/openpilot-lfs.git/info/lfs
locksverify = false
+7 -7
View File
@@ -20,14 +20,14 @@ SetOption('num_jobs', max(1, int(os.cpu_count()/(1 if "CI" in os.environ else 2)
AddOption('--ccflags', action='store', type='string', default='', help='pass arbitrary flags over the command line')
AddOption('--verbose', action='store_true', default=False, help='show full build commands')
release = not os.path.exists(File('#.gitmodules').abspath) # file absent on release branch, see release_files.py
release = os.path.exists(File('#git_src_commit').abspath) # written by release packaging
AddOption('--minimal',
action='store_false',
dest='extras',
default=(not COMMA_HARDWARE and not release),
help='the minimum build to run openpilot. no tests, tools, etc.')
submodule_python_paths = [
dependency_python_paths = [
Dir("#").abspath,
Dir("#msgq_repo").abspath,
Dir("#opendbc_repo").abspath,
@@ -35,12 +35,12 @@ submodule_python_paths = [
Dir("#teleoprtc_repo").abspath,
Dir("#tinygrad_repo").abspath,
]
for p in reversed(submodule_python_paths):
for p in reversed(dependency_python_paths):
if p not in sys.path:
sys.path.insert(0, p)
if external_pythonpath := os.environ.get("PYTHONPATH"):
submodule_python_paths += [p for p in external_pythonpath.split(os.pathsep) if p and p not in submodule_python_paths]
dependency_python_paths += [p for p in external_pythonpath.split(os.pathsep) if p and p not in dependency_python_paths]
# Detect platform
arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
@@ -120,7 +120,7 @@ def _libflags(target, source, env, for_signature):
env = Environment(
ENV={
"PATH": os.environ['PATH'],
"PYTHONPATH": os.pathsep.join(submodule_python_paths),
"PYTHONPATH": os.pathsep.join(dependency_python_paths),
"ACADOS_SOURCE_DIR": acados.DIR,
"ACADOS_PYTHON_INTERFACE_PATH": acados.TEMPLATE_DIR,
"TERA_PATH": acados.TERA_PATH
@@ -256,7 +256,7 @@ common = [_common, 'json11', 'zmq']
Export('common')
# Build messaging (cereal + msgq + socketmaster + their dependencies)
# Enable swaglog include in submodules
# Enable swaglog include in dependency sources
env_swaglog = env.Clone()
env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""')
SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog})
@@ -268,7 +268,7 @@ messaging = [socketmaster, msgq, 'capnp', 'kj',]
Export('messaging')
# Build other submodules
# Build dependency sources
SConscript(['panda/SConscript'])
# Build rednose library
+7 -3
View File
@@ -1,3 +1,7 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9e019ed5af500e8820d05934d47e0380728e1b30e01a179f786dc4edb1eccbd7
size 349
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 128 128">
<style>
path { fill: #000; }
@media (prefers-color-scheme: dark) { path { fill: #fff; } }
</style>
<path transform="translate(28.5)" d="M35.5 0C15.9 0 0 15.9 0 35.5S12 61 30 61c5 0 9-1 13-3 2-1 5 1 5 4-1 23-13 43-40 55l-2 11c35-11 58-38 64-69C77 28 64 0 35.5 0Z"/>
</svg>

Before

Width:  |  Height:  |  Size: 128 B

After

Width:  |  Height:  |  Size: 349 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 129 B

After

Width:  |  Height:  |  Size: 1.9 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 129 B

After

Width:  |  Height:  |  Size: 1.9 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 129 B

After

Width:  |  Height:  |  Size: 2.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 129 B

After

Width:  |  Height:  |  Size: 1.2 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 131 B

After

Width:  |  Height:  |  Size: 118 KiB

+1 -1
View File
@@ -72,7 +72,7 @@ function launch {
ln -sfn "$(pwd)" /data/pythonpath
export PYTHONPATH="$PWD"
# submodule package symlinks for PYTHONPATH imports on device.
# Dependency package symlinks for PYTHONPATH imports on device.
# on PC these come from editable installs via pyproject.toml / uv.
ln -sfn msgq_repo/msgq msgq
ln -sfn opendbc_repo/opendbc opendbc
Submodule msgq_repo deleted from 0e266c1dbc
+24
View File
@@ -0,0 +1,24 @@
name: tests
on:
push:
branches:
- master
pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.ref != 'refs/heads/master' && github.ref || github.run_id }}-${{ github.event_name }}
cancel-in-progress: true
jobs:
test:
name: ./test.sh
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: ['macos-latest', 'ubuntu-latest']
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- run: ./test.sh
+20
View File
@@ -0,0 +1,20 @@
/gen/
*.tmp
*.pyc
__pycache__
.*.swp
.*.swo
*.os
*.so
*.o
*.a
uv.lock
catch2/
msgq.egg-info/
test_runner
libmessaging.*
libmessaging_shared.*
.sconsign.dblite
.mypy_cache/
+54
View File
@@ -0,0 +1,54 @@
# MSGQ: A lock free single producer multi consumer message queue
## What is this library?
MSGQ is a generic high performance IPC pub sub system with a single publisher and multiple subscribers. It uses a ring buffer in shared memory to efficiently read and write data. Each read requires a copy. Writing can be done without a copy, as long as the size of the data is known in advance. This library also provides a spoofed implementation that can be used for deterministic testing, and visionipc, an IPC system specifically for large contiguous buffers (like images/video).
## Storage
The storage for the queue consists of an area of metadata, and the actual buffer. The metadata contains:
1. A counter to the number of readers that are active
2. A pointer to the head of the queue for writing. From now on referred to as *write pointer*
3. A cycle counter for the writer. This counter is incremented when the writer wraps around
4. N pointers, pointing to the current read position for all the readers. From now on referred to as *read pointer*
5. N counters, counting the number of cycles for all the readers
6. N booleans, indicating validity for all the readers. From now on referred to as *validity flag*
The counter and the pointer are both 32 bit values, packed into 64 bit so they can be read and written atomically.
The data buffer is a ring buffer. All messages are prefixed by an 8 byte size field, followed by the data. A size of -1 indicates a wrap-around, and means the next message is stored at the beginning of the buffer.
## Writing
Writing involves the following steps:
1. Check if the area that is to be written overlaps with any of the read pointers, mark those readers as invalid by clearing the validity flag.
2. Write the message
3. Increase the write pointer by the size of the message
In case there is not enough space at the end of the buffer, a special empty message with a prefix of -1 is written. The cycle counter is incremented by one. In this case step 1 will check there are no read pointers pointing to the remainder of the buffer. Then another write cycle will start with the actual message.
There always needs to be 8 bytes of empty space at the end of the buffer. By doing this there is always space to write the -1.
## Reset reader
When the reader is lagging too much behind the read pointer becomes invalid and no longer points to the beginning of a valid message. To reset a reader to the current write pointer, the following steps are performed:
1. Set valid flag
2. Set read cycle counter to that of the writer
3. Set read pointer to write pointer
## Reading
Reading involves the following steps:
1. Read the size field at the current read pointer
2. Read the validity flag
3. Copy the data out of the buffer
4. Increase the read pointer by the size of the message
5. Check the validity flag again
Before starting the copy, the valid flag is checked. This is to prevent a race condition where the size prefix was invalid, and the read could read outside of the buffer. Make sure that step 1 and 2 are not reordered by your compiler or CPU.
If a writer overwrites the data while it's being copied out, the data will be invalid. Therefore the validity flag is also checked after reading it. The order of step 4 and 5 does not matter.
If at steps 2 or 5 the validity flag is not set, the reader is reset. Any data that was already read is discarded. After the reader is reset, the reading starts from the beginning.
If a message with size -1 is encountered, step 3 and 4 are replaced by increasing the cycle counter and setting the read pointer to the beginning of the buffer. After that another read is performed.
+36
View File
@@ -0,0 +1,36 @@
Import('env', 'envCython', 'common')
visionipc_dir = Dir('msgq/visionipc')
# Build msgq
msgq_objects = env.SharedObject([
'msgq/ipc.cc',
'msgq/event.cc',
'msgq/impl_msgq.cc',
'msgq/impl_fake.cc',
'msgq/msgq.cc',
])
msgq = env.Library('msgq', msgq_objects)
msgq_python = envCython.Program('msgq/ipc_pyx.so', 'msgq/ipc_pyx.pyx', LIBS=envCython["LIBS"]+[msgq]+common)
# Build Vision IPC
vipc_files = ['visionipc.cc', 'visionipc_server.cc', 'visionipc_client.cc']
if File('/dev/ion').exists():
vipc_files += ['visionbuf_ion.cc']
else:
vipc_files += ['visionbuf.cc']
vipc_sources = [f'{visionipc_dir.abspath}/{f}' for f in vipc_files]
vipc_objects = env.SharedObject(vipc_sources)
visionipc = env.Library('visionipc', vipc_objects)
vipc_libs = envCython["LIBS"] + [visionipc, msgq] + common
envCython.Program(f'{visionipc_dir.abspath}/visionipc_pyx.so', f'{visionipc_dir.abspath}/visionipc_pyx.pyx',
LIBS=vipc_libs)
if GetOption('extras'):
env.Program('msgq/test_runner', ['msgq/msgq_tests.cc'], LIBS=[msgq]+common)
Export('visionipc', 'msgq', 'msgq_python')
+81
View File
@@ -0,0 +1,81 @@
import os
import platform
import subprocess
import sysconfig
arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
if platform.system() == "Darwin":
arch = "Darwin"
common = []
cpppath = [
"#/",
'#msgq/',
'/usr/lib/include',
sysconfig.get_paths()['include'],
]
AddOption('--minimal',
action='store_false',
dest='extras',
default=True,
help='the minimum build. no tests, tools, etc.')
AddOption('--asan',
action='store_true',
help='turn on ASAN')
AddOption('--ubsan',
action='store_true',
help='turn on UBSan')
ccflags = []
ldflags = []
if GetOption('ubsan'):
flags = [
"-fsanitize=undefined",
"-fno-sanitize-recover=undefined",
]
ccflags += flags
ldflags += flags
elif GetOption('asan'):
ccflags += ["-fsanitize=address", "-fno-omit-frame-pointer"]
ldflags += ["-fsanitize=address"]
env = Environment(
ENV=os.environ,
CCFLAGS=[
"-g",
"-fPIC",
"-O2",
"-Wunused",
"-Werror",
"-Wshadow" if arch == "Darwin" else "-Wshadow=local",
"-Wno-vla-cxx-extension",
"-Wno-unknown-warning-option",
] + ccflags,
LDFLAGS=ldflags,
LINKFLAGS=ldflags,
CFLAGS="-std=gnu11",
CXXFLAGS="-std=c++1z",
CPPPATH=cpppath,
CYTHONCFILESUFFIX=".cpp",
tools=["default", "cython"]
)
Export('env', 'arch', 'common')
envCython = env.Clone(LIBS=[])
envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-cpp", "-Wno-shadow", "-Wno-deprecated-declarations"]
envCython["CCFLAGS"].remove('-Werror')
if arch == "Darwin":
envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"]
else:
envCython["LINKFLAGS"] = ["-pthread", "-shared"]
Export('envCython')
SConscript(['SConscript'])
+32
View File
@@ -0,0 +1,32 @@
output:
- meta # Print lefthook version
- summary # Print summary block (successful and failed steps)
- empty_summary # Print summary heading when there are no steps to run
- success # Print successful steps
- failure # Print failed steps printing
- execution # Print any execution logs
#- execution_out # Print execution output
#- execution_info # Print `EXECUTE > ...` logging
- skips # Print "skip" (i.e. no files matched)
test:
parallel: true
commands:
# *** static analysis ***
ruff:
run: ruff check .
ty:
run: ty check .
codespell:
run: codespell {files} -L ned,stdio,master --builtin clear,rare,informal,usage,code,names,en-GB_to_en-US -S uv.lock,*_pyx.cpp
files: git ls-tree -r HEAD --name-only
cppcheck:
run: cppcheck --error-exitcode=1 --inline-suppr --language=c++ --force --quiet -j4 --check-level=exhaustive $(git ls-files '*.cc' | grep -v -E '(msgq_tests|test_runner)\.cc')
cpplint:
run: cpplint --exclude=msgq/ipc_pyx.cpp --exclude=msgq/visionipc/visionipc_pyx.cpp --recursive --quiet --counting=detailed --linelength=240 --filter=-build,-legal,-readability,-runtime,-whitespace,+build/include_subdir,+build/forward_decl,+build/include_what_you_use,+build/deprecated,+whitespace/comma,+whitespace/line_length,+whitespace/empty_if_body,+whitespace/empty_loop_body,+whitespace/empty_conditional_body,+whitespace/forcolon,+whitespace/parens,+whitespace/semicolon,+whitespace/tab,+readability/braces msgq/
# *** tests ***
test_runner:
run: msgq/test_runner
unittest:
run: python -m unittest discover -t . -s msgq
+1
View File
@@ -0,0 +1 @@
ipc_pyx.cpp
+74
View File
@@ -0,0 +1,74 @@
# must be built with scons
from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
from msgq.ipc_pyx import MultiplePublishersError, IpcError
from typing import Optional, List, Union
__all__ = [
"Context",
"Poller",
"SubSocket",
"PubSocket",
"SocketEventHandle",
"MultiplePublishersError",
"IpcError",
"toggle_fake_events",
"set_fake_prefix",
"get_fake_prefix",
"delete_fake_prefix",
"wait_for_one_event",
"NO_TRAVERSAL_LIMIT",
"context",
"fake_event_handle",
"pub_sock",
"sub_sock",
"drain_sock_raw",
]
NO_TRAVERSAL_LIMIT = 2**64-1
context = Context()
def fake_event_handle(endpoint: str, identifier: Optional[Union[str, bytes]] = None, override: bool = True, enable: bool = False) -> SocketEventHandle:
ident = identifier if identifier is not None else get_fake_prefix()
handle = SocketEventHandle(endpoint, ident, override)
if override:
handle.enabled = enable
return handle
def pub_sock(endpoint: str, segment_size: int = 0) -> PubSocket:
sock = PubSocket()
sock.connect(context, endpoint, segment_size)
return sock
def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1",
conflate: bool = False, timeout: Optional[int] = None, segment_size: int = 0) -> SubSocket:
sock = SubSocket()
sock.connect(context, endpoint, addr.encode('utf8'), conflate, segment_size)
if timeout is not None:
sock.setTimeout(timeout)
if poller is not None:
poller.registerSocket(sock)
return sock
def drain_sock_raw(sock: SubSocket, wait_for_one: bool = False) -> List[bytes]:
"""Receive all message currently available on the queue"""
ret: List[bytes] = []
while 1:
if wait_for_one and len(ret) == 0:
dat = sock.receive()
else:
dat = sock.receive(non_blocking=True)
if dat is None:
break
ret.append(dat)
return ret
+244
View File
@@ -0,0 +1,244 @@
#include <cerrno>
#include <chrono>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <string>
#include <vector>
#include <fcntl.h>
#include <poll.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#include "msgq/event.h"
namespace {
size_t event_fifo_counter = 0;
[[noreturn]] void throw_errno(const std::string& msg) {
throw std::runtime_error(msg + ", errno: " + std::to_string(errno) + " pid: " + std::to_string(getpid()));
}
int open_event_fifo(const char* path) {
if (path[0] == '\0') return -1;
int fd = open(path, O_RDWR | O_NONBLOCK);
if (fd < 0 && errno != ENOENT) throw_errno("Could not open event fifo");
return fd;
}
// poll() that retries on EINTR with a monotonic deadline so signal storms
// don't extend the effective timeout.
int poll_events(pollfd *fds, nfds_t nfds, int timeout_sec) {
auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(timeout_sec);
while (true) {
int timeout_ms = -1;
if (timeout_sec >= 0) {
auto remaining = std::chrono::duration_cast<std::chrono::milliseconds>(
deadline - std::chrono::steady_clock::now()).count();
timeout_ms = remaining > 0 ? static_cast<int>(remaining) : 0;
}
int ret = poll(fds, nfds, timeout_ms);
if (ret >= 0) return ret;
if (errno != EINTR) throw_errno("Event poll failed");
}
}
// macOS limits shm_open names to ~31 chars, so hash the (prefix, identifier, endpoint)
// tuple into a fixed-length name.
std::string event_shm_name(const std::string& endpoint, const std::string& identifier) {
const char* op_prefix = std::getenv("OPENPILOT_PREFIX");
std::string key = std::string(op_prefix ? op_prefix : "") + '\x1f' + identifier + '\x1f' + endpoint;
uint64_t h = 14695981039346656037ULL;
for (char c : key) {
h ^= static_cast<uint8_t>(c);
h *= 1099511628211ULL;
}
char buf[32];
std::snprintf(buf, sizeof(buf), "/msgq_%016llx", static_cast<unsigned long long>(h));
return buf;
}
} // namespace
void event_state_shm_mmap(std::string endpoint, std::string identifier, char **shm_mem, std::string *shm_name_out) {
std::string name = event_shm_name(endpoint, identifier);
int shm_fd = shm_open(name.c_str(), O_RDWR | O_CREAT, 0664);
if (shm_fd < 0) throw_errno("Could not open shared memory");
// macOS rejects ftruncate on an already-sized shm object, so only size it once.
struct stat st;
if (fstat(shm_fd, &st) < 0) {
close(shm_fd);
throw_errno("Could not stat shared memory");
}
if (st.st_size < static_cast<off_t>(sizeof(EventState))) {
if (ftruncate(shm_fd, sizeof(EventState)) < 0) {
close(shm_fd);
throw_errno("Could not truncate shared memory");
}
}
char *mem = reinterpret_cast<char*>(mmap(NULL, sizeof(EventState), PROT_READ | PROT_WRITE, MAP_SHARED, shm_fd, 0));
close(shm_fd);
if (mem == MAP_FAILED) throw_errno("Could not map shared memory");
if (shm_mem != nullptr) *shm_mem = mem;
if (shm_name_out != nullptr) *shm_name_out = name;
}
SocketEventHandle::SocketEventHandle(std::string endpoint, std::string identifier, bool override) {
char *mem;
event_state_shm_mmap(endpoint, identifier, &mem, &this->shm_name);
this->state = (EventState*)mem;
this->owns_fifos = override;
if (override) {
std::string base = "/tmp/msgq_event_" + std::to_string(getpid()) + "_" + std::to_string(event_fifo_counter++);
for (size_t i = 0; i < 2; i++) {
std::string p = base + "." + std::to_string(i);
if (p.size() >= EVENT_PATH_MAX) {
throw std::runtime_error("Event path too long: " + p);
}
unlink(p.c_str());
if (mkfifo(p.c_str(), 0664) < 0) throw_errno("Could not create event fifo");
std::memcpy(this->state->paths[i], p.c_str(), p.size() + 1);
}
this->state->enabled = false;
}
for (size_t i = 0; i < 2; i++) {
this->fds[i] = open_event_fifo(this->state->paths[i]);
}
}
SocketEventHandle::~SocketEventHandle() {
if (this->state == nullptr) return;
for (int fd : this->fds) {
if (fd >= 0) close(fd);
}
if (this->owns_fifos) {
unlink(this->state->paths[RECV_CALLED]);
unlink(this->state->paths[RECV_READY]);
shm_unlink(this->shm_name.c_str());
}
munmap(this->state, sizeof(EventState));
}
bool SocketEventHandle::is_enabled() {
return this->state->enabled;
}
void SocketEventHandle::set_enabled(bool enabled) {
this->state->enabled = enabled;
}
Event SocketEventHandle::recv_called() {
return Event(this->fds[RECV_CALLED]);
}
Event SocketEventHandle::recv_ready() {
return Event(this->fds[RECV_READY]);
}
void SocketEventHandle::toggle_fake_events(bool enabled) {
if (enabled)
setenv("CEREAL_FAKE", "1", true);
else
unsetenv("CEREAL_FAKE");
}
void SocketEventHandle::set_fake_prefix(std::string prefix) {
if (prefix.size() == 0) {
unsetenv("CEREAL_FAKE_PREFIX");
} else {
setenv("CEREAL_FAKE_PREFIX", prefix.c_str(), true);
}
}
std::string SocketEventHandle::fake_prefix() {
const char* prefix = std::getenv("CEREAL_FAKE_PREFIX");
return prefix == nullptr ? "" : std::string(prefix);
}
Event::Event(int fd): event_fd(fd) {}
void Event::set() const {
throw_if_invalid();
char val = 1;
while (true) {
ssize_t count = write(this->event_fd, &val, sizeof(val));
if (count == sizeof(val)) return;
if (errno == EINTR) continue;
if (errno == EAGAIN || errno == EWOULDBLOCK) return;
throw_errno("Event write failed");
}
}
int Event::clear() const {
throw_if_invalid();
int total = 0;
char buf[64];
while (true) {
ssize_t count = read(this->event_fd, buf, sizeof(buf));
if (count > 0) {
total += static_cast<int>(count);
continue;
}
if (count == 0 || errno == EAGAIN || errno == EWOULDBLOCK) return total;
if (errno == EINTR) continue;
throw_errno("Event read failed");
}
}
void Event::wait(int timeout_sec) const {
throw_if_invalid();
pollfd fds = {this->event_fd, POLLIN, 0};
if (poll_events(&fds, 1, timeout_sec) == 0) {
throw std::runtime_error("Event timed out pid: " + std::to_string(getpid()));
}
}
bool Event::peek() const {
throw_if_invalid();
pollfd fds = {this->event_fd, POLLIN, 0};
return poll_events(&fds, 1, 0) > 0;
}
bool Event::is_valid() const {
return event_fd >= 0;
}
int Event::fd() const {
return event_fd;
}
int Event::wait_for_one(const std::vector<Event>& events, int timeout_sec) {
pollfd fds[events.size()];
for (size_t i = 0; i < events.size(); i++) {
fds[i] = {events[i].fd(), POLLIN, 0};
}
if (poll_events(fds, events.size(), timeout_sec) == 0) {
throw std::runtime_error("Event timed out pid: " + std::to_string(getpid()));
}
for (size_t i = 0; i < events.size(); i++) {
if (fds[i].revents & POLLIN) {
return static_cast<int>(i);
}
}
throw std::runtime_error("Event poll failed, no events ready");
}
+64
View File
@@ -0,0 +1,64 @@
#pragma once
#include <cstddef>
#include <stdexcept>
#include <string>
#include <vector>
#define CEREAL_EVENTS_PREFIX std::string("cereal_events")
constexpr size_t EVENT_PATH_MAX = 128;
void event_state_shm_mmap(std::string endpoint, std::string identifier, char **shm_mem, std::string *shm_name);
enum EventPurpose {
RECV_CALLED,
RECV_READY
};
struct EventState {
char paths[2][EVENT_PATH_MAX];
bool enabled;
};
class Event {
private:
int event_fd = -1;
inline void throw_if_invalid() const {
if (!this->is_valid()) {
throw std::runtime_error("Event does not have valid file descriptor.");
}
}
public:
Event(int fd = -1);
void set() const;
int clear() const;
void wait(int timeout_sec = -1) const;
bool peek() const;
bool is_valid() const;
int fd() const;
static int wait_for_one(const std::vector<Event>& events, int timeout_sec = -1);
};
class SocketEventHandle {
private:
std::string shm_name;
EventState* state = nullptr;
bool owns_fifos = false;
int fds[2] = {-1, -1};
public:
SocketEventHandle(std::string endpoint, std::string identifier = "", bool override = true);
~SocketEventHandle();
bool is_enabled();
void set_enabled(bool enabled);
Event recv_called();
Event recv_ready();
static void toggle_fake_events(bool enabled);
static void set_fake_prefix(std::string prefix);
static std::string fake_prefix();
};
+10
View File
@@ -0,0 +1,10 @@
#include <vector>
#include "msgq/impl_fake.h"
void FakePoller::registerSocket(SubSocket *socket) {
this->sockets.push_back(socket);
}
std::vector<SubSocket*> FakePoller::poll(int timeout) {
return this->sockets;
}
+75
View File
@@ -0,0 +1,75 @@
#pragma once
#include <cassert>
#include <iostream>
#include <string>
#include <vector>
#include <filesystem>
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include "msgq/ipc.h"
#include "msgq/event.h"
template<typename TSubSocket>
class FakeSubSocket: public TSubSocket {
private:
EventState *state = nullptr;
int fds[2] = {-1, -1};
void ensure_fifos_open() {
for (size_t i = 0; i < 2; i++) {
if (fds[i] < 0 && state->paths[i][0] != '\0') {
fds[i] = open(state->paths[i], O_RDWR | O_NONBLOCK);
}
}
}
public:
FakeSubSocket(): TSubSocket() {}
~FakeSubSocket() {
for (int fd : fds) {
if (fd >= 0) close(fd);
}
if (state != nullptr) {
munmap(state, sizeof(EventState));
}
}
int connect(Context *context, std::string endpoint, std::string address, bool conflate=false, bool check_endpoint=true, size_t segment_size=0) override {
const char* cereal_prefix = std::getenv("CEREAL_FAKE_PREFIX");
char* mem;
std::string identifier = cereal_prefix != nullptr ? std::string(cereal_prefix) : "";
event_state_shm_mmap(endpoint, identifier, &mem, nullptr);
this->state = (EventState*)mem;
ensure_fifos_open();
return TSubSocket::connect(context, endpoint, address, conflate, check_endpoint, segment_size);
}
Message *receive(bool non_blocking=false) override {
if (this->state->enabled) {
ensure_fifos_open();
Event(fds[EventPurpose::RECV_CALLED]).set();
Event(fds[EventPurpose::RECV_READY]).wait();
Event(fds[EventPurpose::RECV_READY]).clear();
}
return TSubSocket::receive(non_blocking);
}
};
class FakePoller: public Poller {
private:
std::vector<SubSocket*> sockets;
public:
void registerSocket(SubSocket *socket) override;
std::vector<SubSocket*> poll(int timeout) override;
~FakePoller() {}
};
+169
View File
@@ -0,0 +1,169 @@
#include <cassert>
#include <cstring>
#include <iostream>
#include <cstdlib>
#include <cerrno>
#include <string>
#include <vector>
#include "msgq/impl_msgq.h"
void Message::init(size_t sz) {
size = sz;
data = new char[size];
}
void Message::init(char * d, size_t sz) {
size = sz;
data = new char[size];
memcpy(data, d, size);
}
void Message::takeOwnership(char * d, size_t sz) {
size = sz;
data = d;
}
void Message::close() {
if (size > 0){
delete[] data;
}
size = 0;
}
Message::~Message() {
this->close();
}
int MSGQSubSocket::connect(Context *context, std::string endpoint, std::string address, bool conflate, bool check_endpoint, size_t segment_size){
assert(context);
assert(address == "127.0.0.1");
q = new msgq_queue_t;
size_t size = segment_size > 0 ? segment_size : DEFAULT_SEGMENT_SIZE;
int r = msgq_new_queue(q, endpoint.c_str(), size);
if (r != 0){
return r;
}
msgq_init_subscriber(q);
if (conflate){
q->read_conflate = true;
}
timeout = -1;
return 0;
}
Message * MSGQSubSocket::receive(bool non_blocking){
msgq_msg_t msg;
Message *r = NULL;
int rc = msgq_msg_recv(&msg, q);
// Hack to implement blocking read with a poller. Don't use this
while (!non_blocking && rc == 0){
msgq_pollitem_t items[1];
items[0].q = q;
int t = (timeout != -1) ? timeout : 100;
int n = msgq_poll(items, 1, t);
rc = msgq_msg_recv(&msg, q);
// The poll indicated a message was ready, but the receive failed. Try again
if (n == 1 && rc == 0){
continue;
}
if (timeout != -1){
break;
}
}
if (rc > 0){
r = new Message;
r->takeOwnership(msg.data, msg.size);
}
return r;
}
void MSGQSubSocket::setTimeout(int t){
timeout = t;
}
MSGQSubSocket::~MSGQSubSocket(){
if (q != NULL){
msgq_close_queue(q);
delete q;
}
}
int PubSocket::connect(Context *context, std::string endpoint, bool check_endpoint, size_t segment_size){
assert(context);
q = new msgq_queue_t;
size_t size = segment_size > 0 ? segment_size : DEFAULT_SEGMENT_SIZE;
int r = msgq_new_queue(q, endpoint.c_str(), size);
if (r != 0){
return r;
}
msgq_init_publisher(q);
return 0;
}
int PubSocket::sendMessage(Message *message){
msgq_msg_t msg;
msg.data = message->getData();
msg.size = message->getSize();
return msgq_msg_send(&msg, q);
}
int PubSocket::send(char *data, size_t size){
msgq_msg_t msg;
msg.data = data;
msg.size = size;
return msgq_msg_send(&msg, q);
}
bool PubSocket::all_readers_updated() {
return msgq_all_readers_updated(q);
}
PubSocket::~PubSocket(){
if (q != NULL){
msgq_close_queue(q);
delete q;
}
}
void MSGQPoller::registerSocket(SubSocket * socket){
assert(num_polls + 1 < MAX_POLLERS);
polls[num_polls].q = static_cast<MSGQSubSocket*>(socket)->getQueue();
sockets.push_back(socket);
num_polls++;
}
std::vector<SubSocket*> MSGQPoller::poll(int timeout){
std::vector<SubSocket*> r;
msgq_poll(polls, num_polls, timeout);
for (size_t i = 0; i < num_polls; i++){
if (polls[i].revents){
r.push_back(sockets[i]);
}
}
return r;
}
+33
View File
@@ -0,0 +1,33 @@
#pragma once
#include <string>
#include <vector>
#include "msgq/ipc.h"
#include "msgq/msgq.h"
#define MAX_POLLERS 128
class MSGQSubSocket : public SubSocket {
private:
msgq_queue_t * q = NULL;
int timeout;
public:
int connect(Context *context, std::string endpoint, std::string address, bool conflate=false, bool check_endpoint=true, size_t segment_size=0);
void setTimeout(int timeout);
msgq_queue_t * getQueue() {return q;}
Message *receive(bool non_blocking=false);
~MSGQSubSocket();
};
class MSGQPoller : public Poller {
private:
std::vector<SubSocket*> sockets;
msgq_pollitem_t polls[MAX_POLLERS];
size_t num_polls = 0;
public:
void registerSocket(SubSocket *socket);
std::vector<SubSocket*> poll(int timeout);
~MSGQPoller(){}
};
+71
View File
@@ -0,0 +1,71 @@
#include <cassert>
#include <iostream>
#include <string>
#include "msgq/ipc.h"
#include "msgq/impl_msgq.h"
#include "msgq/impl_fake.h"
bool messaging_use_fake(){
char* fake_enabled = std::getenv("CEREAL_FAKE");
return fake_enabled != NULL;
}
Context * Context::create(){
return new Context();
}
SubSocket * SubSocket::create(){
if (messaging_use_fake()) {
return new FakeSubSocket<MSGQSubSocket>();
}
return new MSGQSubSocket();
}
SubSocket * SubSocket::create(Context * context, std::string endpoint, std::string address, bool conflate, bool check_endpoint, size_t segment_size){
SubSocket *s = SubSocket::create();
int r = s->connect(context, endpoint, address, conflate, check_endpoint, segment_size);
if (r == 0) {
return s;
} else {
std::cerr << "Error, failed to connect SubSocket to " << endpoint << ": " << strerror(errno) << std::endl;
delete s;
return nullptr;
}
}
PubSocket * PubSocket::create(){
return new PubSocket();
}
PubSocket * PubSocket::create(Context * context, std::string endpoint, bool check_endpoint, size_t segment_size){
PubSocket *s = PubSocket::create();
int r = s->connect(context, endpoint, check_endpoint, segment_size);
if (r == 0) {
return s;
} else {
std::cerr << "Error, failed to bind PubSocket to " << endpoint << ": " << strerror(errno) << std::endl;
delete s;
return nullptr;
}
}
Poller * Poller::create(){
if (messaging_use_fake()) {
return new FakePoller();
}
return new MSGQPoller();
}
Poller * Poller::create(std::vector<SubSocket*> sockets){
Poller * p = Poller::create();
for (auto s : sockets){
p->registerSocket(s);
}
return p;
}
+69
View File
@@ -0,0 +1,69 @@
#pragma once
#include <cstddef>
#include <map>
#include <string>
#include <vector>
#include <utility>
#include <time.h>
#ifdef __APPLE__
#define CLOCK_BOOTTIME CLOCK_MONOTONIC
#endif
#define MSG_MULTIPLE_PUBLISHERS 100
class Context {
public:
static Context * create();
~Context(){}
};
class Message {
private:
char * data = nullptr;
size_t size = 0;
public:
void init(size_t size);
void init(char * data, size_t size);
void takeOwnership(char * data, size_t size);
void close();
size_t getSize(){return size;}
char * getData(){return data;}
~Message();
};
class SubSocket {
public:
virtual int connect(Context *context, std::string endpoint, std::string address, bool conflate=false, bool check_endpoint=true, size_t segment_size=0) = 0;
virtual void setTimeout(int timeout) = 0;
virtual Message *receive(bool non_blocking=false) = 0;
static SubSocket * create();
static SubSocket * create(Context * context, std::string endpoint, std::string address="127.0.0.1", bool conflate=false, bool check_endpoint=true, size_t segment_size=0);
virtual ~SubSocket(){}
};
class PubSocket {
private:
struct msgq_queue_t * q = nullptr;
public:
int connect(Context *context, std::string endpoint, bool check_endpoint=true, size_t segment_size=0);
int sendMessage(Message *message);
int send(char *data, size_t size);
bool all_readers_updated();
static PubSocket * create();
static PubSocket * create(Context * context, std::string endpoint, bool check_endpoint=true, size_t segment_size=0);
~PubSocket();
};
class Poller {
public:
virtual void registerSocket(SubSocket *socket) = 0;
virtual std::vector<SubSocket*> poll(int timeout) = 0;
static Poller * create();
static Poller * create(std::vector<SubSocket*> sockets);
virtual ~Poller(){}
};
+68
View File
@@ -0,0 +1,68 @@
# distutils: language = c++
#cython: language_level=3
from libcpp.string cimport string
from libcpp.vector cimport vector
from libcpp cimport bool
cdef extern from "msgq/impl_fake.h":
cdef cppclass Event:
@staticmethod
int wait_for_one(vector[Event], int) except +
Event()
Event(int)
void set()
int clear()
void wait(int) except +
bool peek()
int fd()
cdef cppclass SocketEventHandle:
@staticmethod
void toggle_fake_events(bool)
@staticmethod
void set_fake_prefix(string)
@staticmethod
string fake_prefix()
SocketEventHandle(string, string, bool)
bool is_enabled()
void set_enabled(bool)
Event recv_called()
Event recv_ready()
cdef extern from "msgq/ipc.h":
cdef cppclass Context:
@staticmethod
Context * create()
cdef cppclass Message:
void init(size_t)
void init(char *, size_t)
void close()
size_t getSize()
char *getData()
cdef cppclass SubSocket:
@staticmethod
SubSocket * create() nogil
int connect(Context *, string, string, bool, bool, size_t) nogil
Message * receive(bool) nogil
void setTimeout(int) nogil
cdef cppclass PubSocket:
@staticmethod
PubSocket * create()
int connect(Context *, string, bool, size_t)
int sendMessage(Message *)
int send(char *, size_t)
bool all_readers_updated()
cdef cppclass Poller:
@staticmethod
Poller * create()
void registerSocket(SubSocket *)
vector[SubSocket*] poll(int) nogil
+256
View File
@@ -0,0 +1,256 @@
# distutils: language = c++
# cython: c_string_encoding=ascii, language_level=3
import time
from libcpp.string cimport string
from libcpp.vector cimport vector
from libcpp cimport bool
from libc cimport errno
from libc.string cimport strerror
from cython.operator import dereference
from .ipc cimport Context as cppContext
from .ipc cimport SubSocket as cppSubSocket
from .ipc cimport PubSocket as cppPubSocket
from .ipc cimport Poller as cppPoller
from .ipc cimport Message as cppMessage
from .ipc cimport Event as cppEvent, SocketEventHandle as cppSocketEventHandle
class IpcError(Exception):
def __init__(self, endpoint=None):
suffix = f"with {endpoint.decode('utf-8')}" if endpoint else ""
message = f"Messaging failure {suffix}: {strerror(errno.errno).decode('utf-8')}"
super().__init__(message)
class MultiplePublishersError(IpcError):
pass
def toggle_fake_events(bool enabled):
cppSocketEventHandle.toggle_fake_events(enabled)
def set_fake_prefix(string prefix):
cppSocketEventHandle.set_fake_prefix(prefix)
def get_fake_prefix():
return cppSocketEventHandle.fake_prefix()
def delete_fake_prefix():
cppSocketEventHandle.set_fake_prefix(b"")
def wait_for_one_event(list events, int timeout=-1):
cdef vector[cppEvent] items
for event in events:
items.push_back(dereference(<cppEvent*><size_t>event.ptr))
return cppEvent.wait_for_one(items, timeout)
cdef class Event:
cdef cppEvent event;
cdef setEvent(self, cppEvent event):
self.event = event
def set(self):
self.event.set()
def clear(self):
return self.event.clear()
def wait(self, int timeout=-1):
self.event.wait(timeout)
def peek(self):
return self.event.peek()
@property
def fd(self):
return self.event.fd()
@property
def ptr(self):
return <size_t><void*>&self.event
cdef class SocketEventHandle:
cdef cppSocketEventHandle * handle;
def __cinit__(self, string endpoint, string identifier, bool override):
self.handle = new cppSocketEventHandle(endpoint, identifier, override)
def __dealloc__(self):
del self.handle
@property
def enabled(self):
return self.handle.is_enabled()
@enabled.setter
def enabled(self, bool value):
self.handle.set_enabled(value)
@property
def recv_called_event(self):
e = Event()
e.setEvent(self.handle.recv_called())
return e
@property
def recv_ready_event(self):
e = Event()
e.setEvent(self.handle.recv_ready())
return e
cdef class Context:
cdef cppContext * context
def __cinit__(self):
self.context = cppContext.create()
def term(self):
del self.context
self.context = NULL
def __dealloc__(self):
pass
# Deleting the context will hang if sockets are still active
# TODO: Figure out a way to make sure the context is closed last
# del self.context
cdef class Poller:
cdef cppPoller * poller
cdef list sub_sockets
def __cinit__(self):
self.sub_sockets = []
self.poller = cppPoller.create()
def __dealloc__(self):
del self.poller
def registerSocket(self, SubSocket socket):
self.sub_sockets.append(socket)
self.poller.registerSocket(socket.socket)
def poll(self, timeout):
sockets = []
cdef int t = timeout
with nogil:
result = self.poller.poll(t)
for s in result:
socket = SubSocket()
socket.setPtr(s)
sockets.append(socket)
return sockets
cdef class SubSocket:
cdef cppSubSocket * socket
cdef bool is_owner
def __cinit__(self):
with nogil:
self.socket = cppSubSocket.create()
self.is_owner = True
if self.socket == NULL:
raise IpcError
def __dealloc__(self):
if self.is_owner:
with nogil:
del self.socket
cdef setPtr(self, cppSubSocket * ptr):
if self.is_owner:
with nogil:
del self.socket
self.is_owner = False
self.socket = ptr
def connect(self, Context context, string endpoint, string address=b"127.0.0.1", bool conflate=False, size_t segment_size=0):
cdef int r
with nogil:
r = self.socket.connect(context.context, endpoint, address, conflate, True, segment_size)
if r != 0:
if errno.errno == errno.EADDRINUSE:
raise MultiplePublishersError(endpoint)
else:
raise IpcError(endpoint)
def setTimeout(self, int timeout):
with nogil:
self.socket.setTimeout(timeout)
def receive(self, bool non_blocking=False):
cdef cppMessage *msg
with nogil:
msg = self.socket.receive(non_blocking)
if msg == NULL:
return None
else:
sz = msg.getSize()
m = msg.getData()[:sz]
with nogil:
del msg
return m
cdef class PubSocket:
cdef cppPubSocket * socket
def __cinit__(self):
self.socket = cppPubSocket.create()
if self.socket == NULL:
raise IpcError
def __dealloc__(self):
del self.socket
def connect(self, Context context, string endpoint, size_t segment_size=0):
r = self.socket.connect(context.context, endpoint, True, segment_size)
if r != 0:
if errno.errno == errno.EADDRINUSE:
raise MultiplePublishersError(endpoint)
else:
raise IpcError(endpoint)
def send(self, bytes data):
length = len(data)
r = self.socket.send(<char*>data, length)
if r != length:
if errno.errno == errno.EADDRINUSE:
raise MultiplePublishersError
else:
raise IpcError
def all_readers_updated(self):
return self.socket.all_readers_updated()
def wait_for_readers(self, double timeout=1.0, double interval=0.001):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if self.all_readers_updated():
return
time.sleep(interval)
raise TimeoutError("subscriber did not connect")
+21
View File
@@ -0,0 +1,21 @@
#pragma once
#ifdef SWAGLOG
// cppcheck-suppress preprocessorErrorDirective
#include SWAGLOG
#else
#define CLOUDLOG_DEBUG 10
#define CLOUDLOG_INFO 20
#define CLOUDLOG_WARNING 30
#define CLOUDLOG_ERROR 40
#define CLOUDLOG_CRITICAL 50
#define cloudlog(lvl, fmt, ...) printf(fmt "\n", ## __VA_ARGS__)
#define LOGD(fmt, ...) cloudlog(CLOUDLOG_DEBUG, fmt, ## __VA_ARGS__)
#define LOG(fmt, ...) cloudlog(CLOUDLOG_INFO, fmt, ## __VA_ARGS__)
#define LOGW(fmt, ...) cloudlog(CLOUDLOG_WARNING, fmt, ## __VA_ARGS__)
#define LOGE(fmt, ...) cloudlog(CLOUDLOG_ERROR, fmt, ## __VA_ARGS__)
#endif
+504
View File
@@ -0,0 +1,504 @@
#include <iostream>
#include <cassert>
#include <cerrno>
#include <cmath>
#include <cstring>
#include <cstdint>
#include <chrono>
#include <algorithm>
#include <cstdlib>
#include <csignal>
#include <random>
#include <string>
#include <limits>
#include <poll.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/syscall.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdio.h>
#include "msgq/msgq.h"
void sigusr2_handler(int signal) {
assert(signal == SIGUSR2);
}
uint64_t msgq_get_uid(void){
std::random_device rd("/dev/urandom");
std::uniform_int_distribution<uint64_t> distribution(0, std::numeric_limits<uint32_t>::max());
#ifdef __APPLE__
// TODO: this doesn't work
uint64_t uid = distribution(rd) << 32 | getpid();
#else
uint64_t uid = distribution(rd) << 32 | syscall(SYS_gettid);
#endif
return uid;
}
int msgq_msg_init_size(msgq_msg_t * msg, size_t size){
msg->size = size;
msg->data = new(std::nothrow) char[size];
return (msg->data == NULL) ? -1 : 0;
}
int msgq_msg_init_data(msgq_msg_t * msg, char * data, size_t size) {
int r = msgq_msg_init_size(msg, size);
if (r == 0)
memcpy(msg->data, data, size);
return r;
}
int msgq_msg_close(msgq_msg_t * msg){
if (msg->size > 0)
delete[] msg->data;
msg->size = 0;
return 0;
}
void msgq_reset_reader(msgq_queue_t * q){
int id = q->reader_id;
q->read_valids[id]->store(true);
q->read_pointers[id]->store(*q->write_pointer);
}
void msgq_wait_for_subscriber(msgq_queue_t *q){
while (*q->num_readers == 0){
// wait for subscriber
}
}
int msgq_new_queue(msgq_queue_t * q, const char * path, size_t size){
assert(size < 0xFFFFFFFF); // Buffer must be smaller than 2^32 bytes
std::signal(SIGUSR2, sigusr2_handler);
#ifdef __APPLE__
std::string base_path = "/tmp/msgq_";
#else
std::string base_path = "/dev/shm/msgq_";
#endif
const char* prefix = std::getenv("OPENPILOT_PREFIX");
if (prefix) {
base_path += std::string(prefix) + "/";
}
std::string full_path = base_path + path;
auto fd = open(full_path.c_str(), O_RDWR | O_CREAT, 0664);
if (fd < 0) {
std::cout << "Warning, could not open: " << full_path << std::endl;
return -1;
}
int rc = ftruncate(fd, size + sizeof(msgq_header_t));
if (rc < 0){
close(fd);
return -1;
}
int mmap_flags = MAP_SHARED;
#ifdef MAP_POPULATE
if (std::getenv("MSGQ_PREALLOC")) {
mmap_flags |= MAP_POPULATE;
}
#endif
char * mem = (char*)mmap(NULL, size + sizeof(msgq_header_t), PROT_READ | PROT_WRITE, mmap_flags, fd, 0);
close(fd);
if (mem == MAP_FAILED){
return -1;
}
q->mmap_p = mem;
msgq_header_t *header = (msgq_header_t *)mem;
// Setup pointers to header segment
q->num_readers = reinterpret_cast<std::atomic<uint64_t>*>(&header->num_readers);
q->write_pointer = reinterpret_cast<std::atomic<uint64_t>*>(&header->write_pointer);
q->write_uid = reinterpret_cast<std::atomic<uint64_t>*>(&header->write_uid);
for (size_t i = 0; i < NUM_READERS; i++){
q->read_pointers[i] = reinterpret_cast<std::atomic<uint64_t>*>(&header->read_pointers[i]);
q->read_valids[i] = reinterpret_cast<std::atomic<uint64_t>*>(&header->read_valids[i]);
q->read_uids[i] = reinterpret_cast<std::atomic<uint64_t>*>(&header->read_uids[i]);
}
q->data = mem + sizeof(msgq_header_t);
q->size = size;
q->reader_id = -1;
q->endpoint = path;
q->read_conflate = false;
return 0;
}
void msgq_close_queue(msgq_queue_t *q){
if (q->mmap_p != NULL){
munmap(q->mmap_p, q->size + sizeof(msgq_header_t));
}
}
void msgq_init_publisher(msgq_queue_t * q) {
//std::cout << "Starting publisher" << std::endl;
uint64_t uid = msgq_get_uid();
*q->write_uid = uid;
*q->num_readers = 0;
for (size_t i = 0; i < NUM_READERS; i++){
*q->read_valids[i] = false;
*q->read_uids[i] = 0;
}
q->write_uid_local = uid;
}
static void thread_signal(uint32_t tid) {
#ifdef __APPLE__
// macOS doesn't have tkill, rely on polling instead
(void)tid;
#elif !defined(SYS_tkill)
// fallback for systems without tkill
kill(tid, SIGUSR2);
#else
syscall(SYS_tkill, tid, SIGUSR2);
#endif
}
void msgq_init_subscriber(msgq_queue_t * q) {
assert(q != NULL);
assert(q->num_readers != NULL);
uint64_t uid = msgq_get_uid();
// Get reader id
while (true){
uint64_t cur_num_readers = *q->num_readers;
uint64_t new_num_readers = cur_num_readers + 1;
// No more slots available. Reset all subscribers to kick out inactive ones
if (new_num_readers > NUM_READERS){
//std::cout << "Warning, evicting all subscribers!" << std::endl;
*q->num_readers = 0;
for (size_t i = 0; i < NUM_READERS; i++){
*q->read_valids[i] = false;
uint64_t old_uid = *q->read_uids[i];
*q->read_uids[i] = 0;
// Wake up reader in case they are in a poll
thread_signal(old_uid & 0xFFFFFFFF);
}
continue;
}
// Use atomic compare and swap to handle race condition
// where two subscribers start at the same time
if (std::atomic_compare_exchange_strong(q->num_readers,
&cur_num_readers,
new_num_readers)){
q->reader_id = cur_num_readers;
q->read_uid_local = uid;
// We start with read_valid = false,
// on the first read the read pointer will be synchronized with the write pointer
*q->read_valids[cur_num_readers] = false;
*q->read_pointers[cur_num_readers] = 0;
*q->read_uids[cur_num_readers] = uid;
break;
}
}
//std::cout << "New subscriber id: " << q->reader_id << " uid: " << q->read_uid_local << " " << q->endpoint << std::endl;
msgq_reset_reader(q);
}
int msgq_msg_send(msgq_msg_t * msg, msgq_queue_t *q){
// Die if we are no longer the active publisher
if (q->write_uid_local != *q->write_uid){
std::cout << "Killing old publisher: " << q->endpoint << std::endl;
errno = EADDRINUSE;
return -1;
}
uint64_t total_msg_size = ALIGN(msg->size + sizeof(int64_t));
// We need to fit at least three messages in the queue,
// then we can always safely access the last message
assert(3 * total_msg_size <= q->size);
uint64_t num_readers = *q->num_readers;
uint32_t write_cycles, write_pointer;
UNPACK64(write_cycles, write_pointer, *q->write_pointer);
char *p = q->data + write_pointer; // add base offset
// Check remaining space
// Always leave space for a wraparound tag for the next message, including alignment
int64_t remaining_space = q->size - write_pointer - total_msg_size - sizeof(int64_t);
if (remaining_space <= 0){
// Write -1 size tag indicating wraparound
*(int64_t*)p = -1;
// Invalidate all readers that are beyond the write pointer
// TODO: should we handle the case where a new reader shows up while this is running?
for (uint64_t i = 0; i < num_readers; i++){
uint64_t read_pointer = *q->read_pointers[i];
uint64_t read_cycles = read_pointer >> 32;
read_pointer &= 0xFFFFFFFF;
if ((read_pointer > write_pointer) && (read_cycles != write_cycles)) {
*q->read_valids[i] = false;
}
}
// Update global and local copies of write pointer and write_cycles
write_pointer = 0;
write_cycles = write_cycles + 1;
PACK64(*q->write_pointer, write_cycles, write_pointer);
// Set actual pointer to the beginning of the data segment
p = q->data;
}
// Invalidate readers that are in the area that will be written
uint64_t start = write_pointer;
uint64_t end = ALIGN(start + sizeof(int64_t) + msg->size);
for (uint64_t i = 0; i < num_readers; i++){
uint32_t read_cycles, read_pointer;
UNPACK64(read_cycles, read_pointer, *q->read_pointers[i]);
if ((read_pointer >= start) && (read_pointer < end) && (read_cycles != write_cycles)) {
*q->read_valids[i] = false;
}
}
// Write size tag
std::atomic<int64_t> *size_p = reinterpret_cast<std::atomic<int64_t>*>(p);
*size_p = msg->size;
// Copy data
memcpy(p + sizeof(int64_t), msg->data, msg->size);
__sync_synchronize();
// Update write pointer
uint32_t new_ptr = ALIGN(write_pointer + msg->size + sizeof(int64_t));
PACK64(*q->write_pointer, write_cycles, new_ptr);
// Notify readers
for (uint64_t i = 0; i < num_readers; i++){
uint64_t reader_uid = *q->read_uids[i];
thread_signal(reader_uid & 0xFFFFFFFF);
}
return msg->size;
}
int msgq_msg_ready(msgq_queue_t * q){
start:
int id = q->reader_id;
assert(id >= 0); // Make sure subscriber is initialized
if (q->read_uid_local != *q->read_uids[id]){
//std::cout << q->endpoint << ": Reader was evicted, reconnecting" << std::endl;
msgq_init_subscriber(q);
goto start;
}
// Check valid
if (!*q->read_valids[id]){
msgq_reset_reader(q);
goto start;
}
uint32_t read_cycles, read_pointer;
UNPACK64(read_cycles, read_pointer, *q->read_pointers[id]);
UNUSED(read_cycles);
uint32_t write_cycles, write_pointer;
UNPACK64(write_cycles, write_pointer, *q->write_pointer);
UNUSED(write_cycles);
// Check if new message is available
return (read_pointer != write_pointer);
}
int msgq_msg_recv(msgq_msg_t * msg, msgq_queue_t * q){
start:
int id = q->reader_id;
assert(id >= 0); // Make sure subscriber is initialized
if (q->read_uid_local != *q->read_uids[id]){
//std::cout << q->endpoint << ": Reader was evicted, reconnecting" << std::endl;
msgq_init_subscriber(q);
goto start;
}
// Check valid
if (!*q->read_valids[id]){
msgq_reset_reader(q);
goto start;
}
uint32_t read_cycles, read_pointer;
UNPACK64(read_cycles, read_pointer, *q->read_pointers[id]);
uint32_t write_cycles, write_pointer;
UNPACK64(write_cycles, write_pointer, *q->write_pointer);
UNUSED(write_cycles);
char * p = q->data + read_pointer;
// Check if new message is available
if (read_pointer == write_pointer) {
msg->size = 0;
return 0;
}
// Read potential message size
std::atomic<int64_t> *size_p = reinterpret_cast<std::atomic<int64_t>*>(p);
std::int64_t size = *size_p;
// Check if the size that was read is valid
if (!*q->read_valids[id]){
msgq_reset_reader(q);
goto start;
}
// If size is -1 the buffer was full, and we need to wrap around
if (size == -1){
read_cycles++;
PACK64(*q->read_pointers[id], read_cycles, 0);
goto start;
}
// crashing is better than passing garbage data to the consumer
// the size will have weird value if it was overwritten by data accidentally
assert((uint64_t)size < q->size);
assert(size > 0);
uint32_t new_read_pointer = ALIGN(read_pointer + sizeof(std::int64_t) + size);
// If conflate is true, check if this is the latest message, else start over
if (q->read_conflate){
if (new_read_pointer != write_pointer){
// Update read pointer
PACK64(*q->read_pointers[id], read_cycles, new_read_pointer);
goto start;
}
}
// Copy message
if (msgq_msg_init_size(msg, size) < 0)
return -1;
__sync_synchronize();
memcpy(msg->data, p + sizeof(int64_t), size);
__sync_synchronize();
// Update read pointer
PACK64(*q->read_pointers[id], read_cycles, new_read_pointer);
// Check if the actual data that was copied is valid
if (!*q->read_valids[id]){
msgq_msg_close(msg);
msgq_reset_reader(q);
goto start;
}
return msg->size;
}
int msgq_poll(msgq_pollitem_t * items, size_t nitems, int timeout){
int num = 0;
// Check if messages ready
for (size_t i = 0; i < nitems; i++) {
items[i].revents = msgq_msg_ready(items[i].q);
if (items[i].revents) num++;
}
int ms = (timeout == -1) ? 100 : timeout;
#ifdef __APPLE__
// On macOS, signals can't interrupt nanosleep, so poll more frequently
int poll_ms = std::min(ms, 10);
int remaining_ms = ms;
#else
int poll_ms = ms;
#endif
struct timespec ts;
ts.tv_sec = poll_ms / 1000;
ts.tv_nsec = (poll_ms % 1000) * 1000 * 1000;
while (num == 0) {
int ret;
ret = nanosleep(&ts, &ts);
// Check if messages ready
for (size_t i = 0; i < nitems; i++) {
if (items[i].revents == 0 && msgq_msg_ready(items[i].q)){
num += 1;
items[i].revents = 1;
}
}
#ifdef __APPLE__
// exit if we had a timeout and we've exhausted it
if (timeout != -1 && ret == 0){
remaining_ms -= poll_ms;
if (remaining_ms <= 0){
break;
}
poll_ms = std::min(remaining_ms, 10);
ts.tv_sec = poll_ms / 1000;
ts.tv_nsec = (poll_ms % 1000) * 1000 * 1000;
}
#else
// exit if we had a timeout and the sleep finished
if (timeout != -1 && ret == 0){
break;
}
#endif
}
return num;
}
bool msgq_all_readers_updated(msgq_queue_t *q) {
uint64_t num_readers = *q->num_readers;
for (uint64_t i = 0; i < num_readers; i++) {
if (*q->read_valids[i] && *q->write_pointer != *q->read_pointers[i]) {
return false;
}
}
return num_readers > 0;
}
+70
View File
@@ -0,0 +1,70 @@
#pragma once
#include <cstdint>
#include <cstring>
#include <string>
#include <atomic>
#define DEFAULT_SEGMENT_SIZE (1 * 1024 * 1024)
#define NUM_READERS 15
#define ALIGN(n) ((n + (8 - 1)) & -8)
#define UNUSED(x) (void)x
#define UNPACK64(higher, lower, input) do {uint64_t tmp = input; higher = tmp >> 32; lower = tmp & 0xFFFFFFFF;} while (0)
#define PACK64(output, higher, lower) output = ((uint64_t)higher << 32) | ((uint64_t)lower & 0xFFFFFFFF)
struct msgq_header_t {
uint64_t num_readers;
uint64_t write_pointer;
uint64_t write_uid;
uint64_t read_pointers[NUM_READERS];
uint64_t read_valids[NUM_READERS];
uint64_t read_uids[NUM_READERS];
};
struct msgq_queue_t {
std::atomic<uint64_t> *num_readers;
std::atomic<uint64_t> *write_pointer;
std::atomic<uint64_t> *write_uid;
std::atomic<uint64_t> *read_pointers[NUM_READERS];
std::atomic<uint64_t> *read_valids[NUM_READERS];
std::atomic<uint64_t> *read_uids[NUM_READERS];
char * mmap_p;
char * data;
size_t size;
int reader_id;
uint64_t read_uid_local;
uint64_t write_uid_local;
bool read_conflate;
std::string endpoint;
};
struct msgq_msg_t {
size_t size;
char * data;
};
struct msgq_pollitem_t {
msgq_queue_t *q;
int revents;
};
void msgq_wait_for_subscriber(msgq_queue_t *q);
void msgq_reset_reader(msgq_queue_t *q);
int msgq_msg_init_size(msgq_msg_t *msg, size_t size);
int msgq_msg_init_data(msgq_msg_t *msg, char * data, size_t size);
int msgq_msg_close(msgq_msg_t *msg);
int msgq_new_queue(msgq_queue_t * q, const char * path, size_t size);
void msgq_close_queue(msgq_queue_t *q);
void msgq_init_publisher(msgq_queue_t * q);
void msgq_init_subscriber(msgq_queue_t * q);
int msgq_msg_send(msgq_msg_t *msg, msgq_queue_t *q);
int msgq_msg_recv(msgq_msg_t *msg, msgq_queue_t *q);
int msgq_msg_ready(msgq_queue_t * q);
int msgq_poll(msgq_pollitem_t * items, size_t nitems, int timeout);
bool msgq_all_readers_updated(msgq_queue_t *q);
+426
View File
@@ -0,0 +1,426 @@
#include "msgq/test_runner.h"
#include "msgq/msgq.h"
static void cleanup_test_queue() {
#ifdef __APPLE__
remove("/tmp/msgq_test_queue");
#else
remove("/dev/shm/msgq_test_queue");
#endif
}
TEST_CASE("ALIGN")
{
REQUIRE(ALIGN(0) == 0);
REQUIRE(ALIGN(1) == 8);
REQUIRE(ALIGN(7) == 8);
REQUIRE(ALIGN(8) == 8);
REQUIRE(ALIGN(99999) == 100000);
}
TEST_CASE("msgq_msg_init_size")
{
const size_t msg_size = 30;
msgq_msg_t msg;
msgq_msg_init_size(&msg, msg_size);
REQUIRE(msg.size == msg_size);
msgq_msg_close(&msg);
}
TEST_CASE("msgq_msg_init_data")
{
const size_t msg_size = 30;
char *data = new char[msg_size];
for (size_t i = 0; i < msg_size; i++)
{
data[i] = i;
}
msgq_msg_t msg;
msgq_msg_init_data(&msg, data, msg_size);
REQUIRE(msg.size == msg_size);
REQUIRE(memcmp(msg.data, data, msg_size) == 0);
delete[] data;
msgq_msg_close(&msg);
}
TEST_CASE("msgq_init_subscriber")
{
cleanup_test_queue();
msgq_queue_t q;
msgq_new_queue(&q, "test_queue", 1024);
REQUIRE(*q.num_readers == 0);
q.reader_id = 1;
*q.read_valids[0] = false;
*q.read_pointers[0] = ((uint64_t)1 << 32);
*q.write_pointer = 255;
msgq_init_subscriber(&q);
REQUIRE(q.read_conflate == false);
REQUIRE(*q.read_valids[0] == true);
REQUIRE((*q.read_pointers[0] >> 32) == 0);
REQUIRE((*q.read_pointers[0] & 0xFFFFFFFF) == 255);
}
TEST_CASE("msgq_msg_send first message")
{
const size_t msg_sizes[] = {128, 127};
for (size_t msg_size : msg_sizes)
{
cleanup_test_queue();
msgq_queue_t q;
msgq_new_queue(&q, "test_queue", 1024);
msgq_init_publisher(&q);
REQUIRE(*q.write_pointer == 0);
char *data = new char[msg_size];
for (size_t i = 0; i < msg_size; i++)
{
data[i] = i;
}
msgq_msg_t msg;
msgq_msg_init_data(&msg, data, msg_size);
msgq_msg_send(&msg, &q);
REQUIRE(*(int64_t *)q.data == msg_size); // Check size tag
REQUIRE(*q.write_pointer == 128 + sizeof(int64_t));
REQUIRE(memcmp(q.data + sizeof(int64_t), data, msg_size) == 0);
delete[] data;
msgq_msg_close(&msg);
}
}
TEST_CASE("msgq_msg_send test wraparound")
{
cleanup_test_queue();
msgq_queue_t q;
msgq_new_queue(&q, "test_queue", 1024);
msgq_init_publisher(&q);
REQUIRE((*q.write_pointer & 0xFFFFFFFF) == 0);
REQUIRE((*q.write_pointer >> 32) == 0);
const size_t msg_size = 120;
msgq_msg_t msg;
msgq_msg_init_size(&msg, msg_size);
for (int i = 0; i < 8; i++)
{
msgq_msg_send(&msg, &q);
}
// Check 8th message was written at the beginning
REQUIRE((*q.write_pointer & 0xFFFFFFFF) == msg_size + sizeof(int64_t));
// Check cycle count
REQUIRE((*q.write_pointer >> 32) == 1);
// Check wraparound tag
char *tag_location = q.data;
tag_location += 7 * (msg_size + sizeof(int64_t));
REQUIRE(*(int64_t *)tag_location == -1);
msgq_msg_close(&msg);
}
TEST_CASE("msgq_msg_recv test wraparound")
{
const bool keep_up_cases[] = {false, true};
for (bool keep_up_with_writer : keep_up_cases)
{
cleanup_test_queue();
msgq_queue_t q_pub, q_sub;
msgq_new_queue(&q_pub, "test_queue", 1024);
msgq_new_queue(&q_sub, "test_queue", 1024);
msgq_init_publisher(&q_pub);
msgq_init_subscriber(&q_sub);
REQUIRE((*q_pub.write_pointer >> 32) == 0);
REQUIRE((*q_sub.read_pointers[0] >> 32) == 0);
const size_t msg_size = 120;
msgq_msg_t msg1;
msgq_msg_init_size(&msg1, msg_size);
for (int i = 0; i < 8; i++)
{
msgq_msg_send(&msg1, &q_pub);
if (keep_up_with_writer)
{
msgq_msg_t msg2;
msgq_msg_recv(&msg2, &q_sub);
REQUIRE(msg2.size > 0);
msgq_msg_close(&msg2);
}
}
if (!keep_up_with_writer)
{
msgq_msg_t msg2;
msgq_msg_recv(&msg2, &q_sub);
REQUIRE(msg2.size == 0); // Reader had to reset
msgq_msg_close(&msg2);
}
REQUIRE((*q_sub.read_pointers[0] >> 32) == 1);
msgq_msg_close(&msg1);
}
}
TEST_CASE("msgq_msg_send test invalidation")
{
for (int read_pointer_location = 0; read_pointer_location < 3; read_pointer_location++)
{
cleanup_test_queue();
msgq_queue_t q_pub, q_sub;
msgq_new_queue(&q_pub, "test_queue", 1024);
msgq_new_queue(&q_sub, "test_queue", 1024);
msgq_init_publisher(&q_pub);
msgq_init_subscriber(&q_sub);
*q_sub.write_pointer = (uint64_t)1 << 32;
REQUIRE(*q_sub.read_valids[0] == true);
if (read_pointer_location == 0)
{
*q_sub.read_pointers[0] = 0;
}
else if (read_pointer_location == 1)
{
*q_sub.read_pointers[0] = 64;
}
else
{
*q_pub.write_pointer = ((uint64_t)1 << 32) | 1000; // Writer is one cycle ahead
*q_sub.read_pointers[0] = 1020;
}
msgq_msg_t msg;
msgq_msg_init_size(&msg, 128);
msgq_msg_send(&msg, &q_pub);
REQUIRE(*q_sub.read_valids[0] == false);
msgq_msg_close(&msg);
}
}
TEST_CASE("msgq_init_subscriber init 2 subscribers")
{
cleanup_test_queue();
msgq_queue_t q1, q2;
msgq_new_queue(&q1, "test_queue", 1024);
msgq_new_queue(&q2, "test_queue", 1024);
*q1.num_readers = 0;
REQUIRE(*q1.num_readers == 0);
REQUIRE(*q2.num_readers == 0);
msgq_init_subscriber(&q1);
REQUIRE(*q1.num_readers == 1);
REQUIRE(*q2.num_readers == 1);
REQUIRE(q1.reader_id == 0);
msgq_init_subscriber(&q2);
REQUIRE(*q1.num_readers == 2);
REQUIRE(*q2.num_readers == 2);
REQUIRE(q2.reader_id == 1);
}
TEST_CASE("Write 1 msg, read 1 msg")
{
cleanup_test_queue();
const size_t msg_size = 128;
msgq_queue_t writer, reader;
msgq_new_queue(&writer, "test_queue", 1024);
msgq_new_queue(&reader, "test_queue", 1024);
msgq_init_publisher(&writer);
msgq_init_subscriber(&reader);
// Build 128 byte message
msgq_msg_t outgoing_msg;
msgq_msg_init_size(&outgoing_msg, msg_size);
for (size_t i = 0; i < msg_size; i++)
{
outgoing_msg.data[i] = i;
}
REQUIRE(msgq_msg_send(&outgoing_msg, &writer) == msg_size);
msgq_msg_t incoming_msg1;
REQUIRE(msgq_msg_recv(&incoming_msg1, &reader) == msg_size);
REQUIRE(memcmp(incoming_msg1.data, outgoing_msg.data, msg_size) == 0);
// Verify that there are no more messages
msgq_msg_t incoming_msg2;
REQUIRE(msgq_msg_recv(&incoming_msg2, &reader) == 0);
msgq_msg_close(&outgoing_msg);
msgq_msg_close(&incoming_msg1);
msgq_msg_close(&incoming_msg2);
}
TEST_CASE("Write 2 msg, read 2 msg - conflate = false")
{
cleanup_test_queue();
const size_t msg_size = 128;
msgq_queue_t writer, reader;
msgq_new_queue(&writer, "test_queue", 1024);
msgq_new_queue(&reader, "test_queue", 1024);
msgq_init_publisher(&writer);
msgq_init_subscriber(&reader);
// Build 128 byte message
msgq_msg_t outgoing_msg;
msgq_msg_init_size(&outgoing_msg, msg_size);
for (size_t i = 0; i < msg_size; i++)
{
outgoing_msg.data[i] = i;
}
REQUIRE(msgq_msg_send(&outgoing_msg, &writer) == msg_size);
REQUIRE(msgq_msg_send(&outgoing_msg, &writer) == msg_size);
msgq_msg_t incoming_msg1;
REQUIRE(msgq_msg_recv(&incoming_msg1, &reader) == msg_size);
REQUIRE(memcmp(incoming_msg1.data, outgoing_msg.data, msg_size) == 0);
msgq_msg_t incoming_msg2;
REQUIRE(msgq_msg_recv(&incoming_msg2, &reader) == msg_size);
REQUIRE(memcmp(incoming_msg2.data, outgoing_msg.data, msg_size) == 0);
msgq_msg_close(&outgoing_msg);
msgq_msg_close(&incoming_msg1);
msgq_msg_close(&incoming_msg2);
}
TEST_CASE("Write 2 msg, read 2 msg - conflate = true")
{
cleanup_test_queue();
const size_t msg_size = 128;
msgq_queue_t writer, reader;
msgq_new_queue(&writer, "test_queue", 1024);
msgq_new_queue(&reader, "test_queue", 1024);
msgq_init_publisher(&writer);
msgq_init_subscriber(&reader);
reader.read_conflate = true;
// Build 128 byte message
msgq_msg_t outgoing_msg;
msgq_msg_init_size(&outgoing_msg, msg_size);
for (size_t i = 0; i < msg_size; i++)
{
outgoing_msg.data[i] = i;
}
REQUIRE(msgq_msg_send(&outgoing_msg, &writer) == msg_size);
REQUIRE(msgq_msg_send(&outgoing_msg, &writer) == msg_size);
msgq_msg_t incoming_msg1;
REQUIRE(msgq_msg_recv(&incoming_msg1, &reader) == msg_size);
REQUIRE(memcmp(incoming_msg1.data, outgoing_msg.data, msg_size) == 0);
// Verify that there are no more messages
msgq_msg_t incoming_msg2;
REQUIRE(msgq_msg_recv(&incoming_msg2, &reader) == 0);
msgq_msg_close(&outgoing_msg);
msgq_msg_close(&incoming_msg1);
msgq_msg_close(&incoming_msg2);
}
TEST_CASE("1 publisher, 1 slow subscriber")
{
cleanup_test_queue();
msgq_queue_t writer, reader;
msgq_new_queue(&writer, "test_queue", 1024);
msgq_new_queue(&reader, "test_queue", 1024);
msgq_init_publisher(&writer);
msgq_init_subscriber(&reader);
int n_received = 0;
int n_skipped = 0;
for (uint64_t i = 0; i < 1e5; i++)
{
msgq_msg_t outgoing_msg;
msgq_msg_init_data(&outgoing_msg, (char *)&i, sizeof(uint64_t));
msgq_msg_send(&outgoing_msg, &writer);
msgq_msg_close(&outgoing_msg);
if (i % 10 == 0)
{
msgq_msg_t msg1;
msgq_msg_recv(&msg1, &reader);
if (msg1.size == 0)
{
n_skipped++;
}
else
{
n_received++;
}
msgq_msg_close(&msg1);
}
}
// TODO: verify these numbers by hand
REQUIRE(n_received == 8572);
REQUIRE(n_skipped == 1428);
}
TEST_CASE("1 publisher, 2 subscribers")
{
cleanup_test_queue();
msgq_queue_t writer, reader1, reader2;
msgq_new_queue(&writer, "test_queue", 1024);
msgq_new_queue(&reader1, "test_queue", 1024);
msgq_new_queue(&reader2, "test_queue", 1024);
msgq_init_publisher(&writer);
msgq_init_subscriber(&reader1);
msgq_init_subscriber(&reader2);
for (uint64_t i = 0; i < 1024 * 3; i++)
{
msgq_msg_t outgoing_msg;
msgq_msg_init_data(&outgoing_msg, (char *)&i, sizeof(uint64_t));
msgq_msg_send(&outgoing_msg, &writer);
msgq_msg_t msg1, msg2;
msgq_msg_recv(&msg1, &reader1);
msgq_msg_recv(&msg2, &reader2);
REQUIRE(msg1.size == sizeof(uint64_t));
REQUIRE(msg2.size == sizeof(uint64_t));
REQUIRE(*(uint64_t *)msg1.data == i);
REQUIRE(*(uint64_t *)msg2.data == i);
msgq_msg_close(&outgoing_msg);
msgq_msg_close(&msg1);
msgq_msg_close(&msg2);
}
}
+106
View File
@@ -0,0 +1,106 @@
#pragma once
#include <cstddef>
#include <exception>
#include <iostream>
#include <string>
#include <vector>
namespace test_runner {
struct TestCase {
const char *name;
void (*function)();
};
inline std::vector<TestCase> &tests() {
static std::vector<TestCase> registered_tests;
return registered_tests;
}
struct Registrar {
Registrar(const char *name, void (*function)()) {
tests().push_back({name, function});
}
};
struct TestState {
std::size_t assertion_count = 0;
bool failed = false;
std::string failure_message;
};
inline TestState &state() {
static TestState test_state;
return test_state;
}
inline bool require(bool result, const char *expression, const char *file, int line) {
state().assertion_count++;
if (!result) {
state().failed = true;
state().failure_message = std::string(file) + ":" + std::to_string(line) +
": REQUIRE(" + expression + ") failed";
}
return result;
}
inline int run_all() {
if (tests().empty()) {
std::cerr << "[FAIL] no tests registered\n";
return 1;
}
std::size_t passed = 0;
for (const TestCase &test : tests()) {
state() = {};
bool threw = false;
try {
test.function();
} catch (const std::exception &error) {
threw = true;
std::cerr << "[FAIL] " << test.name << "\n " << error.what() << '\n';
} catch (...) {
threw = true;
std::cerr << "[FAIL] " << test.name << "\n unknown exception\n";
}
if (threw) {
continue;
} else if (state().failed) {
std::cerr << "[FAIL] " << test.name << "\n " << state().failure_message << '\n';
} else if (state().assertion_count == 0) {
std::cerr << "[FAIL] " << test.name << "\n no assertions executed\n";
} else {
++passed;
std::cout << "[PASS] " << test.name << '\n';
}
}
std::cout << '\n' << passed << "/" << tests().size() << " tests passed\n";
return passed == tests().size() ? 0 : 1;
}
} // namespace test_runner
#define TEST_RUNNER_JOIN_IMPL(left, right) left##right
#define TEST_RUNNER_JOIN(left, right) TEST_RUNNER_JOIN_IMPL(left, right)
#define TEST_CASE(name) \
static void TEST_RUNNER_JOIN(test_case_, __LINE__)(); \
static const test_runner::Registrar TEST_RUNNER_JOIN(test_registrar_, \
__LINE__)( \
name, TEST_RUNNER_JOIN(test_case_, __LINE__)); \
static void TEST_RUNNER_JOIN(test_case_, __LINE__)()
#define REQUIRE(expression) \
do { \
if (!test_runner::require(static_cast<bool>(expression), #expression, \
__FILE__, __LINE__)) { \
return; \
} \
} while (false)
int main() {
return test_runner::run_all();
}
View File
+208
View File
@@ -0,0 +1,208 @@
import multiprocessing
import unittest
import msgq
from typing import Optional
WAIT_TIMEOUT = 5
def set_event_run(endpoint, identifier):
handle = msgq.fake_event_handle(endpoint, identifier=identifier, override=False)
handle.recv_called_event.set()
def daemon_repub_process_run(prefix: Optional[str] = None):
# Re-apply fake config in the child. Python 3.14+ defaults to forkserver on
# Linux, which does not inherit setenv() calls from the parent.
msgq.toggle_fake_events(True)
if prefix is not None:
msgq.set_fake_prefix(prefix)
else:
msgq.delete_fake_prefix()
pub_sock = msgq.pub_sock("ubloxGnss")
sub_sock = msgq.sub_sock("carState")
frame = -1
while True:
frame += 1
msg = sub_sock.receive(non_blocking=True)
if msg is None:
print("none received")
continue
bts = frame.to_bytes(8, 'little')
pub_sock.send(bts)
class TestEvents(unittest.TestCase):
def test_mutation(self):
handle = msgq.fake_event_handle("carState")
event = handle.recv_called_event
assert not event.peek()
event.set()
assert event.peek()
event.clear()
assert not event.peek()
del event
def test_wait(self):
handle = msgq.fake_event_handle("carState")
event = handle.recv_called_event
event.set()
try:
event.wait(WAIT_TIMEOUT)
assert event.peek()
except RuntimeError:
self.fail("event.wait() timed out")
def test_wait_multiprocess(self):
handle = msgq.fake_event_handle("carState")
event = handle.recv_called_event
p = multiprocessing.Process(target=set_event_run, args=("carState", ""))
p.start()
try:
event.wait(WAIT_TIMEOUT)
assert event.peek()
except RuntimeError:
self.fail("event.wait() timed out")
finally:
p.kill()
p.join()
def test_wait_zero_timeout(self):
handle = msgq.fake_event_handle("carState")
event = handle.recv_called_event
try:
event.wait(0)
self.fail("event.wait() did not time out")
except RuntimeError:
assert not event.peek()
def test_wait_for_one(self):
handles = [msgq.fake_event_handle(s) for s in ("carState", "controlsState")]
handles[1].recv_called_event.set()
assert msgq.wait_for_one_event([h.recv_called_event for h in handles], WAIT_TIMEOUT) == 1
class FakeSocketsTestBase:
prefix: Optional[str] = None
def setUp(self):
msgq.toggle_fake_events(True)
if self.prefix is not None:
msgq.set_fake_prefix(self.prefix)
else:
msgq.delete_fake_prefix()
def tearDown(self):
msgq.toggle_fake_events(False)
msgq.delete_fake_prefix()
def test_event_handle_init(self):
handle = msgq.fake_event_handle("controlsState", override=True)
assert not handle.enabled
assert handle.recv_called_event.fd >= 0
assert handle.recv_ready_event.fd >= 0
def test_non_managed_socket_state(self):
# non managed socket should have no event state
_ = msgq.pub_sock("ubloxGnss")
handle = msgq.fake_event_handle("ubloxGnss", override=False)
assert not handle.enabled
assert handle.recv_called_event.fd == -1
assert handle.recv_ready_event.fd == -1
def test_managed_socket_state(self):
# managed socket should not change anything about the state
handle = msgq.fake_event_handle("ubloxGnss")
handle.enabled = True
expected_enabled = handle.enabled
expected_recv_called_fd = handle.recv_called_event.fd
expected_recv_ready_fd = handle.recv_ready_event.fd
_ = msgq.pub_sock("ubloxGnss")
assert handle.enabled == expected_enabled
assert handle.recv_called_event.fd == expected_recv_called_fd
assert handle.recv_ready_event.fd == expected_recv_ready_fd
def test_sockets_enable_disable(self):
carState_handle = msgq.fake_event_handle("ubloxGnss", enable=True)
recv_called = carState_handle.recv_called_event
recv_ready = carState_handle.recv_ready_event
pub_sock = msgq.pub_sock("ubloxGnss")
sub_sock = msgq.sub_sock("ubloxGnss")
try:
carState_handle.enabled = True
recv_ready.set()
pub_sock.send(b"test")
_ = sub_sock.receive()
assert recv_called.peek()
recv_called.clear()
carState_handle.enabled = False
recv_ready.set()
pub_sock.send(b"test")
_ = sub_sock.receive()
assert not recv_called.peek()
except RuntimeError:
raise AssertionError("event.wait() timed out")
def test_synced_pub_sub(self):
carState_handle = msgq.fake_event_handle("carState", enable=True)
recv_called = carState_handle.recv_called_event
recv_ready = carState_handle.recv_ready_event
pub_sock = msgq.pub_sock("carState")
p = multiprocessing.Process(target=daemon_repub_process_run, args=(self.prefix,))
p.start()
sub_sock = msgq.sub_sock("ubloxGnss")
try:
for i in range(10):
recv_called.wait(WAIT_TIMEOUT)
recv_called.clear()
if i == 0:
sub_sock.receive(non_blocking=True)
bts = i.to_bytes(8, 'little')
pub_sock.send(bts)
recv_ready.set()
recv_called.wait(WAIT_TIMEOUT)
msg = sub_sock.receive(non_blocking=True)
assert msg is not None
assert len(msg) == 8
frame = int.from_bytes(msg, 'little')
assert frame == i
except RuntimeError:
raise AssertionError("event.wait() timed out")
finally:
p.kill()
p.join()
class TestFakeSockets(FakeSocketsTestBase, unittest.TestCase):
pass
class TestFakeSocketsWithPrefix(FakeSocketsTestBase, unittest.TestCase):
prefix = "test"
+57
View File
@@ -0,0 +1,57 @@
import random
import time
import string
import unittest
import msgq
def random_sock():
return ''.join(random.choices(string.ascii_uppercase + string.digits, k=10))
def random_bytes(length=1000):
return bytes([random.randrange(0xFF) for _ in range(length)])
class TestPubSubSockets(unittest.TestCase):
def test_pub_sub(self):
sock = random_sock()
pub_sock = msgq.pub_sock(sock)
sub_sock = msgq.sub_sock(sock, conflate=False, timeout=None)
for _ in range(100):
msg = random_bytes()
pub_sock.send(msg)
recvd = sub_sock.receive()
assert msg == recvd
def test_conflate(self):
sock = random_sock()
pub_sock = msgq.pub_sock(sock)
for conflate in [True, False]:
num_msgs = random.randint(3, 10)
sub_sock = msgq.sub_sock(sock, conflate=conflate, timeout=None)
sent_msgs = []
for __ in range(num_msgs):
msg = random_bytes()
pub_sock.send(msg)
sent_msgs.append(msg)
recvd_msgs = msgq.drain_sock_raw(sub_sock)
if conflate:
assert len(recvd_msgs) == 1
assert recvd_msgs[0] == sent_msgs[-1]
else:
assert len(recvd_msgs) == len(sent_msgs)
for rec_msg, sent_msg in zip(recvd_msgs, sent_msgs):
assert rec_msg == sent_msg
def test_receive_timeout(self):
sock = random_sock()
timeout_ms = 5
sub_sock = msgq.sub_sock(sock, timeout=timeout_ms)
start_time = time.monotonic()
recvd = sub_sock.receive()
elapsed = time.monotonic() - start_time
assert recvd is None
assert elapsed >= timeout_ms / 1000
assert elapsed < 5 # this can be noisy due to other load on the system
+135
View File
@@ -0,0 +1,135 @@
import unittest
import msgq
import concurrent.futures
SERVICE_NAME = 'myService'
def poller():
context = msgq.Context()
p = msgq.Poller()
sub = msgq.SubSocket()
sub.connect(context, SERVICE_NAME)
p.registerSocket(sub)
socks = p.poll(10000)
r = [s.receive(non_blocking=True) for s in socks]
return r
class TestPoller(unittest.TestCase):
def test_poll_once(self):
context = msgq.Context()
pub = msgq.PubSocket()
pub.connect(context, SERVICE_NAME)
with concurrent.futures.ThreadPoolExecutor() as e:
poll = e.submit(poller)
pub.wait_for_readers()
# Send message
pub.send(b"a")
# Wait for poll result
result = poll.result()
del pub
context.term()
assert result == [b"a"]
def test_poll_and_create_many_subscribers(self):
context = msgq.Context()
pub = msgq.PubSocket()
pub.connect(context, SERVICE_NAME)
with concurrent.futures.ThreadPoolExecutor() as e:
poll = e.submit(poller)
pub.wait_for_readers()
c = msgq.Context()
for _ in range(10):
msgq.SubSocket().connect(c, SERVICE_NAME)
# Send message
pub.send(b"a")
# Wait for poll result
result = poll.result()
del pub
context.term()
assert result == [b"a"]
def test_multiple_publishers_exception(self):
context = msgq.Context()
with self.assertRaises(msgq.MultiplePublishersError):
pub1 = msgq.PubSocket()
pub1.connect(context, SERVICE_NAME)
pub2 = msgq.PubSocket()
pub2.connect(context, SERVICE_NAME)
pub1.send(b"a")
del pub1
del pub2
context.term()
def test_multiple_messages(self):
context = msgq.Context()
pub = msgq.PubSocket()
pub.connect(context, SERVICE_NAME)
sub = msgq.SubSocket()
sub.connect(context, SERVICE_NAME)
pub.wait_for_readers()
for i in range(1, 100):
pub.send(b'a'*i)
msg_seen = False
i = 1
while True:
r = sub.receive(non_blocking=True)
if r is not None:
assert b'a'*i == r
msg_seen = True
i += 1
if r is None and msg_seen:
break
del pub
del sub
context.term()
def test_conflate(self):
context = msgq.Context()
pub = msgq.PubSocket()
pub.connect(context, SERVICE_NAME)
sub = msgq.SubSocket()
sub.connect(context, SERVICE_NAME, conflate=True)
pub.wait_for_readers()
pub.send(b'a')
pub.send(b'b')
assert b'b' == sub.receive()
del pub
del sub
context.term()
+2
View File
@@ -0,0 +1,2 @@
visionipc_pyx.cpp
*.so
+8
View File
@@ -0,0 +1,8 @@
from msgq.visionipc.visionipc_pyx import VisionBuf, VisionIpcClient, VisionIpcServer, get_endpoint_name
__all__ = [
"VisionBuf",
"VisionIpcClient",
"VisionIpcServer",
"get_endpoint_name",
]
@@ -0,0 +1,103 @@
import struct
import unittest
from typing import Optional
from msgq.visionipc import VisionIpcServer, VisionIpcClient
class TestVisionIpc(unittest.TestCase):
server: Optional[VisionIpcServer]
client: Optional[VisionIpcClient]
def setUp(self):
self.server = None
self.client = None
def tearDown(self):
self.client = None
self.server = None
def setup_vipc(self, name, *stream_types, num_buffers=1, width=100, height=100, conflate=False):
self.server = VisionIpcServer(name)
for stream_type in stream_types:
self.server.create_buffers(stream_type, num_buffers, width, height)
self.server.start_listener()
if len(stream_types):
self.client = VisionIpcClient(name, stream_types[0], conflate)
assert self.client.connect(True)
else:
self.client = None
return self.server, self.client
def test_connect(self):
self.setup_vipc("camerad", 0)
assert self.client is not None
assert self.client.is_connected()
def test_available_streams(self):
stream_types = (0, 2)
self.setup_vipc("camerad", *stream_types)
available_streams = VisionIpcClient.available_streams("camerad", True)
assert available_streams == set(stream_types)
def test_buffers(self):
width, height, num_buffers = 100, 200, 5
self.setup_vipc("camerad", 0, num_buffers=num_buffers, width=width, height=height)
assert self.client is not None
assert self.client.width == width
assert self.client.height == height
assert self.client.buffer_len is not None and self.client.buffer_len > 0
assert self.client.num_buffers == num_buffers
def test_send_single_buffer(self):
self.setup_vipc("camerad", 0)
assert self.server is not None
assert self.client is not None
assert self.client.buffer_len is not None
buf = bytearray(self.client.buffer_len)
struct.pack_into("<Q", buf, 0, 1234)
self.server.send(0, buf, frame_id=1337)
recv_buf = self.client.recv()
assert recv_buf is not None
data = recv_buf.data
assert isinstance(data, memoryview)
assert struct.unpack_from("<Q", data, 0)[0] == 1234
assert len(data) == self.client.buffer_len
assert data[8:].nbytes == self.client.buffer_len - 8
assert self.client.frame_id == 1337
assert recv_buf.frame_id == 1337
def test_no_conflate(self):
self.setup_vipc("camerad", 0)
assert self.server is not None
assert self.client is not None
assert self.client.buffer_len is not None
buf = bytearray(self.client.buffer_len)
self.server.send(0, buf, frame_id=1)
self.server.send(0, buf, frame_id=2)
recv_buf = self.client.recv()
assert recv_buf is not None
assert self.client.frame_id == 1
recv_buf = self.client.recv()
assert recv_buf is not None
assert self.client.frame_id == 2
def test_conflate(self):
self.setup_vipc("camerad", 0, conflate=True)
assert self.server is not None
assert self.client is not None
assert self.client.buffer_len is not None
buf = bytearray(self.client.buffer_len)
self.server.send(0, buf, frame_id=1)
self.server.send(0, buf, frame_id=2)
recv_buf = self.client.recv()
assert recv_buf is not None
assert self.client.frame_id == 2
recv_buf = self.client.recv(timeout_ms=5)
assert recv_buf is None
+79
View File
@@ -0,0 +1,79 @@
#include "msgq/visionipc/visionbuf.h"
#include <atomic>
#include <stdio.h>
#include <fcntl.h>
#include <assert.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/types.h>
std::atomic<int> offset = 0;
static void *malloc_with_fd(size_t len, int *fd) {
char full_path[0x100];
#ifdef __APPLE__
snprintf(full_path, sizeof(full_path)-1, "/tmp/visionbuf_%d_%d", getpid(), offset++);
#else
snprintf(full_path, sizeof(full_path)-1, "/dev/shm/msgq_visionbuf_%d_%d", getpid(), offset++);
#endif
*fd = open(full_path, O_RDWR | O_CREAT, 0664);
assert(*fd >= 0);
unlink(full_path);
int ret = ftruncate(*fd, len);
assert(ret == 0);
void *addr = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, *fd, 0);
assert(addr != MAP_FAILED);
return addr;
}
void VisionBuf::allocate(size_t length) {
this->len = length;
this->mmap_len = this->len + sizeof(uint64_t);
this->addr = malloc_with_fd(this->mmap_len, &this->fd);
this->frame_id = (uint64_t*)((uint8_t*)this->addr + this->len);
}
void VisionBuf::import(){
assert(this->fd >= 0);
this->addr = mmap(NULL, this->mmap_len, PROT_READ | PROT_WRITE, MAP_SHARED, this->fd, 0);
assert(this->addr != MAP_FAILED);
this->frame_id = (uint64_t*)((uint8_t*)this->addr + this->len);
}
void VisionBuf::init_yuv(size_t init_width, size_t init_height, size_t init_stride, size_t init_uv_offset){
this->width = init_width;
this->height = init_height;
this->stride = init_stride;
this->uv_offset = init_uv_offset;
this->y = (uint8_t *)this->addr;
this->uv = this->y + this->uv_offset;
}
int VisionBuf::sync(int dir) {
return 0;
}
int VisionBuf::free() {
int err = munmap(this->addr, this->mmap_len);
if (err != 0) return err;
err = close(this->fd);
return err;
}
uint64_t VisionBuf::get_frame_id() {
return *frame_id;
}
void VisionBuf::set_frame_id(uint64_t id) {
*frame_id = id;
}
+48
View File
@@ -0,0 +1,48 @@
#pragma once
#include "msgq/visionipc/visionipc.h"
#define VISIONBUF_SYNC_FROM_DEVICE 0
#define VISIONBUF_SYNC_TO_DEVICE 1
// Stream ids are opaque to visionipc. Producers/consumers pick an id when
// creating a stream; the sentinel below is reserved to request the list of
// available streams from a server.
typedef uint32_t VisionStreamType;
constexpr VisionStreamType VISION_STREAM_LIST = 0xffffffff;
constexpr size_t VISIONIPC_MAX_STREAMS = 64;
class VisionBuf {
public:
size_t len = 0;
size_t mmap_len = 0;
void * addr = nullptr;
uint64_t *frame_id;
int fd = 0;
size_t width = 0;
size_t height = 0;
size_t stride = 0;
size_t uv_offset = 0;
// YUV
uint8_t * y = nullptr;
uint8_t * uv = nullptr;
// Visionipc
uint64_t server_id = 0;
size_t idx = 0;
VisionStreamType type;
// ion
int handle = 0;
void allocate(size_t len);
void import();
void init_yuv(size_t width, size_t height, size_t stride, size_t uv_offset);
int sync(int dir);
int free();
void set_frame_id(uint64_t id);
uint64_t get_frame_id();
};
+140
View File
@@ -0,0 +1,140 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <errno.h>
#include <sys/mman.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <linux/ion.h>
#include <linux/msm_ion.h>
#include "msgq/visionipc/visionbuf.h"
// keep trying if x gets interrupted by a signal
#define HANDLE_EINTR(x) \
({ \
decltype(x) ret; \
int try_cnt = 0; \
do { \
ret = (x); \
} while (ret == -1 && errno == EINTR && try_cnt++ < 100); \
ret; \
})
struct IonFileHandle {
IonFileHandle() {
fd = open("/dev/ion", O_RDWR | O_NONBLOCK);
assert(fd >= 0);
}
~IonFileHandle() {
close(fd);
}
int fd = -1;
};
int ion_fd() {
static IonFileHandle fh;
return fh.fd;
}
void VisionBuf::allocate(size_t length) {
struct ion_allocation_data ion_alloc = {0};
ion_alloc.len = length + sizeof(uint64_t);
ion_alloc.align = 4096;
ion_alloc.heap_id_mask = 1 << ION_IOMMU_HEAP_ID;
ion_alloc.flags = ION_FLAG_CACHED;
int err = HANDLE_EINTR(ioctl(ion_fd(), ION_IOC_ALLOC, &ion_alloc));
assert(err == 0);
struct ion_fd_data ion_fd_data = {0};
ion_fd_data.handle = ion_alloc.handle;
err = HANDLE_EINTR(ioctl(ion_fd(), ION_IOC_SHARE, &ion_fd_data));
assert(err == 0);
void *mmap_addr = mmap(NULL, ion_alloc.len,
PROT_READ | PROT_WRITE,
MAP_SHARED, ion_fd_data.fd, 0);
assert(mmap_addr != MAP_FAILED);
memset(mmap_addr, 0, ion_alloc.len);
this->len = length;
this->mmap_len = ion_alloc.len;
this->addr = mmap_addr;
this->handle = ion_alloc.handle;
this->fd = ion_fd_data.fd;
this->frame_id = (uint64_t*)((uint8_t*)this->addr + this->len);
}
void VisionBuf::import(){
int err;
assert(this->fd >= 0);
// Get handle
struct ion_fd_data fd_data = {0};
fd_data.fd = this->fd;
err = HANDLE_EINTR(ioctl(ion_fd(), ION_IOC_IMPORT, &fd_data));
assert(err == 0);
this->handle = fd_data.handle;
this->addr = mmap(NULL, this->mmap_len, PROT_READ | PROT_WRITE, MAP_SHARED, this->fd, 0);
assert(this->addr != MAP_FAILED);
this->frame_id = (uint64_t*)((uint8_t*)this->addr + this->len);
}
void VisionBuf::init_yuv(size_t init_width, size_t init_height, size_t init_stride, size_t init_uv_offset){
this->width = init_width;
this->height = init_height;
this->stride = init_stride;
this->uv_offset = init_uv_offset;
this->y = (uint8_t *)this->addr;
this->uv = this->y + this->uv_offset;
}
int VisionBuf::sync(int dir) {
struct ion_flush_data flush_data = {0};
flush_data.handle = this->handle;
flush_data.vaddr = this->addr;
flush_data.offset = 0;
flush_data.length = this->len;
// ION_IOC_INV_CACHES ~= DMA_FROM_DEVICE
// ION_IOC_CLEAN_CACHES ~= DMA_TO_DEVICE
// ION_IOC_CLEAN_INV_CACHES ~= DMA_BIDIRECTIONAL
struct ion_custom_data custom_data = {0};
assert(dir == VISIONBUF_SYNC_FROM_DEVICE || dir == VISIONBUF_SYNC_TO_DEVICE);
custom_data.cmd = (dir == VISIONBUF_SYNC_FROM_DEVICE) ?
ION_IOC_INV_CACHES : ION_IOC_CLEAN_CACHES;
custom_data.arg = (unsigned long)&flush_data;
return HANDLE_EINTR(ioctl(ion_fd(), ION_IOC_CUSTOM, &custom_data));
}
int VisionBuf::free() {
int err = munmap(this->addr, this->mmap_len);
if (err != 0) return err;
err = close(this->fd);
if (err != 0) return err;
struct ion_handle_data handle_data = {.handle = this->handle};
return HANDLE_EINTR(ioctl(ion_fd(), ION_IOC_FREE, &handle_data));
}
uint64_t VisionBuf::get_frame_id() {
return *frame_id;
}
void VisionBuf::set_frame_id(uint64_t id) {
*frame_id = id;
}
+121
View File
@@ -0,0 +1,121 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <assert.h>
#include <errno.h>
#include <sys/mman.h>
#include <sys/socket.h>
#include <sys/un.h>
#ifdef __APPLE__
#define getsocket() socket(AF_UNIX, SOCK_STREAM, 0)
#else
#define getsocket() socket(AF_UNIX, SOCK_SEQPACKET, 0)
#endif
#include "msgq/visionipc/visionipc.h"
int ipc_connect(const char* socket_path) {
int err;
int sock = getsocket();
if (sock < 0) return -1;
struct sockaddr_un addr = {
.sun_family = AF_UNIX,
};
snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", socket_path);
err = connect(sock, (struct sockaddr*)&addr, sizeof(addr));
if (err != 0) {
close(sock);
return -1;
}
return sock;
}
int ipc_bind(const char* socket_path) {
int err;
unlink(socket_path);
int sock = getsocket();
struct sockaddr_un addr = {
.sun_family = AF_UNIX,
};
snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", socket_path);
err = bind(sock, (struct sockaddr *)&addr, sizeof(addr));
assert(err == 0);
err = listen(sock, 3);
assert(err == 0);
return sock;
}
int ipc_sendrecv_with_fds(bool send, int fd, void *buf, size_t buf_size, int* fds, int num_fds,
int *out_num_fds) {
char control_buf[CMSG_SPACE(sizeof(int) * num_fds)];
memset(control_buf, 0, CMSG_SPACE(sizeof(int) * num_fds));
struct iovec iov = {
.iov_base = buf,
.iov_len = buf_size,
};
struct msghdr msg = {
.msg_iov = &iov,
.msg_iovlen = 1,
};
if (num_fds > 0) {
assert(fds);
msg.msg_control = control_buf;
msg.msg_controllen = CMSG_SPACE(sizeof(int) * num_fds);
}
if (send) {
if (num_fds) {
struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
assert(cmsg);
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_RIGHTS;
cmsg->cmsg_len = CMSG_LEN(sizeof(int) * num_fds);
memcpy(CMSG_DATA(cmsg), fds, sizeof(int) * num_fds);
}
return sendmsg(fd, &msg, 0);
} else {
int r = recvmsg(fd, &msg, 0);
if (r < 0) return r;
int recv_fds = 0;
if (msg.msg_controllen > 0) {
struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
assert(cmsg);
assert(cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS);
recv_fds = (cmsg->cmsg_len - CMSG_LEN(0));
assert(recv_fds > 0 && (recv_fds % sizeof(int)) == 0);
recv_fds /= sizeof(int);
assert(fds && recv_fds <= num_fds);
memcpy(fds, CMSG_DATA(cmsg), sizeof(int) * recv_fds);
}
if (msg.msg_flags) {
for (int i=0; i<recv_fds; i++) {
close(fds[i]);
}
return -1;
}
if (fds) {
assert(out_num_fds);
*out_num_fds = recv_fds;
}
return r;
}
}
+25
View File
@@ -0,0 +1,25 @@
#pragma once
#include <cstdint>
#include <cstddef>
int ipc_connect(const char* socket_path);
int ipc_bind(const char* socket_path);
int ipc_sendrecv_with_fds(bool send, int fd, void *buf, size_t buf_size, int* fds, int num_fds,
int *out_num_fds);
constexpr int VISIONIPC_MAX_FDS = 128;
struct VisionIpcBufExtra {
uint32_t frame_id;
uint64_t timestamp_sof;
uint64_t timestamp_eof;
bool valid;
};
struct VisionIpcPacket {
uint64_t server_id;
size_t idx;
struct VisionIpcBufExtra extra;
};
+52
View File
@@ -0,0 +1,52 @@
# distutils: language = c++
#cython: language_level=3
from libcpp.string cimport string
from libcpp.vector cimport vector
from libcpp.set cimport set
from libc.stdint cimport uint32_t, uint64_t
from libcpp cimport bool, int
cdef extern from "msgq/visionipc/visionbuf.h":
ctypedef uint32_t VisionStreamType "VisionStreamType"
cdef cppclass VisionBuf:
void * addr
int fd
size_t len
size_t width
size_t height
size_t stride
size_t uv_offset
size_t idx
void set_frame_id(uint64_t id)
uint64_t get_frame_id()
cdef extern from "msgq/visionipc/visionipc.h":
struct VisionIpcBufExtra:
uint32_t frame_id
uint64_t timestamp_sof
uint64_t timestamp_eof
bool valid
cdef extern from "msgq/visionipc/visionipc_server.h":
string get_endpoint_name(string, VisionStreamType)
cdef cppclass VisionIpcServer:
VisionIpcServer(string)
void create_buffers(VisionStreamType, size_t, size_t, size_t)
void create_buffers_with_sizes(VisionStreamType, size_t, size_t, size_t, size_t, size_t, size_t)
VisionBuf * get_buffer(VisionStreamType)
void send(VisionBuf *, VisionIpcBufExtra *, bool)
void start_listener()
cdef extern from "msgq/visionipc/visionipc_client.h":
cdef cppclass VisionIpcClient:
int num_buffers
VisionBuf buffers[1]
VisionIpcClient(string, VisionStreamType, bool)
VisionBuf * recv(VisionIpcBufExtra *, int) nogil
bool connect(bool)
bool is_connected()
@staticmethod
set[VisionStreamType] getAvailableStreams(string, bool)
@@ -0,0 +1,162 @@
#include <chrono>
#include <cassert>
#include <iostream>
#include <thread>
#include <string>
#include <set>
#include <unistd.h>
#include "msgq/visionipc/visionipc.h"
#include "msgq/visionipc/visionipc_client.h"
#include "msgq/visionipc/visionipc_server.h"
#include "msgq/logger/logger.h"
static int connect_to_vipc_server(const std::string &name, bool blocking) {
const std::string ipc_path = get_ipc_path(name);
int socket_fd = ipc_connect(ipc_path.c_str());
bool logged_retry = false;
while (socket_fd < 0 && blocking) {
if (!logged_retry) {
std::cout << "VisionIpcClient connecting" << std::endl;
logged_retry = true;
}
std::this_thread::sleep_for(std::chrono::milliseconds(20));
socket_fd = ipc_connect(ipc_path.c_str());
}
return socket_fd;
}
VisionIpcClient::VisionIpcClient(std::string name_, VisionStreamType type_, bool conflate) : name(name_), type(type_) {
msg_ctx = Context::create();
sock = SubSocket::create(msg_ctx, get_endpoint_name(name, type), "127.0.0.1", conflate, false);
poller = Poller::create();
poller->registerSocket(sock);
}
// Connect is not thread safe. Do not use the buffers while calling connect
bool VisionIpcClient::connect(bool blocking) {
connected = false;
// Cleanup old buffers on reconnect
for (size_t i = 0; i < num_buffers; i++) {
if (buffers[i].free() != 0) {
LOGE("Failed to free buffer %zu", i);
}
}
num_buffers = 0;
int socket_fd = connect_to_vipc_server(name, blocking);
if (socket_fd < 0) {
return false;
}
// Send stream type to server to request FDs
int r = ipc_sendrecv_with_fds(true, socket_fd, &type, sizeof(type), nullptr, 0, nullptr);
assert(r == sizeof(type));
// Get FDs
int fds[VISIONIPC_MAX_FDS] = {};
VisionBuf bufs[VISIONIPC_MAX_FDS] = {};
r = ipc_sendrecv_with_fds(false, socket_fd, &bufs, sizeof(bufs), fds, VISIONIPC_MAX_FDS, &num_buffers);
if (r < 0) {
// only expected error is server shutting down
assert(errno == ECONNRESET);
close(socket_fd);
return false;
}
assert(num_buffers >= 0);
assert(r == sizeof(VisionBuf) * num_buffers);
// Import buffers
for (size_t i = 0; i < num_buffers; i++) {
buffers[i] = bufs[i];
buffers[i].fd = fds[i];
buffers[i].import();
buffers[i].init_yuv(buffers[i].width, buffers[i].height, buffers[i].stride, buffers[i].uv_offset);
}
close(socket_fd);
connected = true;
return true;
}
VisionBuf * VisionIpcClient::recv(VisionIpcBufExtra * extra, const int timeout_ms) {
auto p = poller->poll(timeout_ms);
if (!p.size()) {
return nullptr;
}
Message * r = sock->receive(true);
if (r == nullptr) {
return nullptr;
}
// Get buffer
assert(r->getSize() == sizeof(VisionIpcPacket));
VisionIpcPacket *packet = (VisionIpcPacket*)r->getData();
// Check if packet index is out of bounds, indicating server has changed
if (packet->idx >= num_buffers) {
connected = false;
delete r;
return nullptr;
}
VisionBuf * buf = &buffers[packet->idx];
if (buf->server_id != packet->server_id) {
connected = false;
delete r;
return nullptr;
}
if (extra) {
*extra = packet->extra;
}
if (buf->sync(VISIONBUF_SYNC_TO_DEVICE) != 0) {
LOGE("Failed to sync buffer");
}
delete r;
return buf;
}
std::set<VisionStreamType> VisionIpcClient::getAvailableStreams(const std::string &name, bool blocking) {
int socket_fd = connect_to_vipc_server(name, blocking);
if (socket_fd < 0) {
return {};
}
// Send VISION_STREAM_LIST to server to request available streams
VisionStreamType request = VISION_STREAM_LIST;
int r = ipc_sendrecv_with_fds(true, socket_fd, &request, sizeof(request), nullptr, 0, nullptr);
assert(r == sizeof(request));
VisionStreamType available_streams[VISIONIPC_MAX_STREAMS] = {};
r = ipc_sendrecv_with_fds(false, socket_fd, &available_streams, sizeof(available_streams), nullptr, 0, nullptr);
if (r < 0) {
// only expected error is server shutting down
assert(errno == ECONNRESET);
close(socket_fd);
return {};
}
assert(r % sizeof(VisionStreamType) == 0);
close(socket_fd);
return std::set<VisionStreamType>(available_streams, available_streams + r / sizeof(VisionStreamType));
}
VisionIpcClient::~VisionIpcClient() {
for (size_t i = 0; i < num_buffers; i++) {
if (buffers[i].free() != 0) {
LOGE("Failed to free buffer %zu", i);
}
}
delete sock;
delete poller;
delete msg_ctx;
}
@@ -0,0 +1,28 @@
#pragma once
#include <set>
#include <string>
#include "msgq/ipc.h"
#include "msgq/visionipc/visionbuf.h"
class VisionIpcClient {
private:
std::string name;
Context * msg_ctx;
SubSocket * sock;
Poller * poller;
public:
bool connected = false;
VisionStreamType type;
int num_buffers = 0;
VisionBuf buffers[VISIONIPC_MAX_FDS];
VisionIpcClient(std::string name, VisionStreamType type, bool conflate);
~VisionIpcClient();
VisionBuf * recv(VisionIpcBufExtra * extra=nullptr, const int timeout_ms=100);
bool connect(bool blocking=true);
bool is_connected() { return connected; }
static std::set<VisionStreamType> getAvailableStreams(const std::string &name, bool blocking = true);
};
@@ -0,0 +1,10 @@
# distutils: language = c++
#cython: language_level=3
from .visionipc cimport VisionBuf as cppVisionBuf
cdef class VisionBuf:
cdef cppVisionBuf * buf
@staticmethod
cdef create(cppVisionBuf*)
+163
View File
@@ -0,0 +1,163 @@
# distutils: language = c++
# cython: c_string_encoding=ascii, language_level=3
from libc.string cimport memcpy
from libc.stdint cimport uint32_t, uint64_t
from libcpp cimport bool
from libcpp.string cimport string
from .visionipc cimport VisionIpcServer as cppVisionIpcServer
from .visionipc cimport VisionIpcClient as cppVisionIpcClient
from .visionipc cimport VisionBuf as cppVisionBuf
from .visionipc cimport VisionIpcBufExtra, VisionStreamType
from .visionipc cimport get_endpoint_name as cpp_get_endpoint_name
def get_endpoint_name(string name, VisionStreamType stream):
return cpp_get_endpoint_name(name, stream).decode('utf-8')
cdef class VisionBuf:
@staticmethod
cdef create(cppVisionBuf * cbuf):
buf = VisionBuf()
buf.buf = cbuf
return buf
@property
def data(self):
cdef unsigned char[:] data = <unsigned char[:self.buf.len]> self.buf.addr
return memoryview(data)
@property
def width(self):
return self.buf.width
@property
def height(self):
return self.buf.height
@property
def stride(self):
return self.buf.stride
@property
def uv_offset(self):
return self.buf.uv_offset
@property
def idx(self):
return self.buf.idx
@property
def fd(self):
return self.buf.fd
@property
def frame_id(self):
return self.buf.get_frame_id()
cdef class VisionIpcServer:
cdef cppVisionIpcServer * server
def __init__(self, string name):
self.server = new cppVisionIpcServer(name)
def create_buffers(self, VisionStreamType tp, size_t num_buffers, size_t width, size_t height):
self.server.create_buffers(tp, num_buffers, width, height)
def create_buffers_with_sizes(self, VisionStreamType tp, size_t num_buffers, size_t width, size_t height, size_t size, size_t stride, size_t uv_offset):
self.server.create_buffers_with_sizes(tp, num_buffers, width, height, size, stride, uv_offset)
def send(self, VisionStreamType tp, const unsigned char[:] data, uint32_t frame_id=0, uint64_t timestamp_sof=0, uint64_t timestamp_eof=0):
cdef cppVisionBuf * buf = self.server.get_buffer(tp)
# Populate buffer
assert buf.len == len(data)
memcpy(buf.addr, &data[0], len(data))
buf.set_frame_id(frame_id)
cdef VisionIpcBufExtra extra
extra.frame_id = frame_id
extra.timestamp_sof = timestamp_sof
extra.timestamp_eof = timestamp_eof
extra.valid = False
self.server.send(buf, &extra, False)
def start_listener(self):
self.server.start_listener()
def __dealloc__(self):
del self.server
cdef class VisionIpcClient:
cdef cppVisionIpcClient * client
cdef VisionIpcBufExtra extra
def __cinit__(self, string name, VisionStreamType stream, bool conflate):
self.client = new cppVisionIpcClient(name, stream, conflate)
def __dealloc__(self):
del self.client
@property
def width(self):
return self.client.buffers[0].width if self.client.num_buffers else None
@property
def height(self):
return self.client.buffers[0].height if self.client.num_buffers else None
@property
def stride(self):
return self.client.buffers[0].stride if self.client.num_buffers else None
@property
def uv_offset(self):
return self.client.buffers[0].uv_offset if self.client.num_buffers else None
@property
def buffer_len(self):
return self.client.buffers[0].len if self.client.num_buffers else None
@property
def num_buffers(self):
return self.client.num_buffers
@property
def frame_id(self):
return self.extra.frame_id
@property
def timestamp_sof(self):
return self.extra.timestamp_sof
@property
def timestamp_eof(self):
return self.extra.timestamp_eof
@property
def valid(self):
return self.extra.valid
def recv(self, int timeout_ms=100):
cdef cppVisionBuf * buf
# release the GIL, this can block for timeout_ms
with nogil:
buf = self.client.recv(&self.extra, timeout_ms)
if not buf:
return None
return VisionBuf.create(buf)
def connect(self, bool blocking):
return self.client.connect(blocking)
def is_connected(self):
return self.client.is_connected()
@staticmethod
def available_streams(string name, bool block):
return cppVisionIpcClient.getAvailableStreams(name, block)
@@ -0,0 +1,210 @@
#include <iostream>
#include <chrono>
#include <cassert>
#include <random>
#include <limits>
#include <string>
#include <vector>
#include <poll.h>
#include <sys/socket.h>
#include <unistd.h>
#include "msgq/visionipc/visionipc.h"
#include "msgq/visionipc/visionipc_server.h"
#include "msgq/logger/logger.h"
std::string get_endpoint_name(std::string name, VisionStreamType type){
return "visionipc_" + name + "_" + std::to_string(type);
}
std::string get_ipc_path(const std::string& name) {
std::string path = "/tmp/";
if (char* prefix = std::getenv("OPENPILOT_PREFIX")) {
path += std::string(prefix) + "_";
}
return path + "visionipc_" + name;
}
VisionIpcServer::VisionIpcServer(std::string name_) : name(name_) {
msg_ctx = Context::create();
std::random_device rd("/dev/urandom");
std::uniform_int_distribution<uint64_t> distribution(0, std::numeric_limits<uint64_t>::max());
server_id = distribution(rd);
}
void VisionIpcServer::create_buffers(VisionStreamType type, size_t num_buffers, size_t width, size_t height){
// TODO: assert that this type is not created yet
assert(num_buffers < VISIONIPC_MAX_FDS);
size_t size = width * height * 3 / 2;
size_t stride = width;
size_t uv_offset = width * height;
create_buffers_with_sizes(type, num_buffers, width, height, size, stride, uv_offset);
}
void VisionIpcServer::create_buffers_with_sizes(VisionStreamType type, size_t num_buffers, size_t width, size_t height, size_t size, size_t stride, size_t uv_offset) {
// Create map + alloc requested buffers
for (size_t i = 0; i < num_buffers; i++){
VisionBuf* buf = new VisionBuf();
buf->allocate(size);
buf->idx = i;
buf->type = type;
buf->init_yuv(width, height, stride, uv_offset);
buffers[type].push_back(buf);
}
cur_idx[type] = 0;
// Create msgq publisher for each of the `name` + type combos
sockets[type] = PubSocket::create(msg_ctx, get_endpoint_name(name, type), false);
}
void VisionIpcServer::start_listener(){
listener_thread = std::thread(&VisionIpcServer::listener, this);
}
void VisionIpcServer::listener(){
LOGD("Starting listener for: %s", name.c_str());
const std::string ipc_path = get_ipc_path(name);
int sock = ipc_bind(ipc_path.c_str());
assert(sock >= 0);
while (!should_exit){
// Wait for incoming connection
struct pollfd polls[1] = {{0}};
polls[0].fd = sock;
polls[0].events = POLLIN;
int ret = poll(polls, 1, 100);
if (ret < 0) {
if (errno == EINTR || errno == EAGAIN) continue;
std::cout << "poll failed, stopping listener" << std::endl;
break;
}
if (should_exit) break;
if (!polls[0].revents) {
continue;
}
// Handle incoming request
int fd = accept(sock, NULL, NULL);
assert(fd >= 0);
VisionStreamType type = VISION_STREAM_LIST;
int r = ipc_sendrecv_with_fds(false, fd, &type, sizeof(type), nullptr, 0, nullptr);
if (r != sizeof(type)) {
close(fd);
if (should_exit) break;
continue;
}
// send available stream types
if (type == VISION_STREAM_LIST) {
std::vector<VisionStreamType> available_stream_types;
for (auto& [stream_type, _] : buffers) {
available_stream_types.push_back(stream_type);
}
r = ipc_sendrecv_with_fds(true, fd, available_stream_types.data(), available_stream_types.size() * sizeof(VisionStreamType), nullptr, 0, nullptr);
assert(r == available_stream_types.size() * sizeof(VisionStreamType));
close(fd);
continue;
}
if (buffers.count(type) <= 0) {
std::cout << "got request for invalid buffer type: " << type << std::endl;
close(fd);
continue;
}
int fds[VISIONIPC_MAX_FDS] = {};
int num_fds = buffers[type].size();
VisionBuf bufs[VISIONIPC_MAX_FDS] = {};
for (int i = 0; i < num_fds; i++){
fds[i] = buffers[type][i]->fd;
bufs[i] = *buffers[type][i];
// Remove some private ion metadata
bufs[i].handle = 0;
bufs[i].server_id = server_id;
}
r = ipc_sendrecv_with_fds(true, fd, &bufs, sizeof(VisionBuf) * num_fds, fds, num_fds, nullptr);
close(fd);
}
LOGD("Stopping listener for: %s", name.c_str());
close(sock);
unlink(ipc_path.c_str());
}
VisionBuf * VisionIpcServer::get_buffer(VisionStreamType type, int idx){
// Do we want to keep track if the buffer has been sent out yet and warn user?
assert(buffers.count(type));
auto b = buffers[type];
if (idx < 0) {
idx = cur_idx[type]++ % b.size();
} else {
assert(idx < b.size() && idx >= 0);
cur_idx[type] = idx;
}
return b[idx];
}
void VisionIpcServer::send(VisionBuf * buf, VisionIpcBufExtra * extra, bool sync){
if (sync) {
if (buf->sync(VISIONBUF_SYNC_FROM_DEVICE) != 0) {
LOGE("Failed to sync buffer");
}
}
assert(buffers.count(buf->type));
assert(buf->idx < buffers[buf->type].size());
// Send over correct msgq socket
VisionIpcPacket packet = {0};
packet.server_id = server_id;
packet.idx = buf->idx;
packet.extra = *extra;
sockets[buf->type]->send((char*)&packet, sizeof(packet));
}
VisionIpcServer::~VisionIpcServer(){
should_exit = true;
if (listener_thread.joinable()) {
int sock = ipc_connect(get_ipc_path(name).c_str());
if (sock >= 0) {
close(sock);
}
listener_thread.join();
}
// VisionBuf cleanup
for (auto const& [type, buf] : buffers) {
for (VisionBuf* b : buf){
if (b->free() != 0) {
LOGE("Failed to free buffer");
}
delete b;
}
}
// Messaging cleanup
for (auto const& [type, sock] : sockets) {
delete sock;
}
delete msg_ctx;
}
@@ -0,0 +1,40 @@
#pragma once
#include <vector>
#include <string>
#include <thread>
#include <atomic>
#include <map>
#include "msgq/ipc.h"
#include "msgq/visionipc/visionbuf.h"
std::string get_endpoint_name(std::string name, VisionStreamType type);
std::string get_ipc_path(const std::string &name);
class VisionIpcServer {
private:
uint64_t server_id;
std::atomic<bool> should_exit = false;
std::string name;
std::thread listener_thread;
std::map<VisionStreamType, std::atomic<size_t> > cur_idx;
std::map<VisionStreamType, std::vector<VisionBuf*> > buffers;
Context * msg_ctx;
std::map<VisionStreamType, PubSocket*> sockets;
void listener(void);
public:
VisionIpcServer(std::string name);
~VisionIpcServer();
VisionBuf * get_buffer(VisionStreamType type, int idx = -1);
void create_buffers(VisionStreamType type, size_t num_buffers, size_t width, size_t height);
void create_buffers_with_sizes(VisionStreamType type, size_t num_buffers, size_t width, size_t height, size_t size, size_t stride, size_t uv_offset);
void send(VisionBuf * buf, VisionIpcBufExtra * extra, bool sync=true);
void start_listener();
};
+46
View File
@@ -0,0 +1,46 @@
[project]
name = "msgq"
version = "0.0.1"
description = "Lock-free IPC pub/sub message queue"
readme = "README.md"
requires-python = ">=3.11"
license = {text = "MIT"}
authors = [{name = "comma.ai"}]
classifiers = [
"Natural Language :: English",
"Programming Language :: Python :: 3",
"Topic :: System :: Hardware",
]
dependencies = []
[project.optional-dependencies]
dev = [
"setuptools", # for distutils
"Cython",
"scons",
"ruff",
"cppcheck",
"cpplint",
"codespell",
"ty",
"lefthook",
]
[tool.setuptools.package-data]
"*" = ["*.so"]
# https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml
[tool.ruff]
lint.select = ["E", "F", "W", "PIE", "C4", "ISC", "RUF100", "A"]
lint.ignore = ["W292", "E741", "E402", "C408", "ISC003"]
lint.flake8-implicit-str-concat.allow-multiline=false
line-length = 160
target-version="py311"
[tool.ty.src]
exclude = ["site_scons/"]
[tool.ty.rules]
# Cython modules are compiled at build time, not available for static analysis
unresolved-import = "ignore"
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
set -e
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
cd $DIR
if ! command -v uv &>/dev/null; then
echo "'uv' is not installed. Installing 'uv'..."
curl -LsSf https://astral.sh/uv/install.sh | sh
# doesn't require sourcing on all platforms
set +e
source $HOME/.local/bin/env
set -e
fi
export UV_PROJECT_ENVIRONMENT="$DIR/.venv"
uv sync --all-extras
source "$DIR/.venv/bin/activate"
+72
View File
@@ -0,0 +1,72 @@
import re
import SCons
from SCons.Action import Action
from SCons.Scanner import Scanner
pyx_from_import_re = re.compile(r'^from\s+(\S+)\s+cimport', re.M)
pyx_import_re = re.compile(r'^cimport\s+(\S+)', re.M)
cdef_import_re = re.compile(r'^cdef extern from\s+.(\S+).:', re.M)
def pyx_scan(node, env, path, arg=None):
contents = node.get_text_contents()
# from <module> cimport ...
matches = pyx_from_import_re.findall(contents)
# cimport <module>
matches += pyx_import_re.findall(contents)
# Modules can be either .pxd or .pyx files
files = [m.replace('.', '/') + '.pxd' for m in matches]
files += [m.replace('.', '/') + '.pyx' for m in matches]
# cdef extern from <file>
files += cdef_import_re.findall(contents)
# Handle relative imports
cur_dir = str(node.get_dir())
files = [cur_dir + f if f.startswith('/') else f for f in files]
# Filter out non-existing files (probably system imports)
files = [f for f in files if env.File(f).exists()]
return env.File(files)
pyxscanner = Scanner(function=pyx_scan, skeys=['.pyx', '.pxd'], recursive=True)
cythonAction = Action("$CYTHONCOM")
def create_builder(env):
try:
cython = env['BUILDERS']['Cython']
except KeyError:
cython = SCons.Builder.Builder(
action=cythonAction,
emitter={},
suffix=cython_suffix_emitter,
single_source=1
)
env.Append(SCANNERS=pyxscanner)
env['BUILDERS']['Cython'] = cython
return cython
def cython_suffix_emitter(env, source):
return "$CYTHONCFILESUFFIX"
def generate(env):
env["CYTHON"] = "cythonize"
env["CYTHONCOM"] = "$CYTHON $CYTHONFLAGS $SOURCE"
env["CYTHONCFILESUFFIX"] = ".cpp"
c_file, _ = SCons.Tool.createCFileBuilders(env)
c_file.suffix['.pyx'] = cython_suffix_emitter
c_file.add_action('.pyx', cythonAction)
c_file.suffix['.py'] = cython_suffix_emitter
c_file.add_action('.py', cythonAction)
create_builder(env)
def exists(env):
return True
+18
View File
@@ -0,0 +1,18 @@
#!/bin/bash
set -e
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
cd $DIR
source ./setup.sh
# *** build ***
scons -j8
# *** lint + test ***
lefthook run test
# *** all done ***
GREEN='\033[0;32m'
NC='\033[0m'
printf "\n${GREEN}All good!${NC} Finished build, lint, and test in ${SECONDS}s\n"
Submodule opendbc_repo deleted from 35f7e08134
+1
View File
@@ -0,0 +1 @@
* text=auto eol=lf
+49
View File
@@ -0,0 +1,49 @@
name: Car bug report
description: For issues with a particular car make or model
labels: ["car", "bug"]
body:
- type: markdown
attributes:
value: >
Before creating a **car bug report**, please check the following:
* If the issue likely isn't specific to your car model or make, go back and open an **openpilot bug report** instead.
* If the issue is related to the driving or driver monitoring models, you should open a [discussion](https://github.com/commaai/openpilot/discussions/categories/model-feedback) instead.
* If you are fingerprinting your car, refer to the [fingerprinting guide](https://github.com/commaai/openpilot/wiki/Fingerprinting) or visit the [community Discord server](https://discord.comma.ai) to request help.
* If you are inquiring about future car support, check the [community Discord server](https://discord.comma.ai) for progress on your vehicle and help contribute. Issues simply requesting support for new vehicles will be closed.
* Ensure you're running the latest openpilot release.
* Ensure you're using officially supported hardware.
* Ensure there isn't an existing issue for your bug. If there is, leave a comment on the existing issue.
* Ensure you're running stock openpilot. We cannot look into bug reports from forks.
If you're unsure whether you've hit a bug, check out the #installation-help channel in the [community Discord server](https://discord.comma.ai).
- type: textarea
attributes:
label: Describe the car bug
description: Also include a description of how to reproduce the bug
validations:
required: true
- type: input
id: route
attributes:
label: Provide a route where the issue occurs
description: Ensure the route is fully uploaded at https://useradmin.comma.ai. We cannot look into issues without routes, or at least a Dongle ID.
placeholder: 77611a1fac303767|2020-05-11--16-37-07
validations:
required: true
- type: input
id: version
attributes:
label: openpilot version
description: If you're not on release, provide the commit hash
placeholder: 0.8.10
validations:
required: true
- type: textarea
attributes:
label: Additional info
+14
View File
@@ -0,0 +1,14 @@
blank_issues_enabled: false
contact_links:
- name: openpilot bug report
url: https://github.com/commaai/openpilot/issues/new
about: For issues not limited to a particular car make or model, e.g. driving model performance or issues with your comma device.
- name: Join the Discord
url: https://discord.comma.ai
about: The community Discord is for both car port development and experience discussion
- name: Report driving behavior feedback
url: https://discord.com/channels/469524606043160576/1254834193066623017
about: Feedback for the driving and driver monitoring models goes in the #driving-feedback in Discord
- name: Community Wiki
url: https://github.com/commaai/openpilot/wiki
about: Check out our community wiki
+36
View File
@@ -0,0 +1,36 @@
name: Miscellaneous
description: For tools or test bugs, or enhancements
labels: ["bug"]
body:
- type: markdown
attributes:
value: >
* If the issue likely only affects your car model or make, go back and open a **car bug report** instead.
Before creating a **miscellaneous bug report**, please check the following:
* Ensure there isn't an existing issue for your bug. If there is, leave a comment on the existing issue.
- type: textarea
attributes:
label: Describe the bug or enhancement
description: Also include a description of how to reproduce the bug or how the enhancement would work
validations:
required: true
- type: input
id: route
attributes:
label: Provide a route where the issue occurs if applicable
description: Ensure the route is fully uploaded at https://useradmin.comma.ai
placeholder: 77611a1fac303767|2020-05-11--16-37-07
- type: input
id: version
attributes:
label: openpilot version if applicable
description: If you're not on release, provide the commit hash
placeholder: 0.8.10
- type: textarea
attributes:
label: Additional info
+84
View File
@@ -0,0 +1,84 @@
CI / testing:
- changed-files:
- any-glob-to-any-file: '.github/**'
car:
- changed-files:
- any-glob-to-any-file: 'opendbc/car/**'
car safety:
- changed-files:
- any-glob-to-any-file: 'opendbc/safety/**'
can:
- changed-files:
- any-glob-to-any-file: 'opendbc/can/**'
DBC signals:
- changed-files:
- any-glob-to-any-file: 'opendbc/dbc/**'
body:
- changed-files:
- any-glob-to-any-file: 'opendbc/car/body/**'
byd:
- changed-files:
- any-glob-to-any-file: 'opendbc/car/byd/**'
chrysler:
- changed-files:
- any-glob-to-any-file: 'opendbc/car/chrysler/**'
ford:
- changed-files:
- any-glob-to-any-file: 'opendbc/car/ford/**'
gm:
- changed-files:
- any-glob-to-any-file: 'opendbc/car/gm/**'
honda:
- changed-files:
- any-glob-to-any-file: 'opendbc/car/honda/**'
hyundai:
- changed-files:
- any-glob-to-any-file: 'opendbc/car/hyundai/**'
mazda:
- changed-files:
- any-glob-to-any-file: 'opendbc/car/mazda/**'
nissan:
- changed-files:
- any-glob-to-any-file: 'opendbc/car/nissan/**'
rivian:
- changed-files:
- any-glob-to-any-file: 'opendbc/car/rivian/**'
mg:
- changed-files:
- any-glob-to-any-file: 'opendbc/car/mg/**'
subaru:
- changed-files:
- any-glob-to-any-file: 'opendbc/car/subaru/**'
tesla:
- changed-files:
- any-glob-to-any-file: 'opendbc/car/tesla/**'
toyota:
- changed-files:
- any-glob-to-any-file: 'opendbc/car/toyota/**'
volkswagen:
- changed-files:
- any-glob-to-any-file: 'opendbc/car/volkswagen/**'
fingerprint:
- changed-files:
- any-glob-to-all-files: 'opendbc/car/*/fingerprints.py'
+11
View File
@@ -0,0 +1,11 @@
<!--
We need these details to verify your pull request.
Find your device's dongle ID and a route at https://connect.comma.ai.
Ideally, the route is recorded with the exact branch of your pull request.
If you are porting a car with a new harness variant, please add it to https://github.com/commaai/opendbc/blob/master/opendbc/car/docs_definitions.py. Let us know and we can add it to the shop at the time of merge.
-->
Validation
* Dongle ID:
* Route:
+51
View File
@@ -0,0 +1,51 @@
name: "PR review"
on:
pull_request_target:
types: [opened, reopened, synchronize, edited, edited]
jobs:
labeler:
name: review
permissions:
contents: read
pull-requests: write
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
# Label PRs
- uses: actions/labeler@v7.0.0
with:
dot: true
sync-labels: true
configuration-path: .github/labeler.yaml
# Check PR target branch
- name: check branch
uses: Vankka/pr-target-branch-action@5da68a42bcb7b43d39104295a876a6f3f8d7908b
if: github.repository == 'commaai/opendbc'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
target: /^(?!master$).*/
exclude: /commaai:.*/
change-to: ${{ github.base_ref }}
already-exists-action: close_this
already-exists-comment: "Your PR should be made against the `master` branch"
# Welcome comment
- name: "First timers PR"
uses: actions/first-interaction@v3
if: github.event.pull_request.head.repo.full_name != 'commaai/opendbc'
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
issue_message: "This workflow does not handle issues."
pr_message: |
<!-- _(run_id **${{ github.run_id }}**)_ -->
Thanks for contributing to opendbc! In order for us to review your PR as quickly as possible, check the following:
* Convert your PR to a draft unless it's ready to review
* Read the [contributing docs](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md)
* Before marking as "ready for review", ensure:
* the goal is clearly stated in the description
* all the tests are passing
* include a route or your device' dongle ID if relevant
+45
View File
@@ -0,0 +1,45 @@
name: car diff
on:
pull_request_target:
types: [opened, synchronize, reopened]
jobs:
comment:
name: comment
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
pull-requests: write
actions: read
steps:
- name: Wait for car diff
id: wait
continue-on-error: true
uses: lewagon/wait-on-check-action@v1.9.1
with:
ref: ${{ github.event.pull_request.head.sha }}
check-name: car diff
repo-token: ${{ secrets.GITHUB_TOKEN }}
allowed-conclusions: success
wait-interval: 20
- name: Download car diff
if: steps.wait.outcome == 'success'
uses: dawidd6/action-download-artifact@v24
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
workflow: tests.yml
workflow_conclusion: ''
commit: ${{ github.event.pull_request.head.sha }}
name: car_diff_${{ github.event.number }}
path: .
allow_forks: true
- name: Comment car diff on PR
if: steps.wait.outcome == 'success'
uses: thollander/actions-comment-pull-request@v3.0.1
with:
file-path: diff.txt
comment-tag: car_diff
pr-number: ${{ github.event.number }}
github-token: ${{ secrets.GITHUB_TOKEN }}
+18
View File
@@ -0,0 +1,18 @@
name: Re-draft on review
on:
pull_request_review:
types: [submitted]
jobs:
redraft:
# TODO: doesn't work due to perms issue. https://github.com/commaai/opendbc/actions/runs/16079182504/job/45380739128?pr=2385
if: false
#if: github.event.review.state == 'changes_requested'
permissions:
pull-requests: write
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- run: gh pr ready --undo "${{ github.event.pull_request.number }}"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+34
View File
@@ -0,0 +1,34 @@
name: Publish to PyPI
on:
release:
types: [published]
workflow_dispatch:
jobs:
pypi-publish:
name: Upload release to PyPI
runs-on: ubuntu-latest
environment:
name: pypi
url: https://pypi.org/p/opendbc
permissions:
id-token: write # trusted publishing
steps:
- uses: actions/checkout@v7
- name: Build distribution
run: |
./test.sh
source .venv/bin/activate
uv pip install build
python -m build
- name: Publish to Test PyPI
if: github.event_name == 'workflow_dispatch'
uses: pypa/gh-action-pypi-publish@v1.14.2
with:
repository-url: https://test.pypi.org/legacy/
- name: Publish to Production PyPI
if: github.event_name == 'release'
uses: pypa/gh-action-pypi-publish@v1.14.2
+123
View File
@@ -0,0 +1,123 @@
name: tests
on:
push:
branches:
- master
pull_request:
jobs:
tests:
name: ./test.sh (${{ matrix.name }})
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
include:
- os: ${{ github.repository == 'commaai/opendbc' && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
name: Linux
- os: ${{ github.repository == 'commaai/opendbc' && 'namespace-profile-macos-8x14' || 'macos-latest' }}
name: macOS
steps:
- uses: actions/checkout@v7
- uses: commaai/timeout@v1
with:
timeout: ${{ github.repository == 'commaai/opendbc' && '90' || '999' }}
- run: ./test.sh
safety_tests:
name: safety
runs-on: ${{ github.repository == 'commaai/opendbc' && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
steps:
- uses: commaai/timeout@v1
with:
timeout: ${{ github.repository == 'commaai/opendbc' && '60' || '999' }}
- uses: actions/checkout@v7
- name: Run safety tests
run: ./opendbc/safety/tests/test.sh
mutation:
name: Safety mutation tests (${{ matrix.name }})
runs-on: ${{ matrix.os }}
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
include:
- os: ${{ github.repository == 'commaai/opendbc' && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
name: Linux
- os: ${{ github.repository == 'commaai/opendbc' && 'namespace-profile-macos-8x14' || 'macos-latest' }}
name: macOS
steps:
- uses: actions/checkout@v7
- name: Run mutation tests
run: |
source setup.sh
python opendbc/safety/tests/mutation.py
car_diff:
name: car diff
runs-on: ${{ github.repository == 'commaai/opendbc' && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
env:
GIT_REF: ${{ github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.event.before || format('origin/{0}', github.event.repository.default_branch) }}
steps:
- uses: actions/checkout@v7
with:
fetch-depth: 0
- name: Test car diff
if: github.event_name == 'pull_request'
run: source setup.sh && python opendbc/car/tests/car_diff.py | tee diff.txt
- name: Upload diff
if: always() && github.event_name == 'pull_request'
uses: actions/upload-artifact@v7
with:
name: car_diff_${{ github.event.number }}
path: diff.txt
- name: Update refs
if: github.repository == 'commaai/opendbc' && github.ref == 'refs/heads/master'
run: source setup.sh && python opendbc/car/tests/car_diff.py --update-refs
- name: Checkout ci-artifacts
if: github.repository == 'commaai/opendbc' && github.ref == 'refs/heads/master'
uses: actions/checkout@v7
with:
repository: commaai/ci-artifacts
ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
path: ${{ github.workspace }}/ci-artifacts
- name: Push refs
if: github.repository == 'commaai/opendbc' && github.ref == 'refs/heads/master'
working-directory: ${{ github.workspace }}/ci-artifacts
run: |
ls ${{ github.workspace }}/car_diff/*.zst 2>/dev/null || exit 0
git config user.name "GitHub Actions Bot"
git config user.email "<>"
git fetch origin car_diff || true
git checkout car_diff 2>/dev/null || git checkout --orphan car_diff
cp ${{ github.workspace }}/car_diff/*.zst .
git add *.zst
git commit -m "car_diff refs for ${{ github.sha }}" || echo "No changes to commit"
git push origin car_diff
test_models:
name: test models
runs-on: ${{ github.repository == 'commaai/opendbc' && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
strategy:
fail-fast: false
matrix:
job: [0, 1, 2, 3]
env:
CI: 1
steps:
- uses: actions/checkout@v7
- run: ./setup.sh
- name: Cache test routes
id: routes-cache
uses: actions/cache@v6
with:
path: /tmp/comma_download_cache
key: car_models-${{ hashFiles('opendbc/car/tests/test_models.py', 'opendbc/car/tests/routes.py') }}-${{ matrix.job }}
- name: Test car models
timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.routes-cache.outputs.cache-hit == 'true') && 2 || 6 }}
run: source .venv/bin/activate && MAX_EXAMPLES=1 opendbc/car/tests/test_models.py
env:
NUM_JOBS: 4
JOB_ID: ${{ matrix.job }}
+24
View File
@@ -0,0 +1,24 @@
name: Update CARS.md
on:
schedule:
- cron: '0 8 * * *' # 12am PST (8am UTC)
workflow_dispatch:
jobs:
update-cars:
runs-on: ubuntu-24.04
permissions:
contents: write
steps:
- uses: actions/checkout@v7
- name: Generate Car Docs
run: |
pip install -e .
python -m pip install jinja2==3.1.4
python opendbc/car/docs.py
- uses: stefanzweifel/git-auto-commit-action@4a55954c782fc1ea30b9056cd3e7a2b40ca8887d
with:
commit_message: 'docs: Scheduled auto-update CARS.md'
file_pattern: 'docs/CARS.md'
+30
View File
@@ -0,0 +1,30 @@
name: Update uv.lock
on:
schedule:
- cron: "0 14 * * 1" # every Monday at 2am UTC (6am PST)
workflow_dispatch:
jobs:
update-uv-lock:
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
steps:
- uses: actions/checkout@v7
- uses: astral-sh/setup-uv@v10.1.0
- name: Update uv.lock
run: uv lock --upgrade
- name: Create Pull Request
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1
with:
author: Vehicle Researcher <user@comma.ai>
token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
commit-message: "[bot] Update uv.lock"
title: "[bot] Update uv.lock"
body: "Weekly update of uv.lock dependencies"
branch: "update-uv-lock"
base: "master"
delete-branch: true
labels: bot
+29
View File
@@ -0,0 +1,29 @@
.cache/
/build/
*.pyc
*.os
*.o
*.so
*.a
*.tmp
*.dylib
.*.swp
.DS_Store
*.egg-info/
*.html
*.gcda
*.gcno
*.dump
*.gcov
/dist/
.vscode/
__pycache__/
*.profraw
opendbc/can/build/
opendbc/can/obj/
opendbc/dbc/*_generated.dbc
cppcheck-addon-ctu-file-list
opendbc/safety/tests/coverage-out
.mull/
+7
View File
@@ -0,0 +1,7 @@
Copyright (c) 2020, Comma.ai, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+3
View File
@@ -0,0 +1,3 @@
include opendbc/car/car.capnp
include opendbc/car/include/c++.capnp
recursive-include opendbc/safety *.h
+209
View File
@@ -0,0 +1,209 @@
<div align="center" style="text-align: center;">
<h1>opendbc</h1>
<p>
<b>opendbc is a Python API for your car.</b>
<br>
Control the gas, brake, steering, and more. Read the speed, steering angle, and more.
</p>
<h3>
<a href="https://docs.comma.ai">Docs</a>
<span> · </span>
<a href="https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md">Contribute</a>
<span> · </span>
<a href="https://discord.comma.ai">Discord</a>
</h3>
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
[![X Follow](https://img.shields.io/twitter/follow/comma_ai)](https://x.com/comma_ai)
[![Discord](https://img.shields.io/discord/469524606043160576)](https://discord.comma.ai)
<br>
<h3><i>How to Port a Car — Jason Young, COMMA_CON 2023</i></h3>
<a href="https://www.youtube.com/watch?v=XxPS5TpTUnI&t=142s">
<img src="https://github.com/user-attachments/assets/ae89198e-561b-4210-a0d4-ccecd917577d" alt="▶ How to Port a Car - Jason Young, COMMA_CON 2023" width="800">
</a>
<br>
<h3><i>How Do We Control The Car? — Robbe Derks, COMMA_CON 2021</i></h3>
<a href="https://www.youtube.com/watch?v=nNU6ipme878">
<img src="https://github.com/user-attachments/assets/28c40bc0-7884-47e9-b392-f47f03190497" alt="▶ How Do We Control The Car? - Robbe Derks, COMMA_CON 2021" width="800">
</a>
<br>
</div>
---
Most cars since 2016 have electronically-actuatable steering, gas, and brakes thanks to [LKAS](https://en.wikipedia.org/wiki/Lane_departure_warning_system#Lane_keeping_and_next_technologies) and [ACC](https://en.wikipedia.org/wiki/Adaptive_cruise_control).
The goal of this project is to support controlling the steering, gas, and brakes on every single one of those cars.
While the primary focus is on supporting ADAS interfaces for [openpilot](https://github.com/commaai/openpilot), we're also interested in reading and writing as many things as we can (EV charge status, lock/unlocking doors, etc) such that we can build the best vehicle management app ever.
---
This README and the [supported cars list](docs/CARS.md) are all the docs for the opendbc project.
Everything you need to know to use, contribute, and extend opendbc are in these docs.
## Quick start
```bash
git clone https://github.com/commaai/opendbc.git
cd opendbc
# you probably just want to use this. it's an all-in-one for dependency
# installation, compiling, linting, and tests. it's also what runs in CI
./test.sh
# here are the individual commands it runs
pip3 install -e .[testing] # install dependencies
scons -j8 # build with 8 cores
unittest-parallel # run the tests
lefthook run lint # run the linter
```
[`examples/`](examples/) contains small example programs that can read state from the car and control the steering, gas, and brakes.
[`examples/joystick.py`](examples/joystick.py) allows you to control a car with a joystick.
### Project Structure
* [`opendbc/dbc/`](opendbc/dbc/) is a repository of [DBC](https://en.wikipedia.org/wiki/CAN_bus#DBC_(CAN_Database_Files)) files
* [`opendbc/can/`](opendbc/can/) is a library for parsing and building CAN messages from DBC files
* [`opendbc/car/`](opendbc/car/) is a high-level library for interfacing with cars using Python
* [`opendbc/safety/`](opendbc/safety/) is the functional safety for all the cars supported by `opendbc/car/`
## How to Port a Car
This guide covers everything from adding support to a new car all the way to improving existing cars (e.g. adding longitudinal control or radar parsing). If similar cars to yours are already compatible, most of this work is likely already done for you.
At its most basic, a car port will control the steering on a car. A "complete" car port will have all of: lateral control, longitudinal control, good tuning for both lateral and longitudinal, radar parsing (if equipped), fuzzy fingerprinting, and more. The new car support docs will clearly communicate each car's support level.
### Connect to the Car
The first step is to get connected to the car with a comma four and a car harness.
The car harness gets you connected to two different CAN buses and splits one of those buses to send our own actuation messages.
If you're lucky, a harness compatible with your car will already be designed and sold on comma.ai/shop.
If you're not so lucky, start with a "developer harness" from comma.ai/shop and crimp on whatever connector you need.
### Structure of a port
Depending on the brand, most of this basic structure will already be in place.
The entirety of a car port lives in `opendbc/car/<brand>/`:
* `carstate.py`: parses out the relevant information from the CAN stream using the car's DBC file
* `carcontroller.py`: outputs CAN messages to control the car
* `<brand>can.py`: thin Python helpers around the DBC file to build CAN messages
* `fingerprints.py`: database of ECU firmware versions for identifying car models
* `interface.py`: high level class for interfacing with the car
* `radar_interface.py`: parses out the radar
* `values.py`: enumerates the brand's supported cars
### Reverse Engineer CAN messages
Start off by recording a route with lots of interesting events: enable LKAS and ACC, turn the steering wheel both extremes, etc. Then, load up that route in [cabana](https://github.com/commaai/openpilot/tree/master/tools/cabana).
### Tuning
#### Longitudinal
Use the [longitudinal maneuvers](https://github.com/commaai/openpilot/tree/master/tools/longitudinal_maneuvers) report to evaluate your car's longitudinal control and tune it.
## Contributing
All opendbc development is coordinated on GitHub and [Discord](https://discord.comma.ai). Check out the `#dev-opendbc-cars` channel and `Vehicle Specific` section.
### Roadmap
Short term
- [ ] `pip install opendbc`
- [ ] 100% type coverage
- [ ] 100% line coverage
- [ ] Make car ports easier: refactors, tools, tests, and docs
- [ ] Expose the state of all supported cars better: https://github.com/commaai/opendbc/issues/1144
Longer term
- [ ] Extend support to every car with LKAS + ACC interfaces
- [ ] Automatic lateral and longitudinal control/tuning evaluation
- [ ] Auto-tuning for [lateral](https://blog.comma.ai/090release/#torqued-an-auto-tuner-for-lateral-control) and longitudinal control
- [ ] [Automatic Emergency Braking](https://en.wikipedia.org/wiki/Automated_emergency_braking_system)
Contributions towards anything here are welcome.
## Safety Model
When a [panda](https://comma.ai/shop/panda) powers up with [opendbc safety firmware](opendbc/safety), by default it's in `SAFETY_SILENT` mode. While in `SAFETY_SILENT` mode, the CAN buses are forced to be silent. In order to send messages, you have to select a safety mode. Some of safety modes (for example `SAFETY_ALLOUTPUT`) are disabled in release firmwares. In order to use them, compile and flash your own build.
Safety modes optionally support `controls_allowed`, which allows or blocks a subset of messages based on a customizable state in the board.
## Code Rigor
The opendbc safety firmware is written for its use in conjunction with [openpilot](https://github.com/commaai/openpilot) and [panda](https://github.com/commaai/panda). The safety firmware, through its safety model, provides and enforces the
[openpilot safety](https://github.com/commaai/openpilot/blob/master/docs/SAFETY.md). Due to its critical function, it's important that the application code rigor within the `safety` folder is held to high standards.
These are the [CI regression tests](https://github.com/commaai/opendbc/actions) we have in place:
* A generic static code analysis is performed by [cppcheck](https://github.com/danmar/cppcheck/).
* In addition, [cppcheck](https://github.com/danmar/cppcheck/) has a specific addon to check for [MISRA C:2012](https://misra.org.uk/) violations. See [current coverage](opendbc/safety/tests/misra/coverage_table).
* Compiler options are relatively strict: the flags `-Wall -Wextra -Wstrict-prototypes -Werror` are enforced.
* The [safety logic](opendbc/safety) is tested and verified by [unit tests](opendbc/safety/tests) for each supported car variant.
The above tests are themselves tested by:
* a [mutation test](opendbc/safety/tests/misra/test_mutation.py) on the MISRA coverage
* 100% line coverage enforced on the safety unit tests
In addition, we run the [ruff linter](https://github.com/astral-sh/ruff) and [ty](https://github.com/astral-sh/ty) on the car interface library.
### Bounties
Every car port is eligible for a bounty:
* $2000 - [Any car brand / platform port](https://github.com/orgs/commaai/projects/26/views/1?pane=issue&itemId=47913774)
* $250 - [Any car model port](https://github.com/orgs/commaai/projects/26/views/1?pane=issue&itemId=47913790)
* $300 - [Reverse Engineering a new Actuation Message](https://github.com/orgs/commaai/projects/26/views/1?pane=issue&itemId=73445563)
In addition to the standard bounties, we also offer higher value bounties for more popular cars. See those at [comma.ai/bounties](https://comma.ai/bounties).
## FAQ
***How do I use this?*** A [comma four](https://comma.ai/shop/comma-four) is custom-designed to be the best way to run and develop opendbc and openpilot.
***Which cars are supported?*** See the [supported cars list](docs/CARS.md).
***Can I add support for my car?*** Yes, most car support comes from the community. Read the guide [here](https://github.com/commaai/opendbc/blob/master/README.md#how-to-port-a-car).
***Which cars can be supported?*** Any car with LKAS and ACC. More info [here](https://github.com/commaai/openpilot/blob/master/docs/CARS.md#dont-see-your-car-here).
***How does this work?*** In short, we designed hardware to replace your car's built-in lane keep and adaptive cruise features. See [this talk](https://www.youtube.com/watch?v=FL8CxUSfipM) for an in-depth explanation.
***Is there a timeline or roadmap for adding car support?*** No, most car support comes from the community, with comma doing final safety and quality validation. The more complete the community car port is and the more popular the car is, the more likely we are to pick it up as the next one to validate.
### Terms
* **port**: refers to the integration and support of a specific car
* **lateral control**: aka steering control
* **longitudinal control**: aka gas/brakes control
* **fingerprinting**: automatic process for identifying the car
* **[LKAS](https://en.wikipedia.org/wiki/Lane_departure_warning_system)**: lane keeping assist
* **[ACC](https://en.wikipedia.org/wiki/Adaptive_cruise_control)**: adaptive cruise control
* **[harness](https://comma.ai/shop/car-harness)**: car-specific hardware to attach to the car and intercept the ADAS messages
* **[panda](https://github.com/commaai/panda)**: hardware used to get on a car's CAN bus
* **[ECU](https://en.wikipedia.org/wiki/Electronic_control_unit)**: computers or control modules inside the car
* **[CAN bus](https://en.wikipedia.org/wiki/CAN_bus)**: a bus that connects the ECUs in a car
* **[cabana](https://github.com/commaai/openpilot/tree/master/tools/cabana#readme)**: our tool for reverse engineering CAN messages
* **[DBC file](https://en.wikipedia.org/wiki/CAN_bus#DBC)**: contains definitions for messages on a CAN bus
* **[openpilot](https://github.com/commaai/openpilot)**: an ADAS system for cars supported by opendbc
* **[comma](https://github.com/commaai)**: the company behind opendbc
* **[comma four](https://comma.ai/shop/comma-four)**: the hardware used to run openpilot
### More resources
* [*How Do We Control The Car?*](https://www.youtube.com/watch?v=nNU6ipme878&pp=ygUoY29tbWEgY29uIDIwMjEgaG93IGRvIHdlIGNvbnRyb2wgdGhlIGNhcg%3D%3D) by [@robbederks](https://github.com/robbederks) from COMMA_CON 2021
* [*How to Port a Car*](https://www.youtube.com/watch?v=XxPS5TpTUnI&t=142s&pp=ygUPamFzb24gY29tbWEgY29u) by [@jyoung8607](https://github.com/jyoung8607) from COMMA_CON 2023
* [commaCarSegments](https://huggingface.co/datasets/commaai/commaCarSegments): a massive dataset of CAN data from 300 different car models
* [cabana](https://github.com/commaai/openpilot/tree/master/tools/cabana#readme): our tool for reverse engineering CAN messages
* [can_print_changes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/debug/can_print_changes.py): diff the whole CAN bus across two drives, such as one without any LKAS and one with LKAS
* [longitudinal maneuvers](https://github.com/commaai/openpilot/tree/master/tools/longitudinal_maneuvers): a tool for evaluating and tuning longitudinal control
* [opendbc data](https://commaai.github.io/opendbc-data/): a repository of longitudinal maneuver evaluations
## Come work with us -- [comma.ai/jobs](https://comma.ai/jobs)
comma is hiring engineers to work on opendbc and [openpilot](https://github.com/commaai/openpilot). We love hiring contributors.
+41
View File
@@ -0,0 +1,41 @@
Version 0.3.1 (2026-04-22)
========================
* Ship DBCs, capnp schemas, and torque data in the wheel
Version 0.3.0 (2026-04-22)
========================
* Supported car count: 345 → 397
* Tesla Model 3, Model Y, and Model X (HW3 and HW4) support thanks to lukasloetkolben and greatgitsby!
* Rivian R1S and R1T (Gen 1 and Gen 2) support thanks to lukasloetkolben!
* Porsche Macan and Audi Q5 (VW MLB) support thanks to jyoung8607 and Dennis-NL!
* VW MEB (ID.x) architecture support thanks to jyoung8607!
* PSA AEE2010_R3 platform support thanks to elkoled!
* Honda Accord 2023-25, CR-V 2023-25, Pilot 2023-25, Passport 2026, and Acura MDX 2025 support thanks to vanillagorillaa and MVL!
* Honda Odyssey 2021-25 support thanks to csouers and MVL!
* Acura TLX 2021 support thanks to MVL!
* Honda City 2023 support thanks to vanillagorillaa and drFritz!
* Honda N-Box 2018 support thanks to miettal!
* Ford F-150, F-150 Hybrid, Mach-E, and Ranger support
* Ford Escape 2023-24 and Kuga 2024 support thanks to incognitojam!
* Hyundai Nexo 2021 support thanks to sunnyhaibin!
* Kia K7 2017 support thanks to royjr!
* Lexus LS 2018 support thanks to Hacheoy!
* Lexus RC 2023 support thanks to nelsonjchen!
* CANParser and CANPacker rewritten in pure Python — the installed package is pure Python, no compilation required
* Safety code, `isotp.py`/`ccp.py`/`xcp.py`, and `vehicle_model.py` moved into opendbc from panda/openpilot — opendbc is now the self-contained car API package
* `mull` replaced with a faster custom mutation test runner
* Safety hardening: per-brand message-block config, missing RX checks, relay-malfunction config
Version 0.2.1 (2025-02-10)
========================
* Fix missing files making car/ package not importable
Version 0.2.0 (2025-02-10)
========================
* Moved car/ directory from openpilot to opendbc. It comprises the APIs necessary to communicate with 275+ car models
* opendbc is moving towards being a complete self-contained car API package
* Soon all opendbc-related tests from openpilot will be migrated as well
Version 0.1.0 (2024-08-01)
========================
* Initial pre-release package with can/ and dbc/ directories
+476
View File
@@ -0,0 +1,476 @@
<!--- AUTOGENERATED FROM selfdrive/car/CARS_template.md, DO NOT EDIT. --->
# Support Information for 406 Known Cars
|Make|Model|Package|Support Level|
|---|---|---|:---:|
|Acura|ADX 2025-26|All|[Upstream](#upstream)|
|Acura|ILX 2016-18|Technology Plus Package or AcuraWatch Plus|[Upstream](#upstream)|
|Acura|ILX 2019|All|[Upstream](#upstream)|
|Acura|Integra 2023-25|All|[Community](#community)|
|Acura|MDX 2015-16|Advance Package|[Community](#community)|
|Acura|MDX 2017-20|All|[Community](#community)|
|Acura|MDX 2022-24|All|[Upstream](#upstream)|
|Acura|MDX 2025-26|All except Type S|[Upstream](#upstream)|
|Acura|MDX Hybrid 2017-20|All|[Community](#community)|
|Acura|RDX 2016-18|AcuraWatch Plus or Advance Package|[Upstream](#upstream)|
|Acura|RDX 2019-21|All|[Upstream](#upstream)|
|Acura|RDX 2022-24|All|[Dashcam mode](#dashcam)|
|Acura|RDX 2025-26|All|[Community](#community)|
|Acura|RLX 2017|Advance Package or Technology Package|[Community](#community)|
|Acura|TLX 2015-17|Advance Package|[Community](#community)|
|Acura|TLX 2018-20|All|[Community](#community)|
|Acura|TLX 2021-22|All|[Upstream](#upstream)|
|Acura|TLX 2023|All|[Community](#community)|
|Acura|TLX 2025|All|[Upstream](#upstream)|
|Acura|ZDX 2024|All|[Not compatible](#can-bus-security)|
|Audi|A3 2014-19|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Audi|A3 Sportback e-tron 2017-18|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Audi|A4 2016-24|All|[Not compatible](#flexray)|
|Audi|A5 2016-24|All|[Not compatible](#flexray)|
|Audi|Q2 2018|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Audi|Q3 2019-24|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Audi|Q5 2013-17|Adaptive Cruise Control (ACC) & Lane Assist|[Dashcam mode](#dashcam)|
|Audi|Q5 2017-24|All|[Not compatible](#flexray)|
|Audi|RS3 2018|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Audi|S3 2015-17|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|BYD|Atto 3 2022-25|All|[Dashcam mode](#dashcam)|
|Chevrolet|Bolt EUV 2022-23|Premier or Premier Redline Trim, without Super Cruise Package|[Upstream](#upstream)|
|Chevrolet|Bolt EV 2022-23|2LT Trim with Adaptive Cruise Control Package|[Upstream](#upstream)|
|Chevrolet|Equinox 2019-22|Adaptive Cruise Control (ACC)|[Upstream](#upstream)|
|Chevrolet|Silverado 1500 2020-21|Safety Package II|[Upstream](#upstream)|
|Chevrolet|Trailblazer 2021-22|Adaptive Cruise Control (ACC)|[Upstream](#upstream)|
|Chrysler|Pacifica 2017-18|Adaptive Cruise Control (ACC)|[Upstream](#upstream)|
|Chrysler|Pacifica 2019-20|Adaptive Cruise Control (ACC)|[Upstream](#upstream)|
|Chrysler|Pacifica 2021-23|All|[Upstream](#upstream)|
|Chrysler|Pacifica Hybrid 2017-18|Adaptive Cruise Control (ACC)|[Upstream](#upstream)|
|Chrysler|Pacifica Hybrid 2019-25|Adaptive Cruise Control (ACC)|[Upstream](#upstream)|
|comma|body|All|[Upstream](#upstream)|
|CUPRA|Ateca 2018-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|CUPRA|Born 2021-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Dodge|Durango 2020-21|Adaptive Cruise Control (ACC)|[Upstream](#upstream)|
|Ford|Bronco Sport 2021-24|Co-Pilot360 Assist+|[Upstream](#upstream)|
|Ford|Escape 2020-22|Co-Pilot360 Assist+|[Upstream](#upstream)|
|Ford|Escape 2023-24|Co-Pilot360 Assist+|[Upstream](#upstream)|
|Ford|Escape Hybrid 2020-22|Co-Pilot360 Assist+|[Upstream](#upstream)|
|Ford|Escape Hybrid 2023-24|Co-Pilot360 Assist+|[Upstream](#upstream)|
|Ford|Escape Plug-in Hybrid 2020-22|Co-Pilot360 Assist+|[Upstream](#upstream)|
|Ford|Escape Plug-in Hybrid 2023-24|Co-Pilot360 Assist+|[Upstream](#upstream)|
|Ford|Expedition 2022-24|Co-Pilot360 Assist 2.0|[Upstream](#upstream)|
|Ford|Explorer 2020-24|Co-Pilot360 Assist+|[Upstream](#upstream)|
|Ford|Explorer Hybrid 2020-24|Co-Pilot360 Assist+|[Upstream](#upstream)|
|Ford|F-150 2021-23|Co-Pilot360 Assist 2.0|[Upstream](#upstream)|
|Ford|F-150 Hybrid 2021-23|Co-Pilot360 Assist 2.0|[Upstream](#upstream)|
|Ford|Focus 2018-22|Adaptive Cruise Control with Lane Centering|[Upstream](#upstream)|
|Ford|Focus Hybrid 2018-22|Adaptive Cruise Control with Lane Centering|[Upstream](#upstream)|
|Ford|Kuga 2020-23|Adaptive Cruise Control with Lane Centering|[Upstream](#upstream)|
|Ford|Kuga Hybrid 2020-23|Adaptive Cruise Control with Lane Centering|[Upstream](#upstream)|
|Ford|Kuga Hybrid 2024|All|[Upstream](#upstream)|
|Ford|Kuga Plug-in Hybrid 2020-23|Adaptive Cruise Control with Lane Centering|[Upstream](#upstream)|
|Ford|Kuga Plug-in Hybrid 2024|All|[Upstream](#upstream)|
|Ford|Maverick 2022|LARIAT Luxury|[Upstream](#upstream)|
|Ford|Maverick 2023-24|Co-Pilot360 Assist|[Upstream](#upstream)|
|Ford|Maverick Hybrid 2022|LARIAT Luxury|[Upstream](#upstream)|
|Ford|Maverick Hybrid 2023-24|Co-Pilot360 Assist|[Upstream](#upstream)|
|Ford|Mustang Mach-E 2021-24|All|[Upstream](#upstream)|
|Ford|Ranger 2024|Adaptive Cruise Control with Lane Centering|[Upstream](#upstream)|
|Genesis|G70 2018|All|[Upstream](#upstream)|
|Genesis|G70 2019-21|All|[Upstream](#upstream)|
|Genesis|G70 2022-23|All|[Upstream](#upstream)|
|Genesis|G80 2017|All|[Upstream](#upstream)|
|Genesis|G80 2018-19|All|[Upstream](#upstream)|
|Genesis|G80 (2.5T Advanced Trim, with HDA II) 2024|Highway Driving Assist II|[Upstream](#upstream)|
|Genesis|G90 2017-20|All|[Upstream](#upstream)|
|Genesis|GV60 (Advanced Trim) 2023|All|[Upstream](#upstream)|
|Genesis|GV60 (Performance Trim) 2022-23|All|[Upstream](#upstream)|
|Genesis|GV70 (2.5T Trim, without HDA II) 2022-24|All|[Upstream](#upstream)|
|Genesis|GV70 (3.5T Trim, without HDA II) 2022-23|All|[Upstream](#upstream)|
|Genesis|GV70 Electrified (Australia Only) 2022|All|[Upstream](#upstream)|
|Genesis|GV70 Electrified (with HDA II) 2023-24|Highway Driving Assist II|[Upstream](#upstream)|
|Genesis|GV80 2023|All|[Upstream](#upstream)|
|GMC|Sierra 1500 2020-21|Driver Alert Package II|[Upstream](#upstream)|
|GMC|Yukon 2019-20|Adaptive Cruise Control (ACC) & LKAS|[Dashcam mode](#dashcam)|
|Honda|Accord 2016-17|Honda Sensing|[Community](#community)|
|Honda|Accord 2018-22|All|[Upstream](#upstream)|
|Honda|Accord 2023-25|All|[Upstream](#upstream)|
|Honda|Accord Hybrid 2017|All|[Community](#community)|
|Honda|Accord Hybrid 2018-22|All|[Upstream](#upstream)|
|Honda|Accord Hybrid 2023-26|All|[Upstream](#upstream)|
|Honda|City (Brazil only) 2023|All|[Upstream](#upstream)|
|Honda|Civic 2016-18|Honda Sensing|[Upstream](#upstream)|
|Honda|Civic 2019-21|All|[Upstream](#upstream)|
|Honda|Civic 2022-26|All|[Upstream](#upstream)|
|Honda|Civic Hatchback 2017-18|Honda Sensing|[Upstream](#upstream)|
|Honda|Civic Hatchback 2019-21|All|[Upstream](#upstream)|
|Honda|Civic Hatchback 2022-26|All|[Upstream](#upstream)|
|Honda|Civic Hatchback Hybrid 2025-26|All|[Upstream](#upstream)|
|Honda|Civic Hatchback Hybrid (Europe only) 2023|All|[Upstream](#upstream)|
|Honda|Civic Hybrid 2025-26|All|[Upstream](#upstream)|
|Honda|Clarity 2018-21|All|[Community](#community)|
|Honda|CR-V 2015-16|Touring Trim|[Upstream](#upstream)|
|Honda|CR-V 2017-22|Honda Sensing|[Upstream](#upstream)|
|Honda|CR-V 2023-26|All|[Upstream](#upstream)|
|Honda|CR-V Hybrid 2017-22|Honda Sensing|[Upstream](#upstream)|
|Honda|CR-V Hybrid 2023-26|All|[Upstream](#upstream)|
|Honda|e 2020|All|[Upstream](#upstream)|
|Honda|Fit 2018-20|Honda Sensing|[Upstream](#upstream)|
|Honda|Freed 2020|Honda Sensing|[Upstream](#upstream)|
|Honda|HR-V 2019-22|Honda Sensing|[Upstream](#upstream)|
|Honda|HR-V 2023-27|All|[Upstream](#upstream)|
|Honda|Insight 2019-22|All|[Upstream](#upstream)|
|Honda|Inspire 2018|All|[Upstream](#upstream)|
|Honda|N-Box 2018|All|[Upstream](#upstream)|
|Honda|Odyssey 2018-20|Honda Sensing|[Upstream](#upstream)|
|Honda|Odyssey 2021-26|All|[Upstream](#upstream)|
|Honda|Odyssey (Singapore) 2021|Honda Sensing|[Upstream](#upstream)|
|Honda|Odyssey (Taiwan) 2018-19|Honda Sensing|[Upstream](#upstream)|
|Honda|Passport 2019-25|All|[Upstream](#upstream)|
|Honda|Passport 2026|All|[Upstream](#upstream)|
|Honda|Pilot 2016-22|Honda Sensing|[Upstream](#upstream)|
|Honda|Pilot 2023-25|All|[Upstream](#upstream)|
|Honda|Pilot 2026|All|[Upstream](#upstream)|
|Honda|Prelude 2026|All|[Upstream](#upstream)|
|Honda|Prologue 2024-26|All|[Not compatible](#can-bus-security)|
|Honda|Ridgeline 2017-26|Honda Sensing|[Upstream](#upstream)|
|Hyundai|Azera 2022|All|[Upstream](#upstream)|
|Hyundai|Azera Hybrid 2019|All|[Upstream](#upstream)|
|Hyundai|Azera Hybrid 2020|All|[Upstream](#upstream)|
|Hyundai|Custin 2023|All|[Upstream](#upstream)|
|Hyundai|Elantra 2017-18|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Elantra 2019|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Elantra 2021-23|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Elantra GT 2017-20|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Elantra Hybrid 2021-23|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Genesis 2015-16|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|i30 2017-19|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Ioniq 5 (Southeast Asia and Europe only) 2022-24|All|[Upstream](#upstream)|
|Hyundai|Ioniq 5 (with HDA II) 2022-24|Highway Driving Assist II|[Upstream](#upstream)|
|Hyundai|Ioniq 5 (without HDA II) 2022-24|Highway Driving Assist|[Upstream](#upstream)|
|Hyundai|Ioniq 6 (with HDA II) 2023-24|Highway Driving Assist II|[Upstream](#upstream)|
|Hyundai|Ioniq 6 (without HDA II) 2023-24|Highway Driving Assist|[Upstream](#upstream)|
|Hyundai|Ioniq Electric 2019|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Ioniq Electric 2020|All|[Upstream](#upstream)|
|Hyundai|Ioniq Hybrid 2017-19|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Ioniq Hybrid 2020-22|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Ioniq Plug-in Hybrid 2019|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Ioniq Plug-in Hybrid 2020-22|All|[Upstream](#upstream)|
|Hyundai|Kona 2020|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Kona 2022-23|Smart Cruise Control (SCC)|[Dashcam mode](#dashcam)|
|Hyundai|Kona Electric 2018-21|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Kona Electric 2022-23|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Kona Electric (with HDA II, Korea only) 2023|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Kona Hybrid 2020|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Nexo 2021|All|[Upstream](#upstream)|
|Hyundai|Palisade 2020-22|All|[Upstream](#upstream)|
|Hyundai|Palisade 2023-24|Highway Driving Assist II|[Community](#community)|
|Hyundai|Santa Cruz 2022-24|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Santa Fe 2019-20|All|[Upstream](#upstream)|
|Hyundai|Santa Fe 2021-23|All|[Upstream](#upstream)|
|Hyundai|Santa Fe Hybrid 2022-23|All|[Upstream](#upstream)|
|Hyundai|Santa Fe Plug-in Hybrid 2022-23|All|[Upstream](#upstream)|
|Hyundai|Sonata 2018-19|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Sonata 2020-23|All|[Upstream](#upstream)|
|Hyundai|Sonata Hybrid 2020-23|All|[Upstream](#upstream)|
|Hyundai|Staria 2023|All|[Upstream](#upstream)|
|Hyundai|Tucson 2021|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Tucson 2022|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Tucson 2023-24|All|[Upstream](#upstream)|
|Hyundai|Tucson Diesel 2019|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Hyundai|Tucson Hybrid 2022-24|All|[Upstream](#upstream)|
|Hyundai|Tucson Plug-in Hybrid 2024|All|[Upstream](#upstream)|
|Hyundai|Veloster 2019-20|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Jeep|Cherokee 2019-23|Adaptive Cruise Control (ACC)|[Dashcam mode](#dashcam)|
|Jeep|Grand Cherokee 2016-18|Adaptive Cruise Control (ACC)|[Upstream](#upstream)|
|Jeep|Grand Cherokee 2019-21|Adaptive Cruise Control (ACC)|[Upstream](#upstream)|
|Kia|Carnival 2022-24|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Carnival (China only) 2023|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Ceed 2019-21|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Ceed Plug-in Hybrid 2021|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|EV6 (Southeast Asia only) 2022-24|All|[Upstream](#upstream)|
|Kia|EV6 (with HDA II) 2022-24|Highway Driving Assist II|[Upstream](#upstream)|
|Kia|EV6 (without HDA II) 2022-24|Highway Driving Assist|[Upstream](#upstream)|
|Kia|Forte 2019-21|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Forte 2022-23|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|K5 2021-24|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|K5 Hybrid 2020-22|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|K7 2017|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|K8 Hybrid (with HDA II) 2023|Highway Driving Assist II|[Upstream](#upstream)|
|Kia|Niro EV 2019|All|[Upstream](#upstream)|
|Kia|Niro EV 2020|All|[Upstream](#upstream)|
|Kia|Niro EV 2021|All|[Upstream](#upstream)|
|Kia|Niro EV 2022|All|[Upstream](#upstream)|
|Kia|Niro EV (with HDA II) 2024-25|Highway Driving Assist II|[Upstream](#upstream)|
|Kia|Niro EV (without HDA II) 2023-25|All|[Upstream](#upstream)|
|Kia|Niro Hybrid 2018|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Niro Hybrid 2021|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Niro Hybrid 2022|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Niro Hybrid 2023-24|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Niro Plug-in Hybrid 2018-19|All|[Upstream](#upstream)|
|Kia|Niro Plug-in Hybrid 2020|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Niro Plug-in Hybrid 2021|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Niro Plug-in Hybrid 2022|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Optima 2017|Advanced Smart Cruise Control|[Upstream](#upstream)|
|Kia|Optima 2019-20|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Optima Hybrid 2017|Advanced Smart Cruise Control|[Dashcam mode](#dashcam)|
|Kia|Optima Hybrid 2019|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Seltos 2021|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Sorento 2018|Advanced Smart Cruise Control & LKAS|[Upstream](#upstream)|
|Kia|Sorento 2019|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Sorento 2021-23|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Sorento Hybrid 2021-23|All|[Upstream](#upstream)|
|Kia|Sorento Plug-in Hybrid 2022-23|All|[Upstream](#upstream)|
|Kia|Sportage 2023-24|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Sportage Hybrid 2023|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Sportage Plug-in Hybrid 2023|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Stinger 2018-20|Smart Cruise Control (SCC)|[Upstream](#upstream)|
|Kia|Stinger 2022-23|All|[Upstream](#upstream)|
|Kia|Telluride 2020-22|All|[Upstream](#upstream)|
|Kia|Telluride 2023-24|Highway Driving Assist II|[Community](#community)|
|Lexus|CT Hybrid 2017-18|Lexus Safety System+|[Upstream](#upstream)|
|Lexus|ES 2017-18|All|[Upstream](#upstream)|
|Lexus|ES 2019-25|All|[Upstream](#upstream)|
|Lexus|ES Hybrid 2017-18|All|[Upstream](#upstream)|
|Lexus|ES Hybrid 2019-25|All|[Upstream](#upstream)|
|Lexus|GS F 2016|All|[Upstream](#upstream)|
|Lexus|IS 2017-19|All|[Upstream](#upstream)|
|Lexus|IS 2022-24|All|[Upstream](#upstream)|
|Lexus|LC 2024-25|All|[Upstream](#upstream)|
|Lexus|LS 2018|All except Lexus Safety System+ A|[Upstream](#upstream)|
|Lexus|NS 2022-25|All|[Not compatible](#can-bus-security)|
|Lexus|NX 2018-19|All|[Upstream](#upstream)|
|Lexus|NX 2020-21|All|[Upstream](#upstream)|
|Lexus|NX Hybrid 2018-19|All|[Upstream](#upstream)|
|Lexus|NX Hybrid 2020-21|All|[Upstream](#upstream)|
|Lexus|RC 2018-20|All|[Upstream](#upstream)|
|Lexus|RC 2023|All|[Upstream](#upstream)|
|Lexus|RX 2016|Lexus Safety System+|[Upstream](#upstream)|
|Lexus|RX 2017-19|All|[Upstream](#upstream)|
|Lexus|RX 2020-22|All|[Upstream](#upstream)|
|Lexus|RX Hybrid 2016|Lexus Safety System+|[Upstream](#upstream)|
|Lexus|RX Hybrid 2017-19|All|[Upstream](#upstream)|
|Lexus|RX Hybrid 2020-22|All|[Upstream](#upstream)|
|Lexus|UX Hybrid 2019-24|All|[Upstream](#upstream)|
|Lincoln|Aviator 2020-24|Co-Pilot360 Plus|[Upstream](#upstream)|
|Lincoln|Aviator Plug-in Hybrid 2020-24|Co-Pilot360 Plus|[Upstream](#upstream)|
|MAN|eTGE 2020-24|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|MAN|TGE 2017-24|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Mazda|3 2017-18|All|[Dashcam mode](#dashcam)|
|Mazda|6 2017-20|All|[Dashcam mode](#dashcam)|
|Mazda|CX-5 2017-21|All|[Dashcam mode](#dashcam)|
|Mazda|CX-5 2022-25|All|[Upstream](#upstream)|
|Mazda|CX-9 2016-20|All|[Dashcam mode](#dashcam)|
|Mazda|CX-9 2021-23|All|[Upstream](#upstream)|
|MG|5 EV 2021|All|[Dashcam mode](#dashcam)|
|Nissan|Altima 2019-24|ProPILOT Assist|[Upstream](#upstream)|
|Nissan|Leaf 2018-23|ProPILOT Assist|[Upstream](#upstream)|
|Nissan|Rogue 2018-20|ProPILOT Assist|[Upstream](#upstream)|
|Nissan|X-Trail 2017|ProPILOT Assist|[Upstream](#upstream)|
|Peugeot|208 2019-25|Adaptive Cruise Control (ACC) & Lane Assist|[Dashcam mode](#dashcam)|
|Porsche|Macan 2017-24|Adaptive Cruise Control (ACC) & Lane Assist|[Dashcam mode](#dashcam)|
|Ram|1500 2019-24|Adaptive Cruise Control (ACC)|[Upstream](#upstream)|
|Ram|2500 2020-24|Adaptive Cruise Control (ACC)|[Dashcam mode](#dashcam)|
|Ram|3500 2019-22|Adaptive Cruise Control (ACC)|[Dashcam mode](#dashcam)|
|Rivian|R1S 2022-24|All|[Upstream](#upstream)|
|Rivian|R1T 2022-24|All|[Upstream](#upstream)|
|SEAT|Alhambra 2018-20|Adaptive Cruise Control (ACC) & Lane Assist|[Dashcam mode](#dashcam)|
|SEAT|Ateca 2016-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|SEAT|Leon 2014-20|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Subaru|Ascent 2019-21|All|[Upstream](#upstream)|
|Subaru|Ascent 2023|All|[Dashcam mode](#dashcam)|
|Subaru|Crosstrek 2018-19|EyeSight Driver Assistance|[Upstream](#upstream)|
|Subaru|Crosstrek 2020-23|EyeSight Driver Assistance|[Upstream](#upstream)|
|Subaru|Crosstrek Hybrid 2020|EyeSight Driver Assistance|[Dashcam mode](#dashcam)|
|Subaru|Forester 2017-18|EyeSight Driver Assistance|[Dashcam mode](#dashcam)|
|Subaru|Forester 2019-21|All|[Upstream](#upstream)|
|Subaru|Forester 2022-24|All|[Dashcam mode](#dashcam)|
|Subaru|Forester Hybrid 2020|EyeSight Driver Assistance|[Dashcam mode](#dashcam)|
|Subaru|Impreza 2017-19|EyeSight Driver Assistance|[Upstream](#upstream)|
|Subaru|Impreza 2020-22|EyeSight Driver Assistance|[Upstream](#upstream)|
|Subaru|Legacy 2015-18|EyeSight Driver Assistance|[Dashcam mode](#dashcam)|
|Subaru|Legacy 2020-22|All|[Upstream](#upstream)|
|Subaru|Outback 2015-17|EyeSight Driver Assistance|[Dashcam mode](#dashcam)|
|Subaru|Outback 2018-19|EyeSight Driver Assistance|[Dashcam mode](#dashcam)|
|Subaru|Outback 2020-22|All|[Upstream](#upstream)|
|Subaru|Outback 2023|All|[Dashcam mode](#dashcam)|
|Subaru|Solterra 2023-25|All|[Not compatible](#can-bus-security)|
|Subaru|XV 2018-19|EyeSight Driver Assistance|[Upstream](#upstream)|
|Subaru|XV 2020-21|EyeSight Driver Assistance|[Upstream](#upstream)|
|Škoda|Fabia 2022-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Škoda|Kamiq 2021-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Škoda|Karoq 2019-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Škoda|Kodiaq 2017-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Škoda|Octavia 2015-19|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Škoda|Octavia RS 2016|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Škoda|Octavia Scout 2017-19|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Škoda|Scala 2020-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Škoda|Superb 2015-22|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Tesla|Model 3 (with HW2.5) 2019|All|[Upstream](#upstream)|
|Tesla|Model 3 (with HW3) 2019-23|All|[Upstream](#upstream)|
|Tesla|Model 3 (with HW4) 2024-25|All|[Upstream](#upstream)|
|Tesla|Model X (with HW4) 2024|All|[Dashcam mode](#dashcam)|
|Tesla|Model Y (with HW3) 2020-24|All|[Upstream](#upstream)|
|Tesla|Model Y (with HW4) 2023-25|All|[Upstream](#upstream)|
|Toyota|Alphard 2019-20|All|[Upstream](#upstream)|
|Toyota|Alphard Hybrid 2021|All|[Upstream](#upstream)|
|Toyota|Avalon 2016|Toyota Safety Sense P|[Upstream](#upstream)|
|Toyota|Avalon 2017-18|All|[Upstream](#upstream)|
|Toyota|Avalon 2019-21|All|[Upstream](#upstream)|
|Toyota|Avalon 2022|All|[Upstream](#upstream)|
|Toyota|Avalon Hybrid 2019-21|All|[Upstream](#upstream)|
|Toyota|Avalon Hybrid 2022|All|[Upstream](#upstream)|
|Toyota|bZ4x 2023-25|All|[Not compatible](#can-bus-security)|
|Toyota|C-HR 2017-20|All|[Upstream](#upstream)|
|Toyota|C-HR 2021|All|[Upstream](#upstream)|
|Toyota|C-HR Hybrid 2017-20|All|[Upstream](#upstream)|
|Toyota|C-HR Hybrid 2021-22|All|[Upstream](#upstream)|
|Toyota|Camry 2018-20|All|[Upstream](#upstream)|
|Toyota|Camry 2021-24|All|[Upstream](#upstream)|
|Toyota|Camry 2025|All|[Not compatible](#can-bus-security)|
|Toyota|Camry Hybrid 2018-20|All|[Upstream](#upstream)|
|Toyota|Camry Hybrid 2021-24|All|[Upstream](#upstream)|
|Toyota|Corolla 2017-19|All|[Upstream](#upstream)|
|Toyota|Corolla 2020-22|All|[Upstream](#upstream)|
|Toyota|Corolla Cross 2022-25|All|[Not compatible](#can-bus-security)|
|Toyota|Corolla Cross (Non-US only) 2020-23|All|[Upstream](#upstream)|
|Toyota|Corolla Cross Hybrid (Non-US only) 2020-22|All|[Upstream](#upstream)|
|Toyota|Corolla Hatchback 2019-22|All|[Upstream](#upstream)|
|Toyota|Corolla Hybrid 2020-22|All|[Upstream](#upstream)|
|Toyota|Corolla Hybrid (South America only) 2020-23|All|[Upstream](#upstream)|
|Toyota|Highlander 2017-19|All|[Upstream](#upstream)|
|Toyota|Highlander 2020-23|All|[Upstream](#upstream)|
|Toyota|Highlander 2025|All|[Not compatible](#can-bus-security)|
|Toyota|Highlander Hybrid 2017-19|All|[Upstream](#upstream)|
|Toyota|Highlander Hybrid 2020-23|All|[Upstream](#upstream)|
|Toyota|Mirai 2021|All|[Upstream](#upstream)|
|Toyota|Prius 2016|Toyota Safety Sense P|[Upstream](#upstream)|
|Toyota|Prius 2017-20|All|[Upstream](#upstream)|
|Toyota|Prius 2021-22|All|[Upstream](#upstream)|
|Toyota|Prius Prime 2017-20|All|[Upstream](#upstream)|
|Toyota|Prius Prime 2021-22|All|[Upstream](#upstream)|
|Toyota|Prius v 2017|Toyota Safety Sense P|[Upstream](#upstream)|
|Toyota|RAV4 2016|Toyota Safety Sense P|[Upstream](#upstream)|
|Toyota|RAV4 2017-18|All|[Upstream](#upstream)|
|Toyota|RAV4 2019-21|All|[Upstream](#upstream)|
|Toyota|RAV4 2022|All|[Upstream](#upstream)|
|Toyota|RAV4 2023-25|All|[Upstream](#upstream)|
|Toyota|RAV4 Hybrid 2016|Toyota Safety Sense P|[Upstream](#upstream)|
|Toyota|RAV4 Hybrid 2017-18|All|[Upstream](#upstream)|
|Toyota|RAV4 Hybrid 2019-21|All|[Upstream](#upstream)|
|Toyota|RAV4 Hybrid 2022|All|[Upstream](#upstream)|
|Toyota|RAV4 Hybrid 2023-25|All|[Upstream](#upstream)|
|Toyota|RAV4 Prime 2021-23|All|[Custom](#secoc-cars-with-recoverable-keys)|
|Toyota|RAV4 Prime 2024-25|All|[Not compatible](#can-bus-security)|
|Toyota|Sequoia 2023-25|All|[Not compatible](#can-bus-security)|
|Toyota|Sienna 2018-20|All|[Upstream](#upstream)|
|Toyota|Sienna 2021-23|All|[Custom](#secoc-cars-with-recoverable-keys)|
|Toyota|Sienna 2024-25|All|[Not compatible](#can-bus-security)|
|Toyota|Tundra 2022-25|All|[Not compatible](#can-bus-security)|
|Toyota|Venza 2021-25|All|[Not compatible](#can-bus-security)|
|Toyota|Yaris (Non-US only) 2020, 2023|All|[Custom](#secoc-cars-with-recoverable-keys)|
|Volkswagen|Arteon 2018-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Arteon eHybrid 2020-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Arteon R 2020-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Arteon Shooting Brake 2020-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Atlas 2018-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Atlas Cross Sport 2020-22|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Caddy 2019|Adaptive Cruise Control (ACC) & Lane Assist|[Dashcam mode](#dashcam)|
|Volkswagen|Caddy Maxi 2019|Adaptive Cruise Control (ACC) & Lane Assist|[Dashcam mode](#dashcam)|
|Volkswagen|California 2021-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Caravelle 2020|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|CC 2018-22|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Crafter 2017-24|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|e-Crafter 2018-24|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|e-Golf 2014-20|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Golf 2015-20|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Golf Alltrack 2015-19|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Golf GTD 2015-20|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Golf GTE 2015-20|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Golf GTI 2015-21|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Golf R 2015-19|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Golf SportsVan 2015-20|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Grand California 2019-24|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|ID.4 2021-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|ID.4 2024-25|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Jetta 2015-18|Adaptive Cruise Control (ACC) & Lane Assist|[Dashcam mode](#dashcam)|
|Volkswagen|Jetta 2019-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Jetta GLI 2021-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Passat 2015-22|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Passat Alltrack 2015-22|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Passat GTE 2015-22|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Passat NMS 2017-22|Adaptive Cruise Control (ACC) & Lane Assist|[Dashcam mode](#dashcam)|
|Volkswagen|Polo 2018-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Polo GTI 2018-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Sharan 2018-22|Adaptive Cruise Control (ACC) & Lane Assist|[Dashcam mode](#dashcam)|
|Volkswagen|T-Cross 2021|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|T-Roc 2018-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Taos 2022-24|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Teramont 2018-22|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Teramont Cross Sport 2021-22|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Teramont X 2021-22|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Tiguan 2018-24|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Tiguan eHybrid 2021-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
|Volkswagen|Touran 2016-23|Adaptive Cruise Control (ACC) & Lane Assist|[Upstream](#upstream)|
# Types of Support
**opendbc can support many more cars than it currently does.** There are a few reasons your car may not be supported.
If your car doesn't fit into any of the incompatibility criteria here, then there's a good chance it can be supported!
We're adding support for new cars all the time. **We don't have a roadmap for car support**, and in fact, most car
support comes from users like you!
## Upstream
A supported vehicle is one that just works when you install a comma device. All supported cars provide a better
experience than any stock system. Supported vehicles reference the US market unless otherwise specified.
## Under Review
A vehicle under review is one for which software support has been merged into upstream openpilot, but hasn't yet been
tested for drive quality and conformance with [comma safety guidelines](https://github.com/commaai/openpilot/blob/master/docs/SAFETY.md).
This is a normal part of the development and quality assurance process. This vehicle will not work when upstream
openpilot is installed, but custom forks may allow their use.
## Custom
Vehicles in this category are not considered plug-and-play. Software support is included in upstream openpilot, but
these vehicles might not have a harness in the comma store, or the physical install might be at an unusual or cumbersome
location, or they might need unusual configuration after install. These vehicles will not work with release builds of
openpilot, but depending on the situation, development builds or custom forks may allow their use.
### SecOC cars with recoverable keys
For a small subset of SecOC-protected vehicles, tools may be available in the community to recover the SecOC keys. These
tools, and the recovery process, are not part of openpilot. If supplied with a valid SecOC key, development builds or
custom forks may work with these vehicles. Release builds of openpilot don't support SecOC.
## Dashcam
Dashcam vehicles have software support in upstream openpilot, but will go into "dashcam mode" at startup and will not
engage. This may be due to known issues with driving safety or quality, or it may be a work in progress that isn't yet
ready for safety and quality review.
## Community
Although they're not upstream, the community has openpilot running on other makes and models. See the 'Community
Supported Models' section of each make [on our wiki](https://wiki.comma.ai/).
Some notable works-in-progress:
* Honda
* 2022-24 Acura RDX, commaai/opendbc#1967
* Camera ACC stability improvements, commaai/opendbc#2192
* Alpha longitudinal stability improvements, commaai/opendbc#2347 and commaai/opendbc#2165
## Incompatible
### CAN Bus Security
Vehicles with CAN security measures, such as AUTOSAR Secure Onboard Communication (SecOC) are not usable with openpilot
unless the owner can recover the message signing key and implement CAN message signing. Examples include certain newer
Toyota, and the GM Global B platform.
### FlexRay
All the cars that openpilot supports use a [CAN bus](https://en.wikipedia.org/wiki/CAN_bus) for communication between all the car's computers, however a
CAN bus isn't the only way that the computers in your car can communicate. Most, if not all, vehicles from the following
manufacturers use [FlexRay](https://en.wikipedia.org/wiki/FlexRay) instead of a CAN bus: **BMW, Mercedes, Audi, Land Rover, and some Volvo**. These cars
may one day be supported, but we have no immediate plans to support FlexRay.
+117
View File
@@ -0,0 +1,117 @@
#!/usr/bin/env python3
import time
import threading
import argparse
import numpy as np
from pprint import pprint
from inputs import get_gamepad
from kbhit import KBHit
from opendbc.car.structs import CarControl
from opendbc.car.panda_runner import PandaRunner
class Keyboard:
def __init__(self):
self.kb = KBHit()
self.axis_increment = 0.05 # 5% of full actuation each key press
self.axes_map = {'w': 'gb', 's': 'gb',
'a': 'steer', 'd': 'steer'}
self.axes_values = {'gb': 0., 'steer': 0.}
self.axes_order = ['gb', 'steer']
self.cancel = False
def update(self):
key = self.kb.getch().lower()
print(key)
self.cancel = False
if key == 'r':
self.axes_values = {ax: 0. for ax in self.axes_values}
elif key == 'c':
self.cancel = True
elif key in self.axes_map:
axis = self.axes_map[key]
incr = self.axis_increment if key in ['w', 'a'] else -self.axis_increment
self.axes_values[axis] = float(np.clip(self.axes_values[axis] + incr, -1, 1))
else:
return False
return True
class Joystick:
def __init__(self, gamepad=False):
# TODO: find a way to get this from API, perhaps "inputs" doesn't support it
if gamepad:
self.cancel_button = 'BTN_NORTH' # (BTN_NORTH=X, ABS_RZ=Right Trigger)
accel_axis = 'ABS_Y'
steer_axis = 'ABS_RX'
else:
self.cancel_button = 'BTN_TRIGGER'
accel_axis = 'ABS_Y'
steer_axis = 'ABS_RX'
self.min_axis_value = {accel_axis: 0., steer_axis: 0.}
self.max_axis_value = {accel_axis: 255., steer_axis: 255.}
self.axes_values = {accel_axis: 0., steer_axis: 0.}
self.axes_order = [accel_axis, steer_axis]
self.cancel = False
def update(self):
joystick_event = get_gamepad()[0]
event = (joystick_event.code, joystick_event.state)
if event[0] == self.cancel_button:
if event[1] == 1:
self.cancel = True
elif event[1] == 0: # state 0 is falling edge
self.cancel = False
elif event[0] in self.axes_values:
self.max_axis_value[event[0]] = max(event[1], self.max_axis_value[event[0]])
self.min_axis_value[event[0]] = min(event[1], self.min_axis_value[event[0]])
norm = -float(np.interp(event[1], [self.min_axis_value[event[0]], self.max_axis_value[event[0]]], [-1., 1.]))
self.axes_values[event[0]] = norm if abs(norm) > 0.05 else 0. # center can be noisy, deadzone of 5%
else:
return False
return True
def joystick_thread(joystick):
while True:
joystick.update()
def main(joystick):
threading.Thread(target=joystick_thread, args=(joystick,), daemon=True).start()
with PandaRunner() as p:
CC = CarControl(enabled=False)
while True:
CC.actuators.accel = float(4.0*np.clip(joystick.axes_values['gb'], -1, 1))
CC.actuators.torque = float(np.clip(joystick.axes_values['steer'], -1, 1))
pprint(CC)
p.read()
p.write(CC)
# 100Hz
time.sleep(0.01)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Test the car interface with a joystick. Uses keyboard by default.',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--mode', choices=['keyboard', 'gamepad', 'joystick'], default='keyboard')
args = parser.parse_args()
print()
joystick: Keyboard | Joystick
if args.mode == 'keyboard':
print('Gas/brake control: `W` and `S` keys')
print('Steering control: `A` and `D` keys')
print('Buttons')
print('- `R`: Resets axes')
print('- `C`: Cancel cruise control')
joystick = Keyboard()
else:
joystick = Joystick(gamepad=(args.mode == 'gamepad'))
main(joystick)
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python3
import sys
import termios
import atexit
from select import select
STDIN_FD = sys.stdin.fileno()
class KBHit:
def __init__(self) -> None:
self.set_kbhit_terminal()
def set_kbhit_terminal(self) -> None:
# Save the terminal settings
self.old_term = termios.tcgetattr(STDIN_FD)
self.new_term = self.old_term.copy()
# New terminal setting unbuffered
self.new_term[3] &= ~(termios.ICANON | termios.ECHO)
termios.tcsetattr(STDIN_FD, termios.TCSAFLUSH, self.new_term)
# Support normal-terminal reset at exit
atexit.register(self.set_normal_term)
def set_normal_term(self) -> None:
termios.tcsetattr(STDIN_FD, termios.TCSAFLUSH, self.old_term)
@staticmethod
def getch() -> str:
return sys.stdin.read(1)
@staticmethod
def getarrow() -> int:
c = sys.stdin.read(3)[2]
vals = [65, 67, 66, 68]
return vals.index(ord(c))
@staticmethod
def kbhit():
''' Returns True if keyboard character was hit, False otherwise.
'''
return select([sys.stdin], [], [], 0)[0] != []
if __name__ == "__main__":
kb = KBHit()
print('Hit any key, or ESC to exit')
while True:
if kb.kbhit():
c = kb.getch()
if c == '\x1b': # ESC
break
print(c)
kb.set_normal_term()
+30
View File
@@ -0,0 +1,30 @@
output:
- meta # Print lefthook version
- summary # Print summary block (successful and failed steps)
- empty_summary # Print summary heading when there are no steps to run
- success # Print successful steps
- failure # Print failed steps printing
- execution # Print any execution logs
#- execution_out # Print execution output
#- execution_info # Print `EXECUTE > ...` logging
- skips # Print "skip" (i.e. no files matched)
test:
parallel: true
commands:
# *** static analysis
ruff:
run: ruff check .
ty:
run: ty check
codespell:
run: codespell {files} -L tge,stdio -S *.dbc
files: git ls-tree -r HEAD --name-only
cpplint:
run: cpplint --exclude=opendbc/can/*_pyx.cpp --recursive --quiet --counting=detailed --linelength=240 --filter=-build,-legal,-readability,-runtime,-whitespace,+build/include_subdir,+build/forward_decl,+build/include_what_you_use,+build/deprecated,+whitespace/comma,+whitespace/line_length,+whitespace/empty_if_body,+whitespace/empty_loop_body,+whitespace/empty_conditional_body,+whitespace/forcolon,+whitespace/parens,+whitespace/semicolon,+whitespace/tab,+readability/braces opendbc/
misra:
run: opendbc/safety/tests/misra/test_misra.sh
# *** tests ***
unittest:
run: unittest-parallel -j4
+17
View File
@@ -0,0 +1,17 @@
import os
DBC_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dbc')
# -I include path for e.g. "#include <opendbc/safety/safety.h>"
INCLUDE_PATH = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../"))
_generated_dbc_cache: dict[str, str] | None = None
def get_generated_dbcs() -> dict[str, str]:
"""Lazily generate all *_generated DBC content in memory.
Returns {name: content} where name has no .dbc extension."""
global _generated_dbc_cache
if _generated_dbc_cache is None:
from opendbc.dbc.generator.generator import generate_all
_generated_dbc_cache = generate_all()
return _generated_dbc_cache
+8
View File
@@ -0,0 +1,8 @@
from opendbc.can.packer import CANPacker
from opendbc.can.parser import CANParser, CANDefine
__all__ = [
"CANDefine",
"CANParser",
"CANPacker",
]
+269
View File
@@ -0,0 +1,269 @@
import re
import os
from collections.abc import Callable
from dataclasses import dataclass
from functools import cache
from opendbc import DBC_PATH, get_generated_dbcs
# TODO: these should just be passed in along with the DBC file
from opendbc.car.honda.hondacan import honda_checksum
from opendbc.car.toyota.toyotacan import toyota_checksum
from opendbc.car.subaru.subarucan import subaru_checksum
from opendbc.car.chrysler.chryslercan import chrysler_checksum, fca_giorgio_checksum
from opendbc.car.hyundai.hyundaicanfd import hkg_can_fd_checksum
from opendbc.car.volkswagen.mlbcan import volkswagen_mlb_checksum
from opendbc.car.volkswagen.mqbcan import volkswagen_meb_alt_crc_checksum, volkswagen_mqb_meb_checksum, xor_checksum
from opendbc.car.tesla.teslacan import tesla_checksum
from opendbc.car.body.bodycan import body_checksum
from opendbc.car.byd.bydcan import byd_checksum
from opendbc.car.psa.psacan import psa_checksum
from opendbc.car.rivian.riviancan import rivian_checksum
from opendbc.car.ford.fordcan import FORD_CHECKSUM_FIELDS, ford_checksum
from opendbc.car.mg.mgcan import mg_checksum
class SignalType:
DEFAULT = 0
COUNTER = 1
HONDA_CHECKSUM = 2
TOYOTA_CHECKSUM = 3
BODY_CHECKSUM = 4
VOLKSWAGEN_MQB_MEB_CHECKSUM = 5
XOR_CHECKSUM = 6
SUBARU_CHECKSUM = 7
CHRYSLER_CHECKSUM = 8
HKG_CAN_FD_CHECKSUM = 9
FCA_GIORGIO_CHECKSUM = 10
TESLA_CHECKSUM = 11
PSA_CHECKSUM = 12
VOLKSWAGEN_MLB_CHECKSUM = 13
BYD_CHECKSUM = 14
RIVIAN_CHECKSUM = 15
FORD_CHECKSUM = 16
MG_CHECKSUM = 17
@dataclass
class Signal:
name: str
start_bit: int
msb: int
lsb: int
size: int
is_signed: bool
factor: float
offset: float
is_little_endian: bool
type: int = SignalType.DEFAULT
calc_checksum: 'Callable[[int, Signal, bytearray], int] | None' = None
checksum_fields: tuple['Signal', ...] = ()
def get_raw_value(self, dat: bytes | bytearray) -> int:
ret = 0
i = self.msb // 8
bits = self.size
while 0 <= i < len(dat) and bits > 0:
lsb = self.lsb if (self.lsb // 8) == i else i * 8
msb = self.msb if (self.msb // 8) == i else (i + 1) * 8 - 1
size = msb - lsb + 1
d = (dat[i] >> (lsb - (i * 8))) & ((1 << size) - 1)
ret |= d << (bits - size)
bits -= size
i = i - 1 if self.is_little_endian else i + 1
return ret
@dataclass
class Msg:
name: str
address: int
size: int
sigs: dict[str, Signal]
@dataclass
class Val:
name: str
address: int
def_val: str
BO_RE = re.compile(r"^BO_ (\w+) (\w+) *: (\w+) (\w+)")
SG_RE = re.compile(r"^SG_ (\w+) : (\d+)\|(\d+)@(\d)([+-]) \(([0-9.+\-eE]+),([0-9.+\-eE]+)\) \[[0-9.+\-eE]+\|[0-9.+\-eE]+\] \".*\" .*")
SGM_RE = re.compile(r"^SG_ (\w+) (\w+) *: (\d+)\|(\d+)@(\d)([+-]) \(([0-9.+\-eE]+),([0-9.+\-eE]+)\) \[[0-9.+\-eE]+\|[0-9.+\-eE]+\] \".*\" .*")
VAL_RE = re.compile(r"^VAL_ (\w+) (\w+) (.*);")
VAL_SPLIT_RE = re.compile(r'["]+')
@cache
class DBC:
def __init__(self, name: str):
if os.path.exists(name):
self._parse_file(name)
else:
dbc_path = os.path.join(DBC_PATH, name + ".dbc")
if content := get_generated_dbcs().get(name):
self._parse_content(name, content)
elif os.path.exists(dbc_path):
self._parse_file(dbc_path)
else:
raise FileNotFoundError(f"DBC not found: {name}")
def _parse_file(self, path: str):
self.name = os.path.basename(path).replace(".dbc", "")
with open(path) as f:
lines = f.readlines()
self._parse_lines(lines)
def _parse_content(self, name: str, content: str):
self.name = name
lines = content.splitlines(keepends=True)
self._parse_lines(lines)
def _parse_lines(self, lines: list[str]):
checksum_state = get_checksum_state(self.name)
be_bits = [j + i * 8 for i in range(64) for j in range(7, -1, -1)]
self.msgs: dict[int, Msg] = {}
self.addr_to_msg: dict[int, Msg] = {}
self.name_to_msg: dict[str, Msg] = {}
self.vals: list[Val] = []
address = 0
signals_temp: dict[int, dict[str, Signal]] = {}
for line_num, line in enumerate(lines, 1):
line = line.strip()
if line.startswith("BO_ "):
m = BO_RE.match(line)
if not m:
continue
address = int(m.group(1), 0)
msg_name = m.group(2)
size = int(m.group(3), 0)
sigs = {}
self.msgs[address] = Msg(msg_name, address, size, sigs)
self.addr_to_msg[address] = self.msgs[address]
self.name_to_msg[msg_name] = self.msgs[address]
signals_temp[address] = sigs
elif line.startswith("SG_ "):
m = SG_RE.search(line)
offset = 0
if not m:
m = SGM_RE.search(line)
if not m:
continue
offset = 1
sig_name = m.group(1)
start_bit = int(m.group(2 + offset))
size = int(m.group(3 + offset))
is_little_endian = m.group(4 + offset) == "1"
is_signed = m.group(5 + offset) == "-"
factor = float(m.group(6 + offset))
offset_val = float(m.group(7 + offset))
if is_little_endian:
lsb = start_bit
msb = start_bit + size - 1
else:
idx = be_bits.index(start_bit)
lsb = be_bits[idx + size - 1]
msb = start_bit
sig = Signal(sig_name, start_bit, msb, lsb, size, is_signed, factor, offset_val, is_little_endian)
set_signal_type(sig, checksum_state, self.name, line_num, address)
signals_temp[address][sig_name] = sig
elif line.startswith("VAL_ "):
m = VAL_RE.search(line)
if not m:
continue
val_addr = int(m.group(1), 0)
sgname = m.group(2)
defs = m.group(3)
words = [w.strip() for w in VAL_SPLIT_RE.split(defs) if w.strip()]
words = [w.upper().replace(" ", "_") for w in words]
val_def = " ".join(words).strip()
self.vals.append(Val(sgname, val_addr, val_def))
for addr, sigs in signals_temp.items():
self.msgs[addr].sigs = sigs
if checksum_state and checksum_state.checksum_fields and addr in checksum_state.checksum_fields:
fields = tuple(sigs[name] for name in checksum_state.checksum_fields[addr])
for sig in sigs.values():
if sig.calc_checksum:
sig.checksum_fields = fields
# ***** checksum functions *****
def tesla_setup_signal(sig: Signal, dbc_name: str, line_num: int) -> None:
if sig.name.endswith("Counter"):
sig.type = SignalType.COUNTER
elif sig.name.endswith("Checksum"):
sig.type = SignalType.TESLA_CHECKSUM
sig.calc_checksum = tesla_checksum
def ford_setup_signal(sig: Signal, dbc_name: str, line_num: int) -> None:
if sig.name in ("VehVActlBrk_No_Cnt", "VehVActlEng_No_Cnt", "VehRollYaw_No_Cnt"):
sig.type = SignalType.COUNTER
@dataclass
class ChecksumState:
checksum_type: int
calc_checksum: Callable[[int, Signal, bytearray], int] | None
setup_signal: Callable[[Signal, str, int], None] | None = None
checksum_pattern: str = r"^CHECKSUM$"
checksum_fields: dict[int, tuple[str, ...]] | None = None
def get_checksum_state(dbc_name: str) -> ChecksumState | None:
if dbc_name.startswith(("honda_", "acura_")):
return ChecksumState(SignalType.HONDA_CHECKSUM, honda_checksum)
elif dbc_name.startswith(("toyota_", "lexus_")):
return ChecksumState(SignalType.TOYOTA_CHECKSUM, toyota_checksum)
elif dbc_name.startswith("hyundai_canfd_generated"):
return ChecksumState(SignalType.HKG_CAN_FD_CHECKSUM, hkg_can_fd_checksum)
elif dbc_name.startswith("vw_meb_2024"):
return ChecksumState(SignalType.VOLKSWAGEN_MQB_MEB_CHECKSUM, volkswagen_meb_alt_crc_checksum)
elif dbc_name.startswith(("vw_mqb", "vw_mqbevo", "vw_meb")):
return ChecksumState(SignalType.VOLKSWAGEN_MQB_MEB_CHECKSUM, volkswagen_mqb_meb_checksum)
elif dbc_name.startswith("vw_mlb"):
return ChecksumState(SignalType.VOLKSWAGEN_MLB_CHECKSUM, volkswagen_mlb_checksum)
elif dbc_name.startswith("vw_pq"):
return ChecksumState(SignalType.XOR_CHECKSUM, xor_checksum)
elif dbc_name.startswith("subaru_global_"):
return ChecksumState(SignalType.SUBARU_CHECKSUM, subaru_checksum)
elif dbc_name.startswith("chrysler_"):
return ChecksumState(SignalType.CHRYSLER_CHECKSUM, chrysler_checksum)
elif dbc_name.startswith("fca_giorgio"):
return ChecksumState(SignalType.FCA_GIORGIO_CHECKSUM, fca_giorgio_checksum)
elif dbc_name.startswith("comma_body"):
return ChecksumState(SignalType.BODY_CHECKSUM, body_checksum)
elif dbc_name.startswith("tesla_model3_party"):
return ChecksumState(SignalType.TESLA_CHECKSUM, tesla_checksum, tesla_setup_signal)
elif dbc_name.startswith("psa_"):
return ChecksumState(SignalType.PSA_CHECKSUM, psa_checksum)
elif dbc_name.startswith("byd_"):
return ChecksumState(SignalType.BYD_CHECKSUM, byd_checksum)
elif dbc_name == "ford_lincoln_base_pt":
# Other _Cs fields are optional or use model-specific checksum algorithms.
return ChecksumState(SignalType.FORD_CHECKSUM, ford_checksum, ford_setup_signal, checksum_pattern=r"_Cs$",
checksum_fields=FORD_CHECKSUM_FIELDS)
elif dbc_name == "mg":
return ChecksumState(SignalType.MG_CHECKSUM, mg_checksum, checksum_pattern=r"Chksm|^ChLKARespToqPVHSC2$")
elif dbc_name == "rivian_primary_actuator":
return ChecksumState(SignalType.RIVIAN_CHECKSUM, rivian_checksum, checksum_pattern=r"_Checksum$")
return None
def set_signal_type(sig: Signal, chk: ChecksumState | None, dbc_name: str, line_num: int, address: int) -> None:
sig.calc_checksum = None
if chk:
if chk.setup_signal:
chk.setup_signal(sig, dbc_name, line_num)
if (re.search(chk.checksum_pattern, sig.name) and
(chk.checksum_fields is None or address in chk.checksum_fields)):
sig.type = chk.checksum_type
sig.calc_checksum = chk.calc_checksum
elif sig.name == "COUNTER":
sig.type = SignalType.COUNTER
+71
View File
@@ -0,0 +1,71 @@
import math
from opendbc.car.carlog import carlog
from opendbc.can.dbc import DBC, Signal, SignalType
class CANPacker:
def __init__(self, dbc_name: str):
self.dbc = DBC(dbc_name)
self.counters: dict[int, int] = {}
def pack(self, address: int, values: dict[str, float]) -> bytearray:
msg = self.dbc.addr_to_msg.get(address)
if msg is None:
carlog.error(f"msg not found for {address=}")
return bytearray()
dat = bytearray(msg.size)
counter_set = False
for name, value in values.items():
sig = msg.sigs.get(name)
if sig is None:
carlog.error(f"unknown signal {name=} in {msg.name}")
continue
ival = int(math.floor((value - sig.offset) / sig.factor + 0.5))
if ival < 0:
ival = (1 << sig.size) + ival
set_value(dat, sig, ival)
if sig.type == SignalType.COUNTER or sig.name == "COUNTER":
self.counters[address] = int(value)
counter_set = True
sig_counter = next((s for s in msg.sigs.values() if s.type == SignalType.COUNTER or s.name == "COUNTER"), None)
if sig_counter and not counter_set:
if address not in self.counters:
self.counters[address] = 0
set_value(dat, sig_counter, self.counters[address])
self.counters[address] = (self.counters[address] + 1) % (1 << sig_counter.size)
sig_checksum = next((s for s in msg.sigs.values() if s.type > SignalType.COUNTER), None)
if sig_checksum and sig_checksum.calc_checksum:
checksum = sig_checksum.calc_checksum(address, sig_checksum, dat)
set_value(dat, sig_checksum, checksum)
return dat
def make_can_msg(self, name_or_addr, bus: int, values: dict[str, float]):
if isinstance(name_or_addr, int):
addr = name_or_addr
else:
msg = self.dbc.name_to_msg.get(name_or_addr)
if msg is None:
carlog.error(f"msg not found for {name_or_addr=}")
return 0, b'', bus
addr = msg.address
dat = self.pack(addr, values)
if len(dat) == 0:
return 0, b'', bus
return addr, bytes(dat), bus
def set_value(msg: bytearray, sig: Signal, ival: int) -> None:
i = sig.lsb // 8
bits = sig.size
if sig.size < 64:
ival &= (1 << sig.size) - 1
while 0 <= i < len(msg) and bits > 0:
shift = sig.lsb % 8 if (sig.lsb // 8) == i else 0
size = min(bits, 8 - shift)
mask = ((1 << size) - 1) << shift
msg[i] &= ~mask
msg[i] |= (ival & ((1 << size) - 1)) << shift
bits -= size
ival >>= size
i = i + 1 if sig.is_little_endian else i - 1
+259
View File
@@ -0,0 +1,259 @@
import math
import numbers
from collections import defaultdict, deque
from dataclasses import dataclass, field
from opendbc.car.carlog import carlog
from opendbc.can.dbc import DBC, Signal
MAX_BAD_COUNTER = 5
CAN_INVALID_CNT = 5
@dataclass
class MessageState:
address: int
name: str
size: int
signals: list[Signal]
ignore_alive: bool = False
ignore_checksum: bool = False
ignore_counter: bool = False
frequency: float = 0.0
timeout_threshold: float = 1e5 # default to 1Hz threshold
vals: list[float] = field(default_factory=list)
all_vals: list[list[float]] = field(default_factory=list)
timestamps: deque[int] = field(default_factory=lambda: deque(maxlen=500))
counter: int = 0
counter_fail: int = 0
first_seen_nanos: int = 0
last_warning_log_nanos: int = 0
def rate_limited_log(self, last_update_nanos: int, msg: str) -> None:
if (last_update_nanos - self.last_warning_log_nanos) >= 1_000_000_000:
carlog.warning(f"CANParser: {hex(self.address)} {self.name} {msg}")
self.last_warning_log_nanos = last_update_nanos
def parse(self, nanos: int, dat: bytes) -> bool:
tmp_vals: list[float] = [0.0] * len(self.signals)
checksum_failed = False
counter_failed = False
if self.first_seen_nanos == 0:
self.first_seen_nanos = nanos
for i, sig in enumerate(self.signals):
tmp = sig.get_raw_value(dat)
if sig.is_signed:
tmp -= ((tmp >> (sig.size - 1)) & 0x1) * (1 << sig.size)
if not self.ignore_checksum and sig.calc_checksum is not None:
expected_checksum = sig.calc_checksum(self.address, sig, bytearray(dat))
if tmp != expected_checksum:
checksum_failed = True
self.rate_limited_log(nanos, f"checksum failed: received {hex(tmp)}, calculated {hex(expected_checksum)}")
if not self.ignore_counter and sig.type == 1: # COUNTER
if not self.update_counter(tmp, sig.size):
counter_failed = True
tmp_vals[i] = tmp * sig.factor + sig.offset
# must have good counter and checksum to update data
if checksum_failed or counter_failed:
return False
if not self.vals:
self.vals = [0.0] * len(self.signals)
self.all_vals = [[] for _ in self.signals]
for i, v in enumerate(tmp_vals):
self.vals[i] = v
self.all_vals[i].append(v)
self.timestamps.append(nanos)
if self.frequency < 1e-5 and len(self.timestamps) >= 3:
dt = (self.timestamps[-1] - self.timestamps[0]) * 1e-9
if (dt > 1.0 or (self.timestamps.maxlen is not None and len(self.timestamps) >= self.timestamps.maxlen)) and dt != 0:
self.frequency = min(len(self.timestamps) / dt, 100.0)
self.timeout_threshold = (1_000_000_000 / self.frequency) * 10
return True
def update_counter(self, cur_count: int, cnt_size: int) -> bool:
if ((self.counter + 1) & ((1 << cnt_size) - 1)) != cur_count:
self.counter_fail = min(self.counter_fail + 1, MAX_BAD_COUNTER)
elif self.counter_fail > 0:
self.counter_fail -= 1
self.counter = cur_count
return self.counter_fail < MAX_BAD_COUNTER
def valid(self, current_nanos: int, bus_timeout: bool) -> bool:
if self.ignore_alive:
return True
if not self.timestamps:
return False
if (current_nanos - self.timestamps[-1]) > self.timeout_threshold:
return False
return True
class VLDict(dict):
def __init__(self, parser):
super().__init__()
self.parser = parser
def __getitem__(self, key):
if key not in self:
self.parser._add_message(key)
return super().__getitem__(key)
class CANParser:
def __init__(self, dbc_name: str, messages: list[tuple[str | int, int]], bus: int):
self.dbc_name: str = dbc_name
self.bus: int = bus
self.dbc = DBC(dbc_name)
self.vl: dict[int | str, dict[str, float]] = VLDict(self)
self.vl_all: dict[int | str, dict[str, list[float]]] = {}
self.ts_nanos: dict[int | str, dict[str, int]] = {}
self.addresses: set[int] = set()
self.message_states: dict[int, MessageState] = {}
for name_or_addr, freq in messages:
if isinstance(name_or_addr, numbers.Number):
msg = self.dbc.addr_to_msg.get(int(name_or_addr))
else:
msg = self.dbc.name_to_msg.get(name_or_addr)
if msg is None:
raise RuntimeError(f"could not find message {name_or_addr!r} in DBC {dbc_name}")
if msg.address in self.addresses:
raise RuntimeError("Duplicate Message Check: %d" % msg.address)
self._add_message(name_or_addr, freq)
self.can_invalid_cnt: int = CAN_INVALID_CNT
self.last_nonempty_nanos: int = 0
self._last_update_nanos: int = 0
def _add_message(self, name_or_addr: str | int, freq: int | None = None) -> None:
if isinstance(name_or_addr, numbers.Number):
msg = self.dbc.addr_to_msg.get(int(name_or_addr))
else:
msg = self.dbc.name_to_msg.get(name_or_addr)
assert msg is not None
assert msg.address not in self.addresses
self.addresses.add(msg.address)
signal_names = list(msg.sigs.keys())
signals_dict = {s: 0.0 for s in signal_names}
dict.__setitem__(self.vl, msg.address, signals_dict)
dict.__setitem__(self.vl, msg.name, signals_dict)
self.vl_all[msg.address] = defaultdict(list)
self.vl_all[msg.name] = self.vl_all[msg.address]
self.ts_nanos[msg.address] = {s: 0 for s in signal_names}
self.ts_nanos[msg.name] = self.ts_nanos[msg.address]
state = MessageState(
address=msg.address,
name=msg.name,
size=msg.size,
signals=list(msg.sigs.values()),
ignore_alive=freq is not None and math.isnan(freq),
)
if freq is not None and freq > 0:
state.frequency = freq
else:
# if frequency not specified, assume 1Hz until we learn it
freq = 1
state.timeout_threshold = (1_000_000_000 / freq) * 10
self.message_states[msg.address] = state
@property
def bus_timeout(self) -> bool:
ignore_alive = all(s.ignore_alive for s in self.message_states.values())
bus_timeout_threshold = 500 * 1_000_000
for st in self.message_states.values():
if st.timeout_threshold > 0:
bus_timeout_threshold = min(bus_timeout_threshold, st.timeout_threshold)
return ((self._last_update_nanos - self.last_nonempty_nanos) > bus_timeout_threshold) and not ignore_alive
@property
def can_valid(self) -> bool:
valid = True
counters_valid = True
bus_timeout = self.bus_timeout
for state in self.message_states.values():
if state.counter_fail >= MAX_BAD_COUNTER:
counters_valid = False
state.rate_limited_log(self._last_update_nanos, f"counter invalid, {state.counter_fail=} {MAX_BAD_COUNTER=}")
if not state.valid(self._last_update_nanos, bus_timeout):
valid = False
state.rate_limited_log(self._last_update_nanos, "not valid (timeout or missing)")
# TODO: probably only want to increment this once per update() call
self.can_invalid_cnt = 0 if valid else min(self.can_invalid_cnt + 1, CAN_INVALID_CNT)
return self.can_invalid_cnt < CAN_INVALID_CNT and counters_valid
def update(self, strings, sendcan: bool = False):
if strings and not isinstance(strings[0], list | tuple):
strings = [strings]
for addr in self.addresses:
for k in self.vl_all[addr]:
self.vl_all[addr][k].clear()
updated_addrs: set[int] = set()
for entry in strings:
t = entry[0]
frames = entry[1]
bus_empty = True
for address, dat, src in frames:
if src != self.bus:
continue
bus_empty = False
state = self.message_states.get(address)
if state is None or len(dat) > 64:
continue
if state.parse(t, dat):
updated_addrs.add(address)
vl_addr = self.vl[address]
vl_all_addr = self.vl_all[address]
ts_addr = self.ts_nanos[address]
for i, sig in enumerate(state.signals):
vl_addr[sig.name] = state.vals[i]
vl_all_addr[sig.name] = state.all_vals[i]
ts_addr[sig.name] = state.timestamps[-1]
if not bus_empty:
self.last_nonempty_nanos = t
self._last_update_nanos = t
return updated_addrs
class CANDefine:
def __init__(self, dbc_name: str):
dbc = DBC(dbc_name)
dv = defaultdict(dict)
for val in dbc.vals:
sgname = val.name
address = val.address
msg = dbc.addr_to_msg.get(address)
if msg is None:
raise KeyError(address)
msgname = msg.name
parts = val.def_val.split()
values = [int(v) for v in parts[::2]]
defs = parts[1::2]
dv[address][sgname] = dict(zip(values, defs, strict=True))
dv[msgname][sgname] = dv[address][sgname]
self.dv = dict(dv)
@@ -0,0 +1 @@
*.bz2
@@ -0,0 +1,9 @@
import glob
import os
from opendbc import DBC_PATH, get_generated_dbcs
static_dbcs = [os.path.basename(dbc).split('.')[0] for dbc in
glob.glob(f"{DBC_PATH}/*.dbc")]
ALL_DBCS = sorted(set(static_dbcs + list(get_generated_dbcs().keys())))
TEST_DBC = os.path.abspath(os.path.join(os.path.dirname(__file__), "test.dbc"))
+46
View File
@@ -0,0 +1,46 @@
#!/usr/bin/env python3
import time
from opendbc.can import CANPacker, CANParser
def _benchmark(checks, n):
parser = CANParser('toyota_new_mc_pt_generated', checks, 0)
packer = CANPacker('toyota_new_mc_pt_generated')
t1 = time.process_time_ns()
can_msgs = []
for i in range(10000):
values = {"ACC_CONTROL": {"ACC_TYPE": 1, "ALLOW_LONG_PRESS": 3}}
msgs = [packer.make_can_msg(k, 0, v) for k, v in values.items()]
can_msgs.append([int(0.01 * i * 1e9), msgs])
t2 = time.process_time_ns()
pack_dt = t2 - t1
ets = []
for _ in range(25):
if n > 1:
strings = []
for i in range(0, len(can_msgs), n):
strings.append(can_msgs[i:i + n])
t1 = time.process_time_ns()
for m in strings:
parser.update(m)
t2 = time.process_time_ns()
else:
t1 = time.process_time_ns()
for m in can_msgs:
parser.update([m])
t2 = time.process_time_ns()
ets.append(t2 - t1)
et = sum(ets) / len(ets)
avg_nanos = et / len(can_msgs)
print('[%d] %.1fms to pack, %.1fms to parse %s messages, avg: %dns' % (n, pack_dt/1e6, et/1e6, len(can_msgs), avg_nanos))
if __name__ == "__main__":
# python -m cProfile -s cumulative benchmark.py
_benchmark([('ACC_CONTROL', 10)], 1)
_benchmark([('ACC_CONTROL', 10)], 5)
_benchmark([('ACC_CONTROL', 10)], 10)
+27
View File
@@ -0,0 +1,27 @@
CM_ "This DBC is used for the CAN parser and packer tests.";
BO_ 228 STEERING_CONTROL: 5 EON
SG_ STEER_TORQUE_REQUEST : 23|1@0+ (1,0) [0|1] "" EPS
SG_ SET_ME_X00 : 22|7@0+ (1,0) [0|127] "" EPS
SG_ SET_ME_X00_2 : 31|8@0+ (1,0) [0|0] "" EPS
SG_ STEER_TORQUE : 7|16@0- (1,0) [-4096|4096] "" EPS
SG_ STEER_DOWN_TO_ZERO : 38|1@0+ (1,0) [0|1] "" EPS
SG_ COUNTER : 37|2@0+ (1,0) [0|3] "" EPS
SG_ CHECKSUM : 35|4@0+ (1,0) [0|15] "" EPS
BO_ 316 Brake_Status: 8 XXX
SG_ CHECKSUM : 0|8@1+ (1,0) [0|255] "" XXX
SG_ COUNTER : 8|4@1+ (1,0) [0|15] "" XXX
SG_ Signal1 : 12|46@1+ (1,0) [0|1] "" XXX
SG_ ES_Brake : 58|1@1+ (1,0) [0|1] "" XXX
SG_ Signal2 : 59|3@1+ (1,0) [0|1] "" XXX
SG_ Brake : 62|1@1+ (1,0) [0|1] "" XXX
SG_ Signal3 : 63|1@1+ (1,0) [0|1] "" XXX
BO_ 245 CAN_FD_MESSAGE: 32 XXX
SG_ COUNTER : 7|8@0+ (1,0) [0|1] "" XXX
SG_ SIGNED : 22|16@0- (1,0) [0|1] "" XXX
SG_ 64_BIT_LE : 159|64@1+ (1,0) [0|1] "" XXX
SG_ 64_BIT_BE : 80|64@0+ (1,0) [0|1] "" XXX
VAL_ 80 NON_EXISTENT_ADDR 0 "test";
@@ -0,0 +1,557 @@
import copy
import unittest
from opendbc.can import CANPacker, CANParser
class TestCanChecksums(unittest.TestCase):
def verify_checksum(self, dbc_file: str, msg_name: str, msg_addr: int, test_messages: list[bytes],
checksum_field: str = 'CHECKSUM', counter_field = 'COUNTER'):
"""
Verify that opendbc calculates payload CRCs/checksums matching those received in known-good sample messages
Depends on all non-zero bits in the sample message having a corresponding DBC signal, add UNKNOWN signals if needed
"""
parser = CANParser(dbc_file, [(msg_name, 0)], 0)
packer = CANPacker(dbc_file)
for data in test_messages:
expected_msg = (msg_addr, data, 0)
parser.update([0, [expected_msg]])
expected = copy.deepcopy(parser.vl[msg_name])
modified = copy.deepcopy(expected)
modified.pop(checksum_field, None)
modified_msg = packer.make_can_msg(msg_name, 0, modified)
parser.update([0, [modified_msg]])
tested = parser.vl[msg_name]
with self.subTest(counter=expected[counter_field]):
assert tested[checksum_field] == expected[checksum_field]
def verify_fca_giorgio_crc(self, msg_name: str, msg_addr: int, test_messages: list[bytes]):
"""Test modified SAE J1850 CRCs, with special final XOR cases for EPS messages"""
assert len(test_messages) == 3
self.verify_checksum("fca_giorgio", msg_name, msg_addr, test_messages)
def test_fca_giorgio_eps_1(self):
self.verify_fca_giorgio_crc("EPS_1", 0xDE, [
b'\x17\x51\x97\xcc\x00\xdf',
b'\x17\x51\x97\xc9\x01\xa3',
b'\x17\x51\x97\xcc\x02\xe5',
])
def test_fca_giorgio_eps_2(self):
self.verify_fca_giorgio_crc("EPS_2", 0x106, [
b'\x7c\x43\x57\x60\x00\x00\xa1',
b'\x7c\x63\x58\xe0\x00\x01\xd5',
b'\x7c\x63\x58\xe0\x00\x02\xf2',
])
def test_fca_giorgio_eps_3(self):
self.verify_fca_giorgio_crc("EPS_3", 0x122, [
b'\x7b\x30\x00\xf8',
b'\x7b\x10\x01\x90',
b'\x7b\xf0\x02\x6e',
])
def test_fca_giorgio_abs_2(self):
self.verify_fca_giorgio_crc("ABS_2", 0xFE, [
b'\x7e\x38\x00\x7d\x10\x31\x80\x32',
b'\x7e\x38\x00\x7d\x10\x31\x81\x2f',
b'\x7e\x38\x00\x7d\x20\x31\x82\x20',
])
def test_honda_checksum(self):
"""Test checksums for Honda standard and extended CAN ids"""
# TODO: refactor to use self.verify_checksum()
dbc_file = "honda_civic_hatchback_ex_2017_can_generated"
msgs = [("LKAS_HUD", 0), ("LKAS_HUD_A", 0)]
parser = CANParser(dbc_file, msgs, 0)
packer = CANPacker(dbc_file)
values = {
'LKAS_READY': 1,
'LKAS_STATE_CHANGE': 1,
'STEERING_REQUIRED': 1,
'SOLID_LANES': 1,
'BEEP': 0,
}
# known correct checksums according to the above values
checksum_std = [11, 10, 9, 8]
checksum_ext = [4, 3, 2, 1]
for std, ext in zip(checksum_std, checksum_ext, strict=True):
msgs = [
packer.make_can_msg("LKAS_HUD", 0, values),
packer.make_can_msg("LKAS_HUD_A", 0, values),
]
parser.update([0, msgs])
assert parser.vl['LKAS_HUD']['CHECKSUM'] == std
assert parser.vl['LKAS_HUD_A']['CHECKSUM'] == ext
def verify_volkswagen_mqb_crc(self, msg_name: str, msg_addr: int, test_messages: list[bytes], counter_field: str = 'COUNTER'):
"""Test AUTOSAR E2E Profile 2 CRCs"""
assert len(test_messages) == 16 # All counter values must be tested
self.verify_checksum("vw_mqb", msg_name, msg_addr, test_messages, counter_field=counter_field)
def test_volkswagen_mqb_crc_lwi_01(self):
self.verify_volkswagen_mqb_crc("LWI_01", 0x86, [
b'\x6b\x00\xbd\x00\x00\x00\x00\x00',
b'\xee\x01\x0a\x00\x00\x00\x00\x00',
b'\xd8\x02\xa9\x00\x00\x00\x00\x00',
b'\x03\x03\xbe\xa2\x12\x00\x00\x00',
b'\x7b\x04\x31\x20\x03\x00\x00\x00',
b'\x8b\x05\xe2\x85\x09\x00\x00\x00',
b'\x63\x06\x13\x21\x00\x00\x00\x00',
b'\x66\x07\x05\x00\x00\x00\x00\x00',
b'\x49\x08\x0d\x00\x00\x00\x00\x00',
b'\x5f\x09\x7e\x60\x01\x00\x00\x00',
b'\xaf\x0a\x72\x20\x00\x00\x00\x00',
b'\x59\x0b\x1b\x00\x00\x00\x00\x00',
b'\xa8\x0c\x06\x00\x00\x00\x00\x00',
b'\xbc\x0d\x72\x20\x00\x00\x00\x00',
b'\xf9\x0e\x0f\x00\x00\x00\x00\x00',
b'\x60\x0f\x62\xc0\x00\x00\x00\x00',
])
def test_volkswagen_mqb_crc_airbag_01(self):
self.verify_volkswagen_mqb_crc("Airbag_01", 0x40, [
b'\xaf\x00\x00\x80\xc0\x00\x20\x3e',
b'\x54\x01\x00\x80\xc0\x00\x20\x1a',
b'\x54\x02\x00\x80\xc0\x00\x60\x00',
b'\x31\x03\x00\x80\xc0\x00\x60\xf2',
b'\xe0\x04\x00\x80\xc0\x00\x60\xcc',
b'\xb3\x05\x00\x80\xc0\x00\x40\xde',
b'\xa4\x06\x00\x80\xc0\x00\x40\x18',
b'\x94\x07\x00\x80\xc0\x00\x20\x38',
b'\x2d\x08\x00\x80\xc0\x00\x60\xae',
b'\xc2\x09\x00\x80\xc0\x00\x00\x1c',
b'\x1f\x0a\x00\x80\xc0\x00\x60\x2c',
b'\x7f\x0b\x00\x80\xc0\x00\x00\x00',
b'\x03\x0c\x00\x80\xc0\x00\x40\xd6',
b'\x56\x0d\x00\x80\xc0\x00\x20\x50',
b'\x4a\x0e\x00\x80\xc0\x00\x20\xf2',
b'\xe5\x0f\x00\x80\xc0\x00\x40\xf6',
])
def test_volkswagen_mqb_crc_lh_eps_03(self):
self.verify_volkswagen_mqb_crc("LH_EPS_03", 0x9F, [
b'\x11\x30\x2e\x00\x05\x1c\x80\x30',
b'\x5b\x31\x8e\x03\x05\x53\x00\x30',
b'\xcb\x32\xd3\x06\x05\x73\x00\x30',
b'\xf2\x33\x28\x00\x05\x26\x00\x30',
b'\x0b\x34\x44\x00\x05\x5b\x80\x30',
b'\xed\x35\x80\x00\x03\x34\x00\x30',
b'\xf0\x36\x88\x00\x05\x3d\x80\x30',
b'\x9e\x37\x44\x03\x05\x41\x00\x30',
b'\x68\x38\x06\x01\x05\x18\x80\x30',
b'\x87\x39\x51\x00\x05\x11\x80\x30',
b'\x8c\x3a\x29\x00\x05\xac\x00\x30',
b'\x08\x3b\xbd\x00\x05\x8e\x00\x30',
b'\xd4\x3c\x19\x00\x05\x05\x80\x30',
b'\x29\x3d\x54\x00\x05\x5b\x00\x30',
b'\xa1\x3e\x49\x01\x03\x04\x80\x30',
b'\xe2\x3f\x05\x00\x05\x0a\x00\x30',
])
def test_volkswagen_mqb_crc_getriebe_11(self):
self.verify_volkswagen_mqb_crc("Getriebe_11", 0xAD, [
b'\xf8\xe0\xbf\xff\x5f\x20\x20\x20',
b'\xb0\xe1\xbf\xff\xc6\x98\x21\x80',
b'\xd2\xe2\xbf\xff\x5f\x20\x20\x20',
b'\x00\xe3\xbf\xff\xaa\x20\x20\x10',
b'\xf1\xe4\xbf\xff\x5f\x20\x20\x20',
b'\xc4\xe5\xbf\xff\x5f\x20\x20\x20',
b'\xda\xe6\xbf\xff\x5f\x20\x20\x20',
b'\x85\xe7\xbf\xff\x5f\x20\x20\x20',
b'\x12\xe8\xbf\xff\x5f\x20\x20\x20',
b'\x45\xe9\xbf\xff\xaa\x20\x20\x10',
b'\x03\xea\xbf\xff\xcc\x20\x20\x10',
b'\xfc\xeb\xbf\xff\x5f\x20\x21\x20',
b'\xfe\xec\xbf\xff\xad\x20\x20\x10',
b'\xbd\xed\xbf\xff\xaa\x20\x20\x10',
b'\x67\xee\xbf\xff\xaa\x20\x20\x10',
b'\x36\xef\xbf\xff\xaa\x20\x20\x10',
], counter_field="COUNTER_DISABLED") # see opendbc#1235
def test_volkswagen_mqb_crc_esp_21(self):
self.verify_volkswagen_mqb_crc("ESP_21", 0xFD, [
b'\x66\xd0\x1f\x80\x45\x05\x00\x00',
b'\x87\xd1\x1f\x80\x52\x05\x00\x00',
b'\xcd\xd2\x1f\x80\x50\x06\x00\x00',
b'\xfd\xd3\x1f\x80\x35\x02\x00\x00',
b'\xfa\xd4\x1f\x80\x22\x05\x00\x00',
b'\xfd\xd5\x1f\x80\x84\x04\x00\x00',
b'\x2e\xd6\x1f\x80\xf0\x03\x00\x00',
b'\x9f\xd7\x1f\x80\x00\x00\x00\x00',
b'\x1e\xd8\x1f\x80\xb3\x03\x00\x00',
b'\x61\xd9\x1f\x80\x6d\x05\x00\x00',
b'\x44\xda\x1f\x80\x47\x02\x00\x00',
b'\x86\xdb\x1f\x80\x3a\x02\x00\x00',
b'\x39\xdc\x1f\x80\xcb\x01\x00\x00',
b'\x19\xdd\x1f\x80\x00\x00\x00\x00',
b'\x8c\xde\x1f\x80\xba\x04\x00\x00',
b'\xfb\xdf\x1f\x80\x46\x00\x00\x00',
])
def test_volkswagen_mqb_crc_esp_02(self):
self.verify_volkswagen_mqb_crc("ESP_02", 0x101, [
b'\xf2\x00\x7e\xff\xa1\x2a\x40\x00',
b'\xd3\x01\x7d\x00\xa2\x0c\x02\x00',
b'\x03\x02\x7a\x06\xa2\x49\x42\x00',
b'\xfd\x03\x70\xfb\xa1\xde\x00\x00',
b'\x8e\x04\x7b\xf7\xa1\xd2\x01\x00',
b'\x0f\x05\x7d\xfd\xa1\x31\x40\x00',
b'\xb6\x06\x7d\x01\xa2\x0a\x40\x00',
b'\xe8\x07\x7e\xfd\xa1\x12\x40\x00',
b'\x74\x08\x7a\x01\xa2\x40\x01\x00',
b'\xe3\x09\x81\x00\xa2\xb5\x01\x00',
b'\xab\x0a\x74\x09\xa2\x9f\x42\x00',
b'\xf3\x0b\x80\x12\xa2\x94\x00\x00',
b'\x88\x0c\x7f\x07\xa2\x46\x00\x00',
b'\x6f\x0d\x7f\xff\xa1\x53\x40\x00',
b'\x38\x0e\x73\xd6\xa1\x6a\x40\x00',
b'\x49\x0f\x85\x12\xa2\xf6\x01\x00',
])
def test_volkswagen_mqb_crc_esp_05(self):
self.verify_volkswagen_mqb_crc("ESP_05", 0x106, [
b'\x90\x80\x64\x00\x00\x00\xe7\x10',
b'\xf4\x81\x64\x00\x00\x00\xe7\x10',
b'\x90\x82\x63\x00\x00\x00\xe8\x10',
b'\xa0\x83\x63\x00\x00\x00\xe6\x10',
b'\xe7\x84\x63\x00\x00\x00\xe8\x10',
b'\x2e\x85\x78\x04\x00\x00\xea\x30',
b'\x7b\x86\x63\x00\x00\x00\xe6\x10',
b'\x71\x87\x79\x04\x00\x00\xd0\x30',
b'\x50\x88\x79\x04\x00\x00\xea\x30',
b'\x81\x89\x64\x00\x00\x00\xe1\x10',
b'\x6a\x8a\x68\x00\x00\x04\xd0\x10',
b'\x17\x8b\x6a\x04\x00\x00\xe6\x10',
b'\xc7\x8c\x63\x00\x00\x00\xd1\x10',
b'\x53\x8d\x64\x04\x00\x00\xe2\x10',
b'\x24\x8e\x63\x00\x00\x00\xe7\x10',
b'\x3f\x8f\x82\x04\x00\x00\xe6\x30',
])
def test_volkswagen_mqb_crc_esp_10(self):
self.verify_volkswagen_mqb_crc("ESP_10", 0x116, [
b'\x2d\x00\xd5\x98\x9f\x26\x25\x0f',
b'\x24\x01\x60\x63\x2c\x5e\x3b\x0f',
b'\x08\x02\xb2\x2f\xee\x9a\x29\x0f',
b'\x7c\x03\x17\x07\x1d\xe5\x8c\x0f',
b'\xaa\x04\xd6\xe3\xeb\x98\xe8\x0f',
b'\x4e\x05\xbb\xd9\x65\x43\xca\x0f',
b'\x59\x06\x78\xbd\x25\xc6\xf2\xff',
b'\xaf\x07\x42\x85\x53\xbe\xbe\x0f',
b'\x2a\x08\xa6\xcd\x95\x8c\x12\x0f',
b'\xce\x09\x6e\x17\x6d\x1b\x2f\x0f',
b'\x60\x0a\xd3\xe6\x3a\x8d\xf0\x0f',
b'\xc5\x0b\xfc\x69\x57\x50\x21\x0f',
b'\x70\x0c\xde\xf3\x9d\xe9\x6b\xff',
b'\x62\x0d\xc4\x1a\xdb\x61\x7a\x0f',
b'\x76\x0e\x79\x69\xe3\x32\x67\x0f',
b'\x15\x0f\x51\x59\x56\x35\xb1\x0f',
])
def test_volkswagen_mqb_crc_acc_10(self):
self.verify_volkswagen_mqb_crc("ACC_10", 0x117, [
b'\x9b\x00\x00\x40\x68\x00\x00\xff',
b'\xff\x01\x00\x40\x68\x00\x00\xff',
b'\x53\x02\x00\x40\x68\x00\x00\xff',
b'\x37\x03\x00\x40\x68\x00\x00\xff',
b'\x24\x04\x00\x40\x68\x00\x00\xff',
b'\x40\x05\x00\x40\x68\x00\x00\xff',
b'\xec\x06\x00\x40\x68\x00\x00\xff',
b'\x88\x07\x00\x40\x68\x00\x00\xff',
b'\xca\x08\x00\x40\x68\x00\x00\xff',
b'\xae\x09\x00\x40\x68\x00\x00\xff',
b'\x02\x0a\x00\x40\x68\x00\x00\xff',
b'\x66\x0b\x00\x40\x68\x00\x00\xff',
b'\x75\x0c\x00\x40\x68\x00\x00\xff',
b'\x11\x0d\x00\x40\x68\x00\x00\xff',
b'\xbd\x0e\x00\x40\x68\x00\x00\xff',
b'\xd9\x0f\x00\x40\x68\x00\x00\xff',
])
def test_volkswagen_mqb_crc_tsk_06(self):
self.verify_volkswagen_mqb_crc("TSK_06", 0x120, [
b'\xc1\x00\x00\x02\x00\x08\xff\x21',
b'\x34\x01\x00\x02\x00\x08\xff\x21',
b'\xcc\x02\x00\x02\x00\x08\xff\x21',
b'\x1e\x03\x00\x02\x00\x08\xff\x21',
b'\x48\x04\x00\x02\x00\x08\xff\x21',
b'\x4a\x05\x00\x02\x00\x08\xff\x21',
b'\xa5\x06\x00\x02\x00\x08\xff\x21',
b'\xa7\x07\x00\x02\x00\x08\xff\x21',
b'\xfe\x08\x00\x02\x00\x08\xff\x21',
b'\xa8\x09\x00\x02\x00\x08\xff\x21',
b'\x73\x0a\x00\x02\x00\x08\xff\x21',
b'\xdf\x0b\x00\x02\x00\x08\xff\x21',
b'\x05\x0c\x00\x02\x00\x08\xff\x21',
b'\xb5\x0d\x00\x02\x00\x08\xff\x21',
b'\xde\x0e\x00\x02\x00\x08\xff\x21',
b'\x0b\x0f\x00\x02\x00\x08\xff\x21',
])
def test_volkswagen_mqb_crc_motor_20(self):
self.verify_volkswagen_mqb_crc("Motor_20", 0x121, [
b'\xb9\x00\x00\xc0\x39\x46\x7e\xfe',
b'\x85\x31\x20\x00\x1a\x46\x7e\xfe',
b'\xc7\x12\x00\x40\x1a\x46\x7e\xfe',
b'\x53\x93\x00\x00\x19\x46\x7e\xfe',
b'\xa4\x34\x00\x80\x1a\x46\x7e\xfe',
b'\x0e\x55\x20\x60\x18\x46\x7e\xfe',
b'\x3f\x06\x00\xc0\x37\x4c\x7e\xfe',
b'\x0c\x07\x00\x40\x39\x46\x7e\xfe',
b'\x2a\x08\x00\x00\x3a\x46\x7e\xfe',
b'\x7f\x49\x20\x80\x1a\x46\x7e\xfe',
b'\x2f\x0a\x00\xc0\x39\x46\x7e\xfe',
b'\x70\xbb\x00\x00\x17\x46\x7e\xfe',
b'\x06\x0c\x00\x00\x39\x46\x7e\xfe',
b'\x4b\x9d\x20\xe0\x16\x4c\x7e\xfe',
b'\x73\xfe\x00\x40\x16\x46\x7e\xfe',
b'\xaf\x0f\x20\x80\x39\x4c\x7e\xfe',
])
def test_volkswagen_mqb_crc_acc_06(self):
self.verify_volkswagen_mqb_crc("ACC_06", 0x122, [
b'\x14\x80\x00\xfe\x07\x00\x00\x18',
b'\x9f\x81\x00\xfe\x07\x00\x00\x18',
b'\x0a\x82\x00\xfe\x07\x00\x00\x28',
b'\x40\x83\x00\xfe\x07\x00\x00\x18',
b'\x2d\x84\x00\xfe\x07\x00\x00\x28',
b'\xdb\x85\x00\xfe\x07\x00\x00\x18',
b'\x4d\x86\x00\xfe\x07\x00\x00\x28',
b'\x35\x87\x00\xfe\x07\x00\x00\x18',
b'\x23\x88\x00\xfe\x07\x00\x00\x28',
b'\x4a\x89\x00\xfe\x07\x00\x00\x28',
b'\xe1\x8a\x00\xfe\x07\x00\x00\x28',
b'\x30\x8b\x00\xfe\x07\x00\x00\x28',
b'\x60\x8c\x00\xfe\x07\x00\x00\x28',
b'\x0d\x8d\x00\xfe\x07\x00\x00\x18',
b'\x8c\x8e\x00\xfe\x07\x00\x00\x18',
b'\x6f\x8f\x00\xfe\x07\x00\x00\x28',
])
def test_volkswagen_mqb_crc_hca_01(self):
self.verify_volkswagen_mqb_crc("HCA_01", 0x126, [
b'\x00\x30\x0d\xc0\x05\xfe\x07\x00',
b'\x3e\x31\x54\xc0\x05\xfe\x07\x00',
b'\xa7\x32\xbb\x40\x05\xfe\x07\x00',
b'\x96\x33\x29\xc0\x05\xfe\x07\x00',
b'\x5f\x34\x00\x00\x03\xfe\x07\x00',
b'\x3b\x35\xae\x40\x05\xfe\x07\x00',
b'\xc7\x36\x7a\x40\x05\xfe\x07\x00',
b'\x6f\x37\x76\x40\x05\xfe\x07\x00',
b'\xb1\x38\x00\x00\x03\xfe\x07\x00',
b'\xd5\x39\x00\x00\x03\xfe\x07\x00',
b'\xba\x3a\x69\xc0\x05\xfe\x07\x00',
b'\x65\x3b\x10\x40\x05\xfe\x07\x00',
b'\x49\x3c\x72\xc0\x05\xfe\x07\x00',
b'\xc6\x3d\xdf\x40\x05\xfe\x07\x00',
b'\x1d\x3e\x2c\xc1\x05\xfe\x07\x00',
b'\x9b\x3f\x20\x40\x05\xfe\x07\x00',
])
def test_volkswagen_mqb_crc_gra_acc_01(self):
self.verify_volkswagen_mqb_crc("GRA_ACC_01", 0x12B, [
b'\x86\x40\x80\x2a\x00\x00\x00\x00',
b'\xf4\x41\x80\x2a\x00\x00\x00\x00',
b'\x50\x42\x80\x2a\x00\x00\x00\x00',
b'\x08\x43\x80\x2a\x00\x00\x00\x00',
b'\x88\x44\x80\x2a\x00\x00\x00\x00',
b'\x2d\x45\x80\x2a\x00\x00\x00\x00',
b'\x34\x46\x80\x2a\x00\x00\x00\x00',
b'\x11\x47\x80\x2a\x00\x00\x00\x00',
b'\xc4\x48\x80\x2a\x00\x00\x00\x00',
b'\xcc\x49\x80\x2a\x00\x00\x00\x00',
b'\xdc\x4a\x80\x2a\x00\x00\x00\x00',
b'\x79\x4b\x80\x2a\x00\x00\x00\x00',
b'\x3c\x4c\x80\x2a\x00\x00\x00\x00',
b'\x68\x4d\x80\x2a\x00\x00\x00\x00',
b'\x27\x4e\x80\x2a\x00\x00\x00\x00',
b'\x0d\x4f\x80\x2a\x00\x00\x00\x00',
])
def test_volkswagen_mqb_crc_acc_07(self):
self.verify_volkswagen_mqb_crc("ACC_07", 0x12E, [
b'\xac\xe0\x7f\x00\xfe\x00\xc0\xff',
b'\xa2\xe1\x7f\x00\xfe\x00\xc0\xff',
b'\x6b\xe2\x7f\x00\xfe\x00\xc0\xff',
b'\xf2\xe3\x7f\x00\xfe\x00\xc0\xff',
b'\xd5\xe4\x7f\x00\xfe\x00\xc0\xff',
b'\x35\xe5\x7f\x00\xfe\x00\xc0\xff',
b'\x7f\xe6\x7f\x00\xfe\x00\xc0\xff',
b'\x6c\xe7\x7f\x00\xfe\x00\xc0\xff',
b'\x05\xe8\x7f\x00\xfe\x00\xc0\xff',
b'\x79\xe9\x7f\x00\xfe\x00\xc0\xff',
b'\x25\xea\x7f\x00\xfe\x00\xc0\xff',
b'\xd1\xeb\x7f\x00\xfe\x00\xc0\xff',
b'\x72\xec\x7f\x00\xfe\x00\xc0\xff',
b'\x58\xed\x7f\x00\xfe\x00\xc0\xff',
b'\x82\xee\x7f\x00\xfe\x00\xc0\xff',
b'\x85\xef\x7f\x00\xfe\x00\xc0\xff',
])
def test_volkswagen_mqb_crc_motor_ev_01(self):
self.verify_volkswagen_mqb_crc("Motor_EV_01", 0x187, [
b'\x70\x80\x15\x00\x00\x00\x00\xF0',
b'\x07\x81\x15\x00\x00\x00\x00\xF0',
b'\x7A\x82\x15\x00\x00\x00\x00\xF0',
b'\x26\x83\x15\x00\x00\x00\x00\xF0',
b'\xBE\x84\x15\x00\x00\x00\x00\xF0',
b'\x5A\x85\x15\x00\x00\x00\x00\xF0',
b'\xFC\x86\x15\x00\x00\x00\x00\xF0',
b'\x9E\x87\x15\x00\x00\x00\x00\xF0',
b'\xAF\x88\x15\x00\x00\x00\x00\xF0',
b'\x35\x89\x15\x00\x00\x00\x00\xF0',
b'\xC5\x8A\x15\x00\x00\x00\x00\xF0',
b'\x11\x8B\x15\x00\x00\x00\x00\xF0',
b'\xD0\x8C\x15\x00\x00\x00\x00\xF0',
b'\xE8\x8D\x15\x00\x00\x00\x00\xF0',
b'\xF5\x8E\x15\x00\x00\x00\x00\xF0',
b'\x00\x8F\x15\x00\x00\x00\x00\xF0',
])
def test_volkswagen_mqb_crc_esp_33(self):
self.verify_volkswagen_mqb_crc("ESP_33", 0x1AB, [
b'\x64\x00\x80\x02\x00\x00\x00\x00',
b'\x19\x01\x00\x00\x00\x00\x00\x00',
b'\xfc\x02\x00\x10\x01\x00\x00\x00',
b'\x8b\x03\x80\x02\x00\x00\x00\x00',
b'\xa4\x04\x00\x10\x01\x00\x00\x00',
b'\x97\x05\x00\x02\x00\x00\x01\x00',
b'\xd5\x06\x80\x02\x00\x00\x01\x00',
b'\xa0\x07\x80\x02\x00\x00\x01\x00',
b'\x89\x08\x00\x00\x00\x00\x00\x00',
b'\xe3\x09\x00\x00\x00\x00\x00\x00',
b'\x0e\x0a\x00\x00\x00\x00\x00\x00',
b'\x90\x0b\x00\x00\x00\x00\x00\x00',
b'\x32\x0c\x00\x10\x01\x00\x00\x00',
b'\x30\x0d\x00\x00\x00\x00\x00\x00',
b'\xc2\x0e\x00\x10\x01\x00\x00\x00',
b'\x68\x0f\x80\x02\x00\x00\x00\x00',
])
def test_volkswagen_mqb_crc_acc_02(self):
self.verify_volkswagen_mqb_crc("ACC_02", 0x30C, [
b'\x82\xf0\x3f\x00\x40\x30\x00\x40',
b'\xe6\xf1\x3f\x00\x40\x30\x00\x40',
b'\x4a\xf2\x3f\x00\x40\x30\x00\x40',
b'\x2e\xf3\x3f\x00\x40\x30\x00\x40',
b'\x3d\xf4\x3f\x00\x40\x30\x00\x40',
b'\x59\xf5\x3f\x00\x40\x30\x00\x40',
b'\xf5\xf6\x3f\x00\x40\x30\x00\x40',
b'\x91\xf7\x3f\x00\x40\x30\x00\x40',
b'\xd3\xf8\x3f\x00\x40\x30\x00\x40',
b'\xb7\xf9\x3f\x00\x40\x30\x00\x40',
b'\x1b\xfa\x3f\x00\x40\x30\x00\x40',
b'\x7f\xfb\x3f\x00\x40\x30\x00\x40',
b'\x6c\xfc\x3f\x00\x40\x30\x00\x40',
b'\x08\xfd\x3f\x00\x40\x30\x00\x40',
b'\xa4\xfe\x3f\x00\x40\x30\x00\x40',
b'\xc0\xff\x3f\x00\x40\x30\x00\x40',
])
def test_volkswagen_mqb_crc_swa_01(self):
self.verify_volkswagen_mqb_crc("SWA_01", 0x30F, [
b'\x10\x00\x10\x00\x00\x00\x00\x00',
b'\x74\x01\x10\x00\x00\x00\x00\x00',
b'\xD8\x02\x10\x00\x00\x00\x00\x00',
b'\xBC\x03\x10\x00\x00\x00\x00\x00',
b'\xAF\x04\x10\x00\x00\x00\x00\x00',
b'\xCB\x05\x10\x00\x00\x00\x00\x00',
b'\x67\x06\x10\x00\x00\x00\x00\x00',
b'\x03\x07\x10\x00\x00\x00\x00\x00',
b'\x41\x08\x10\x00\x00\x00\x00\x00',
b'\x25\x09\x10\x00\x00\x00\x00\x00',
b'\x89\x0A\x10\x00\x00\x00\x00\x00',
b'\xED\x0B\x10\x00\x00\x00\x00\x00',
b'\xFE\x0C\x10\x00\x00\x00\x00\x00',
b'\x9A\x0D\x10\x00\x00\x00\x00\x00',
b'\x36\x0E\x10\x00\x00\x00\x00\x00',
b'\x52\x0F\x10\x00\x00\x00\x00\x00',
])
def test_volkswagen_mqb_crc_acc_04(self):
self.verify_volkswagen_mqb_crc("ACC_04", 0x324, [
b'\xba\x00\x00\x00\x00\x00\x00\x10',
b'\xde\x01\x00\x00\x00\x00\x00\x10',
b'\x72\x02\x00\x00\x00\x00\x00\x10',
b'\x16\x03\x00\x00\x00\x00\x00\x10',
b'\x05\x04\x00\x00\x00\x00\x00\x10',
b'\x44\x05\x00\x00\x00\x00\x00\x00',
b'\xe8\x06\x00\x00\x00\x00\x00\x00',
b'\xa9\x07\x00\x00\x00\x00\x00\x10',
b'\xeb\x08\x00\x00\x00\x00\x00\x10',
b'\x8f\x09\x00\x00\x00\x00\x00\x10',
b'\x06\x0a\x00\x00\x00\x00\x00\x00',
b'\x47\x0b\x00\x00\x00\x00\x00\x10',
b'\x71\x0c\x00\x00\x00\x00\x00\x00',
b'\x15\x0d\x00\x00\x00\x00\x00\x00',
b'\xb9\x0e\x00\x00\x00\x00\x00\x00',
b'\xdd\x0f\x00\x00\x00\x00\x00\x00',
])
def test_volkswagen_mqb_crc_klemmen_status_01(self):
self.verify_volkswagen_mqb_crc("Klemmen_Status_01", 0x3C0, [
b'\x74\x00\x03\x00',
b'\xc1\x01\x03\x00',
b'\x31\x02\x03\x00',
b'\x84\x03\x03\x00',
b'\xfe\x04\x03\x00',
b'\x4b\x05\x03\x00',
b'\xbb\x06\x03\x00',
b'\x0e\x07\x03\x00',
b'\x4f\x08\x03\x00',
b'\xfa\x09\x03\x00',
b'\x0a\x0a\x03\x00',
b'\xbf\x0b\x03\x00',
b'\xc5\x0c\x03\x00',
b'\x70\x0d\x03\x00',
b'\x80\x0e\x03\x00',
b'\x35\x0f\x03\x00',
])
def test_volkswagen_mqb_crc_licht_anf_01(self):
self.verify_volkswagen_mqb_crc("Licht_Anf_01", 0x3D5, [
b'\xc8\x00\x00\x04\x00\x00\x00\x00',
b'\x9f\x01\x00\x04\x00\x00\x00\x00',
b'\x5e\x02\x00\x04\x00\x00\x00\x00',
b'\x52\x03\x00\x04\x00\x00\x00\x00',
b'\xf2\x04\x00\x04\x00\x00\x00\x00',
b'\x79\x05\x00\x04\x00\x00\x00\x00',
b'\xe6\x06\x00\x04\x00\x00\x00\x00',
b'\xfd\x07\x00\x04\x00\x00\x00\x00',
b'\xf8\x08\x00\x04\x00\x00\x00\x00',
b'\xc6\x09\x00\x04\x00\x00\x00\x00',
b'\xf5\x0a\x00\x04\x00\x00\x00\x00',
b'\x1a\x0b\x00\x04\x00\x00\x00\x00',
b'\x65\x0c\x00\x04\x00\x00\x00\x00',
b'\x41\x0d\x00\x04\x00\x00\x00\x00',
b'\x7f\x0e\x00\x04\x00\x00\x00\x00',
b'\x98\x0f\x00\x04\x00\x00\x00\x00',
])
def test_volkswagen_mqb_crc_esp_20(self):
self.verify_volkswagen_mqb_crc("ESP_20", 0x65D, [
b'\x98\x30\x2b\x10\x00\x00\x22\x81',
b'\xc8\x31\x2b\x10\x00\x00\x22\x81',
b'\x9d\x32\x2b\x10\x00\x00\x22\x81',
b'\x1f\x33\x2b\x10\x00\x00\x22\x81',
b'\x6e\x34\x2b\x10\x00\x00\x22\x81',
b'\x61\x35\x2b\x10\x00\x00\x22\x81',
b'\x6f\x36\x2b\x10\x00\x00\x22\x81',
b'\xe5\x37\x2b\x10\x00\x00\x22\x81',
b'\xf8\x38\x2b\x10\x00\x00\x22\x81',
b'\xe1\x39\x2b\x10\x00\x00\x22\x81',
b'\xaa\x3a\x2b\x10\x00\x00\x22\x81',
b'\xe6\x3b\x2b\x10\x00\x00\x22\x81',
b'\xef\x3c\x2b\x10\x00\x00\x22\x81',
b'\xbb\x3d\x2b\x10\x00\x00\x22\x81',
b'\x9b\x3e\x2b\x10\x00\x00\x22\x81',
b'\x72\x3f\x2b\x10\x00\x00\x22\x81',
])
@@ -0,0 +1,29 @@
import unittest
from opendbc.can import CANDefine, CANPacker, CANParser
from opendbc.can.tests import TEST_DBC
class TestCanParserPackerExceptions(unittest.TestCase):
def test_civic_exceptions(self):
dbc_file = "honda_civic_touring_2016_can_generated"
dbc_invalid = dbc_file + "abcdef"
msgs = [("STEERING_CONTROL", 50)]
with self.assertRaises(FileNotFoundError):
CANParser(dbc_invalid, msgs, 0)
with self.assertRaises(FileNotFoundError):
CANPacker(dbc_invalid)
with self.assertRaises(FileNotFoundError):
CANDefine(dbc_invalid)
with self.assertRaises(KeyError):
CANDefine(TEST_DBC)
parser = CANParser(dbc_file, msgs, 0)
with self.assertRaises(IndexError):
parser.update([b''])
# Everything is supposed to work below
CANParser(dbc_file, msgs, 0)
CANParser(dbc_file, [], 0)
CANPacker(dbc_file)
CANDefine(dbc_file)

Some files were not shown because too many files have changed in this diff Show More