From 9b6b53396fff6ec7541a5415901d892e57756b91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Sch=C3=A4fer?= Date: Tue, 23 May 2023 20:44:17 +0200 Subject: [PATCH] Custom events reserved for forks (#449) * Custom events reserved for forks * Fixes * Custom updates * Update README.md * little more * add identifiers * Language * little more * rm that --------- Co-authored-by: Adeeb Shihadeh --- README.md | 33 +++++++++++++++++++-------------- SConscript | 2 +- __init__.py | 1 + custom.capnp | 39 +++++++++++++++++++++++++++++++++++++++ log.capnp | 13 +++++++++++++ 5 files changed, 73 insertions(+), 15 deletions(-) create mode 100644 custom.capnp diff --git a/README.md b/README.md index a07953b..737534e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -What is cereal? [![cereal tests](https://github.com/commaai/cereal/workflows/tests/badge.svg?event=push)](https://github.com/commaai/cereal/actions) [![codecov](https://codecov.io/gh/commaai/cereal/branch/master/graph/badge.svg)](https://codecov.io/gh/commaai/cereal) ----- +# What is cereal? [![cereal tests](https://github.com/commaai/cereal/workflows/tests/badge.svg?event=push)](https://github.com/commaai/cereal/actions) [![codecov](https://codecov.io/gh/commaai/cereal/branch/master/graph/badge.svg)](https://codecov.io/gh/commaai/cereal) cereal is both a messaging spec for robotics systems as well as generic high performance IPC pub sub messaging with a single publisher and multiple subscribers. @@ -9,29 +8,35 @@ Imagine this use case: * A localization process subscribes to the `sensorEvents` packet to use the IMU also -Messaging Spec ----- +## Messaging Spec -You'll find the message types in [log.capnp](log.capnp). It uses [Cap'n proto](https://capnproto.org/capnp-tool.html) and defines one struct called Event. +You'll find the message types in [log.capnp](log.capnp). It uses [Cap'n proto](https://capnproto.org/capnp-tool.html) and defines one struct called `Event`. -All Events have a `logMonoTime` and a `valid`. Then a big union defines the packet type. +All `Events` have a `logMonoTime` and a `valid`. Then a big union defines the packet type. - -Message definition Best Practices ----- +### Best Practices - **All fields must describe quantities in SI units**, unless otherwise specified in the field name. - - In the context of the message they are in, field names should be completely unambiguous. - - All values should be easy to plot and be human-readable with minimal parsing. +### Maintaining backwards-compatibility +When making changes to the messaging spec you want to maintain backwards-compatability, such that old logs can +be parsed with a new version of cereal. Adding structs and adding members to structs is generally safe, most other +things are not. Read more details [here](https://capnproto.org/language.html). -Pub Sub Backends ----- +### Custom forks -cereal supports two backends, one based on [zmq](https://zeromq.org/) and another called msgq, a custom pub sub based on shared memory that doesn't require the bytes to pass through the kernel. +Forks of [openpilot](https://github.com/commaai/openpilot) might want to add things to the messaging +spec, however this could conflict with future changes made in mainline cereal/openpilot. Rebasing against mainline openpilot +then means breaking backwards-compatibility with all old logs of your fork. So we added reserved events in +[custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your +fork will remain backwards-compatible with all versions of mainline cereal/openpilot and your fork.** + +## Pub Sub Backends + +cereal supports two backends, one based on [zmq](https://zeromq.org/) and another called [msgq](messaging/msgq.cc), a custom pub sub based on shared memory that doesn't require the bytes to pass through the kernel. Example --- diff --git a/SConscript b/SConscript index dc3f1e0..192e42d 100644 --- a/SConscript +++ b/SConscript @@ -8,7 +8,7 @@ messaging_dir = Dir('messaging') # Build cereal -schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp'] +schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp', 'custom.capnp'] env.Command(["gen/c/include/c++.capnp.h"], [], "mkdir -p " + gen_dir.path + "/c/include && touch $TARGETS") env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files], schema_files, diff --git a/__init__.py b/__init__.py index 88a181c..4e26ffb 100644 --- a/__init__.py +++ b/__init__.py @@ -7,3 +7,4 @@ capnp.remove_import_hook() log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp")) car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp")) +custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp")) diff --git a/custom.capnp b/custom.capnp new file mode 100644 index 0000000..369222a --- /dev/null +++ b/custom.capnp @@ -0,0 +1,39 @@ +using Cxx = import "./include/c++.capnp"; +$Cxx.namespace("cereal"); + +@0xb526ba661d550a59; + +# custom.capnp: a home for empty structs reserved for custom forks +# These structs are guaranteed to remain reserved and empty in mainline +# cereal, so use these if you want custom events in your fork. + +# you can rename the struct, but don't change the identifier +struct CustomReserved0 @0x81c2f05a394cf4af { +} + +struct CustomReserved1 @0xaedffd8f31e7b55d { +} + +struct CustomReserved2 @0xf35cc4560bbf6ec2 { +} + +struct CustomReserved3 @0xda96579883444c35 { +} + +struct CustomReserved4 @0x80ae746ee2596b11 { +} + +struct CustomReserved5 @0xa5cd762cd951a455 { +} + +struct CustomReserved6 @0xf98d843bfd7004a3 { +} + +struct CustomReserved7 @0xb86e6369214c01c8 { +} + +struct CustomReserved8 @0xf416ec09499d9d19 { +} + +struct CustomReserved9 @0xa1680744031fdb2d { +} diff --git a/log.capnp b/log.capnp index 03aadfb..ff871e0 100644 --- a/log.capnp +++ b/log.capnp @@ -3,6 +3,7 @@ $Cxx.namespace("cereal"); using Car = import "car.capnp"; using Legacy = import "legacy.capnp"; +using Custom = import "custom.capnp"; @0xf3b1f17e25a4285b; @@ -2199,6 +2200,18 @@ struct Event { wideRoadEncodeData @88 :EncodeData; qRoadEncodeData @89 :EncodeData; + # *********** Custom: reserved for forks *********** + customReserved0 @107 :Custom.CustomReserved0; + customReserved1 @108 :Custom.CustomReserved1; + customReserved2 @109 :Custom.CustomReserved2; + customReserved3 @110 :Custom.CustomReserved3; + customReserved4 @111 :Custom.CustomReserved4; + customReserved5 @112 :Custom.CustomReserved5; + customReserved6 @113 :Custom.CustomReserved6; + customReserved7 @114 :Custom.CustomReserved7; + customReserved8 @115 :Custom.CustomReserved8; + customReserved9 @116 :Custom.CustomReserved9; + # *********** legacy + deprecated *********** model @9 :Legacy.ModelData; # TODO: rename modelV2 and mark this as deprecated liveMpcDEPRECATED @36 :LiveMpcData;