alexander/bproto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Init Commit: Moved bproto to seperate repo

Browse Source
This commit is contained in:
AlexanderHD27
2025-04-14 14:43:03 +02:00
commit 45bfc724fc
125 changed files with 10822 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
docs/bprotoDocumentationGraphics.drawio Normal file
View File

@@ -0,0 +1,57 @@
<mxfile host="Electron" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/26.0.3 Chrome/130.0.6723.137 Electron/33.2.1 Safari/537.36" version="26.0.3" pages="2">
<diagram name="Core Concept" id="uytGOHupnUP-L8cOnyTT">
<mxGraphModel dx="512" dy="301" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
<root>
<mxCell id="0" />
<mxCell id="1" parent="0" />
<mxCell id="-I73Jp0letHz3FIaNn9U-9" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;targetPerimeterSpacing=6;sourcePerimeterSpacing=6;strokeWidth=2;" edge="1" parent="1" source="-I73Jp0letHz3FIaNn9U-2" target="-I73Jp0letHz3FIaNn9U-7">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="-I73Jp0letHz3FIaNn9U-2" value="&lt;b&gt;.bproto&lt;/b&gt;" style="whiteSpace=wrap;html=1;shape=mxgraph.basic.document" vertex="1" parent="1">
<mxGeometry x="120" y="125" width="60" height="70" as="geometry" />
</mxCell>
<mxCell id="-I73Jp0letHz3FIaNn9U-3" value="Protocol Description" style="text;html=1;align=center;verticalAlign=top;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
<mxGeometry x="110" y="210" width="80" height="60" as="geometry" />
</mxCell>
<mxCell id="-I73Jp0letHz3FIaNn9U-7" value="" style="whiteSpace=wrap;html=1;aspect=fixed;" vertex="1" parent="1">
<mxGeometry x="240" y="120" width="80" height="80" as="geometry" />
</mxCell>
<mxCell id="-I73Jp0letHz3FIaNn9U-6" value="" style="sketch=0;outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#232F3D;strokeColor=none;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;pointerEvents=1;shape=mxgraph.aws4.gear;" vertex="1" parent="1">
<mxGeometry x="250" y="129" width="59" height="59" as="geometry" />
</mxCell>
<mxCell id="-I73Jp0letHz3FIaNn9U-8" value="bproto-Compiler" style="text;html=1;align=center;verticalAlign=top;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
<mxGeometry x="240" y="210" width="80" height="60" as="geometry" />
</mxCell>
<mxCell id="-I73Jp0letHz3FIaNn9U-14" value="&lt;b&gt;.py&lt;/b&gt;" style="whiteSpace=wrap;html=1;shape=mxgraph.basic.document" vertex="1" parent="1">
<mxGeometry x="380" y="170" width="40" height="50" as="geometry" />
</mxCell>
<mxCell id="-I73Jp0letHz3FIaNn9U-16" value="&lt;b&gt;.c/.h&lt;/b&gt;" style="whiteSpace=wrap;html=1;shape=mxgraph.basic.document" vertex="1" parent="1">
<mxGeometry x="380" y="100" width="40" height="50" as="geometry" />
</mxCell>
<mxCell id="-I73Jp0letHz3FIaNn9U-17" value="Protocol Implementation" style="text;html=1;align=center;verticalAlign=top;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
<mxGeometry x="355" y="230" width="90" height="60" as="geometry" />
</mxCell>
<mxCell id="-I73Jp0letHz3FIaNn9U-18" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;targetPerimeterSpacing=6;sourcePerimeterSpacing=6;strokeWidth=2;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;" edge="1" parent="1" source="-I73Jp0letHz3FIaNn9U-7" target="-I73Jp0letHz3FIaNn9U-16">
<mxGeometry relative="1" as="geometry">
<mxPoint x="196" y="170" as="sourcePoint" />
<mxPoint x="244" y="170" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="-I73Jp0letHz3FIaNn9U-20" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;targetPerimeterSpacing=6;sourcePerimeterSpacing=6;strokeWidth=2;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;" edge="1" parent="1" source="-I73Jp0letHz3FIaNn9U-7" target="-I73Jp0letHz3FIaNn9U-14">
<mxGeometry relative="1" as="geometry">
<mxPoint x="330" y="170" as="sourcePoint" />
<mxPoint x="390" y="125" as="targetPoint" />
</mxGeometry>
</mxCell>
</root>
</mxGraphModel>
</diagram>
<diagram id="zBquDBVsXlGvDgQ8uFh9" name="Compadibilty Desition Tree">
<mxGraphModel dx="717" dy="422" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
<root>
<mxCell id="0" />
<mxCell id="1" parent="0" />
</root>
</mxGraphModel>
</diagram>
</mxfile>

75
docs/compatibility.md Normal file
View File

@@ -0,0 +1,75 @@
# Compatibility and Versioning
This document describes how bproto handles compatibility and versioning of protocols and messages.
When receiving a message, the receiver must know if the data is compatible with the hard-coded protocol implementation.
There are multiple mechanisms to ensure this.
The worst-case scenario is that the receiver gets a message that is not compatible with the protocol implementation -> meaning something has changed in the protocol and the receiver is not aware of it.
## Compatibility
**A protocol/message is compatible if**
- ✅ protocol and message definitions are the same
- ✅ the order of declaration of fields or messages has changed (but not the ID)
**A protocol/message is incompatible if**
- ❌ If the protocol version has changed
- ❌ If a message has been removed or added
- ❌ If the protocol, message, bitfield, enum, or field name has changed
- ❌ If Message or Field ID has changed
- ❌ the number, type, order, or array size of fields in a message has changed
- ❌ If the order of declaration of enums or bitfields has changed
=> In general, the receiver must have the same protocol implementation build as the sender.
## Naming Conventions
Use descriptive names that clearly indicate the purpose of the protocol. Avoid generic names. Also include a hint about the fact that this is for transmitting data like "protocol" or "datastream".
## Versioning Strategies
The version number is used to differentiate between different iterations of the protocol. Different versions of the same protocol are incompatible. By changing the protocol version, the developer can indicate a major change in structure, design, or purpose of the protocol/messages.Evalia
**Example:**
```bproto
protocol robotControlProtocol version 1
```
More about bproto's syntax can be found in the [syntax documentation](/docs/syntax.md#protocol-tag-line).
## Compatibility Validation Process in Detail
This is the message validation process that should prevent incompatible messages from being processed, causing undefined behavior.
**Step #1 - Message Size Check:**
The first byte of the message is the message ID.
The receiver can with the message ID derive the expected message size. (If the message ID is unknown, the size is set to 0)
The receiver can then check if the received message is of the expected size.
If the size is not as expected, the message is discarded.
-> This prevents messages with changed field/message sizes from being processed.
**Step #2 - CRC-16 Check:**
The CRC-16 is calculated over the message data and compared to the last 2 bytes of the message.
If the CRC-16 is not correct, the message is discarded.
The CRC-16 is initialized by first prepending a SHA-256 hash of a text form of the protocol definition to the message data.
This text form is a human-readable representation of the protocol definition but also represents the internal structure of the protocol used by the bproto-compiler to generate other protocol implementations.
If something changes in this internal structure, a potential incompatibility is introduced.
More details about how the CRC-16 is calculated can be found in the [serialization documentation](/docs/serialization.md#crc-16-calculation).
This text form can be generated by the bproto-compiler when choosing the backend "txt".
This will also display the CRC-16 initialization vector and SHA-256 hash of the protocol definition.
There are two possible reasons for a failed CRC check:
1. The message has been corrupted during transmission.
2. The message data is fine but the protocol definition has changed, and therefore the SHA-256 hash is different.
=> The CRC-16 protects against data corruption and protocol changes.
**Step #3 - Message ID Check:**
The receiver checks if the message ID is known.
If the message ID is unknown, the message is discarded.
After Step #2 the receiver should already have discarded the message if it is incompatible with the protocol implementation. Nevertheless, this step is an additional line of defense.

BIN
docs/img/bprotoDocumentationGraphics.drawio.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 35 KiB

66
docs/serialization.md Normal file
View File

@@ -0,0 +1,66 @@
# Message Encoding
This section describes how messages are encoded/serialized and decoded/deserialized.
bproto is binary data with a well-defined structure.
## Message Structure
Every bproto message looks like this:
```
[ 1 ] [ N ] [ 2 ]
[Message ID] [Field Data] [CRC-16]
```
- **Message ID:** The first byte of the message is the message ID. This is used to identify the message type for decoding. Every message is declared with a unique ID in the protocol definition. See [Message](/docs/syntax.md#message)
- **Field Data:** The field data is the serialized data of the message fields. See [Message Field Data Encoding](#message-field-data-encoding)
- **CRC-16:** The last 2 bytes of the message are the CRC-16 checksum of the message data. This is used to verify the integrity of the message. See [CRC-16 Check](/docs/compatibility.md#step-2---crc-16-check) and [CRC-16 Calculation](/docs/serialization.md#crc-16-calculation)
### Message Field Data Encoding
The order, length, and type of the fields are defined in the protocol definition.
This means that bproto message structure is fixed and can be decoded without any additional delimiters or markers.
The fields are just concatenated together in the order of their field ID.
Field IDs are declared in the protocol definition and are declaration order independent (see [Syntax of Fields](/docs/syntax.md#fields)).
All numeric data types are serialized in big-endian byte order.
Data types are serialized as follows:
- For floating point numbers, the IEEE 754 standard is used.
- Signed integers are serialized as two's complement.
- Unsigned integers are serialized as is.
- Char and string are serialized as ASCII characters.
- Bool is serialized as 0 for false and 1 for true.
- Enums are serialized as 8-bit unsigned integers.
- Bitfields are serialized as a single (or multiple) byte with each bit representing a boolean value. The order of the bits is from the least significant bit to the most significant bit and from the first byte to the last byte.
Datatype sizes are fixed and are documented in the datatypes-table in the [Datatypes](/docs/syntax.md#datatypes) section.
compatiblity
For arrays, the elements are serialized in order, with the first element first, the second element second, and so on.
### Example Message Encoding
An example of a message and serialization with 3 fields:
```
message [1] ExampleMessage {
[0] a : uint8,
[1] b : uint16,
[2] c : bool
}
```
with the values a=42, b=12345, c=true would be serialized as: `012a390501`
```
[Message ID] [Field a] [Field b] [Field c] [CRC16]
[ 01 ] [ 2a ] [ 3905 ] [ 01 ] [31 06]
```
### CRC-16 Calculation
The CRC-16 has the purpose of verifying the integrity of the message and preventing incompatible messages/protocol versions from being processed.
When performing the CRC-16 calculation, the CRC-16 is initialized by first prepending a SHA-256 hash of a text form of the protocol definition to the message data.
This text hash is a fingerprint of the protocol definition and is used to detect changes in the protocol definition. See [Compatibility Validation Process in Detail](/docs/compatibility.md#compatibility-validation-process-in-detail)
Then CRC-16 is calculated over the message data and compared to the last 2 bytes of the message.
If the CRC-16 is not correct, the message should be discarded.

221
docs/syntax.md Normal file
View File

@@ -0,0 +1,221 @@
# Syntax Documentation
bproto uses a custom syntax `bproto` (file extension `.bproto`).
Each protocol is described in a single bproto file.
For Beginners, a simple example of a protocol definition and usage can be found [here](/docs/quickstart.md).
The protocol definitions consist of the following basic components: **messages**, **enums**, **bitfields**, and the **protocol tag line**.
## Table of Contents
- [Syntax Documentation](#syntax-documentation)
- [Table of Contents](#table-of-contents)
- [Names and IDs](#names-and-ids)
- [Protocol Tag Line](#protocol-tag-line)
- [Message](#message)
- [Fields](#fields)
- [Datatypes](#datatypes)
- [Bitfields](#bitfields)
- [Enumerations](#enumerations)
- [Comments](#comments)
## Names and IDs
Names of enums, bitfields, messages, and fields support any letter (lower and uppercase), numbers, and underscores.
Regular expression: `[a-zA-Z0-9_]+`
Numbers/IDs are decimal encoded numbers. Trailing zeros are ignored.
Regular expression: `[0-9]+`
## Protocol Tag Line
This is a short description of the protocol consisting of a name and a version number.
This must be the first line of any bproto definition file.
**Syntax:**
```bproto
protocol <name> version <version>
```
**Example:**
```bproto
protocol myCustomAirplaneProtocol version 2
```
The name should identify the protocol in the implementation context (like `robotControlProtocol` or `weatherStationTelemetryDataStream`).
The version number is used to differentiate between different iterations of the protocol. Different versions of the same protocol are incompatible.
For detailed information on naming conventions and versioning strategies, please refer to the [compatibility documentation](/docs/compatibility.md#naming-conventions).
## Message
A message is a collection of fields that can be serialized and deserialized to be sent over the wire or channel.
A message should only be a transport container and should not, if possible, be stored permanently.
**Syntax:**
```bproto
message [<Message ID>] <Message Name> {
// List of fields
[<field ID #0>] <field name #0> : <field type #0>,
[<field ID #1>] <field name #1> : <field type #1>,
....
[<field ID #N>] <field name #N> : <field type #N>,
}
```
**Example:**
```bproto
message [1] EngineSensorData {
[0] engine_thrust_left : float32,
[1] engine_thrust_right : float32,
[2] fuel_tanks_fill : uint32[8], // Fill levels of 8 fuel tanks
[3] engine_fault_left : bool,
[4] engine_fault_right : bool
}
```
- **Message ID:** Unique identifier within a protocol definition used to differentiate different messages from each other. The Message ID should be between 0 and 255.
- **Message Name:** Unique name given to this message. This will be used to identify a field in the protocol implementation.
### Fields
A message can have as many fields as necessary. These are comparable to members in a class or fields in a C-struct.
A field has an `ID`, `Name`, and a `datatype`.
**Syntax:**
`[<ID>] <Name> : <datatype>`
**Example:**
``[2] fanSpeed : float32``
- **Name:** This is a unique name given to the field. This will be used to identify a field in the protocol implementation.
- **ID:** A unique identifier used to order the fields in the encoding/decoding. More about serialization can be found [here](/docs/serialization.md#message-field-data-encoding).
- **datatype:** What type of data to store in this field.
### Datatypes
Datatypes in bproto are independent of language and are translated into the target language-specific type.
Every Field in a message must have a datatype.
Here is a list of datatypes available:
| bproto Datatype | Encode Size (bytes) | Python | C | Value Range | Description |
| --------------- | ------------------- | ------- | ---------- | ------------------------------------------------------- | ---------------------------- |
| `uint8` | 1 | `int` | `uint8_t` | 0 to 255 | Unsigned 8-bit integer |
| `uint16` | 2 | `int` | `uint16_t` | 0 to 65,535 | Unsigned 16-bit integer |
| `uint32` | 4 | `int` | `uint32_t` | 0 to 4,294,967,295 | Unsigned 32-bit integer |
| `int64` | 8 | `int` | `uint64_t` | 0 to 18,446,744,073,709,551,615 | Unsigned 64-bit integer |
| `int8` | 1 | `int` | `int8_t` | -128 to 127 | Signed 8-bit integer |
| `int16` | 2 | `int` | `int16_t` | -32,768 to 32,767 | Signed 16-bit integer |
| `int32` | 4 | `int` | `int32_t` | -2,147,483,648 to 2,147,483,647 | Signed 32-bit integer |
| `int64` | 8 | `int` | `int64_t` | -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 | Signed 64-bit integer |
| `float32` | 4 | `float` | `float` | 1.2E-38 to 3.4E+38 | 32-bit floating point number |
| `float64` | 8 | `float` | `double` | 2.3E-308 to 1.7E+308 | 64-bit floating point number |
| `string[<N>]` | N | `str` | `char[]` | N/A | Fixed-size string |
| `char` | 1 | `str` | `char` | Single character | Single character |
| `bool` | 1 | `bool` | `bool` | true or false | Boolean value |
Every datatype in this list can be used in a fixed-size array by using this format:
**Syntax:**
`<datatype>[<array_size>]`
**Example:**
`float[]`
A special case is `string`.
Strings _must_ be declared as an array because their size must be known from the start.
There are also `bits` and `enum` corresponding to **bitfields** and **enums**.
They are a bit special and are not able to be used as an array.
Bitfields and enums can be declared inline.
More details can be found in the sections about [enums](#enumerations) and [bitfields](#bitfields).
How different datatypes are encoded can be found [here](/docs/serialization.md#message-field-data-encoding).
## Bitfields
A compact representation of multiple boolean flags within a single byte or group of bytes, optimizing space for binary communication.
Bitfields are treated as datatypes.
**Syntax:**
```bproto
bits <bitfield name> {
<1st bit>, <2nd bit>, ...
}
```
**Example:**
```bproto
bits engine_enables {
engine_left, engine_right
}
```
Each bit can be either 0 or 1 (true or false), represented as a bool in the target language.
More details on how Bitfields are encoded in the corresponding language can be found [here](/docs/datatype_language_details.md#bitfields).
Bitfields, like Enumerations, can be declared inline in a [message](#message).
**Syntax:**
```bproto
[<Field ID>] <Field Name> : bits {
<1st bit>, <2nd bit>, ...
}
```
**Example:**
```bproto
[1] lightsEnable: bits {
Front, Spot, Key, BackLight
}
```
Bitfield naming when using them inline is found [here](/docs/datatype_language_details.md#inline).
## Enumerations
A set of named integer values that represent discrete states or options.
Enums enhance code clarity by using descriptive names instead of raw numeric values.
Enumerations or Enums in bproto are comparable to C or Python enums.
More details on how Enums are encoded in the corresponding language can be found [here](/docs/datatype_language_details.md#enums).
Enums are treated as datatypes in bproto.
Each Enum item represents a positive (or zero) numeric value.
These can be implicitly generated by bproto, incrementing by 1.
The value of the Enum value therefore depends on the declaration order.
This may lead to bugs and issues when updating one client/receiver's protocol but not the counterparts.
For more info about this problem, see [here](/docs/serialization.md#message-field-data-encoding).
Enum values can also be set explicitly, which eliminates this problem.
**Syntax:**
```bproto
enum <enum name> {
<Name #0> (= <Value #0>),
<Name #1> (= <Value #1>),
...
<Name #N> (= <Value #N>),
}
```
**Example:**
```bproto
dangerLvl: enum {
WARNING,
MELTDOWN,
ERROR = 10,
}
```
**Syntax:**
```bproto
[<Field ID>] <Field Name> : enum {
<Name #0> (= <Value #0>),
<Name #1> (= <Value #1>),
...
<Name #N> (= <Value #N>),
}
```
**Example:**
```bproto
[6] dangerLvl: enum {
WARNING,
MELTDOWN = 5,
ERROR,
}
```
Enumeration naming when using them inline is found [here](/docs/datatype_language_details.md#inline).
## Comments
bproto supports C-style comments (with `// ...` and `/* ... */`).