diff --git a/README.md b/README.md index b50fa00a0..62b8cdd16 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ where * `py` : Python * `ts` : Typescript * `go` : Go + * `md` : Markdown (Documentation) `java` is the default value if no language is specified. diff --git a/generator.py b/generator.py index 311c2db57..d4d834980 100755 --- a/generator.py +++ b/generator.py @@ -111,19 +111,28 @@ print("Hazelcast Client Binary Protocol version %s" % (protocol_versions[-1])) env = create_environment(lang, namespace_arg) -codec_template = env.get_template("codec-template.%s.j2" % lang_str_arg) - -generate_codecs(protocol_defs, codec_template, codec_output_dir, lang) -print('Generated codecs are at \'%s\'' % os.path.abspath(codec_output_dir)) +if lang != SupportedLanguages.MD: + codec_template = env.get_template("codec-template.%s.j2" % lang_str_arg) + generate_codecs(protocol_defs, codec_template, codec_output_dir, lang) + print('Generated codecs are at \'%s\'' % os.path.abspath(codec_output_dir)) if custom_protocol_defs is not None: - custom_codec_template = env.get_template("custom-codec-template.%s.j2" % lang_str_arg) - relative_custom_codec_output_dir = out_dir_arg if out_dir_arg is not None else custom_codec_output_directories[lang] - custom_codec_output_dir = os.path.join(root_dir, relative_custom_codec_output_dir) - generate_custom_codecs(custom_protocol_defs, custom_codec_template, custom_codec_output_dir, file_extensions[lang]) - print('Generated custom codecs are at \'%s\'' % custom_codec_output_dir) + if lang != SupportedLanguages.MD: + custom_codec_template = env.get_template("custom-codec-template.%s.j2" % lang_str_arg) + if out_dir_arg: + relative_custom_codec_output_dir = out_dir_arg + else: + relative_custom_codec_output_dir = custom_codec_output_directories[lang] + custom_codec_output_dir = os.path.join(root_dir, relative_custom_codec_output_dir) + generate_custom_codecs(custom_protocol_defs, custom_codec_template, custom_codec_output_dir, + file_extensions[lang]) + print('Generated custom codecs are at \'%s\'' % custom_codec_output_dir) + else: + documentation_template = env.get_template('documentation-template.j2') + generate_documentation(protocol_defs, custom_protocol_defs, documentation_template, codec_output_dir, + file_extensions[lang]) -if not no_binary_arg: +if not no_binary_arg and lang != SupportedLanguages.MD: relative_test_output_dir = test_out_dir_arg if test_out_dir_arg is not None else test_output_directories[lang] relative_binary_output_dir = bin_out_dir_arg if bin_out_dir_arg is not None else binary_output_directories[lang] test_output_dir = os.path.join(root_dir, relative_test_output_dir) diff --git a/md/__init__.py b/md/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/md/documentation-template.j2 b/md/documentation-template.j2 new file mode 100644 index 000000000..c48117327 --- /dev/null +++ b/md/documentation-template.j2 @@ -0,0 +1,1260 @@ +{% macro resolve_partition_identifier(identifier) %} + {% if identifier == 'random' %} +a random partition ID from 0 to PARTITION_COUNT(exclusive) + {% elif identifier == -1 %} +-1 + {% elif identifier == 'partitionId' %} +the value passed in to the partitionId parameter + {% else %} +Murmur hash of {{ identifier }} % PARTITION_COUNT + {%- endif %} +{% endmacro -%} + +{%- macro convert_type(type) -%} + {%- if type.startswith(('List_', 'ListCN_')) -%} +Array of {{ convert_type(item_type(param_name, type)) }} + {%- elif type.startswith('EntryList_') -%} +Array of {{ convert_type(key_type(param_name, type)) }} to {{ convert_type(value_type(param_name, type)) }} mappings + {%- else -%} +{{ type }} + {%- endif -%} +{%- endmacro -%} + + +# Hazelcast Open Binary Client Protocol + +**Revision History** + +{# Changelog for each version added. #} +| Date | Document Version | Change Description | +| ---- | ---------------- | ------------------ | +| 15 January 2020 | 2.0 | Initial release of the Hazelcast Open Binary Client Protocol version 2. | + +**Table of Contents** +{# List of manually written sections #} +- [1. Introduction](#1-introduction) +- [2. Data Format Details](#2-data-format-details) + - [2.1. Client Message](#21-client-message) + - [2.1.1. Frame](#211-frame) + - [2.1.2. Initial Frame](#212-initial-frame) + - [2.1.2.1. Message Type](#2121-message-type) + - [2.1.2.1.1. Request Message Type](#21211-request-message-type) + - [2.1.2.1.2. Response Message Type](#21212-response-message-type) + - [2.1.2.1.3. Event Response Message Type](#21213-event-response-message-type) + - [2.1.2.1.4. Error Message Type](#21214-error-message-type) + - [2.1.2.2. Correlation ID](#2122-correlation-id) + - [2.1.2.3. Partition ID](#2123-partition-id) + - [2.1.2.4. Backup Acks Count](#2124-backup-acks-count) + - [2.1.3. Encoding of Variable Sized Fields](#213-encoding-of-variable-sized-fields) + - [2.1.3.1. Encoding of String Fields](#2131-encoding-of-string-fields) + - [2.1.3.2. Encoding of Custom Type Fields](#2132-encoding-of-custom-type-fields) + - [2.1.3.3. Encoding of Array Fields](#2133-encoding-of-array-fields) + - [2.1.3.3.1. Encoding of Array of Fix Sized Fields](#21331-encoding-of-array-of-fix-sized-fields) + - [2.1.3.3.2. Encoding of Array of Variable Sized Fields](#21332-encoding-of-array-of-variable-sized-fields) + - [2.1.3.3.3. Encoding of Array of Map Fields](#21333-encoding-of-array-of-map-fields) + - [2.1.4. Client Message Fragmentation](#214-client-message-fragmentation) + - [2.1.5. Client Message Boundaries](#215-client-message-boundaries) + - [2.1.6. Backward Compatibility of the Client Messages](#216-backward-compatibility-of-the-client-messages) + - [2.1.7. Augmented Backus-Naur Format Representation of the Client Messages](#217-augmented-backusnaur-format-representation-of-the-client-messages) + - [3. Client Protocol Data Types](#3-client-protocol-data-types) + - [4. Connection Guide](#4-connection-guide) + - [4.1. Opening a Connection](#41-opening-a-connection) + - [4.2. Connection Initialization](#42-connection-initialization) + - [4.3. Authentication](#43-authentication) + - [4.4. Communication via Messages](#44-communication-via-messages) + - [4.5. Closing Connections](#45-closing-connections) + - [5. Requests and Responses](#5-requests-and-responses) + - [5.1. Distributed Objects](#51-distributed-objects) + - [5.2. Operation Messages and Responses](#52-operation-messages-and-responses) + - [5.3. Proxies](#53-proxies) + - [5.3.1. Proxy Creation](#531-proxy-creation) + - [5.3.2. List Example](#532-list-example) + - [5.3.3. Fenced Lock Example](#533-fenced-lock-example) + - [5.3.4. Map Example](#534-map-example) + - [5.3.5. Queue Example](#535-queue-example) + - [5.3.6. Set Example](#536-set-example) + - [5.4. Multiple Responses to a Single Request](#54-multiple-responses-to-a-single-request) + - [5.5. Listeners](#55-listeners) + - [5.6. Cluster View Listener](#56-cluster-view-listener) + - [5.7. Timeouts and Retry](#57-timeouts-and-retry) + - [5.8. Error Codes](#58-error-codes) + - [6. Miscellaneous](#6-miscellaneous) + - [6.1. Smart Client and Unisocket Client](#61-smart-client-and-unisocket-client) + - [6.2. Serialization](#62-serialization) + - [6.3. Security](#63-security) +{# Offset to start listing protocol messages. Should be set to number of manually listed sections + 1 #} +{% set protocol_message_offset = 7 %} + - [{{ protocol_message_offset }}. Protocol Messages](#{{ protocol_message_offset }}-protocol-messages) + - [{{ protocol_message_offset }}.1. Custom Data Types Used In The Protocol](#{{ protocol_message_offset }}1-custom-data-types-used-in-the-protocol) +{% for definition in custom_definitions %} +{% set custom_types = definition.get('customTypes', None) %} +{% if custom_types is not none %} +{% for custom_type in custom_types %} + - [{{ protocol_message_offset }}.1.{{ loop.index }}. {{ custom_type.name }}](#{{ protocol_message_offset }}1{{ loop.index }}-{{ custom_type.name|lower }}) +{% endfor %} +{% endif %} +{% endfor %} +{% for service in services %} + {% set service_id = service.id + 2 %} + - [{{ protocol_message_offset }}.{{ service_id }}. {{ service.name }}](#{{ protocol_message_offset }}{{ service_id }}-{{ service.name|lower }}) + {% for method in service.methods %} + - [{{ protocol_message_offset }}.{{ service_id }}.{{ method.id }}. {{ service.name }}.{{ method.name|capital }}](#{{ protocol_message_offset }}{{ service_id }}{{ method.id }}-{{ service.name|lower }}{{ method.name|lower }}) + {% endfor %} +{% endfor %} + +{# START of manually written sections #} +## 1. Introduction +This document explains the new binary protocol that Hazelcast uses in order to communicate with the clients. +This document is not a guide to implement a client that will interact with Hazelcast; rather, it specifies the wire +data format for the messages exchanged between a client and a Hazelcast member node. Any client that wants to +communicate with the Hazelcast cluster should obey the data format and communication details explained in this document. + +The protocol is designed to be strict enough to ensure a standardization in the communication, but flexible enough +that developers may expand upon the protocol to implement custom features. + +General guidelines: +- This document uses terms MUST, MUST NOT, MAY, SHOULD and SHOULD NOT as described in the IETF RFC 2119. +- Client refers to the entity which communicates with a Hazelcast member node. +- Member or server refers to the Hazelcast node to which the client connects. + +## 2. Data Format Details +Hazelcast provides a communication interface to access distributed objects through client protocol. This interface is +a TCP socket listening for request messages. Currently, TCP socket communication is the only way a client can connect to +a member. The client MUST connect to the port that Hazelcast member is listening to for new connections. Because of +this, there is no fixed port to which client must connect. + +Protocol communication is built on sending and receiving messages. Client protocol defines a simple entity called +client message for communication. It is the only data format defined by this protocol. + +Before diving into the details of the communication, lets define what the client message is. + +### 2.1. Client Message +A client message is a transmission data unit composed of frames which are array of bytes. Its main purpose is to +encapsulate a unit of data to be transferred from one entity the another. It may represent a request, a response or an +event response. A client message can be fragmented into multiple client messages and sent in order one-by-one. See the +[Client Message Fragmentation](#214-client-message-fragmentation) section for details. + +#### 2.1.1. Frame +As said above, frames are building blocks of a client message. A frame is an array of bytes consisted of frame length, +flags and payload bytes as shown below. + +| Frame Length | Flags | Payload | +| ------------ | ---- | ------- | +| ← 4 bytes → | ← 2 bytes → | ← Payload size → | + +Frame length includes the length of itself, flags and payload bytes. Hence, the minimum size of a frame is 6 bytes in +case of an empty payload (4 bytes for frame length and 2 bytes for flags). Payload bytes store the actual data carried +over the frame. + +Flag bits have the structure shown below. + +| Flag Bit | Name | Description | +| -------- | ---- | ----------- | +| 15 | BEGIN_FRAGMENT_FLAG | Set to 1 if the frame is the begin frame of a fragmented client message | +| 14 | END_FRAGMENT_FLAG | Set to 1 if the frame is the end frame of a fragmented client message | +| 13 | IS_FINAL_FLAG | Set to 1 if the frame is the last frame of a client message | +| 12 | BEGIN_DATA_STRUCTURE_FLAG | Set to 1 if the frame is the begin frame of a custom type or array of variable sized types | +| 11 | END_DATA_STRUCTURE_FLAG | Set to 1 if the frame is the end frame of a custom type or array of variable sized types | +| 10 | IS_NULL_FLAG | Set to 1 if the frame represents a null field | +| 9 | IS_EVENT_FLAG | Set to 1 if the frame is the initial frame of a client message that represents a event response from a member | +| 8 | BACKUP_AWARE_FLAG | Set to 1 if the client enabled receiving backup acks directly from members that backups are applied to | +| 7 | BACKUP_EVENT_FLAG | Set to 1 if the frame is the initial frame of a client message that represents a backup event response from a member | +| 6 to 0 | Reserved | Reserved for future usage | + +#### 2.1.2. Initial Frame +Each client message starts with a special frame called initial frame. It is special in a sense that it includes all the +fix sized fields of a client message. Fix sized fields are fields of a request, a response or an event response +message which the sizes of the fields in bytes can be known in advance. These types of fields are listed below. + +| Type | Size in bytes | +| ---- | ------------- | +| byte (int8 or uint8) | 1 | +| boolean | 1 | +| int(int32 or uint32) | 4 | +| long(int64 or uint64) | 8 | +| UUID* | 17 | + +> *: UUID is described by two longs. Since UUID can be null, a boolean flag is also used to distinguish null UUIDs from non-null ones. That makes the length 17 bytes (1 + 8 + 8) in total. + +The overall structure of the initial frame shown below. + +For requests and event responses, the overall structure of the initial frame is shown below. + +| Frame length | Flags | Message type | Correlation ID | Partition ID | Fix sized field 1 | Fix sized field 2 | ... | Fix sized field n | +| ------------ | ----- | ------------ | -------------- | -------------------- | ----------------- | ----------------- | --- | ----------------- | +| ← 4 bytes → | ← 2 bytes → | ← 4 bytes → | ← 8 bytes → | ← 4 bytes → | ← Field 1 size in bytes → | ← Field 2 size in bytes → | ... | ← Field n size in bytes → | + +For responses, the overall structure of the initial frame is shown below. + +| Frame length | Flags | Message type | Correlation ID | Backup Acks Count | Fix sized field 1 | Fix sized field 2 | ... | Fix sized field n | +| ------------ | ----- | ------------ | -------------- | -------------------- | ----------------- | ----------------- | --- | ----------------- | +| ← 4 bytes → | ← 2 bytes → | ← 4 bytes → | ← 8 bytes → | ← 1 byte → | ← Field 1 size in bytes → | ← Field 2 size in bytes → | ... | ← Field n size in bytes → | + +Initial frame of the client messages has one in BEGIN_FRAGMENT_FLAG and END_FRAGMENT_FLAG bits. + +For the details of frame length and flags, see the section [above](#211-frame). + +Payload bytes of the initial frame consist of message type, correlation ID, partition ID or the backup acks count +depending on the message type and fix sized fields. + +##### 2.1.2.1. Message Type +Message type corresponds to a unique operation of a distributed object such as map.put request, list.get response or +an event response for a registered listener. + +| Message type bit | Description | +| ---------------- | ----------- | +| 0 | Unused, set to 0 | +| 1 | Service ID | +| 2 | Method ID | +| 3 | Request, response or event response ID | + +Service ID represents the unique ID assigned to managed services provided by Hazelcast such as map, list, client etc. +It is in the range of 0 to 255. + +Method ID represents the unique IDs of methods provided by the service. It is in the range of 1 to 255. + +The last bit of the message type represents whether this client message is a request, a response or an event response. +It is equal to zero for requests, one for responses and two plus event ID for event responses. + +For example: +- *0x010200* is the message type for the request(*00*) for the get method(*02*) of the map service(*01*). +- *0x050F01* is the message type for the response(*01*) for the get method(0F) of the list service(*05*). +- *0x011C02* is the message type for the event response(*02*) for the addEntryListener method(*1C*) of the map service(*01*). + +For the error messages that is send by the member node to the client, message type is set to *0x000000*. + +A full list of message types can be found in the [Protocol Messages](#{{ protocol_message_offset }}-protocol-messages) section. + +If the Hazelcast member receives a message with a unsupported message type, it will return the "unsupported operation" +error to the client with the message type of *0x000000*. The client is guaranteed to receive only the messages listed +in the [Protocol Messages](#{{ protocol_message_offset }}-protocol-messages) and the error messages. + +The details of the different message types are described in the next sections. + +##### 2.1.2.1.1. Request Message Type +Each distributed object defines various operations. Each operation corresponds to a well defined request message to +be sent to the cluster. For each request message, client will ge a response message from the cluster. Request messages +MUST be sent from the client to the server. + +The request fields are binary encoded entirely within the payload bytes of the frames that constitute the client message. + +##### 2.1.2.1.2. Response Message Type +Once a request is received and processed on the member side, the member produces a response message and send it to +the client. Each request message type defines response message that can be sent. The correlation ID relates all instances +of the response messages to their requests. + +The response fields are binary encoded entirely within the payload bytes of the frames that constitute the client messages. + +##### 2.1.2.1.3. Event Response Message Type +An event response message is a special kind of the response message. A client can register to a specific listener by +sending a request message with the message type of adding a listener. When an event is triggered that the client is +listening for, the member will send a message to the client using the same correlation ID as the original request message. +The payload bytes of the frames of event message carries the specific event object. The possible event message types +for a registration request are documented in the "Event Messages" section of each request in the +[Protocol Messages](#{{ protocol_message_offset }}-protocol-messages) section. + +For these messages, IS_EVENT_FLAG bit of the initial frame of the client message is set to one. + +The member will continue to send the client event updates until the client unregisters from that event or the connection +is broken. + +##### 2.1.2.1.4. Error Message Type +The member may return an error response to the client for the requests it made. For this message, message type is set +to 0x000000000. The payload of the member's response message contains the error message along with the error code. +You may choose to provide the error codes directly to the user or you may use some other technique, such as exceptions, +to delegate the error to the user. See the "ErrorHolder" custom type and the list of "Error Messages" for details. + +##### 2.1.2.2. Correlation ID +This ID correlates the request to responses. It should unique to identify one message in the communication. This ID +is used to track the request response cycle of a client operation. Members send response messages with the same ID as +the request message. The uniqueness is per connection. If the client receives the response to a request and the request +is not a multi-response request (i.e. not a request for event transmission), then the correlation ID for the request can +be reused by the subsequent requests. Note that once a correlation ID is used to register for an event, it SHOULD NOT +be used again unless the client unregisters (stops listening) for that event. + +##### 2.1.2.3. Partition ID +The partition ID defines the partition against which the operation is executed. The client can get the partition list +of the cluster. This information tells the client which member handles which partition. The client can use this +information to send requests to the responsible member directly for processing. The client gets this information from +the "Partitions View" event of the "Add Cluster View Listener" request. (see the +[Protocol Messages](#{{ protocol_message_offset }}-protocol-messages)) + +To determine the partition ID of an operation, compute the Murmur Hash (version 3, 32-bit, see +[https://en.wikipedia.org/wiki/MurmurHash](https://en.wikipedia.org/wiki/MurmurHash) and +[https://github.com/aappleby/smhasher/wiki/MurmurHash3](https://github.com/aappleby/smhasher/wiki/MurmurHash3)) of a +certain byte array (which is identified for each message in the description section) and take the modulus of the result +over the total number of partitions. The seed for the Murmur Hash SHOULD be 0x0100193. Most operations with a key +parameter use the key parameter byte array as the data for the hash calculation. + +Some operations are not required to be executed on a specific partition, but can be run on global execution pool. For +these operations, the partition ID is set to a negative value. No hash calculation is required in this case. + +##### 2.1.2.4 Backup Acks Count +When the client performs an operation on a distributed object that requires backups to be created when a change is made, +the client only receives the response of the operation when acks from the member nodes that participated in the +backup process is seen. + +Hazelcast offers two different ways to perform operations that involve backups. + +If the client is a [smart client](#61-smart-client-and-unisocket-client), it can mark the requests it sends as backup +aware by setting the "BACKUP_AWARE_FLAG" to one. When a Hazelcast member receives such a request, it sends a response +message that carries information about how many backup operations must be performed along with the actual response in +this part of the initial frame. In this case, the client is notified about the successful backups with event responses +coming from the member nodes that created the backups in their partitions. To do so, client must register listeners to +all member nodes that it is connected to using the "Local Backup Listener" message. The client SHOULD wait until it +receives event responses marked with "BACKUP_EVENT_FLAG" from that many Hazelcast member nodes before resolving the +response of the request. + +However, if the client is a [unisocket client](#61-smart-client-and-unisocket-client) or the requests going out from +it are not marked with the "BACKUP_AWARE_FLAG", the member node that receives the request from the client only sends +response back when it receives acks from other cluster members which are participated in the backup process. + +The former way is faster in a sense that it results in fewer serial network hops. + +#### 2.1.3. Encoding of Variable Sized Fields + +The fields of the client message that have variable size, that are not listed in the fix sized types described +in the [Initial Frame](#212-initial-frame) section, such as String, array of primitive or custom types etc. are +encoded following the initial frame in their respective frames. A variable sized field can be encoded into one or more +frames based on its type. For the sections below, following special frames will be used while describing the encoding +process of variable sized fields. + +- NULL_FRAME: A frame that has one in its IS_NULL_FLAG. It is used to represent fields that have null value. It has empty payload bytes. +- BEGIN_FRAME: A frame that has one in its BEGIN_DATA_STRUCTURE_FLAG. It is used to mark the beginning of the field encodings that cannot fit into a single frame. It has empty payload bytes. +- END_FRAME: A frame that has one in its END_DATA_STRUCTURE_FLAG. It is used to mark the ending of the field encodings that cannot fit into a single frame. It has empty payload bytes. + +For the encodings described below, if the field is of a variable sized type and its value is null, it is encoded as +NULL_FRAME. + +##### 2.1.3.1. Encoding of String Fields +Each string field of the client message can be encoded into its own single frame. String fields are expected to +be encoded according to UTF-8 standard described in the RFC 3629. Encoded string data must be placed in the payload +bytes of the frame. Below is the sample structure of a string frame. + +| Frame length | Flags | UTF-8 encoded string data | +| ------------ | ----- | ------------------------- | +| ← 4 bytes → | ← 2 bytes → | ← Size of the payload → | + +##### 2.1.3.2. Encoding of Custom Type Fields +Custom types, which are the fields of the client messages that contain primitive, array or other custom types +in their definitions are encoded in between BEGIN_FRAME and END_FRAME. Overall, the structure of custom type encodings +is below. + +| BEGIN_FRAME | Payload Frame 1 | Payload Frame 2 | ... | Payload Frame n | END_FRAME | +| ----------- | --------------- | --------------- | --- | --------------- | --------- | + +BEGIN_FRAME and END_FRAME is used to identify the boundaries of different custom type encodings. While reading +frames of a client message, when a BEGIN_FRAME is encountered, it means that the custom type encoding is started and +it is safe to read frames until the END_FRAME is encountered. All of the frames in between those two will carry the +actual data stored inside the custom type. + +Payload frames follow a similar schema to the initial frame and variable sized data frame structure described above. All +of the fix sized fields of the custom object is encoded in the initial payload frame that comes after the BEGIN_FRAME +and all the other variable sized or the custom fields are encoded in the following payload frames in the same way +described in the [Encoding of Variable Sized Fields](#213-encoding-of-variable-sized-fields) section. Therefore, each custom type encoding +consists of at least three and possible more frames depending on the types of the fields of the custom object. + +For example, if the custom type has the fields of the type integer, long, string and another custom type that has +boolean and string fields, then the encoded structure of the custom object will be as below. + +| BEGIN_FRAME | Payload frame for the fix sized fields | Payload frame for the string field | BEGIN_FRAME | Payload frame for the fix sized fields of the custom type field | Payload frame for the string field of the custom type field | END_FRAME | END_FRAME | +| ----------- | -------------------------------------- | ---------------------------------- | ----------- | ----------------------------------------------------------------- | ------------------------------------------------------------- | --------- | --------- | +| BEGIN_FRAME of the custom type | Payload frame for the integer and long fields | Payload frame for the string field | BEGIN_FRAME of the custom type field | Payload frame for the boolean field of the custom type field | Payload frame for the string field of the custom type field | END_FRAME for the custom type field | END_FRAME for the custom type | + +As depicted above, fix sized fields of the custom type which are integer and long fields, are encoded in the initial +frame that follows the BEGIN_FRAME of the custom type. Then, payload frame for the string field comes. It is encoded +in the same way described in the [Encoding of String Fields](#2131-encoding-of-string-fields) section. Custom types can also contain other +custom type fields. They are encoded in the same way as we describe in the beginning of the section. Payload frames +of the inner custom type, which are the frames for the boolean field and the string field, are encoded in its +respective BEGIN_FRAME and END_FRAME. Finally, the END_FRAME at the end signals the finish of the custom type +encoding. + +##### 2.1.3.3. Encoding of Array Fields + +Client messages may also contain array of fix sized or variable sized types. The encoding of the array frames changes +according to the type of the array elements. + +##### 2.1.3.3.1. Encoding of Array of Fix Sized Fields + +Since the byte size of the fix sized fields and the element count of the array can be known in advance, content +of the array can be fit into a single frame. For these types of arrays, payload size is calculated as +`ELEMENT_COUNT * ELEMENT_SIZE_IN_BYTES` and elements are encoded at the offsets depending on their indexes on arrays. +Assuming zero-based indexing, element offsets can be calculated as `ELEMENT_INDEX * ELEMENT_SIZE_IN_BYTES`. + +For example, array of integers can be encoded into a single frame as follows: + +| Frame length | Flags | int-0 | int-1 | ... | int-n | +| ------------ | ----- | ----- | ----- | --- | ----- | +| ← 4 bytes → | ← 2 bytes → | ← 4 bytes → | ← 4 bytes → | ... | ← 4 bytes → | + +Writing the elements of an array into a single frame puts an upper limit on the maximum number of elements that the array +contains. The number of elements that can be fit into a single frame can be calculated as `(2^31 - 6)/(size of the primitive type)`. +For example, for int64, a maximum of 268435455 (around 268 million) entries per array is supported by the protocol. + +##### 2.1.3.3.2. Encoding of Array of Variable Sized Fields +Array or variable sized fields, just like [custom type fields](#2132-encoding-of-custom-type-fields), are encoded in between BEGIN_FRAME and +END_FRAME. Each element of the array are encoded in their respective frames consecutively following the BEGIN_FRAME. Depending on +the type of array elements, each element may be encoded into one or more frames. In fact, encoding of array of variable sized +fields is very similar to encoding of custom types. + +For example, array of string objects can be encoded as follows: + +| BEGIN_FRAME | string-0 | string-1 | ... | string-n | END_FRAME | +| ----------- | -------- | -------- | --- | -------- | --------- | +| Begin frame of the array | Frame containing UTF-8 encoded bytes of string-0 | Frame containing UTF-8 encoded bytes of string-1 | ... | Frame containing UTF-8 encoded bytes of string-n | End frame of the array | + +Note that, elements of the array must be of same type. + +##### 2.1.3.3.3. Encoding of Array of Map Fields +Array of map entries can encoded in different ways depending on the types of keys and values. + +If both are fix sized fields as described above, map entries can be encoded into a single frame since the size of a map entry can be +known in advance. For these map entries, payload size of the frame can be calculated as `ENTRY_COUNT * (SIZE_OF_KEY + SIZE_OF_VALUE)`. +Map entries are encoded in the offset positions depending on their iteration order. Offset of the keys and values can be calculated as +`ENTRY_INDEX * (SIZE_OF_KEY + SIZE_OF_VALUE)` and `ENTRY_INDEX * (SIZE_OF_KEY + SIZE_OF_VALUE) + SIZE_OF_KEY` respectively, +assuming zero-based indexing. + +For example, map entries of int32 to int64 mappings can be encoded as below. + +| Frame length | Flags | int32-0 | int64-0 | int32-1 | int64-1 | ... | int32-n | int64-n | +| ------------ | ----- | ------- | ------- | ------- | ------- | --- | ------- | ------- | +| ← 4 bytes → | ← 2 bytes → | ←4 bytes → | ← 8 bytes → | ←4 bytes → | ← 8 bytes → | ... | ←4 bytes → | ← 8 bytes → | + +If one of them are fixed sized and the other is variable sized, map entries are encoded in between BEGIN_FRAME and END_FRAME. +Each key or value of the entry set that is variable sized is encoded in its respective frames consecutively following the BEGIN_FRAME. +As described above, this encoding may result in one or more frames depending on the type of the variable sized key or value. +Each key or value of the entry set that is fix sized is encoded into a single frame as described in the [Encoding of Array of Fix Sized Fields](#21331-encoding-of-array-of-fix-sized-fields) section. + +For example, map entries of string to int32 mappings can be encoded as below. + +| BEGIN_FRAME | string-0 | string-1 | ... | string-n | array of int32 | END_FRAME | +| ----------- | -------- | -------- | --- | -------- | -------------- | --------- | +| Begin frame of the map entries | Frame containing UTF-8 encoded bytes of string-0 (key-0) | Frame containing UTF-8 encoded bytes of string-1 (key-1) | ... | Frame containing UTF-8 encoded bytes of string-n (key-n) | Frame containing array of int32 (values) | End frame of the map entries | + +However, if both of them are variable sized, map entries are encoded in between BEGIN_FRAME and END_FRAME. Each key or value +of the entry set is encoded in its respective frames consecutively following the BEGIN_FRAME. + +For example, map entries of string to array of int32 mappings can be encoded as below. + +| BEGIN_FRAME | string-0 | array of int32s-0 | string-1 | array of int32-1 | ... | string-n | array of int32-n | END_FRAME | +| ----------- | -------- | ----------------- | -------- | ----------------- | --- | -------- | ----------------- | --------- | +| Begin frame of the map entries | Frame containing UTF-8 encoded bytes of string-0 (key-0) | Frame containing array of int32-0 (value-0) | Frame containing UTF-8 encoded bytes of string-1 (key-1) | Frame containing array of int32-1 (value-1) | ... | Frame containing UTF-8 encoded bytes of string-n (key-n) | Frame containing array of int32-n (value-n) | End frame of the map entries | + +### 2.1.4. Client Message Fragmentation +A fragment is a part of a client message where the client message is too large and it is split into multiple client messages. +It is used to interleave large client messages so that small but urgent client messages can be sent without waiting the transmission +of the large client message. + +Fragmentation is handled through BEGIN_FRAGMENT_FLAG and END_FRAGMENT_FLAG bits of the frame flags. Unfragmented messages have +one in both flag bits. For fragmented client messages, first fragment has one in BEGIN_FRAGMENT_FLAG and zero in END_FRAGMENT_FLAG, +last fragment has zero in BEGIN_FRAGMENT_FLAG and one in END_FRAGMENT_FLAG and middle fragments have zero in both of the flag bits. + +Fragments of different client messages are identified by the 8 bytes long fragment ID. Fragment ID is encoded into the payload bytes. + +Initial frames of the fragmented client messages have the following structure. + +**First Fragment Initial Frame** + +| Frame length | Flags | Payload | +| ------------ | ----- | ------- | +| Frame length | BEGIN_FRAGMENT_FLAG = 1, END_FRAGMENT_FLAG = 0 | Fragment ID | +| ← 4 bytes → | ← 2 bytes → | ← 8 bytes → | + +**Middle Fragment Initial Frame** + +| Frame length | Flags | Payload | +| ------------ | ----- | ------- | +| Frame length | BEGIN_FRAGMENT_FLAG = 0, END_FRAGMENT_FLAG = 0 | Fragment ID | +| ← 4 bytes → | ← 2 bytes → | ← 8 bytes → | + +**Last Fragment Initial Frame** + +| Frame length | Flags | Payload | +| ------------ | ----- | ------- | +| Frame length | BEGIN_FRAGMENT_FLAG = 0, END_FRAGMENT_FLAG = 1 | Fragment ID | +| ← 4 bytes → | ← 2 bytes → | ← 8 bytes → | + +Then, visual representation of the possible fragments of a client message with N frames can be as below: + +**First Fragment** + +| First Fragment Initial Frame | client message - 1st frame | client message - 2nd frame | ... | client message - ith frame | +| -------------------------- | -------------------------- | -------------------------- | --- | -------------------------- | + +**Middle Fragments** + +| Middle Fragment Initial Frame | client message - (i+1)th frame | client message - (i+2)th frame | ... | client message - jth frame | +| --------------------------- | ------------------------------ | ------------------------------ | --- | -------------------------- | + +**Last Fragment** + +| Last Fragment Initial Frame | client message - (j+1)th frame | client message - (j+2)th frame | ... | client message - nth frame | +| ------------------------- | ------------------------------ | ------------------------------ | --- | -------------------------- | + +### 2.1.5. Client Message Boundaries +As described in the [Initial Frame](#212-initial-frame) and [Client Message Fragmentation](#214-client-message-fragmentation) sections, +initial frame of the client messages can be identified with the BEGIN_FRAGMENT_FLAG and END_FRAGMENT_FLAG bits. + +Last frame of a client message can be identified by checking the IS_FINAL_FLAG bit. If set to one, it signals that the client +message is ended. + +### 2.1.6. Backward Compatibility of the Client Messages +Hazelcast Open Binary Protocol guarantees backward compatibility for all major 2.x versions. Therefore, developments done in the +protocol MUST NOT result in deletion of services, methods or any fields. However, new services, methods or fields MAY be added. + +For addition of fix sized fields to service methods, additional fields can be detected by checking the frame length of the initial frame. +An old reader reads and uses old fields and simply skips the bytes that contain the additional fields. + +On the other hand, addition of variable sized fields can be detected using the END_FRAME. An old reader reads and uses old +frames and simple skips the additional frames until it detects the END_FRAME. + +### 2.1.7. Augmented Backus–Naur Format Representation of the Client Messages +Below is the representation of the client messages used within the protocol as described with the rules defined in +RFC 5234 that specifies the Augmented Backus–Naur format. Although the definition below is ambiguous, it clearly +reflects the structure of the client message. + +``` +client-message = message-initial-frame *var-sized-param +message-initial-frame = request-initial-frame / response-initial-frame +request-initial-frame = frame-length flags message-type correlation-id partition-id *fix-sized-param +response-initial-frame = normal-response-initial-frame / event-response-initial-frame / error-response-initial-frame +normal-response-initial-frame = frame-length flags message-type correlation-id backup-acks-count *fix-sized-param +event-response-initial-frame = frame-length flags message-type correlation-id partition-id *fix-sized-param +error-response-initial-frame = frame-length flags message-type correlation-id backup-acks-count error-code *error-holder + +frame-length = int32 +message-type = int32 +correlation-id = int64 +partition-id = int32 +backup-acks-count = int8 +error-code = int32 + +error-holder = begin-frame frame-length flags cause-error-code error-class-name *1error-message *stack-trace-element end-frame +cause-error-code = int32 +error-class-name = string-frame +error-message = string-frame +stack-trace-element = begin-frame frame-length flags line-number class-name method-name *1file-name end-frame +class-name = string-frame +method-name = string-frame +file-name = string-frame + +var-sized-param = string-frame / custom-type-frames / list-frames / map-frames / null-frame +list-frames = var-sized-param-list-frames / fix-sized-param-list-frame +map-frames = map-fix-sized-to-fix-sized-frame / map-var-sized-to-var-sized-frames / + / map-fix-sized-to-var-sized-frames / map-var-sized-to-fix-sized-frames + +var-sized-param-list-frames = begin-frame *var-sized-param end-frame ; All elements should be of same type +fix-sized-param-list-frame = frame-length flags *fix-sized-param ; All elements should be of same type + +map-fix-sized-to-fix-sized-frame = frame-length flags *fix-sized-entry +fix-sized-entry = fix-sized-param fix-sized-param +map-var-sized-to-var-sized-frames = begin-frame *var-sized-entry end-frame +var-sized-entry = var-sized-param var-sized-param +map-fix-sized-to-var-sized-frames = begin-frame *var-sized-entry fix-sized-param-frame end-frame +fix-sized-param-frame = frame-length flags fix-sized-param +map-var-sized-to-fix-sized-frames = begin-frame *var-sized-entry fix-sized-param-frame end-frame + + +string-frame = frame-length flags *OCTET ; Contains UTF-8 encoded octets +custom-type-frames = begin-frame *1custom-type-initial-frame *var-sized-param end-frame +custom-type-initial-frame = frame-length flags *fix-sized-param + +null-frame = frame-length flags ; IS_NULL flag is set to one +begin-frame = frame-length flags ; BEGIN_DATA_STRUCTURE_FLAG is set to one +end-frame = frame-length flags ; END_DATA_STRUCTURE_FLAG is set to one + +flags = begin-fragment end-fragment is-final begin-data-structure end-data-structure is-null is-event backup-aware backup-event 7reserved +begin-fragment = BIT ; Used in message fragmentation +end-fragment = BIT ; Used in message fragmentation +is-final = BIT ; Set to 1 if the frame is the last frame of a client message +begin-data-structure = BIT ; Set to 1 if the frame is the begin frame of a custom type or array of variable sized types +end-data-structure = BIT ; Set to 1 if the frame is the end frame of a custom type or array of variable sized types +is-null = BIT ; Set to 1 if the frame represents a null field +backup-aware = BIT ; Set to 1 if the client enabled receiving backup acks directly from members that backups are applied to +backup-event = BIT ; Set to 1 if the frame is the initial frame of a client message that represents a backup event response from a member +reserved = BIT ; Reserved for future usage + +fix-sized-param = *OCTET / boolean / int8 / int16 / int32 / int64 / UUID +boolean = %x00 / %x01 +int8 = 8BIT +int16 = 16BIT +int32 = 32BIT +int64 = 64BIT +UUID = int64 int64 +``` + +For the fragmented client messages, ABNF definition is below. + +``` +fragmented-message = begin-fragment *middle-fragment end-fragment / client-message +begin-fragment = frame-length flags fragment-id 1*frame ; begin-fragment is set to 1, end-fragment is set to 0, is_final of last frame set to 1 +middle-fragment = frame-length flags fragment-id 1*frame ; begin-fragment is set to 0, end-fragment is set to 0, is_final of last frame set to 1 +end-fragment = frame-length flags fragment-id 1*frame ; begin-fragment is set to 0, end-fragment is set to 1, is_final of last frame set to 1 +``` + +## 3. Client Protocol Data Types + +| Type | Description | Size | Min Value | Max Value | +| ---- | ----------- | ---- | --------- | --------- | +| uint8 | unsigned 8 bit integer | 8 bit | 0 | 2^8 - 1 | +| uint16 | unsigned 16 bit integer | 16 bit | 0 | 2^16 - 1 | +| uint32 | unsigned 32 bit integer | 32 bit | 0 | 2^32 - 1 | +| uint64 | unsigned 64 bit integer | 64 bit | 0 | 2^64 - 1 | +| int8 | signed 8 bit integer in 2's complement | 8 bit | -2^7 | 2^7 - 1 | +| int16 | signed 16 bit integer in 2's complement | 16 bit | -2^15 | 2^15 - 1 | +| int32 | signed 32 bit integer in 2's complement | 32 bit | -2^31 | 2^31 - 1 | +| int64 | signed 64 bit integer in 2's complement | 64 bit | -2^63 | 2^63 - 1 | +| float | single precision IEEE 754 floating point number | 32 bit | | | +| double | double precision IEEE 754 floating point number | 64 bit | | | +| boolean | same as uint8 with special meanings. 0 is "false", any other value is "true" | 8 bit | | | +| string | String encoded as a byte array with UTF-8 encoding as described in RFC 3629 | variable | | + +Data types are consistent with those defined in The Open Group Base Specification Issue 7 IEEE Std 1003.1, 2013 Edition. +Data types are in **Little Endian** format. + +## 4. Connection Guide + +### 4.1. Opening a Connection +TCP socket communication is use for clint-to-member communication. Each member has a socket listening for incoming connections. +This TCP socket is used both for member-to-member and client-to-member communications. + +As the first step of client-to-member communication, the client MUST open a TCP socket connection to the member. + +A client needs to establish a single connection to each member node if it is a smart client. If it is a unisocket client, +a single connection is enough for a particular client. For details, see [Smart Client versus Unisocket Client](#61-smart-client-and-unisocket-client). + +#### 4.2. Connection Initialization +After successfully connecting to the member TCP socket, the client MUST send three bytes of initialization data to identify +the connection type to the member. + +For any client, the three byte initializer data is [0x43, 0x50, 0x32], which is the string "CP2" in UTF-8 encoding. + +### 4.3. Authentication +The first message sent through an initialized connection must be an authentication message. Any other type of message +will fail with an authorization error unless the authentication is complete. Authentication is the process of sending +an authentication message and receiving an authentication response. + +Upon successful authentication, the client will receive a response from the member with the member's IP address, UUID +that uniquely identify the client-to-member connection and cluster UUID along with the other response fields described +in "Client.Authentication". The status field in the authentication response should be checked for the authentication status. +There are three possible statuses: + +1. Authentication is successful. +2. Credentials failed. The provided credentials (e.g. cluster name, username or password may be incorrect) +3. Serialization version mismatch. The requested serialization version and the serialization version used at the member side +are different. The client get the member's serialization version from the "serverHazelcastVersion" field of the response. +It is suggested that the client tries to reconnect using the matching serialization version assuming that the client +implements the version for serialization. + +There are two types authentications: +1. Username/Password authentication: "Client.Authentication" message is used for this authentication type which contains +username and password for the client if exist along with the cluster name. +2. Custom credentials authentication: "Client.CustomAuthentication" message is used for this authentication type. Custom +authentication credentials are sent as a byte array. + +### 4.4. Communication via Messages + +After a successful authentication, a client may send request messages to the member to access distributed objects +or perform other operations on the cluster. This step is the actual communication step. + +Once connected, a client can do the followings: +1. Send periodic updates. +2. Get updates on cluster state view which consists of partition table and member list. +3. Send operation messages and receive responses. + +All request messages will be sent to the member and all responses and event responses will be sent to the client. + +See [Protocol Messages](#{{ protocol_message_offset }}-protocol-messages) for details. + +### 4.5. Closing Connections +To end the communication, the network socket that was opened in the "Open Connection" step should be closed. This will +result in releasing resources on the member side specific to this connection. + +## 5. Requests and Responses +### 5.1. Distributed Objects +To access distributed object information, use the "Get Distributed Object" message type. + +To add a listener for adding distributed objects, use the "Add Distributed Object Listener" message type. + +To remove a formerly added listener, use the "Remove Distributed Object Listener" message type. + +### 5.2. Operation Messages And Responses +Operational messages are the messages where a client can expect exactly one response for a given request. The client +knows which request te response correlates to via the correlation ID. An example of one of these operations is a +"map put" operation. + +To execute a particular operation, set the message type ID to the corresponding operation type and encode the fields +as described in the [Client Message](#21-client-message) section. + +### 5.3. Proxies +Before using a distributed object, you SHOULD first create a proxy for the object. Do this by using the "Create Proxy" +request message. + +To destroy a proxy, yse tge "Destroy Proxy" request message. + +#### 5.3.1. Proxy Creation + +**Java Example:** +```java +HazelcastInstance client = HazelcastClient.newHazelcastClient(); +IMap map = client.getMap("map-name"); +``` + +**Python Example** +```python +client = HazelcastClient() +map = client.get_map("map-name") +``` +Raw bytes for the create proxy request and response is below. + +**Client Request** +``` +// Initial frame +0x16 0x00 0x00 0x00 // Frame length +0x00 0xc0 // Flags +0x00 0x04 0x00 0x00 // Message type +0x30 0x00 0x00 0x00 0x00 0x00 0x00 0x00 // Correlation ID +0xff 0xff 0xff 0xff // Partition ID +// Frame for the "map-name" string +0x0e 0x00 0x00 0x00 // Frame length +0x00 0x00 // Flags +0x6d 0x61 0x70 0x2d 0x6e 0x61 0x6d 0x65 // UTF-8 encoded data of the "map-name" string +// Frame for the name of the map service (which is "hz:impl:mapService") +0x18 0x00 0x00 0x00 // Frame length +0x00 0x20 // Flags +0x68 0x7a 0x3a 0x69 0x6d 0x70 0x6c 0x3a 0x6d 0x61 0x70 0x53 0x65 0x72 0x76 0x69 0x63 0x65 // UTF-8 encoded data of the "hz:impl:mapService" string +``` + +**Member Response** +``` +// Initial frame +0x13 0x00 0x00 0x00 // Frame length +0x00 0xe0 // Flags +0x01 0x04 0x00 0x00 // Message type +0x30 0x00 0x00 0x00 0x00 0x00 0x00 0x00 // Correlation ID +0x00 // Backup acks count +``` + +For a request with a key, the client SHOULD send the request to the cluster member that houses the data for the key. +A client can do this by using the partition ID. For the "Create Proxy" request above, since the proxy creation is meant +to be sent to a random cluster member, partition ID is given as -1. + +The response to a request message is always one of the following: +1. Regular response message: The response is the message as listed in the protocol specification for the specific request message type. +2. An error message: See the [Error Codes](#58-error-codes) section. + +We give examples of operations on various data structures below. + +#### 5.3.2. List Example + +**Java Example** +```java +IList myList = client.getList("list"); // Create proxy +System.out.println(myList.get(3)); +``` + +**Python Example** +```python +my_list = client.get_list("list") # Create proxy +print(my_list.get(3)) +``` +Raw bytes for get request and response: +**Client Request** +``` +// Initial frame +0x1a 0x00 0x00 0x00 // Frame length +0x00 0xc0 // Flags +0x00 0x0f 0x05 0x00 // Message type +0x05 0x00 0x00 0x00 0x00 0x00 0x00 0x00 // Correlation ID +0x7e 0x00 0x00 0x00 // Partition ID +0x03 0x00 0x00 0x00 // Item index: 3 +// Frame for the list name +0x0a 0x00 0x00 0x00 // Frame length +0x00 0x20 // Flags +0x6c 0x69 0x73 0x74 // UTF-8 encoded data of the "list" string +``` + +**Member Response** +``` +// Initial frame +0x13 0x00 0x00 0x00 // Frame length +0x00 0xc0 // Flags +0x01 0x0f 0x05 0x00 // Message type +0x05 0x00 0x00 0x00 0x00 0x00 0x00 0x00 // Correlation ID +0x00 // Backup acks count +// Frame for the nullable Data frame +0x18 0x00 0x00 0x00 +0x00 0x20 +0x00 0x0c 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0xff 0xff 0xff 0xf9 0x00 0x00 0x00 0x04 // Nullable Data for the returned value +``` + +#### 5.3.3. Fenced Lock Example +**Java Example** +```java +FencedLock myLock = client.getCPSubsystem().getLock("lock"); // Create proxy +myLock.lock(); +``` + +Raw bytes for the lock request and response: +**Client Request** +``` +// Initial frame +0x37 0x00 0x00 0x00 // Frame length +0x00 0xc0 // Flags +0x00 0x01 0x07 0x00 // Message type +0x07 0x00 0x00 0x00 0x00 0x00 0x00 0x00 // Correlation ID +0x11 0x00 0x00 0x00 // Partition ID +0x7b 0x00 0x00 0x00 0x00 0x00 0x00 0x00 // Session ID: 123 +0x60 0x00 0x00 0x00 0x00 0x00 0x00 0x00 // Thread ID: 96 +0x00 0x15 0xcd 0x5b 0x07 0x00 0x00 0x00 0x00 0xb1 0x68 0xde 0x3a 0x00 0x00 0x00 0x00 // Invocation UUID: UUID(123456789, 987654321) +// Frame for the RaftGroupID +// Begin frame for the RaftGroupId frame +0x06 0x00 0x00 0x00 // Frame length +0x00 0x10 // Flags +// Initial frame for the RaftGroupId +0x16 0x00 0x00 0x00 // Frame length +0x00 0x00 // Flags +0x36 0x00 0x00 0x00 0x00 0x00 0x00 0x00 // Seed: 54 +0x40 0x00 0x00 0x00 0x00 0x00 0x00 0x00 // Id: 64 +// String name field for the RaftGroupId +0x10 0x00 0x00 0x00 // Frame length +0x00 0x00 // Flags +0x72 0x61 0x66 0x74 0x2d 0x67 0x72 0x6f 0x75 0x70 // UTF-8 encoded name of the RaftGroup: "raft-group" +// End frame for the RaftGroupId +0x06 0x00 0x00 0x00 +0x00 0x08 +// String frame for the lock instance +0x0a 0x00 0x00 0x00 // Frame length +0x00 0x20 // Flags +0x6c 0x6f 0x63 0x6b // UTF-8 encoded name of the lock: "lock" +``` + +**Member Response** +``` +// Initial frame +0x1b 0x00 0x00 0x00 // Frame length +0x00 0xe0 // Flags +0x01 0x01 0x07 0x00 // Message type +0x07 0x00 0x00 0x00 0x00 0x00 0x00 0x00 // Correlation ID +0x00 // Backup acks count +0x4e 0x61 0xbc 0x00 0x00 0x00 0x00 0x00 // Fence token: 12345678 +``` + +#### 5.3.4. Map Example + +**Java Example** +```java +String key = "key1"; +int value = 54; +IMap myMap = client.getMap("map"); // Create proxy +myMap.put(key, value); +``` +**Python Example** +```python +key = "key1" +value = 54 +my_map = client.get_map("map") # Create proxy +my_map.put(key, value) +``` +Raw bytes for the map put request and response + +**Client Request** +``` +// Initial frame +0x26 0x00 0x00 0x00 // Frame length +0x00 0xc0 // Flags +0x00 0x01 0x01 0x00 // Message type +0x15 0x02 0x00 0x00 0x00 0x00 0x00 0x00 // Correlation ID +0x11 0x01 0x00 0x00 // Partition ID +0xf1 0xfb 0x90 0x00 0x00 0x00 0x00 0x00 // Thread ID: 654321 +0x10 0x27 0x00 0x00 0x00 0x00 0x00 0x00 // TTL for the map: 10000 +// String frame for the map name +0x09 0x00 0x00 0x00 // Frame length +0x00 0x00 // Flags +0x6d 0x61 0x70 // UTF-8 encoded name of the map: "map" +// Data frame for the key +0x16 0x00 0x00 0x00 // Frame length +0x00 0x00 // Flags +0x00 0x00 0x00 0x00 0xff 0xff 0xff 0xf5 0x00 0x00 0x00 0x04 0x6b 0x65 0x79 0x31 // Key data bytes +// Data frame for the value +0x12 0x00 0x00 0x00 // Frame length +0x00 0x00 // Flags +0x00 0x00 0x00 0x00 0xff 0xff 0xff 0xf9 0x00 0x00 0x00 0x36 // Value data bytes +``` + +**Member Response** +``` +// Initial frame +0x13 0x00 0x00 0x00 // Frame length +0x00 0xc0 // Flags +0x01 0x01 0x01 0x00 // Message type +0x15 0x02 0x00 0x00 0x00 0x00 0x00 0x00 // Correlation ID +0x00 // Backup acks count +// Nullable response frame. Assuming there were no value associated with this key, response is set to null as below +0x06 0x00 0x00 0x00 // Frame length +0x00 0x40 // Flags +``` + +#### 5.3.5. Queue Example + +**Java Example** +```java +IQueue myQueue = client.getQueue("queue"); // Create proxy +System.out.println(myQueue.size()); +``` + +**Python Example** +```python +my_queue = client.get_queue("queue") +print(my_queue.size()) +``` + +Raw bytes for the queue size request and response: + +**Client Request** +``` +// Initial frame +0x16 0x00 0x00 0x00 // Frame length +0x00 0xc0 // Flags +0x00 0x03 0x03 0x00 // Message type +0x12 0x33 0x00 0x00 0x00 0x00 0x00 0x00 // Correlation ID +0x01 0x01 0x00 0x00 // Partition ID +// String frame for the queue name +0x0b 0x00 0x00 0x00 // Frame length +0x00 0x20 // Flags +0x71 0x75 0x65 0x75 0x65 // UTF-8 encoded name of the queue: "queue" +``` + +**Member Response** +``` +// Initial frame +0x17 0x00 0x00 0x00 // Frame length +0x00 0xe0 // Flags +0x01 0x03 0x03 0x00 // Message type +0x12 0x33 0x00 0x00 0x00 0x00 0x00 0x00 // Correlation ID +0x00 // Backup acks count +0x19 0x00 0x00 0x00 // Queue size: 25 +``` + +#### 5.3.6. Set Example + +**Java Example** +```java +ISet set = client.getSet("set"); // Create proxy +set.clear(); +``` + +```python +set = client.get_set("set") # Create proxy +set.clear() +``` + +Raw bytes for the set clear request and response: + +**Client Request** +``` +// Initial frame +0x16 0x00 0x00 0x00 // Frame length +0x00 0xc0 // Flags +0x00 0x09 0x06 0x00 // Message type +0x0a 0x01 0xb5 0x00 0x00 0x00 0x00 0x00 // Correlation ID +0x03 0x02 0x00 0x00 // Partition ID +// String frame for the set name +0x09 0x00 0x00 0x00 // Frame length +0x00 0x20 // Flags +0x73 0x65 0x74 // UTF-8 encoded name of the set: "set" +``` + +**Member Response** +``` +// Initial frame +0x13 0x00 0x00 0x00 // Frame length +0x00 0xe0 // Flags +0x01 0x09 0x06 0x00 // Message type +0x0a 0x01 0xb5 0x00 0x00 0x00 0x00 0x00 // Correlation ID +0x00 // Backup acks count +``` + +### 5.4. Multiple Responses to a Single Request +The client can listener for updates on a member or when specific actions are taken on the cluster. This is managed by the event listener mechanism. +The event messages have the IS_EVENT_FLAG bit is set in the initial frame and they use the same correlation ID as used in the original +registration request for all the subsequent event update messages. The registration message and possible event messages sent are +described in the Events section of the message descriptions. + +### 5.5. Listeners +Listeners are a mean to communicate multiple responses to a client. The client uses one of the listener registration messages to listen +for updates at the cluster. Listeners are specific to a data structure. For example, there are specific listener for map entries and queue +entries. To see how these listeners are explicitly encoded, see the relevant message in [Protocol Messages](#{{ protocol_message_offset }}-protocol-messages) section. + +Because the same correlation ID is reused for every event response for a given request, the correlation ID MUST NOT be reused from event +requests unless the client unregisters the listener. + +One can send "Remove Listener" request message specific to the registration request to remove the listener that the client has +registered to. + +### 5.6. Cluster View Listener +Cluster view consists of views of partition and member lists. Client gets the updates of these views as event responses +after registering a cluster view listener to one of the members of the cluster. + +Partition list view tells the client which members handles which partition key. The client can use this information +to send the related requests to the responsible member (for the request key if it exists) directly for processing. + +The event response for the partition list view consists of the member-partition ownership information. + +To create a listener for the case of a partition being lost, use the "Add Partition Lost Listener" request message. + +To remove the listener, use the "Remove Partition Lost Listener" message. + +Other part of the cluster view is member list view. This view is the list of members connected to the cluster. With this +information and the previous member list that the client has, member updates on cluster such as member addition or removal +can be seen. This information is needed especially if the client operates as a smart client. + +### 5.7. Timeouts and Retry +It is recommended that the client should be able to handle situations where the member may not be able to return the response +in an expected time interval. Even if the response to a specific message is not received, the user may or may not retry the +request. If the client retries the request, they SHOULD NOT use the same correlation ID. + +If no message has been sent in the member's heartbeat time, the member will automatically disconnect from the client. To prevent +this from occurring, a client SHOULD submit a "ping" request to the member periodically. A ping message is only sent from the +client to the member; the member does not perform any ping request. + +### 5.8. Error Codes +The list of errors along with the error code and description is provided below. Note that there may be error messages with +an error code which is not listed in the table. The client can handle this situation differently based on the particular +implementation. (e.g. throw an unknown error code exception) + +| Error Name | Error Code | Description | +| ---------- | ---------- | ----------- | +| ARRAY_INDEX_OUT_OF_BOUNDS | 1 | Thrown to indicate that an array has been accessed with an illegal index. The index is either negative or greater than or equal to the size of the array. | +| ARRAY_STORE | 2 | Thrown to indicate that an attempt has been made to store the wrong type of object into an array of objects. For example, the following code generates an ArrayStoreException:
Object x[] = new String[3];
x[0] = new Integer(0); | +| AUTHENTICATION | 3 | The authentication failed. | +| CACHE | 4 | Thrown to indicate an exception has occurred in the Cache | +| CACHE_LOADER | 5 | An exception to indicate a problem has occurred executing a CacheLoader | +| CACHE_NOT_EXISTS | 6 | This exception class is thrown while creating com.hazelcast.cache.impl.CacheRecordStore instances but the cache config does not exist on the node to create the instance on. This can happen in either of two cases: the cache's config is not yet distributed to the node, or the cache has been already destroyed. For the first option, the caller can decide to just retry the operation a couple of times since distribution is executed in a asynchronous way. | +| CACHE_WRITER | 7 | An exception to indicate a problem has occurred executing a CacheWriter | +| CALLER_NOT_MEMBER | 8 | A Retryable Hazelcast Exception that indicates that an operation was sent by a machine which isn't member in the cluster when the operation is executed. | +| CANCELLATION | 9 | Exception indicating that the result of a value-producing task, such as a FutureTask, cannot be retrieved because the task was cancelled. | +| CLASS_CAST | 10 | The class conversion (cast) failed. | +| CLASS_NOT_FOUND | 11 | The class does not exists in the loaded jars at the member. | +| CONCURRENT_MODIFICATION | 12 | The code is trying to modify a resource concurrently which is not allowed. | +| CONFIG_MISMATCH | 13 | Thrown when 2 nodes want to join, but their configuration doesn't match. | +| DISTRIBUTED_OBJECT_DESTROYED | 14 | The distributed object that you are trying to access is destroyed and does not exist. | +| EOF | 15 | End of file is reached (May be for a file or a socket) | +| ENTRY_PROCESSOR | 16 | An exception to indicate a problem occurred attempting to execute an EntryProcessor against an entry | +| EXECUTION | 17 | Thrown when attempting to retrieve the result of a task that aborted by throwing an exception. | +| HAZELCAST | 18 | General internal error of Hazelcast. | +| HAZELCAST_INSTANCE_NOT_ACTIVE | 19 | The Hazelcast member instance is not active, the server is possibly initialising. | +| HAZELCAST_OVERLOAD | 20 | Thrown when the system won't handle more load due to an overload. This exception is thrown when backpressure is enabled. | +| HAZELCAST_SERIALIZATION | 21 | Error during serialization/de-serialization of data. | +| IO | 22 | An IO error occurred. | +| ILLEGAL_ARGUMENT | 23 | Thrown to indicate that a method has been passed an illegal or inappropriate argument | +| ILLEGAL_ACCESS_EXCEPTION | 24 | An IllegalAccessException is thrown when an application tries to reflectively create an instance (other than an array), set or get a field, or invoke a method, but the currently executing method does not have access to the definition of the specified class, field, method or constructor | +| ILLEGAL_ACCESS_ERROR | 25 | Thrown if an application attempts to access or modify a field, or to call a method that it does not have access to | +| ILLEGAL_MONITOR_STATE | 26 | When an operation on a distributed object is being attempted by a thread which did not initially own the lock on the object. | +| ILLEGAL_STATE | 27 | Signals that a method has been invoked at an illegal or inappropriate time | +| ILLEGAL_THREAD_STATE | 28 | Thrown to indicate that a thread is not in an appropriate state for the requested operation. | +| INDEX_OUT_OF_BOUNDS | 29 | Thrown to indicate that an index of some sort (such as to a list) is out of range. | +| INTERRUPTED | 30 | Thrown when a thread is waiting, sleeping, or otherwise occupied, and the thread is interrupted, either before or during the activity | +| INVALID_ADDRESS | 31 | Thrown when given address is not valid. | +| INVALID_CONFIGURATION | 32 | An InvalidConfigurationException is thrown when there is an Invalid configuration. Invalid configuration can be a wrong Xml Config or logical config errors that are found at real time. | +| MEMBER_LEFT | 33 | Thrown when a member left during an invocation or execution. | +| NEGATIVE_ARRAY_SIZE | 34 | The provided size of the array can not be negative but a negative number is provided. | +| NO_SUCH_ELEMENT | 35 | The requested element does not exist in the distributed object. | +| NOT_SERIALIZABLE | 36 | The object could not be serialized | +| NULL_POINTER | 37 | The server faced a null pointer exception during the operation. | +| OPERATION_TIMEOUT | 38 | An unchecked version of java.util.concurrent.TimeoutException. Some of the Hazelcast operations may throw an . Hazelcast OperationTimeoutException uses OperationTimeoutException to pass TimeoutException up through interfaces that don't have TimeoutException in their signatures | +| PARTITION_MIGRATING | 39 | Thrown when an operation is executed on a partition, but that partition is currently being moved around. | +| QUERY | 40 | Error during query. | +| QUERY_RESULT_SIZE_EXCEEDED | 41 | Thrown when a query exceeds a configurable result size limit. | +| SPLIT_BRAIN_PROTECTION | 42 | An exception thrown when the cluster size is below the defined threshold. | +| REACHED_MAX_SIZE | 43 | Exception thrown when a write-behind MapStore rejects to accept a new element. | +| REJECTED_EXECUTION | 44 | Exception thrown by an Executor when a task cannot be accepted for execution. | +| RESPONSE_ALREADY_SENT | 45 | There is some kind of system error causing a response to be send multiple times for some operation. | +| RETRYABLE_HAZELCAST | 46 | The operation request can be retried. | +| RETRYABLE_IO | 47 | Indicates that an operation can be retried. E.g. if map.get is send to a partition that is currently migrating, a subclass of this exception is thrown, so the caller can deal with it (e.g. sending the request to the new partition owner). | +| RUNTIME | 48 | Exceptions that can be thrown during the normal operation of the Java Virtual Machine | +| SECURITY | 49 | There is a security violation | +| SOCKET | 50 | There is an error in the underlying TCP protocol | +| STALE_SEQUENCE | 51 | Thrown when accessing an item in the Ringbuffer using a sequence that is smaller than the current head sequence. This means that the and old item is read, but it isn't available anymore in the ringbuffer. | +| TARGET_DISCONNECTED | 52 | Indicates that an operation is about to be sent to a non existing machine. | +| TARGET_NOT_MEMBER | 53 | Indicates operation is sent to a machine that isn't member of the cluster. | +| TIMEOUT | 54 | Exception thrown when a blocking operation times out | +| TOPIC_OVERLOAD | 55 | Thrown when a publisher wants to write to a topic, but there is not sufficient storage to deal with the event. This exception is only thrown in combination with the reliable topic. | +| TRANSACTION | 56 | Thrown when something goes wrong while dealing with transactions and transactional data-structures. | +| TRANSACTION_NOT_ACTIVE | 57 | Thrown when an a transactional operation is executed without an active transaction. | +| TRANSACTION_TIMED_OUT | 58 | Thrown when a transaction has timed out. | +| URI_SYNTAX | 59 | Thrown to indicate that a string could not be parsed as a URI reference | +| UTF_DATA_FORMAT | 60 | Signals that a malformed string in modified UTF-8 format has been read in a data input stream or by any class that implements the data input interface | +| UNSUPPORTED_OPERATION | 61 | The message type id for the operation request is not a recognised id. | +| WRONG_TARGET | 62 | An operation is executed on the wrong machine. | +| XA | 63 | An error occurred during an XA operation. | +| ACCESS_CONTROL | 64 | Indicates that a requested access to a system resource is denied. | +| LOGIN | 65 | Basic login exception. | +| UNSUPPORTED_CALLBACK | 66 | Signals that a CallbackHandler does not recognize a particular Callback. | +| NO_DATA_MEMBER | 67 | Thrown when there is no data member in the cluster to assign partitions. | +| REPLICATED_MAP_CANT_BE_CREATED | 68 | Thrown when replicated map create proxy request is invoked on a lite member. | +| MAX_MESSAGE_SIZE_EXCEEDED | 69 | Thrown when client message size exceeds Integer.MAX_VALUE. | +| WAN_REPLICATION_QUEUE_FULL | 70 | Thrown when the wan replication queues are full. | +| ASSERTION_ERROR | 71 | Thrown to indicate that an assertion has failed. | +| OUT_OF_MEMORY_ERROR | 72 | Thrown when the Java Virtual Machine cannot allocate an object because it is out of memory, and no more memory could be made available by the garbage collector. | +| STACK_OVERFLOW_ERROR | 73 | Thrown when a stack overflow occurs because an application recurses too deeply. | +| NATIVE_OUT_OF_MEMORY_ERROR | 74 | Thrown when Hazelcast cannot allocate required native memory. | +| SERVICE_NOT_FOUND | 75 | An exception that indicates that a requested client service doesn't exist. | +| STALE_TASK_ID | 76 | Thrown when retrieving the result of a task via DurableExecutorService if the result of the task is overwritten. This means the task is executed but the result isn't available anymore | +| DUPLICATE_TASK | 77 | thrown when a task's name is already used before for another (or the same, if re-attempted) schedule. | +| STALE_TASK | 78 | Exception thrown by the IScheduledFuture during any operation on a stale (=previously destroyed) task. | +| LOCAL_MEMBER_RESET | 79 | An exception provided to MemberLeftException as a cause when the local member is resetting itself | +| INDETERMINATE_OPERATION_STATE | 80 | Thrown when result of an invocation becomes indecisive. | +| FLAKE_ID_NODE_ID_OUT_OF_RANGE_EXCEPTION | 81 | Thrown from member if that member is not able to generate IDs using Flake ID generator because its node ID is too big. | +| TARGET_NOT_REPLICA_EXCEPTION | 82 | Exception that indicates that the receiver of a CRDT operation is not a CRDT replica. | +| MUTATION_DISALLOWED_EXCEPTION | 83 | Exception that indicates that the state found on this replica disallows mutation. | +| CONSISTENCY_LOST_EXCEPTION | 84 | Exception that indicates that the consistency guarantees provided by some service has been lost. The exact guarantees depend on the service. | +| SESSION_EXPIRED_EXCEPTION | 85 | Thrown when an operation is attached to a Raft session is no longer active | +| WAIT_KEY_CANCELLED_EXCEPTION | 86 | Thrown when a wait key is cancelled and means that the corresponding operation has not succeeded | +| LOCK_ACQUIRE_LIMIT_REACHED_EXCEPTION | 87 | Thrown when the current lock holder could not acquired the lock reentrantly because the configured lock acquire limit is reached. | +| LOCK_OWNERSHIP_LOST_EXCEPTION | 88 | Thrown when an endpoint (either a Hazelcast member or a client) interacts with a FencedLock instance after its CP session is closed in the underlying CP group and its lock ownership is cancelled. | +| CP_GROUP_DESTROYED_EXCEPTION | 89 | Thrown when a request is sent to a destroyed CP group. | +| CANNOT_REPLICATE_EXCEPTION | 90 | Thrown when an entry cannot be replicated | +| LEADER_DEMOTED_EXCEPTION | 91 | Thrown when an appended but not-committed entry is truncated by the new leader. | +| STALE_APPEND_REQUEST_EXCEPTION | 92 | Thrown when a Raft leader node appends an entry to its local Raft log, but demotes to the follower role before learning the commit status of the entry. | +| NOT_LEADER_EXCEPTION | 93 | Thrown when a leader-only request is received by a non-leader member. | +| VERSION_MISMATCH_EXCEPTION | 94 | Indicates that the version of a joining member is not compatible with the cluster version | + +## 6. Miscellaneous + +### 6.1. Smart Client and Unisocket Client +The client can work as a smart or as a unisocket client. In both cases, a client SHOULD calculate which partition ID +is responsible for the requests and put this information in the partition ID offset of the initial frame. + +- **Smart Client:** A smart client sends the request directly to the cluster member that is responsible for the related key. +In order to do so, the client determines the address of the cluster member that handles the calculated partition ID. +The request message will be sent on this cluster member connection. + +- **Unisocket Client:** The client sends the request to any cluster member that it is connected to, regardless of the key +for the request. The cluster member will in turn redirect the request to the correct member in cluster that handles +the request for the provided key. + +The biggest difference between the two types of clients is that a smart client must be connected to all of the members and +must constantly update its partition tables so it knows which connection to use to submit a request. Both clients are compliant +with the protocol defined in this document. + +### 6.2. Serialization +While mostly an implementation detail, serialization plays a crucial role in the protocol. In order for a client to execute +an operation on the member that involves data structure, such as putting some entry in a map or queue, the client must +be aware of how object are serialized and deserialized so that the client can process the bytes it receives accordingly. +The member and client should use the same serialization versions in order to communicate. The version is negotiated in the +connection authentication phase. In general, one need to use serialization to serialize byte-array type fields in the messages as specified +in the [Protocol Messages](#{{ protocol_message_offset }}-protocol-messages) section. The following are examples of such objects that must be serialized +before being sent over the wire: +- Key +- Value +- Old value +- New value +- Callable (Executor Service) +- IFunction (Atomics) +- EntryProcessor (JCache) +- ExpiryPolicy (JCache) +- CacheConfig (JCache) +- ListenerConfig (JCache) +- Interceptor (Map) + +A client may follow Hazelcast's native serialization or it may implement its own custom serialization solution. For more +information on how Hazelcast serializes its objects, see the official [Reference Manual](https://docs.hazelcast.org/docs/latest/manual/html-single/#serialization) serialization section. + +For all byte-array fields, the API use should implement the following, depending on the operation type: +- If the operation is such that no member side deserialization is needed for the field, then the user can just use +any serialization and there is no need to any implementation for the member side. +- If the operation processing at the member requires deserialization of the byte-array field, then the user should +use a Java object implementing one of the Hazelcast serializations and may need to do some member side implementations +depending on the chosen serialization type. Furthermore, the serializer must be registered in the serialization configuration +of the member as described in the serialization section of the [Reference Manual](https://docs.hazelcast.org/docs/latest/manual/html-single/#serialization). + +The client authentication request message contains serialization version field. The client and the member decide the serialization +version to be used using this information. Hazelcast serialization versions will be matched to provide a compatible serialization. +There are two cases that can occur: +- Client may have higher serialization version then the member. In that case, client will auto configure itself during authentication +to match the member serialization version. +- Client may have lower serialization version then the member. In that case, member should be configured with the system property +to downgrade the member serialization version. + +### 6.3. Security +Most of the security is configured on the member side in a Hazelcast cluster. A client must authenticate itself, which in turn +lets the member establish an endpoint for the client. The member can restrict what a client can and cannot access. Current protocol +does not provide an explicit support for encryption. For more information, see the [Security](https://docs.hazelcast.org/docs/latest/manual/html-single/#security) chapter of the Hazelcast Reference Manual. + +{# END of manually written sections #} +## {{ protocol_message_offset }}. Protocol Messages +### {{ protocol_message_offset }}.1. Custom Data Types Used In The Protocol +{% for definition in custom_definitions %} +{% set custom_types = definition.get('customTypes', None) %} +{% if custom_types is not none %} +{% for custom_type in custom_types %} +#### {{ protocol_message_offset }}.1.{{ loop.index }}. {{ custom_type.name }} +**Fields** + +| Name | Type | Nullable | Available Since | +| -------------- | ---- | -------- | --------------- | + {% for param in custom_type.params %} +| {{ param.name }} | {{ convert_type(param.type) }} | {{ param.nullable }} | {{ param.since }} | + {% endfor %} + +{% endfor %} +{% endif %} +{% endfor %} +{% for service in services %} + {% set service_id = service.id + 2 %} + +### {{ protocol_message_offset }}.{{ service_id }}. {{ service.name }} + {% for method in service.methods %} + +#### {{ protocol_message_offset }}.{{ service_id }}.{{ method.id }}. {{ service.name }}.{{ method.name|capital }} + {% for line in method.doc.splitlines() %} +{{ line }} + {% endfor %} + +**Available since:** {{ method.since }} + +#### Request Message +**Message Type:** {{ "0x%02x%02x%02x"|format(service.id, method.id, 0) }} + {% if method.request.params|length > 0 %} + +**Partition Identifier:** {{ resolve_partition_identifier(method.request.partitionIdentifier) }} + +| Name | Type | Nullable | Description | Available Since | +| ---- | ---- | -------- | ----------- | --------------- | + {% for param in method.request.params %} +| {{ param.name }} | {{ convert_type(param.type) }} | {{ param.nullable }} | {{ param.doc.splitlines()|join("
")|safe }} | {{ param.since }} | + {% endfor %} + {% else %} +Header only request message, no message body exist. + {% endif %} + +#### Response Message +**Message Type:** {{ "0x%02x%02x%02x"|format(service.id, method.id, 1) }} + {% if method.response.params|length > 0 %} + +| Name | Type | Nullable | Description | Available Since | +| ---- | ---- | -------- | ----------- | --------------- | + {% for param in method.response.params %} +| {{ param.name }} | {{ convert_type(param.type) }} | {{ param.nullable }} | {{ param.doc.splitlines()|join("
")|safe }} | {{ param.since }} | + {% endfor %} + {% else %} +Header only response message, no message body exist. + {% endif %} + {% if method.events|length > 0 %} + +#### Event Message + {% for event in method.events %} + +##### {{ event.name }} +**Message Type:** {{ "0x%02x%02x%02x"|format(service.id, method.id, 2 + loop.index) }} + +| Name | Type | Nullable | Description | Available Since | +| ---- | ---- | -------- | ----------- | --------------- | + {% for param in event.params %} +| {{ param.name }} | {{ convert_type(param.type) }} | {{ param.nullable }} | {{ param.doc.splitlines()|join("
")|safe }} | {{ param.since }} | + {% endfor %} + {% endfor %} + {% endif %} + {% endfor %} +{% endfor %} diff --git a/protocol-definitions/ContinuousQuery.yaml b/protocol-definitions/ContinuousQuery.yaml index 5de0e2632..e6815bf6c 100644 --- a/protocol-definitions/ContinuousQuery.yaml +++ b/protocol-definitions/ContinuousQuery.yaml @@ -216,7 +216,7 @@ methods: nullable: false since: 2.0 doc: | - Source that dispathces this batch event. + Source that dispatches this batch event. - name: partitionId type: int nullable: false diff --git a/protocol-definitions/Map.yaml b/protocol-definitions/Map.yaml index d9c6b4489..d3b00b6b8 100644 --- a/protocol-definitions/Map.yaml +++ b/protocol-definitions/Map.yaml @@ -2677,7 +2677,7 @@ methods: Updates TTL (time to live) value of the entry specified by {@code key} with a new TTL value. New TTL value is valid from this operation is invoked, not from the original creation of the entry. If the entry does not exist or already expired, then this call has no effect. -

+ The entry will expire and get evicted after the TTL. If the TTL is 0, then the entry lives forever. If the TTL is negative, then the TTL from the map configuration will be used (default: forever). @@ -2685,7 +2685,7 @@ methods: If there is no entry with key {@code key}, this call has no effect. Warning: -

+ Time resolution for TTL is seconds. The given TTL value is rounded to the next closest second value. request: retryable: false diff --git a/protocol-definitions/PNCounter.yaml b/protocol-definitions/PNCounter.yaml index fc1badd08..526c4ef61 100644 --- a/protocol-definitions/PNCounter.yaml +++ b/protocol-definitions/PNCounter.yaml @@ -6,7 +6,7 @@ methods: since: 2.0 doc: | Query operation to retrieve the current value of the PNCounter. -

+ The invocation will return the replica timestamps (vector clock) which can then be sent with the next invocation to keep session consistency guarantees. @@ -61,7 +61,7 @@ methods: doc: | Adds a delta to the PNCounter value. The delta may be negative for a subtraction. -

+ The invocation will return the replica timestamps (vector clock) which can then be sent with the next invocation to keep session consistency guarantees. diff --git a/protocol-definitions/Ringbuffer.yaml b/protocol-definitions/Ringbuffer.yaml index e00b9a951..ad9664477 100644 --- a/protocol-definitions/Ringbuffer.yaml +++ b/protocol-definitions/Ringbuffer.yaml @@ -6,7 +6,7 @@ methods: since: 2.0 doc: | Returns number of items in the ringbuffer. If no ttl is set, the size will always be equal to capacity after the - head completed the first looparound the ring. This is because no items are getting retired. + head completed the first loop around the ring. This is because no items are getting retired. request: retryable: true partitionIdentifier: name @@ -129,7 +129,7 @@ methods: will return the sequence of the written item. If there is no space, it depends on the overflow policy what happens: OverflowPolicy OVERWRITE we just overwrite the oldest item in the ringbuffer and we violate the ttl OverflowPolicy FAIL we return -1. The reason that FAIL exist is to give the opportunity to obey the ttl. -

+ This sequence will always be unique for this Ringbuffer instance so it can be used as a unique id generator if you are publishing items on this Ringbuffer. However you need to take care of correctly determining an initial id when any node uses the ringbuffer for the first time. The most reliable way to do that is to write a dummy item into the ringbuffer and diff --git a/util.py b/util.py index 4fd79a1a9..569535288 100644 --- a/util.py +++ b/util.py @@ -123,6 +123,14 @@ def generate_custom_codecs(services, template, output_dir, extension): print("[%s] contains missing type mapping so ignoring it." % codec_file_name) +def generate_documentation(services, custom_definitions, template, output_dir, extension): + os.makedirs(output_dir, exist_ok=True) + content = template.render(services=services, custom_definitions=custom_definitions) + file_name = os.path.join(output_dir, 'documentation.' + extension) + with open(file_name, 'w', newline='\n') as file: + file.writelines(content) + + def item_type(lang_name, param_type): if param_type.startswith("List_") or param_type.startswith("ListCN_"): return lang_name(param_type.split('_', 1)[1]) @@ -327,6 +335,7 @@ class SupportedLanguages(Enum): # PY = 'py' # TS = 'ts' # GO = 'go' + MD = 'md' codec_output_directories = { @@ -336,6 +345,7 @@ class SupportedLanguages(Enum): # SupportedLanguages.PY: 'hazelcast/protocol/codec/', # SupportedLanguages.TS: 'src/codec/', # SupportedLanguages.GO: 'internal/proto/' + SupportedLanguages.MD: 'documentation' } custom_codec_output_directories = { @@ -354,28 +364,34 @@ class SupportedLanguages(Enum): # SupportedLanguages.PY: 'py', # SupportedLanguages.TS: 'ts', # SupportedLanguages.GO: 'go' + SupportedLanguages.MD: 'md' } language_specific_funcs = { 'lang_types_encode': { SupportedLanguages.JAVA: java_types_encode, SupportedLanguages.CS: cs_types_encode, + SupportedLanguages.MD: lambda x: x, }, 'lang_types_decode': { SupportedLanguages.JAVA: java_types_decode, SupportedLanguages.CS: cs_types_decode, + SupportedLanguages.MD: lambda x: x, }, 'lang_name': { SupportedLanguages.JAVA: java_name, SupportedLanguages.CS: cs_name, + SupportedLanguages.MD: lambda x: x, }, 'param_name': { SupportedLanguages.JAVA: param_name, SupportedLanguages.CS: param_name, + SupportedLanguages.MD: lambda x: x, }, 'escape_keyword': { SupportedLanguages.JAVA: lambda x: x, SupportedLanguages.CS: cs_escape_keyword, + SupportedLanguages.MD: lambda x: x, }, } @@ -407,10 +423,10 @@ def create_environment(lang, namespace): env.globals["item_type"] = item_type env.globals["key_type"] = key_type env.globals["value_type"] = value_type + env.globals["namespace"] = namespace env.globals["lang_types_encode"] = language_specific_funcs['lang_types_encode'][lang] env.globals["lang_types_decode"] = language_specific_funcs['lang_types_decode'][lang] env.globals["lang_name"] = language_specific_funcs['lang_name'][lang] - env.globals["namespace"] = namespace env.globals["param_name"] = language_specific_funcs['param_name'][lang] env.globals["escape_keyword"] = language_specific_funcs['escape_keyword'][lang]