Skip to content

Commit

Permalink
template cleanup, variable name escaping, service filter (#296)
Browse files Browse the repository at this point in the history
  • Loading branch information
@asimarslan
asimarslan authored Jan 20, 2020
1 parent 014c929 commit 81b91c3
Show file tree
Hide file tree
Showing 5 changed files with 77 additions and 109 deletions.
69 changes: 46 additions & 23 deletions cs/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
"unchecked", "unsafe", "ushort", "using", "value", "var", "virtual", "void", "volatile", "where", "while",
"yield"]

cs_ignore_service_list = [7, 8, 9, 10, 11, 12, 19, 20, 22, 24, 25, 26, 27, 28, 30, 31, 32]


def cs_types_encode(key):
try:
Expand Down Expand Up @@ -78,6 +80,7 @@ def cs_escape_keyword(value):
"AttributeConfig": "NA",
"ListenerConfigHolder": "NA",
"CacheSimpleEntryListenerConfig": "NA",
"ClientBwListEntry": "NA",

"Map_String_String": "IDictionary<string, string>"
}
Expand All @@ -87,58 +90,78 @@ def cs_escape_keyword(value):
"QueryCacheEventData": "NA",
"ScheduledTaskHandler": "NA",
"Xid": "NA",
"Member": "Hazelcast.Core.Member",

"List_Long": "IEnumerable<long>",
"List_UUID": "IEnumerable<Guid>",
"List_String": "IEnumerable<string>",
"ClientBwListEntry": "NA",
"MemberInfo": "Hazelcast.Core.MemberInfo",
"MemberVersion": "Hazelcast.Core.MemberVersion",
"MCEvent": "NA",
"AnchorDataListHolder": "NA",
"PagingPredicateHolder": "NA",

"List_Long": "ICollection<long>",
"List_Integer": "ICollection<int>",
"List_UUID": "ICollection<Guid>",
"List_String": "ICollection<string>",
"List_Xid": "NA",
"List_Data": "IEnumerable<IData>",
"ListCN_Data": "IEnumerable<IData>",
"List_Member": "IEnumerable<Hazelcast.Core.Member>",
"List_Data": "ICollection<IData>",
"ListCN_Data": "ICollection<IData>",
"List_MemberInfo": "ICollection<Hazelcast.Core.MemberInfo>",
"List_ScheduledTaskHandler": "NA",
"List_CacheEventData": "NA",
"List_QueryCacheConfigHolder": "NA",
"List_DistributedObjectInfo": "IEnumerable<Hazelcast.Client.DistributedObjectInfo>",
"List_DistributedObjectInfo": "ICollection<Hazelcast.Client.DistributedObjectInfo>",
"List_QueryCacheEventData": "NA",
"List_IndexConfig": "IEnumerable<Hazelcast.Config.IndexConfig>",
"List_IndexConfig": "ICollection<Hazelcast.Config.IndexConfig>",
"List_AttributeConfig": "NA",
"List_ListenerConfigHolder": "NA",
"List_CacheSimpleEntryListenerConfig": "NA",
"List_StackTraceElement": "IEnumerable<Hazelcast.Util.StackTraceElement>",

"EntryList_String_String": "IEnumerable<KeyValuePair<string, string>>",
"EntryList_String_byteArray": "IEnumerable<KeyValuePair<string, byte[]>>",
"EntryList_Long_byteArray": "IEnumerable<KeyValuePair<long, byte[]>>",
"EntryList_Integer_UUID": "IEnumerable<KeyValuePair<int, Guid>>",
"EntryList_Integer_Long": "IEnumerable<KeyValuePair<int, long>>",
"EntryList_Integer_Integer": "IEnumerable<KeyValuePair<int, int>>",
"EntryList_UUID_Long": "IEnumerable<KeyValuePair<Guid, long>>",
"EntryList_String_EntryList_Integer_Long": "IEnumerable<KeyValuePair<string, IEnumerable<KeyValuePair<int, long>>>>",
"EntryList_UUID_List_Integer": "IEnumerable<KeyValuePair<Guid, IEnumerable<int>>>",
"EntryList_Data_Data": "IEnumerable<KeyValuePair<IData, IData>>",
"List_StackTraceElement": "ICollection<Hazelcast.Util.StackTraceElement>",
"List_ClientBwListEntry": "NA",
"List_MCEvent": "NA",

"EntryList_String_String": "ICollection<KeyValuePair<string, string>>",
"EntryList_String_byteArray": "ICollection<KeyValuePair<string, byte[]>>",
"EntryList_Long_byteArray": "ICollection<KeyValuePair<long, byte[]>>",
"EntryList_Integer_UUID": "ICollection<KeyValuePair<int, Guid>>",
"EntryList_Integer_Long": "ICollection<KeyValuePair<int, long>>",
"EntryList_Integer_Integer": "ICollection<KeyValuePair<int, int>>",
"EntryList_UUID_Long": "ICollection<KeyValuePair<Guid, long>>",
"EntryList_String_EntryList_Integer_Long": "ICollection<KeyValuePair<string, ICollection<KeyValuePair<int, long>>>>",
"EntryList_UUID_List_Integer": "ICollection<KeyValuePair<Guid, IList<int>>>",
"EntryList_Data_Data": "ICollection<KeyValuePair<IData, IData>>",
}

_cs_types_decode = {
"CacheEventData": "NA",
"QueryCacheEventData": "NA",
"ScheduledTaskHandler": "NA",
"Xid": "NA",
"ClientBwListEntry": "NA",
"MemberInfo": "Hazelcast.Core.MemberInfo",
"MemberVersion": "Hazelcast.Core.MemberVersion",
"MCEvent": "NA",
"AnchorDataListHolder": "NA",
"PagingPredicateHolder": "NA",

"List_Long": "IList<long>",
"List_Integer": "IList<int>",
"List_UUID": "IList<Guid>",
"List_Xid": "NA",
"List_String": "IList<string>",
"List_Data": "IList<IData>",
"ListCN_Data": "IList<IData>",
"List_MemberInfo": "IList<Hazelcast.Core.MemberInfo>",
"List_CacheEventData": "NA",
"List_QueryCacheConfigHolder": "NA",
"List_DistributedObjectInfo": "IList<Hazelcast.Client.DistributedObjectInfo>",
"List_DistributedObjectInfo": "NA",
"List_QueryCacheEventData": "NA",
"List_IndexConfig": "IList<Hazelcast.Config.IndexConfig>",
"List_AttributeConfig": "NA",
"List_ListenerConfigHolder": "NA",
"List_CacheSimpleEntryListenerConfig": "NA",
"List_StackTraceElement": "IList<Hazelcast.Util.StackTraceElement>",
"List_ClientBwListEntry": "NA",
"List_MCEvent": "NA",
"List_ScheduledTaskHandler": "NA",

"EntryList_String_String": "IList<KeyValuePair<string, string>>",
"EntryList_String_byteArray": "IList<KeyValuePair<string, byte[]>>",
Expand Down
87 changes: 10 additions & 77 deletions cs/codec-template.cs.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,31 +110,17 @@ namespace Hazelcast.Client.Protocol.Codec
private const int Event{{event.name|capital}}MessageType = {{ event.id }};
{% endfor %}

{#REQUEST PARAMETERS#}
public class RequestParameters
{
{% for param in method.request.params %}

/// <summary>
{% for line in param.doc.splitlines() %}
/// {{ line }}
{% endfor %}
///</summary>
public {{ lang_types_decode(param.type) }} {{ param.name|capital }};
{% endfor %}
}

{#REQUEST PARAMETERS# is not required of client side}
{#RequestENCODE#}
public static ClientMessage EncodeRequest({% for param in method.request.params %}{{ lang_types_encode(param.type) }} {{param.name}}{% if not loop.last %}, {% endif %}{% endfor %})
public static ClientMessage EncodeRequest({% for param in method.request.params %}{{ lang_types_encode(param.type) }} {{escape_keyword(param.name)}}{% if not loop.last %}, {% endif %}{% endfor %})
{
var clientMessage = CreateForEncode();
clientMessage.IsRetryable = {{ method.request.retryable|lower }};
clientMessage.AcquiresResource = {{ method.request.acquiresResource|lower }};
clientMessage.OperationName = "{{ service_name|capital }}.{{ method.name|capital }}";
var initialFrame = new Frame(new byte[RequestInitialFrameSize], UnfragmentedMessage);
EncodeInt(initialFrame.Content, TypeFieldOffset, RequestMessageType);
{% for param in fixed_params(method.request.params) %}
Encode{{ lang_types_decode(param.type)|capital }}(initialFrame.Content, Request{{param.name|capital}}FieldOffset, {{ param.name }});
Encode{{ lang_types_decode(param.type)|capital }}(initialFrame.Content, Request{{param.name|capital}}FieldOffset, {{escape_keyword(param.name)}});
{% endfor %}
clientMessage.Add(initialFrame);
{% for param in var_size_params(method.request.params) %}
Expand All @@ -143,26 +129,7 @@ namespace Hazelcast.Client.Protocol.Codec
return clientMessage;
}

{#RequestDECODE#}
public static RequestParameters DecodeRequest(ClientMessage clientMessage)
{
var iterator = clientMessage.GetIterator();
var request = new RequestParameters();
{% if fixed_params(method.request.params)|length != 0 %}
var initialFrame = iterator.Next();
{% else %}
//empty initial frame
iterator.Next();
{% endif %}
{% for param in fixed_params(method.request.params) %}
request.{{ param.name|capital }} = Decode{{ lang_types_decode(param.type)|capital }}(initialFrame.Content, Request{{param.name|capital}}FieldOffset);
{% endfor %}
{% for param in var_size_params(method.request.params) %}
request.{{ param.name|capital }} = {{ decode_var_sized(param) }};
{% endfor %}
return request;
}

{#RequestDECODE# is not required of client side}
{#RESPONSE PARAMETERS#}
public class ResponseParameters
{
Expand All @@ -177,23 +144,7 @@ namespace Hazelcast.Client.Protocol.Codec
{% endfor %}
}

{#RESPONSE ENCODE#}
public static ClientMessage EncodeResponse({% for param in method.response.params %}{{ lang_types_encode(param.type) }} {{escape_keyword(param.name)}}{% if not loop.last %}, {% endif %}{% endfor %})
{
var clientMessage = CreateForEncode();
var initialFrame = new Frame(new byte[ResponseInitialFrameSize], UnfragmentedMessage);
EncodeInt(initialFrame.Content, TypeFieldOffset, ResponseMessageType);
clientMessage.Add(initialFrame);

{% for param in fixed_params(method.response.params) %}
Encode{{ lang_types_decode(param.type)|capital }}(initialFrame.Content, Response{{param.name|capital}}FieldOffset, {{ escape_keyword(param.name) }});
{% endfor %}
{% for param in var_size_params(method.response.params) %}
{{ encode_var_sized(param) }};
{% endfor %}
return clientMessage;
}

{#RESPONSE ENCODE# is not required of client side}
{#RESPONSE DECODE#}
public static ResponseParameters DecodeResponse(ClientMessage clientMessage)
{
Expand All @@ -213,27 +164,9 @@ namespace Hazelcast.Client.Protocol.Codec
{% endfor %}
return response;
}

{# EVENTS#}
{% if method.events|length != 0 %}
{% for event in method.events%}

public static ClientMessage Encode{{ event.name|capital }}Event({% for param in event.params %}{{ lang_types_encode(param.type) }} {{param.name}}{% if not loop.last %}, {% endif %}{% endfor %})
{
var clientMessage = CreateForEncode();
var initialFrame = new Frame(new byte[Event{{ event.name|capital}}InitialFrameSize], UnfragmentedMessage);
initialFrame.Flags |= IsEventFlag;
EncodeInt(initialFrame.Content, TypeFieldOffset, Event{{ (event.name|capital)}}MessageType);
{% for param in fixed_params(event.params) %}
Encode{{ lang_types_decode(param.type)|capital }}(initialFrame.Content, Event{{ (event.name|capital)}}{{(param.name|capital)}}FieldOffset, {{ param.name }});
{% endfor %}
clientMessage.Add(initialFrame);
{% for param in var_size_params(event.params) %}
{{ encode_var_sized(param) }};
{% endfor %}
return clientMessage;
}
{% endfor %}

public static class EventHandler
{
public static void HandleEvent(ClientMessage clientMessage{% for event in method.events%}, Handle{{ event.name|capital }}Event handle{{ event.name|capital }}Event{% endfor %})
Expand All @@ -249,19 +182,19 @@ namespace Hazelcast.Client.Protocol.Codec
iterator.Next();
{% endif %}
{% for param in fixed_params(event.params) %}
{{ lang_types_decode(param.type) }} {{param.name}} = Decode{{ lang_types_decode(param.type)|capital }}(initialFrame.Content, Event{{(event.name|capital)}}{{param.name|capital}}FieldOffset);
{{ lang_types_decode(param.type) }} {{escape_keyword(param.name)}} = Decode{{ lang_types_decode(param.type)|capital }}(initialFrame.Content, Event{{(event.name|capital)}}{{param.name|capital}}FieldOffset);
{% endfor %}
{% for param in var_size_params(event.params) %}
{{ lang_types_decode(param.type) }} {{param.name}} = {{ decode_var_sized(param) }};
{{ lang_types_decode(param.type) }} {{escape_keyword(param.name)}} = {{ decode_var_sized(param) }};
{% endfor %}
handle{{ event.name|capital }}Event({% for param in event.params %}{{param.name}}{% if not loop.last %}, {% endif %}{% endfor %});
handle{{ event.name|capital }}Event({% for param in event.params %}{{escape_keyword(param.name)}}{% if not loop.last %}, {% endif %}{% endfor %});
return;
}
{% endfor %}
Logger.GetLogger(typeof(EventHandler)).Finest("Unknown message type received on event handler :" + messageType);
}
{% for event in method.events%}
public delegate void Handle{{ event.name|capital }}Event({% for param in event.params %}{{ lang_types_encode(param.type) }} {{param.name}}{% if not loop.last %}, {% endif %}{% endfor %});
public delegate void Handle{{ event.name|capital }}Event({% for param in event.params %}{{ lang_types_encode(param.type) }} {{escape_keyword(param.name)}}{% if not loop.last %}, {% endif %}{% endfor %});
{% endfor %}
}
{% endif %}
Expand Down
10 changes: 5 additions & 5 deletions cs/custom-codec-template.cs.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ namespace Hazelcast.Client.Protocol.Codec.Custom

var initialFrame = new Frame(new byte[InitialFrameSize]);
{% endif %}
Encode{{ lang_types_decode(param.type)|capital }}(initialFrame.Content, {{ param.name|capital}}FieldOffset, {{ param_name(codec.name)}}.{% if param.type == 'boolean' %}Is{% else %}{% endif %}{{ param.name|capital }});
Encode{{ lang_types_decode(param.type)|capital }}(initialFrame.Content, {{ param.name|capital}}FieldOffset, {% if param.type == 'enum' %}(int) {% endif %}{{ param_name(codec.name)}}.{% if param.type == 'boolean' %}Is{% else %}{% endif %}{{ param.name|capital }});
{% if loop.last %}
clientMessage.Add(initialFrame);
{% endif %}
Expand All @@ -106,21 +106,21 @@ namespace Hazelcast.Client.Protocol.Codec.Custom

var initialFrame = iterator.Next();
{% endif %}
var {{ param.name }} = Decode{{ lang_types_decode(param.type)|capital }}(initialFrame.Content, {{ param.name|capital }}FieldOffset);
var {{ escape_keyword(param.name) }} = Decode{{ lang_types_decode(param.type)|capital }}(initialFrame.Content, {{ param.name|capital }}FieldOffset);
{% endfor %}
{% for param in var_size_params(codec.params) %}
{% if loop.first %}

{% endif %}
var {{ param.name }} = {{ decode_var_sized(param) }};
var {{ escape_keyword(param.name) }} = {{ decode_var_sized(param) }};
{% endfor %}

CodecUtil.FastForwardToEndFrame(iterator);

{% if codec.returnWithFactory %}
return CustomTypeFactory.Create{{ codec.name }}({% for param in codec.params %}{{ param.name }}{% if not loop.last %}, {% endif %}{% endfor %});
return CustomTypeFactory.Create{{ codec.name }}({% for param in codec.params %}{{ escape_keyword(param.name) }}{% if not loop.last %}, {% endif %}{% endfor %});
{% else %}
return new {{ lang_types_decode(codec.name) }}({% for param in codec.params %}{{ param.name }}{% if not loop.last %}, {% endif %}{% endfor %});
return new {{ lang_types_decode(codec.name) }}({% for param in codec.params %}{{ escape_keyword(param.name) }}{% if not loop.last %}, {% endif %}{% endfor %});
{% endif %}
}
}
Expand Down
2 changes: 1 addition & 1 deletion generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@

codec_template = env.get_template("codec-template.%s.j2" % lang_str_arg)

generate_codecs(protocol_defs, codec_template, codec_output_dir, file_extensions[lang])
generate_codecs(protocol_defs, codec_template, codec_output_dir, lang)
print('Generated codecs are at \'%s\'' % os.path.abspath(codec_output_dir))

if os.path.exists(custom_protocol_defs_path):
Expand Down
18 changes: 15 additions & 3 deletions util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

from binary import FixedLengthTypes, FixedListTypes, FixedEntryListTypes, FixedMapTypes
from java import java_types_encode, java_types_decode
from cs import cs_types_encode, cs_types_decode, cs_escape_keyword
from cs import cs_types_encode, cs_types_decode, cs_escape_keyword, cs_ignore_service_list


def java_name(type_name):
Expand Down Expand Up @@ -48,10 +48,13 @@ def var_size_params(params):
return [p for p in params if not is_fixed_type(p)]


def generate_codecs(services, template, output_dir, extension):
def generate_codecs(services, template, output_dir, lang):
os.makedirs(output_dir, exist_ok=True)
id_fmt = "0x%02x%02x%02x"
for service in services:
if service["id"] in language_service_ignore_list[lang]:
print("[%s] is in ignore list so ignoring it." % service["name"])
continue
if "methods" in service:
methods = service["methods"]
if methods is None:
Expand All @@ -64,7 +67,7 @@ def generate_codecs(services, template, output_dir, extension):
for i in range(len(events)):
method["events"][i]["id"] = int(id_fmt % (service["id"], method["id"], i + 2), 16)

codec_file_name = capital(service["name"]) + capital(method["name"]) + 'Codec.' + extension
codec_file_name = capital(service["name"]) + capital(method["name"]) + 'Codec.' + file_extensions[lang]
try:
content = template.render(service_name=service["name"], method=method)
save_file(os.path.join(output_dir, codec_file_name), content)
Expand Down Expand Up @@ -245,6 +248,15 @@ class SupportedLanguages(Enum):
},
}

language_service_ignore_list = {
SupportedLanguages.JAVA: [],
# SupportedLanguages.CPP: [],
SupportedLanguages.CS: cs_ignore_service_list,
# SupportedLanguages.PY: [],
# SupportedLanguages.TS: [],
# SupportedLanguages.GO: []
}


def create_environment(lang, namespace):
env = Environment(loader=PackageLoader(lang.value, '.'))
Expand Down

0 comments on commit 81b91c3

Please sign in to comment.