From a57131d33b27550e4f6191337cb916a13c3b0562 Mon Sep 17 00:00:00 2001 From: deng Date: Fri, 23 Apr 2021 19:56:35 +0800 Subject: [PATCH 01/21] docs: add http api rfc --- fedb/api-server.md | 59 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 fedb/api-server.md diff --git a/fedb/api-server.md b/fedb/api-server.md new file mode 100644 index 0000000..b5ae97d --- /dev/null +++ b/fedb/api-server.md @@ -0,0 +1,59 @@ +- Start Date: 2021-04-23 +- Target Major Version: (2.2.0) +- Reference Issues: +- Implementation PR: + +# Add API Server + +## Summary + +HTTP interface is one of the most friendly interface and many developer like using http interface to develop their program. HTTP API also help us to show demo simply. So we want to support uing http api to access FEDB. + +## Detailed design + +### Interface design +HTTP URL: http://ip:port/sql + +HTTP Method: POST + +#### Put +``` +{ + "method": "put", + "db": "db_name", + "table": "table_name", + "value": { + "field1": "value1", + "field2": 111, + "field3": 1.4 + } + +} +``` + +#### Execute procedure +``` +{ + "method": "execute_procedure", + "db": "db_name", + "table": "table_name", + "value": { + "name": "procedure_name" + } + +} +``` +#### Execute SQL + +``` +{ + "method": "execute_sql", + "db": "db_name", + "value": { + "sql": "select * from t1" + } + +} +``` + +### Server design \ No newline at end of file From 49c79eea4610355f2ea8e0899ff3a847ee6a6a48 Mon Sep 17 00:00:00 2001 From: deng Date: Fri, 23 Apr 2021 20:19:22 +0800 Subject: [PATCH 02/21] docs: add server design --- fedb/api-server.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fedb/api-server.md b/fedb/api-server.md index b5ae97d..c2eb1b0 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -56,4 +56,6 @@ HTTP Method: POST } ``` -### Server design \ No newline at end of file +### Server design + +We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/docs/en/http_service.md) framework to tackle the http request. So what need to do is forwarding the reques to FEDB server with fedb sdk in brpc http service implementation. From 710b4bb150ffb1be85b9c0e43ff6f7da05dbc051 Mon Sep 17 00:00:00 2001 From: deng Date: Sun, 25 Apr 2021 20:24:21 +0800 Subject: [PATCH 03/21] docs: add json lib --- fedb/api-server.md | 114 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 105 insertions(+), 9 deletions(-) diff --git a/fedb/api-server.md b/fedb/api-server.md index c2eb1b0..edc46e3 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -5,18 +5,20 @@ # Add API Server -## Summary +# Summary HTTP interface is one of the most friendly interface and many developer like using http interface to develop their program. HTTP API also help us to show demo simply. So we want to support uing http api to access FEDB. -## Detailed design +# Detailed design -### Interface design + +## Interface design HTTP URL: http://ip:port/sql HTTP Method: POST -#### Put +### Put +request ``` { "method": "put", @@ -30,32 +32,126 @@ HTTP Method: POST } ``` +response +``` +{ + "code": 0, + "msg": "ok" +} +``` -#### Execute procedure +### Execute procedure +reqeust ``` { "method": "execute_procedure", "db": "db_name", "table": "table_name", "value": { - "name": "procedure_name" + "name": "procedure_name", + "input": { + "id":123, + "field1": "value1" + } } } ``` -#### Execute SQL +response +``` +{ + "code": 0, + "msg": "ok", + "data": [{"field1" : "value1"}, {"field1": "value2"}] +} +``` +### Execute SQL +request ``` { "method": "execute_sql", "db": "db_name", - "value": { + "data": { "sql": "select * from t1" + "input": { + "id":123, + "field1": "value1" + } } } ``` +response +``` +{ + "code": 0, + "msg": "ok", + "data": [{"field1" : "value1"}, {"field1": "value2"}] +} +``` -### Server design +## Server design We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/docs/en/http_service.md) framework to tackle the http request. So what need to do is forwarding the reques to FEDB server with fedb sdk in brpc http service implementation. + +1. Design proto file. + ``` + option cc_generic_services = true; + + message HttpRequest { }; + message HttpResponse { }; + + service APIService { + rpc ForwardSQL(HttpRequest) returns (HttpResponse); + } + ``` +2. Implement the service + ``` + class APIServiceImpl : public APIService { + public: + bool Init() { + // init the sdk + ... + } + virtual void ForwardSQL(google::protobuf::RpcController* controller, + const HttpRequest* request, + HttpResponse* response, + google::protobuf::Closure* done) { + // parse request attachment + ... + } + }; + ``` +3. Dispatch and forward request + + ``` + if (method == "put") { + // sdk put + } else if (method == "execute_procedure") { + // sdk execute procedure + } else if (method == "execute_sql") { + // sdk execute sql + } eles { + LOG(ERROR) << "unsupport method"; + } + ``` + +### JSON lib +We use [RapidJSON](https://github.com/Tencent/rapidjson) as json parser lib for the following reasons: +- RapidJSON is fast. +- RapidJSON is self-contained and header-only. It does not depend on external libraries such as BOOST. + +Benchmark details can be read [here](https://rawgit.com/miloyip/nativejson-benchmark/master/sample/performance_Corei7-4980HQ@2.80GHz_mac64_clang7.0.html#1.%20Parse) + +# Alternative +Set parameters in url, ex [postgrest](https://github.com/PostgREST/postgrest). +``` +GET /people?age=gte.18&student=is.true HTTP/1.1 +``` +For complete document, refer to [here](https://postgrest.org/en/stable/api.html#horizontal-filtering-rows). + +This method make API Interfaces more complicated. + +# Adoption stratege +If we implement this proposal, there is no impact on existing interfaces. \ No newline at end of file From 5b839ff55483b3f91a947b0ebc244425d7aa5b52 Mon Sep 17 00:00:00 2001 From: deng Date: Mon, 26 Apr 2021 19:29:12 +0800 Subject: [PATCH 04/21] docs: use muti methods in one service --- fedb/api-server.md | 58 +++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/fedb/api-server.md b/fedb/api-server.md index edc46e3..0973eda 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -7,23 +7,21 @@ # Summary -HTTP interface is one of the most friendly interface and many developer like using http interface to develop their program. HTTP API also help us to show demo simply. So we want to support uing http api to access FEDB. +HTTP interface is one of the most friendly interface and many developers like using http interface to develop their program. REST API also help us to show demo simply. So we want to support uing http api to access FEDB. # Detailed design ## Interface design -HTTP URL: http://ip:port/sql +HTTP URL: http://ip:port/sql/{method_name}?db={db_name}&table={table_name} HTTP Method: POST ### Put -request +reqeust url: http://ip:port/sql/put?db={db_name}&table={table_name} +request body: ``` { - "method": "put", - "db": "db_name", - "table": "table_name", "value": { "field1": "value1", "field2": 111, @@ -41,12 +39,10 @@ response ``` ### Execute procedure -reqeust +reqeust url: http://ip:port/sql/execute_procedure?db={db_name}&table={table_name} +request body: ``` { - "method": "execute_procedure", - "db": "db_name", - "table": "table_name", "value": { "name": "procedure_name", "input": { @@ -66,12 +62,11 @@ response } ``` ### Execute SQL -request +reqeust url: http://ip:port/sql/execute_sql?db={db_name}&table={table_name} +request body: ``` { - "method": "execute_sql", - "db": "db_name", "data": { "sql": "select * from t1" "input": { @@ -103,7 +98,9 @@ We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/ message HttpResponse { }; service APIService { - rpc ForwardSQL(HttpRequest) returns (HttpResponse); + rpc Put(HttpRequest) returns (HttpResponse); + rpc ExeSQL(HttpRequest) returns (HttpResponse); + rpc ExeProcedure(HttpRequest) returns (HttpResponse); } ``` 2. Implement the service @@ -114,7 +111,21 @@ We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/ // init the sdk ... } - virtual void ForwardSQL(google::protobuf::RpcController* controller, + virtual void Put(google::protobuf::RpcController* controller, + const HttpRequest* request, + HttpResponse* response, + google::protobuf::Closure* done) { + // parse request attachment + ... + } + virtual void ExeSQL(google::protobuf::RpcController* controller, + const HttpRequest* request, + HttpResponse* response, + google::protobuf::Closure* done) { + // parse request attachment + ... + } + virtual void ExeProcedure(google::protobuf::RpcController* controller, const HttpRequest* request, HttpResponse* response, google::protobuf::Closure* done) { @@ -123,17 +134,16 @@ We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/ } }; ``` -3. Dispatch and forward request +3. Add mapping ``` - if (method == "put") { - // sdk put - } else if (method == "execute_procedure") { - // sdk execute procedure - } else if (method == "execute_sql") { - // sdk execute sql - } eles { - LOG(ERROR) << "unsupport method"; + if (server.AddService(&api_svc, + brpc::SERVER_DOESNT_OWN_SERVICE, + "/sql/put => Put," + "/sql/execute_sql => ExeSQL," + "/sql/execute_procedure => ExeProcedure") != 0) { + LOG(ERROR) << "Fail to add api service"; + return -1; } ``` From 6632214099e3e9848eedeb48db42bcab97f42cab Mon Sep 17 00:00:00 2001 From: deng Date: Wed, 28 Apr 2021 19:16:20 +0800 Subject: [PATCH 05/21] docs: add get procedure --- fedb/api-server.md | 69 +++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 40 deletions(-) diff --git a/fedb/api-server.md b/fedb/api-server.md index 0973eda..25bcb88 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -13,20 +13,20 @@ HTTP interface is one of the most friendly interface and many developers like us ## Interface design -HTTP URL: http://ip:port/sql/{method_name}?db={db_name}&table={table_name} - -HTTP Method: POST ### Put -reqeust url: http://ip:port/sql/put?db={db_name}&table={table_name} +reqeust url: http://ip:port/sql/put/{db_name}/{table_name} request body: ``` { - "value": { - "field1": "value1", - "field2": 111, - "field3": 1.4 - } + "value": [{ + "field1": "value1", //string + "field2": 111, // int + "field3": 1.4, // float/double + "field4": "2021-04-27" //date + "field5": "2021-04-27T08:03:15+00:00" //timestamp + "field6": true // bool + }] } ``` @@ -39,18 +39,13 @@ response ``` ### Execute procedure -reqeust url: http://ip:port/sql/execute_procedure?db={db_name}&table={table_name} +reqeust url: http://ip:port/sql/execute_procedure/{db_name}/{procedure_name} request body: ``` { - "value": { - "name": "procedure_name", - "input": { - "id":123, - "field1": "value1" - } - } - + "common_cols":["value1", "value2"], + "input": [["value1", "value2"],["value1", "value2"]], + "need_schema": true } ``` response @@ -58,34 +53,28 @@ response { "code": 0, "msg": "ok", - "data": [{"field1" : "value1"}, {"field1": "value2"}] + "schema": [{"field1":"bool"}, {"field2":"int32"}], + "data": [["value1", "value2"], [...]] } ``` -### Execute SQL -reqeust url: http://ip:port/sql/execute_sql?db={db_name}&table={table_name} -request body: - -``` -{ - "data": { - "sql": "select * from t1" - "input": { - "id":123, - "field1": "value1" - } - } -} -``` +### Get Procedure +request url: http://ip:port/sql/get_procedure/{db_name}/{procedure_name} response ``` { "code": 0, "msg": "ok", - "data": [{"field1" : "value1"}, {"field1": "value2"}] + "name": "procedure_name", + "procedure": "xxxxx", + "common_col": ["field1", "field2"], + "input_schema": [{"field1":"bool"}, {"field2":"int"}], + "output_schema": [{"field1": "bool"}, {"field2": "int"}], + "tables": ["table1", "table2"] } ``` + ## Server design We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/docs/en/http_service.md) framework to tackle the http request. So what need to do is forwarding the reques to FEDB server with fedb sdk in brpc http service implementation. @@ -99,8 +88,8 @@ We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/ service APIService { rpc Put(HttpRequest) returns (HttpResponse); - rpc ExeSQL(HttpRequest) returns (HttpResponse); rpc ExeProcedure(HttpRequest) returns (HttpResponse); + rpc GetProcedure(HttpRequest) returns (HttpResponse); } ``` 2. Implement the service @@ -118,14 +107,14 @@ We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/ // parse request attachment ... } - virtual void ExeSQL(google::protobuf::RpcController* controller, + virtual void ExeProcedure(google::protobuf::RpcController* controller, const HttpRequest* request, HttpResponse* response, google::protobuf::Closure* done) { // parse request attachment ... } - virtual void ExeProcedure(google::protobuf::RpcController* controller, + virtual void GetProcedure(google::protobuf::RpcController* controller, const HttpRequest* request, HttpResponse* response, google::protobuf::Closure* done) { @@ -140,8 +129,8 @@ We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/ if (server.AddService(&api_svc, brpc::SERVER_DOESNT_OWN_SERVICE, "/sql/put => Put," - "/sql/execute_sql => ExeSQL," - "/sql/execute_procedure => ExeProcedure") != 0) { + "/sql/execute_procedure => ExeProcedure," + "/sql/get_procedure => GetProcedure") != 0) { LOG(ERROR) << "Fail to add api service"; return -1; } From f778265d461b692aabcb56f6b27eff5c74e4b6ff Mon Sep 17 00:00:00 2001 From: deng Date: Thu, 6 May 2021 20:06:16 +0800 Subject: [PATCH 06/21] docs: udpate restful api --- fedb/api-server.md | 124 +++++++++++++++++++++++++++++++-------------- 1 file changed, 86 insertions(+), 38 deletions(-) diff --git a/fedb/api-server.md b/fedb/api-server.md index 25bcb88..832e0eb 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -9,21 +9,43 @@ HTTP interface is one of the most friendly interface and many developers like using http interface to develop their program. REST API also help us to show demo simply. So we want to support uing http api to access FEDB. -# Detailed design - - -## Interface design +# Detailed Design + +## Interface Design + +### Schema Mapping +|FEDB type|Json type| +|---|---| +|string|String| +|smallint|Number| +|int|Number| +|bigint|Number| +|float|Number| +|double|Number| +|bool|Boolean| +|null|null| +|date|String| +|timestamp|String| + +JSON does not have a built-in type for date/timestamp values. We store the date/timestamp value as a string in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) format, e.g 2021-05-06, 2021-05-06T00:02:32Z. + +### HTTP Status +|code|message| +|--|--| +|200|ok| +|404|page not found| ### Put -reqeust url: http://ip:port/sql/put/{db_name}/{table_name} +reqeust url: http://ip:port/db/{db_name}/table/{table_name} +http method: PUT request body: ``` { "value": [{ - "field1": "value1", //string + "field1": "value1", // string "field2": 111, // int "field3": 1.4, // float/double - "field4": "2021-04-27" //date + "field4": "2021-04-27" // date "field5": "2021-04-27T08:03:15+00:00" //timestamp "field6": true // bool }] @@ -38,8 +60,14 @@ response } ``` -### Execute procedure -reqeust url: http://ip:port/sql/execute_procedure/{db_name}/{procedure_name} +|code|message| +|--|--| +|0|ok| +|1|get insert information failed| + +### Execute Procedure +reqeust url: http://ip:port/db/{db_name}/procedure/{procedure_name} +http method: POST request body: ``` { @@ -53,27 +81,39 @@ response { "code": 0, "msg": "ok", - "schema": [{"field1":"bool"}, {"field2":"int32"}], - "data": [["value1", "value2"], [...]] + "data": { + "schema": [{"field1":"bool"}, {"field2":"int32"}], + "data": [["value1", "value2"], [...]] + } } ``` +|code|message| +|--|--| +|0|ok| +|-1|execute procedure failed| ### Get Procedure -request url: http://ip:port/sql/get_procedure/{db_name}/{procedure_name} +request url: http://ip:port/db/{db_name}/procedure/{procedure_name} +http method: Get response ``` { "code": 0, "msg": "ok", - "name": "procedure_name", - "procedure": "xxxxx", - "common_col": ["field1", "field2"], - "input_schema": [{"field1":"bool"}, {"field2":"int"}], - "output_schema": [{"field1": "bool"}, {"field2": "int"}], - "tables": ["table1", "table2"] + "data":{ + "name": "procedure_name", + "procedure": "xxxxx", + "common_col": ["field1", "field2"], + "input_schema": [{"name":"field1", "type":"bool"}, {"name":"filed1", "type":"int32"}], + "output_schema": [{"field1": "bool"}, {"field2": "int32"}], + "tables": ["table1", "table2"] + } } ``` - +|code|message| +|--|--| +|0|ok| +|-1|procedure not found| ## Server design @@ -87,9 +127,8 @@ We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/ message HttpResponse { }; service APIService { - rpc Put(HttpRequest) returns (HttpResponse); - rpc ExeProcedure(HttpRequest) returns (HttpResponse); - rpc GetProcedure(HttpRequest) returns (HttpResponse); + rpc ProcessTable(HttpRequest) returns (HttpResponse); + rpc ProcessProcedure(HttpRequest) returns (HttpResponse); } ``` 2. Implement the service @@ -100,26 +139,36 @@ We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/ // init the sdk ... } - virtual void Put(google::protobuf::RpcController* controller, + virtual void ProcessTable(google::protobuf::RpcController* controller, const HttpRequest* request, HttpResponse* response, google::protobuf::Closure* done) { - // parse request attachment - ... + swich (cntl->http_request().method()) { + + case HTTP_METHOD_PUT): + // parse request attachment and put data + ... + break; + default: + // log error + break; + } } - virtual void ExeProcedure(google::protobuf::RpcController* controller, + virtual void ProcessProcedure(google::protobuf::RpcController* controller, const HttpRequest* request, HttpResponse* response, google::protobuf::Closure* done) { - // parse request attachment - ... - } - virtual void GetProcedure(google::protobuf::RpcController* controller, - const HttpRequest* request, - HttpResponse* response, - google::protobuf::Closure* done) { - // parse request attachment - ... + switch (cntl->http_request().method()) { + case brpc::HTTP_METHOD_GET: + // get procedure info + break; + case brpc::HTTP_METHOD_POST: + // execute procedure + break; + default: + // log error + break; + } } }; ``` @@ -128,9 +177,8 @@ We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/ ``` if (server.AddService(&api_svc, brpc::SERVER_DOESNT_OWN_SERVICE, - "/sql/put => Put," - "/sql/execute_procedure => ExeProcedure," - "/sql/get_procedure => GetProcedure") != 0) { + "/db/*/table => ProcessTable," + "/db/*/procedure => ProcessProcedure") != 0) { LOG(ERROR) << "Fail to add api service"; return -1; } From 89b4ab4b81c344cbaaab5db22d077505df64d3fc Mon Sep 17 00:00:00 2001 From: deng Date: Fri, 7 May 2021 11:27:42 +0800 Subject: [PATCH 07/21] docs: add http code --- fedb/api-server.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/fedb/api-server.md b/fedb/api-server.md index 832e0eb..0501397 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -34,25 +34,26 @@ JSON does not have a built-in type for date/timestamp values. We store the date/ |--|--| |200|ok| |404|page not found| +|408|request timeout| ### Put reqeust url: http://ip:port/db/{db_name}/table/{table_name} -http method: PUT +http method: PUT request body: ``` { "value": [{ - "field1": "value1", // string - "field2": 111, // int - "field3": 1.4, // float/double - "field4": "2021-04-27" // date - "field5": "2021-04-27T08:03:15+00:00" //timestamp - "field6": true // bool + "field1": "value1", + "field2": 111, + "field3": 1.4, + "field4": "2021-04-27", + "field5": "2021-04-27T08:03:15+00:00", + "field6": true }] } ``` -response +response: ``` { "code": 0, @@ -76,7 +77,7 @@ request body: "need_schema": true } ``` -response +response: ``` { "code": 0, @@ -94,8 +95,8 @@ response ### Get Procedure request url: http://ip:port/db/{db_name}/procedure/{procedure_name} -http method: Get -response +http method: Get +response: ``` { "code": 0, From 0544e427018095e9cff209e502c507be2367f039 Mon Sep 17 00:00:00 2001 From: deng Date: Fri, 7 May 2021 15:58:22 +0800 Subject: [PATCH 08/21] docs: update rpc mapping --- fedb/api-server.md | 39 ++++++--------------------------------- 1 file changed, 6 insertions(+), 33 deletions(-) diff --git a/fedb/api-server.md b/fedb/api-server.md index 0501397..a239aba 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -73,7 +73,7 @@ request body: ``` { "common_cols":["value1", "value2"], - "input": [["value1", "value2"],["value1", "value2"]], + "input": [["value0", "value3"],["value0", "value3"]], "need_schema": true } ``` @@ -106,7 +106,7 @@ response: "procedure": "xxxxx", "common_col": ["field1", "field2"], "input_schema": [{"name":"field1", "type":"bool"}, {"name":"filed1", "type":"int32"}], - "output_schema": [{"field1": "bool"}, {"field2": "int32"}], + "output_schema": [{"name":"field1", "type":"bool"}, {"name":"field2", "type":"int32"}], "tables": ["table1", "table2"] } } @@ -128,8 +128,7 @@ We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/ message HttpResponse { }; service APIService { - rpc ProcessTable(HttpRequest) returns (HttpResponse); - rpc ProcessProcedure(HttpRequest) returns (HttpResponse); + rpc ProcessSQL(HttpRequest) returns (HttpResponse); } ``` 2. Implement the service @@ -140,36 +139,11 @@ We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/ // init the sdk ... } - virtual void ProcessTable(google::protobuf::RpcController* controller, + virtual void ProcessSQL(google::protobuf::RpcController* controller, const HttpRequest* request, HttpResponse* response, google::protobuf::Closure* done) { - swich (cntl->http_request().method()) { - - case HTTP_METHOD_PUT): - // parse request attachment and put data - ... - break; - default: - // log error - break; - } - } - virtual void ProcessProcedure(google::protobuf::RpcController* controller, - const HttpRequest* request, - HttpResponse* response, - google::protobuf::Closure* done) { - switch (cntl->http_request().method()) { - case brpc::HTTP_METHOD_GET: - // get procedure info - break; - case brpc::HTTP_METHOD_POST: - // execute procedure - break; - default: - // log error - break; - } + // parse url and dispach } }; ``` @@ -178,8 +152,7 @@ We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/ ``` if (server.AddService(&api_svc, brpc::SERVER_DOESNT_OWN_SERVICE, - "/db/*/table => ProcessTable," - "/db/*/procedure => ProcessProcedure") != 0) { + "/db/* => ProcessSQL" != 0) { LOG(ERROR) << "Fail to add api service"; return -1; } From 22a42413aedb3eebc8f822e3f6a018b955cf2da5 Mon Sep 17 00:00:00 2001 From: deng Date: Fri, 7 May 2021 17:36:03 +0800 Subject: [PATCH 09/21] dosc: use plural in url --- fedb/api-server.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fedb/api-server.md b/fedb/api-server.md index a239aba..15212ba 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -37,7 +37,7 @@ JSON does not have a built-in type for date/timestamp values. We store the date/ |408|request timeout| ### Put -reqeust url: http://ip:port/db/{db_name}/table/{table_name} +reqeust url: http://ip:port/dbs/{db_name}/tables/{table_name} http method: PUT request body: ``` @@ -67,7 +67,7 @@ response: |1|get insert information failed| ### Execute Procedure -reqeust url: http://ip:port/db/{db_name}/procedure/{procedure_name} +reqeust url: http://ip:port/dbs/{db_name}/procedures/{procedure_name} http method: POST request body: ``` @@ -94,7 +94,7 @@ response: |-1|execute procedure failed| ### Get Procedure -request url: http://ip:port/db/{db_name}/procedure/{procedure_name} +request url: http://ip:port/dbs/{db_name}/procedures/{procedure_name} http method: Get response: ``` @@ -152,7 +152,7 @@ We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/ ``` if (server.AddService(&api_svc, brpc::SERVER_DOESNT_OWN_SERVICE, - "/db/* => ProcessSQL" != 0) { + "/dbs/* => ProcessSQL" != 0) { LOG(ERROR) << "Fail to add api service"; return -1; } From d2840fd66d4f3afd0daa4bc1507624abaf3bc4f8 Mon Sep 17 00:00:00 2001 From: dl239 Date: Sat, 8 May 2021 13:29:36 +0800 Subject: [PATCH 10/21] feat: add output common cols --- fedb/api-server.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fedb/api-server.md b/fedb/api-server.md index 15212ba..60cc5d4 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -84,7 +84,8 @@ response: "msg": "ok", "data": { "schema": [{"field1":"bool"}, {"field2":"int32"}], - "data": [["value1", "value2"], [...]] + "data": [["value1", "value2"], [...]], + "common_cols": ["value1"] } } ``` @@ -104,9 +105,10 @@ response: "data":{ "name": "procedure_name", "procedure": "xxxxx", - "common_col": ["field1", "field2"], "input_schema": [{"name":"field1", "type":"bool"}, {"name":"filed1", "type":"int32"}], + "input_common_cols": ["field1", "field2"], "output_schema": [{"name":"field1", "type":"bool"}, {"name":"field2", "type":"int32"}], + "output_common_cols":["field"] "tables": ["table1", "table2"] } } From 450a9b2f4e023b606f2d56bb4f7885eedcc77a3c Mon Sep 17 00:00:00 2001 From: dl239 Date: Sat, 8 May 2021 13:32:04 +0800 Subject: [PATCH 11/21] feat: rename common_cols_data --- fedb/api-server.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedb/api-server.md b/fedb/api-server.md index 60cc5d4..2713858 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -85,7 +85,7 @@ response: "data": { "schema": [{"field1":"bool"}, {"field2":"int32"}], "data": [["value1", "value2"], [...]], - "common_cols": ["value1"] + "common_cols_data": ["value1"] } } ``` From a2a36c6999e8b68c9a854435074d3cea9540cd93 Mon Sep 17 00:00:00 2001 From: dl239 Date: Sat, 8 May 2021 19:10:12 +0800 Subject: [PATCH 12/21] docs: update output_common_cols --- fedb/api-server.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fedb/api-server.md b/fedb/api-server.md index 2713858..7dbcc60 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -25,9 +25,9 @@ HTTP interface is one of the most friendly interface and many developers like us |bool|Boolean| |null|null| |date|String| -|timestamp|String| +|timestamp|Number| -JSON does not have a built-in type for date/timestamp values. We store the date/timestamp value as a string in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) format, e.g 2021-05-06, 2021-05-06T00:02:32Z. +JSON does not have a built-in type for date/timestamp values. We store the date value as a String in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) format and store the timestamp as a Number, e.g '2021-05-06', 1620471840256. ### HTTP Status |code|message| @@ -47,7 +47,7 @@ request body: "field2": 111, "field3": 1.4, "field4": "2021-04-27", - "field5": "2021-04-27T08:03:15+00:00", + "field5": 1620471840256, "field6": true }] @@ -83,9 +83,9 @@ response: "code": 0, "msg": "ok", "data": { - "schema": [{"field1":"bool"}, {"field2":"int32"}], + "schema": [{"field1":"bool"}, {"field2":"int"}, {"name":"field3", "type":"bigint"}], "data": [["value1", "value2"], [...]], - "common_cols_data": ["value1"] + "common_cols_data": ["value3"] } } ``` @@ -105,10 +105,10 @@ response: "data":{ "name": "procedure_name", "procedure": "xxxxx", - "input_schema": [{"name":"field1", "type":"bool"}, {"name":"filed1", "type":"int32"}], + "input_schema": [{"name":"field1", "type":"bool"}, {"name":"filed1", "type":"int"}], "input_common_cols": ["field1", "field2"], - "output_schema": [{"name":"field1", "type":"bool"}, {"name":"field2", "type":"int32"}], - "output_common_cols":["field"] + "output_schema": [{"name":"field1", "type":"bool"}, {"name":"field2", "type":"int32"}, {"name":"field3", "type":"bigint"}], + "output_common_cols":["field3"] "tables": ["table1", "table2"] } } From 97f12d7cacbc5ab398dbc91e169a0639447bdecb Mon Sep 17 00:00:00 2001 From: dl239 Date: Mon, 10 May 2021 16:34:45 +0800 Subject: [PATCH 13/21] docs: update http status --- fedb/api-server.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/fedb/api-server.md b/fedb/api-server.md index 7dbcc60..fa09cef 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -30,11 +30,7 @@ HTTP interface is one of the most friendly interface and many developers like us JSON does not have a built-in type for date/timestamp values. We store the date value as a String in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) format and store the timestamp as a Number, e.g '2021-05-06', 1620471840256. ### HTTP Status -|code|message| -|--|--| -|200|ok| -|404|page not found| -|408|request timeout| +API Server will return 200 in most case and store the error in the code of reponse's json. ### Put reqeust url: http://ip:port/dbs/{db_name}/tables/{table_name} From 2731b3a7da22fd5a5ea3e40a5bac8f11f0d3e948 Mon Sep 17 00:00:00 2001 From: dl239 Date: Wed, 12 May 2021 17:30:42 +0800 Subject: [PATCH 14/21] docs: rename function name --- fedb/api-server.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fedb/api-server.md b/fedb/api-server.md index fa09cef..7d41068 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -137,11 +137,11 @@ We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/ // init the sdk ... } - virtual void ProcessSQL(google::protobuf::RpcController* controller, + virtual void Process(google::protobuf::RpcController* controller, const HttpRequest* request, HttpResponse* response, google::protobuf::Closure* done) { - // parse url and dispach + // parse url and dispatch } }; ``` @@ -150,7 +150,7 @@ We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/ ``` if (server.AddService(&api_svc, brpc::SERVER_DOESNT_OWN_SERVICE, - "/dbs/* => ProcessSQL" != 0) { + "/dbs/* => Process" != 0) { LOG(ERROR) << "Fail to add api service"; return -1; } From b9083f4f9612d90a7f670e60b3ada173d2fb3338 Mon Sep 17 00:00:00 2001 From: dl239 Date: Thu, 13 May 2021 11:12:29 +0800 Subject: [PATCH 15/21] docs: use array in put request body --- fedb/api-server.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fedb/api-server.md b/fedb/api-server.md index 7d41068..73c08f7 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -38,14 +38,14 @@ http method: PUT request body: ``` { - "value": [{ - "field1": "value1", - "field2": 111, - "field3": 1.4, - "field4": "2021-04-27", - "field5": 1620471840256, - "field6": true - }] + "value": [[ + "value1", + 111, + 1.4, + "2021-04-27", + 1620471840256, + true + ]] } ``` From 4defb33344a69e2391be4cf8c5d8b46de07119c9 Mon Sep 17 00:00:00 2001 From: dl239 Date: Tue, 25 May 2021 10:57:11 +0800 Subject: [PATCH 16/21] fix: fix sp response schema --- fedb/api-server.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedb/api-server.md b/fedb/api-server.md index 73c08f7..6cebc16 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -79,7 +79,7 @@ response: "code": 0, "msg": "ok", "data": { - "schema": [{"field1":"bool"}, {"field2":"int"}, {"name":"field3", "type":"bigint"}], + "schema": [{"name": "field1", "type": "bool"}, {"name": "field2", "type": "int"}, {"name":"field3", "type": "bigint"}], "data": [["value1", "value2"], [...]], "common_cols_data": ["value3"] } From a28ae21e150624ac29e839505d5022b3338025a3 Mon Sep 17 00:00:00 2001 From: dl239 Date: Tue, 25 May 2021 15:44:36 +0800 Subject: [PATCH 17/21] docs: add note desciption --- fedb/api-server.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fedb/api-server.md b/fedb/api-server.md index 6cebc16..79f6d6f 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -90,6 +90,8 @@ response: |0|ok| |-1|execute procedure failed| +Note: If there is no common col, the reponse json will not contain common_cols_data field + ### Get Procedure request url: http://ip:port/dbs/{db_name}/procedures/{procedure_name} http method: Get @@ -114,6 +116,8 @@ response: |0|ok| |-1|procedure not found| +Note: If there is no common col, the reponse json will not contain input_common_cols/output_common_cols field + ## Server design We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/docs/en/http_service.md) framework to tackle the http request. So what need to do is forwarding the reques to FEDB server with fedb sdk in brpc http service implementation. From 946493543f216fe0346f3774fe2c512f392a6a23 Mon Sep 17 00:00:00 2001 From: dl239 Date: Fri, 28 May 2021 16:52:20 +0800 Subject: [PATCH 18/21] docs: update dispatch --- fedb/api-server.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedb/api-server.md b/fedb/api-server.md index 79f6d6f..9d161b7 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -154,7 +154,7 @@ We use [brpc HTTP Service](https://github.com/apache/incubator-brpc/blob/master/ ``` if (server.AddService(&api_svc, brpc::SERVER_DOESNT_OWN_SERVICE, - "/dbs/* => Process" != 0) { + "/* => Process" != 0) { LOG(ERROR) << "Fail to add api service"; return -1; } From f55e94119e79b61030af6b039642a6aebf6a3ddb Mon Sep 17 00:00:00 2001 From: dl239 Date: Fri, 28 May 2021 16:54:56 +0800 Subject: [PATCH 19/21] docs: update need_schema value --- fedb/api-server.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedb/api-server.md b/fedb/api-server.md index 9d161b7..c12b34b 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -70,7 +70,7 @@ request body: { "common_cols":["value1", "value2"], "input": [["value0", "value3"],["value0", "value3"]], - "need_schema": true + "need_schema": false } ``` response: From 7ef60ae8440dbf370012dde0ce5b01d9ee4d0011 Mon Sep 17 00:00:00 2001 From: dl239 Date: Fri, 28 May 2021 17:03:05 +0800 Subject: [PATCH 20/21] docs: add note --- fedb/api-server.md | 1 + 1 file changed, 1 insertion(+) diff --git a/fedb/api-server.md b/fedb/api-server.md index c12b34b..9b1119e 100644 --- a/fedb/api-server.md +++ b/fedb/api-server.md @@ -61,6 +61,7 @@ response: |--|--| |0|ok| |1|get insert information failed| +Note: Put multi records to ApiServer is not supported. If put multi records in value array, the error will be return. ### Execute Procedure reqeust url: http://ip:port/dbs/{db_name}/procedures/{procedure_name} From d5403861bcf5dbd124dd2f18671c1b5794dab59d Mon Sep 17 00:00:00 2001 From: dl239 Date: Fri, 25 Jun 2021 18:37:30 +0800 Subject: [PATCH 21/21] docs: add memory pool rfc --- hybridse/using-zetasql.md | 4 +- .../image-hybridse-new-parser.png | Bin .../image-zetasql-workflow.png | Bin images/memory_pool.png | Bin 0 -> 14035 bytes {fedb => openmldb}/api-server.md | 0 openmldb/memory-pool.md | 84 ++++++++++++++++++ 6 files changed, 86 insertions(+), 2 deletions(-) rename {hybridse/images => images}/image-hybridse-new-parser.png (100%) rename {hybridse/images => images}/image-zetasql-workflow.png (100%) create mode 100644 images/memory_pool.png rename {fedb => openmldb}/api-server.md (100%) create mode 100644 openmldb/memory-pool.md diff --git a/hybridse/using-zetasql.md b/hybridse/using-zetasql.md index cd11a22..3f7ad23 100644 --- a/hybridse/using-zetasql.md +++ b/hybridse/using-zetasql.md @@ -38,7 +38,7 @@ ZetaSQL is a SQL Analyzer Framework from Google. We are considering using it as We do some survey on ZetaSQL. ZetaSQL provides many APIs and services. -![zetasql analyzer workflow](./images/image-zetasql-workflow.png) +![zetasql analyzer workflow](../images/image-zetasql-workflow.png) The figure above demonstrate the zetasql's analyze workflow. But we **only** use its parser module, since It can fullfill our requirements. @@ -163,7 +163,7 @@ ZetaSQL supports standard ANSI SQL. We are going to integrate zetasql's parser 2. Transform `ASTStatement` into `LogicalPlan` . 3. Convert `ASTExpression` to `ExprNode` -![Figure2-use-zetasql-parser](./images/image-hybridse-new-parser.png) +![Figure2-use-zetasql-parser](../images/image-hybridse-new-parser.png) **Pros**: diff --git a/hybridse/images/image-hybridse-new-parser.png b/images/image-hybridse-new-parser.png similarity index 100% rename from hybridse/images/image-hybridse-new-parser.png rename to images/image-hybridse-new-parser.png diff --git a/hybridse/images/image-zetasql-workflow.png b/images/image-zetasql-workflow.png similarity index 100% rename from hybridse/images/image-zetasql-workflow.png rename to images/image-zetasql-workflow.png diff --git a/images/memory_pool.png b/images/memory_pool.png new file mode 100644 index 0000000000000000000000000000000000000000..740367664fe23f20a873e6b5e94ab7c9c0f29476 GIT binary patch literal 14035 zcmeHud0bQ1y0*1AS3QU5IzTbEMAlYkY?X}js z*89HCv!1m%;YIRTx!hv;f&~j!;;?ASf&~k`7cBVfhoy^wJN5gLegVEdlTbWd7Zl## z_Bh!bB3OH4JTSXBQk1v5UnG=7@s@l3l`3DR2w; zT_9p{+1yapd>uQ89n=|y@%^@BN~LSuvQT@avw?ZjXv zS11)p!mMGgKp7{LhK2**z-{1@>;ruG0)J4*L8$#f_;n$U{;MYD zEDDtdbLOpKh1x364pbxo!^bG7_8jg(GGpF~C}V4%CI)K1%VEwGJ8PIbFm=W`0$So4 zXT{SGjiO`da?jv{WGI@*VTUPb^X&+O1)=kOc>xn+3v@r%2$CMdV!|c%7%vEv%~xP? zII1)T6$TZ+;1WByfRCgpLh*Qx9gh(M52HsTt?ht`bCvpfc=)(M&_0l83LfU*>n(_( zFuj>X1f4}hxuQ5|59Kp?*XKBRB?6U~m;&AsH4K27?Mo zcs_BlF*z?d(1=ih%L5fBq1YA0Y0SR#jx`>o{`g&p@5~x1|CBe#pvD1US$%L>d za(Fn_jm1H^1*73ELAqGalPy{%EyBHV6MR@^+ zxI#I6k&m3KfMDDNNNzYW3h&`XaD^$j#IRs`aEP}(DIDtRNoC=sSbMaL>h2BqMEe4> z?&>Y@_H-e53ZZB!59;R+EDu#KU=f*OdwY5`1c)}w+}iNOk{F37QiKM&cEDoX*}*a# zPR7OYqGSwFG$xwj1!2(OXcm^}>+K#2W074&ZWJViB=GZ*;-nGKV1_*l!a|a$7&~7i z$&Z3$!H7^SEg~c)iX!rGQ1B5rFG3_vO7ZlfW8HndNfa-@16}R?L@c76JClV%c==LE zVeTxH9Ocdshy)V8Llo1`Gnnlc77@;%M24{47(RF`FjP2~?gk~edBa8i7#tfF92LyO z3!{iQVYnm`8R_ris^EKjM-kCb3J(t?2logCLK+>yqtfv>K?v1PfFsz4iAY!m2kFHV zQSIDf!e~e&9?Azi7aOU7hD)eKo*g4JCfEZGkHnC?Sn^;}ls!`sO<}oW{9L6tB+Z@( zac6KsqPW5C1bRr6yNfR;L>z`t#9*Nm7c?du%W}XnLVwwP$(x)qrK&4V83O5m-1(f`~8PGYaJXq{7CKTcoIag5- z8tvi9^A?AA`AONq4j3OEgGgp!=UgyE<`EJd#&RVDyF$rGcNbuqNt_tA3oy4_cqk>3 z5>1R`a%24bfNLBM@G}gSO@QJM1f-Xn7!PO5U0hI7l$1`Eh4|utU*RYpj1&cr_03ItB%$h$DawJ)>aW(hyo?G+G!D zA;e3FC;~;mrlQHNFbEGzp`uuR7zC9a!R5r1ND-B(Kt>6o1XLPbE_INzDKH7u!68hc2vJ1vXb70Xg(ZaZVMHp>PIL^A z!7#ots3-!82V4ooa8eXf0pr;P`$j0p&d;buH05ykYI1#Yi7*Y&} zu5bD*6_=K5IMWd9wQZwIy0@8o$bvkU-JAR!CF7j{c1w z0KRw4uK{o7zo_q*(kj}jSBa`B^%2iZ>{kg-zDi!vG3_J%0k4|9;ItZ!xj zfl(jg`E8K2g?DP-4z7@mrq&@_uifv{bs`cfc6n|2ttDUbGvnptv<{>TM|#75m-$h) zcoM09qP8r&Qysn)75%N(T8=0Uk8QA{Pr^ z{KVZeO||+PP>Tpc$^ap^Mb5*WJ+_S83bN=3eWctOFe+YhCtyt6}rG#nB_!k=Hpv zA40@Tit;fE*L}YWHQ8v0)s^A+`r#|5(WNJo-+~}b{OFF22d0lc%AcaoIFE3e*Kc0d zq?)tSBKwKMlU=B7nyRHTyr{db1;)Rbx5v#rn~~66^329!MaSyuymy*4m!_`Yq_2G5 zzO?Nj4o>d%lA*MNAFKkmHd;6Du{R%IO^#zev44nOwlW#YJE&ZB%@(!O(V9won;iT{X_P#ZPzarejxX~NxfV!{!>LQb=K<1Zf~;&O7=dp3)V}JV#4OZvVb_} z_nw()1dt=Bh)1ZiOoXS7{l##@P;Xd|vyBd?^m+}XO)7ElvF2WMsQC5~9Iv8s9Wdf8UlK0c)WNwl`9#yW|5Frw(iGhS9<2H*@xR!Do*de#;qnV_1=Q zt^MP>a(QmAI=tI%SY$VSKB*tHOltm3UunR>FH*Bo+8b>uH{I-5mVa%|Z>H=5j~pov z8eR8t5;Q0$(ZLYiE-dUujbI#3uiOE7H5i|`F-}?i z_NGhAFZcg@GS7ckn))BCJoQ!bD~bG6QC+=ZO7zRx2Ew6TM@WG!%qZxpXVSoy&z*0c zsQC7Jgm7a+%%;0qS{~w$WxM@vhTYye;rME3e7W7L=~ar@>GVbeYhs2O60&PHwxln7 zi9B$80Qu_W;^mm|mj+|{yv zOx78{+o!*My!v6=lfHk+&QkLSXqoxQ9%IVRR(=oCt~}{yv^7DvFf~``8U!}x@|$ZO zpi5n#r;Hi?0nyo9PyB7rR2u=*Mr>!VF{b{?Wnq^pm&r3swJ|{L#dmia!ND2%8Ig+o zw#eLAyK`ySrOxbR6MVb^WuF)T8Dyr~8-UsyzA7=TZnv4eclVFgBqPRzS!SQ$9SvD) zG9wKzBkkvoWkx#%Ky86n(>9|Se`j?4FHMd&nm?wF4;so?jfP7sFPRr&OqHbEd-nC$ zI;1K#pl`ylfpDTY)WkC6(3c6TJGiI z`x@P6IKFMO`ksn+Rr}D#tbzFKcKToU>el1p*l-^gFH4(eip@{Tg>N^<0O`2y?C{HX zUA8R;Gha`D^sFx`-?l7XEnxLJ)!N80kM_8pzqTQ#Oc$g*!EYb6ixiG9=5q6CO}z{? z3?#m~C2z6Xvz@!V%`o2e<4z}sJ2&p{6V{#BgJ_(+(SX{R)i|o+LEaJu-ggD-Ug~zM zMW9nEt*we|B};pol!G~*c}}X`0uM?KcZ#=3^eEr?afXofBqr!X?wU&O ztiEW~UKOd)r+n?;V+6iXu(RAM(Gg^0?tbX$qBF~!@%YU%&`>bR1w_2HG zw0>O_DxbHVmy^qzFc5V=>56lN34!O+GY%tAV2vpMe2euGFc>Uze3sAZG6-;b5Dp#& z%xDo&h%LlwS7D=k_m=i0@_vjWfmB7-`myRAngFHV5SVG7mo&+ulz(>tv>(3<*SV>s zxT$U{>RZi`t(!f0Pdp1|CxHxCR*!Sr_kC6I>|I@-@?lsD=&>_ff^3GN5eVnJIxdEHu~+P_>o z6VmgYrq^{ZO!M#xX*|C=!i!Qqtxc%1)2u1%jE6gK0K@&?PZ@sB!Ca`^M23qJtr_5x zrnpf%*Ad%Rm*{!fS}k`HM_EbLTjJpM<6T#^78%-`^zJWnEBB+s<87(?2co+4IAur; z)UTXp(DHPt#}{=VHxE7^Ymma}G`K5`=&uZ-nd|SpV?o^y-Cv$=|Ep!5_wyHOrE02a z|5<<;wFFK-_?U5G+k-^F;GIs-LO&Ti(X2msdYY<4%kru=tZ2jf!1tf8{My%&vD#`# zYf%!Qp5CyV)OVxwcC3OKUn2K^8WXfyL_8DbR5ZSdPb>MO9 zYx}BXgNMrvpX1VL zE_qCv+0u_e*|nt|w(ntIHn$E35|mfJAC%komZRd@3NG)3pY-7tw9DavM9*v5jva}h z@q^BXt^LkYFGu-MEPZID83n2>ZE9+PB!E)BWM3*(KlYPuNY|{3R{Ixeg_D7*r)S%? zw{mfCI{cCTKwlwYH-2BnfpdgtC)vgcP@bwl-9DS4?R#$*u;v@b1d9ZPIHZmHre{hn z|D~{BK3TSEBF8^1_d*%!oW(;feNfsQFj4*|5qDQ|V;w%p>H`S=NWy#+jvYvH4z1LJ zUO!^>vqB&FGx~HSoZyI@(p6NsL$l@zI}T)io-Nyx=~&|2EqrRn6=c``-2b)vp#zHw z*LRpl_l#12NZ;Mt{ydY)H;2En)U44Awlgn}f5yEcyB zH0A&FKI*?7H#){Yp0QU44_%t2<=@C$s2;4S)i;vo$=K+uBK=-R@5helV-o*E&(rr+ z$`uiL^n%W3oU*Womlhp;3h<~7o8PiOaWctfwYvL5;)xR#v%?)*i}He|7uP(^E;{NW zD(V>BG#1Z~J-Hy^{=yBic%99^tVssoP8qBw6O$qUge7%xBi`g~B)}O-c6}x?bQK_4 z#G@~)O}+{STp7H6)Wm060j_lQ_PZ}lo~{N0L4A7C)EF1&tMysT2~(WlfH1-}95w}^ zJ!uAe9rm&F$Q6*DBvy6>F0EWo3*tpvibbEl9+@nh- z8@)9dAbIeWXD*mR_V3KT4k^lb{CxZ%XGWkqqjTSsl%jicwD=L;pMH2-tBp!zHBr_y z94&6i|7)e;W;`Rjt|~fW5EWvr}`C_G=G3 zjXCgS#Fy9>uiKiIyN(^Zg;a>smXY6+lE5vv__U^+_rH5yvzkVDdxU2As6SU}IUt6$ zV~4t}TW6yl)&q>>wAtpSiXL_P*)|Rx_O%%hz6n>I;ocvRnQhnyOx-<;mcffbSz+>h zxw;P}7u^C7@81)S0C`u759ky9Stlq~^Zt@5BZ1y}uEJ%erf0Q9P;O*bpNr)$Ty z#j4A#w~$F>dw|Hz4g7stF4Jx7#^Tm@a;M!WWCcufb?~}j#Bf1}RJ-H!Yjt2%B{3RL zx$H*v+@nvl*iAret~=j&Ibd!4k*h7?PDODClAc~@J2*0o@6>;nGMT|j%xQhM@m$Bi z&jHvh`q5NZZggM8rzKk6_CHI%#$uhWu`|?aUf(Lby_bs@(KgHTR*YU)ctvXGN3|LZF-7T zZq+C1`6d#k+l*`8Z?CDe)?fyHWp%K)V*1FboPiUpP>Llk&3ar}(e=(LDU@ecaHdpf zJL5MpX?UL_KRxc-n7{P2+F9wUpJ!waHKX^>FG$EV^Y@75eRb`Tf59lPGW8sZRz%xO zJcFxMAKYeIBk1{t)@%7g{b9C{gPvmlBOO8l?KteN2;tUEH&O& zTUES#UQS*`Q(#I^(&0T7+f&}g3^=?c(yezBd4@Nel!oWgrLwE+sk-3tDrNFki0eZ5 z!g0f|xvfiUxT}D%tOP@#r>e9MGvd@1uPdNjghR?34dNcf5p2`7GqAZ=#R7QO@Rh5M znHCA1c9XQ-8~j^BL5u=zyx>mzCHGTnzYy0Q?FlNf__PP_Xps|0QL|VAA%Eu2_xt*9 zcDTL0`P8ehpR@$lI(f8q)#93(pcejv4+TS4`bNe(Y?LpTHl1`H#I|Keds$LC3K}tk z6AT;2{DVrL?#{t$K)1(>FP=2z=rJqEhPo32+{U%0JEx0*G<~jjZ=-RC`~^Uz;Pwhf z&^$I`{D9)`!l+%A_h=ke#zxkdKgmp{HmMH$Os032O> z*ae-(AIR@O(K~)(qoc8G55UobS3}A3`14IDP~2V@d&a0Le3J^;88#M0n8%;LH~UaX zSnTHBP2@Af8pqM~%ja3zG10)%0>p))x-}uv-1$-dr_|xUXx#s~iAq3OLZF;8ti!8V z;=?3;$|~$UPy1YD(uctg>smy5Ep5Zc>uYWOZyx&j!=|zKibdU>fXnb$yaOl8O zm*MCKU1^G6hx)&m+9RmDppgtJdcp?6B_>W|_WzEs^}kciI~3fP@t7UoJS7tc6f?!% zcDZ}DZ&2XwHDvxacG9pxS7ST+A>%2`rg|5up!e&pops#4p{%yZ^GCNq&Me&h8?N`< zNA^Z?530oxkbbIT@yCz2>d`e9BfmMdA<-sabKZ&0-RoKAM`9?Alf{NwK*RwjERhsXSaGbzQ}l#IF3Nmh zx6kUWtcZOkb1cFf*wKrx?H>8~G9_L_AvRK<0p@-4iz*ezs=$&euXN$1y}6Ql zi#;6*nqd^<#jKI1hH-$N2p4v|{E=^^o!NZLQiI`D0bRKRgWq1umwZgFng$I#-Zt^CcX(kB3B;{zHrB1)xrN?D55>ttpN>>CYqil;$3$ZdAPv+=3s7I<^h66CiV?FB;+pSKR#zYqVi>D!*-A+SCD? zh11K^+lH5a9@I1Lbm&23&w+9fFEH&m?6SqhDYhbJw5S?#ob3*|x#d7J&oQmfu&v6;z{`cJd=vJ}GOO-VN|PVR>Li+xGSi zY8hYMaN1hgI2f?jp#~ZVusWp|O-;Yd(4)u2PJZE@`huFNz_x>6crkq?aCkV+XdCve z{_Vv(9R?TImxbMz+16heA2xFzz2=q2vz+*~oBvu|*#FtlGavc}b~nUpF_$X!-O5G0 z98GGLXZP~_vw2HYQ8eX~0^N*q9eDI{=d-;v(7Gp1wnC?(;{}Ikgwtu_WcQwx2%S&he9V4L!r~%rx=&L6+EqByAG7h{Utk^leF{cB`nC zvXEkau|3LW(w#!_GCcU{tnQ!b>#x#7U^IZS@{>Lf1WIe$WTAh43z$USqIfI zRA1xZJft<;jz;TxvSZR^FHXOw+*%f!oq9o=S_`SzTY=K_U2V_P2E<>minfCPF7>A< zX-X;lLWNE1Jrxd3kF+Ud#C8_ zAAC}o*+1y$`0FT7a4D#>&w+=&=2y@PFT$ky58B$2XLhyXRdbwBgUNaFtj#vzcPWkd z=!)*X?)RM1!X|Zws&i%}r|h7NMzQp4o+5yDG!ICfqS*~V>>fY_Fun2pIP3`kVA50(5|ZSIPn-f8;Qd z4SQgMvOVKD%2@o~IT{cPqAl?z4esikSnzYQi%C@51#I_)tYs!O;46Soef1^G1a92` zJL^{NG}TVt2Iv&7A&ZTTXIBA;QgznaRNDujc0Sl96E~8s1GEBUv6*RPCjoM{{KR@A zaY%RzP`kI7BvWm|oXGJl^r%Uu2n1w`?%%#N)&35s{hbxtWJV>xDl0em{b14|+yMsr z)309`&G>tx34|18L--29{r9WFkMt~?Cw@JF!jgE`!hSz++2_V5|BLvU*26T@rp5gm zzBjfN^6-w`+V%PMhC@gWpSks!WY1zzv5%GWp)s?yXL`O(oLN)kU&=e0^{a93Q(B|L zO+O2-J8Nz}b?phFEDo6NChDQfDTQHsO|ItvENR1Jcl6u&p6X8nz6ZNnvTfddAol?W zh>V}eFzp1GfTx~*osTtA{Sv^r(tG3jjXRtsU%(-ldu+a(H|NFN7N$SCx5BuEfdPA# z=H&Y)ja~g;)%99-v$osgc3xCT6>j~=)Ye3whDLv!B zPb^*15p=$-pn*5Ci=0Zkci2wTbq=r2vc6z7bcW;Cckt)*pR2+0?V1-K4^tXv zA}E7@Yd>(zvG%^2w?n%|0YN#*-Ngq+Vn%*vW#wQ_2C1u2sQsEw*reB^@JonhX+c1# zyx50Q_mhH6xs1;k@RMJ}Bo34vghwzy@Lye|7vo`1qr&-01Urk^6_=5TZ_yah=O2&@_ zxUVs9&{vFgq4ij5d1ciNXQe%D)JGGL02hxtKPuEhtKgceS9o^ZP1dnf0~IUcJN^-C z;U)8|-q`ANI&YniMQfG8`(HIb{>%H;oj141Ie-uuq`RwI0;r0D@}nCx0g`d%xP^;$ zO}6%>E>CSi)J`4;(aNnlz+g}n2(K9&yPBKy!RpjP&=ns&|GbBqsVcIvQSsWepK)gvYs*S# zBS&-QBq-G0$uJ77bseZ2j0E=*c{SR~m(e(}2jwzhyz9z8lAr>>1+$K{X36j2rj`4n z#(DZecnJ+&St6oX?nQ!{7lYM%X-cc1^!)G=Df|dt^%4i{MN{9#AX*le5#sgtjmH;bN?*)i!mlnq@fNz6$ zAK`Wgj(j@yTdVyC$9~DmUiA1)<_gm?{QsjE{U<^h1bTCC;_w-8<3wA=H` z@PEi9>YEmke%GG&)~dzVzn)+1hM@Y@_ZnxuTwu&xO=zG;bNHYUewn)g;QD}HVB$IE z0O}Fi^4*N6R|BA4%mwG^VRvQftom4EXv-%=J0x6ox^w?uBzV#E8Md6hKVlkj!=)oA zxn@(wfhv~?9>)mg2{N;?`=d17w}VYtr^v__mpJ(3Ao|7Lb-mv<&L0Rh5T1RF>dmso z_rH=4t6nHLVx~BLZWr-sd;Nqj)a}DpFD2p*JbGHKtlt!o2sj8}Q6jK+rahd)Go)RiW^8;MtoSng@z2FPM(fUf6gM z+e#WI#xhI<`Vf1e9xZSxp%s{-^P7VJx*B_doLV?@+?=zfOY`c|J+Xy`vh=ki@94@l}WcX!%&~KZ=86vW^dGS z;wOhW26{I8KJ?!Rq;KnDkDPULs(lIBuX-H~&#ZGwWLVdcYBS%YQ}Y z%CCV%h8Ax_IXvvG?I` zweoy6aDY~uW09+^yfSf0?=xcrs43tCbNnv++)v<=4rRdmY7TQs2LNA=Mch=h&QkYU zCsJvr)s+GUYP+U)zy8RHa#?X8g_c-aYa|y_UjYp)L;QI6NM>B^_9uH$c5*s}wtw?% z@JN#(0C+!)mplWlaFj~`jQ=?I+Ud;ePD7S@YKRK=873Y0{L_KoTn_$d{>SBX3fx`B zf19PMBa`sj89NQ;bzx3w!R*h}8h~L$tp<-z`Qb`t?9CIc6G2Gpa| 40 | +| Node| 32 | +| std::atomic*> node0[12] | 96 | 8 * height + +By analyzing the above data structure, we found that the memory alloction has the following situations and we can use different alloction strategy. + +| - | - | struct / class | alloction strategy | +| --- | --- | --- | --- | +| Normal | Fixed length | KeyEntry | boost::pool +| Normal | Variable length| Key | Tcmalloc/Var-length pool +| Timeserise | Fixed length | Node | TimeSerisePool +| Timeserise | Variable length | Value | TimeSerisePool + + +TimeSerisePool is composed of multiple TimeBuckets as followers. + +
+ +```C++ +class TimeBucket { + public: + struct Block { + Block* next; + char data[]; + }; + Clear(); // free the block list + void* Alloc(uint32_t size) { + object_num_++; + if (current_offset_ + size <= block_size_ - sizeof(Block)) { + void* addr = head_->data + current_offset_; + current_offset_ += size; + return addr; + } else { + Block* block = (Block*)malloc(block_size_); + current_offset_ = size; + block->next = head_->next; + head_ = block; + return head_->data; + } + } + private: + uint32_t block_size_; + uint32_t current_offset_; + uint32_t object_num_; + Block* head_; +}; +``` + +```C++ +class TimeSerisePool { + public: + Alloc(uint32_t size, uint64_t time); + Free(void*, uint64_t time); + private: + // key is the time / (60 * 60 * 1000) + std::map> pool_; +}; +``` + +Note: TimeSerisePool does not guarantee thread safety. + +# Adoption stratege +If we implement this proposal, there is no impact on existing interfaces \ No newline at end of file