Merge pull request #117 from resilientdb/mvcc

support version-based key-value service.

README.md

@@ -55,7 +55,7 @@ The latest ResilientDB documentation, including a programming guide, is availabl
 </div>
 
 ## OS Requirements
-Ubuntu 20.*
+Ubuntu 20+
 
 ---
 
@@ -78,29 +78,219 @@ Build Interactive Tools:
 
     bazel build service/tools/kv/api_tools/kv_service_tools
 
-Run tools to set a value by a key (for example, set the value with key "test" and value "test_value"):
+## Functions ##
+ResilientDB supports two types of functions: version-based and non-version-based.
+Version-based functions will leverage versions to protect each update, versions must be obtained before updating a key.
 
-    bazel-bin/service/tools/kv/api_tools/kv_service_tools service/tools/config/interface/service.config set test test_value
-
-You will see the following result if successful:
+***Note***: Version-based functions are not compatible with non-version-based functions. Do not use both in your applications.
 
-    client set ret = 0
+We show the functions below and show how to use [kv_service_tools](service/tools/kv/api_tools/kv_service_tools.cpp) to test the function.
 
-Run tools to get value by a key (for example, get the value with key "test"):
+### Version-Based Functions ###
+#### Get ####
+Obtain the value of `key` with a specific version `v`.
 
-    bazel-bin/service/tools/kv/api_tools/kv_service_tools service/tools/config/interface/service.config get test 
-
-You will see the following result if successful:
+      kv_service_tools --config config_file --cmd get_with_version --key key --version v
+
+|  parameters   |  descriptions |
+|  ----  | ----  |
+| config  | the path of the client config which points to the db entrance |
+| cmd  | get_with_version |
+| key  | the key you want to obtain |
+| version | the version you want to obtain. (If the `v` is 0, it will return the latest version |
+
+
+Example:
+
+      bazel-bin/service/tools/kv/api_tools/kv_service_tools --config service/tools/config/interface/service.config --cmd get_with_version --key key1 --version 0
+
+Results:
+> get key = key1, value = value: "v2"
+> version: 2
+
+#### Set ####
+Set `value` to the key `key` based on version `v`.
+
+      kv_service_tools --config config_file --cmd set_with_version --key key --version v --value value
+
+|  parameters   |  descriptions |
+|  ----  | ----  |
+| config  | the path of the client config which points to the db entrance |
+| cmd  | set_with_version |
+| key  | the key you want to set |
+| version | the version you have obtained. (If the version has been changed during the update, the transaction will be ignored) |
+| value | the new value |
+
+Example:
+
+      bazel-bin/service/tools/kv/api_tools/kv_service_tools --config service/tools/config/interface/service.config --cmd set_with_version --key key1 --version 0 --value v1
+
+Results:
+> set key = key1, value = v3, version = 2 done, ret = 0
+> 
+> current value = value: "v3"
+> version: 3
+
+#### Get Key History ####
+Obtain the update history of key `key` within the versions [`v1`, `v2`].
+
+      kv_service_tools --config config_file --cmd get_history --key key --min_version v1 --max_version v2
+
+
+|  parameters   |  descriptions |
+|  ----  | ----  |
+| config  | the path of the client config which points to the db entrance |
+| cmd  | get_history |
+| key  | the key you want to obtain |
+| min_version | the minimum version you want to obtain |
+| max_version | the maximum version you want to obtain |
+
+Example:
+
+      bazel-bin/service/tools/kv/api_tools/kv_service_tools --config service/tools/config/interface/service.config --cmd get_history --key key1 --min_version 1 --max_version 2
+
+Results:
+
+> get history key = key1, min version = 1, max version = 2 <br>
+>  value = <br>
+> item { <br>
+>  &ensp; key: "key1" <br>
+>  &ensp; value_info { <br>
+>  &ensp;&ensp; value: "v1" <br>
+>  &ensp;&ensp; version: 2 <br>
+> &ensp;} <br>
+> } <br>
+> item { <br>
+> &ensp; key: "key1" <br>
+> &ensp; value_info { <br>
+> &ensp;&ensp; value: "v0" <br>
+> &ensp;&ensp; version: 1 <br>
+> &ensp;} <br>
+> } 
+
+#### Get Top ####
+Obtain the recent `top_number` history of the key `key`.
+
+      kv_service_tools --config config_path --cmd get_top --key key --top top_number
+
+|  parameters   |  descriptions |
+|  ----  | ----  |
+| config  | the path of the client config which points to the db entrance |
+| cmd  | get_top |
+| key  | the key you want to obtain |
+| top | the number of the recent updates |
+
+Example:
+
+      bazel-bin/service/tools/kv/api_tools/kv_service_tools --config service/tools/config/interface/service.config --cmd get_top --key key1 --top 1
+
+Results:
+
+>key = key1, top 1 <br>
+> value = <br>
+> item { <br>
+>&ensp;key: "key1" <br>
+>  &ensp;value_info { <br>
+>  &ensp;&ensp;  value: "v2" <br>
+>  &ensp;&ensp;  version: 3 <br>
+>  &ensp;} <br>
+>}
+
+#### Get Key Range ####
+Obtain the values of the keys in the ranges [`key1`, `key2`]. Do not use this function in your practice code
 
-    client get value = test_value
+      kv_service_tools --config config_file --cmd get_key_range_with_version --min_key key1 --max_key key2
 
-Run tools to get all values that have been set:
+|  parameters   |  descriptions |
+|  ----  | ----  |
+| config  | the path of the client config which points to the db entrance |
+| cmd  | get_key_range_with_version |
+| min_key  | the minimum key  |
+| max_key | the maximum key |
 
-    bazel-bin/service/tools/kv/api_tools/kv_service_tools service/tools/config/interface/service.config getallvalues
+Example:
 
-You will see the following result if successful:
+      bazel-bin/service/tools/kv/api_tools/kv_service_tools --config service/tools/config/interface/service.config --cmd get_key_range_with_version --min_key key1 --max_key key3
+
+Results:
+
+>min key = key1 max key = key2 <br>
+> getrange value = <br>
+> item { <br>
+> &ensp; key: "key1" <br>
+> &ensp; value_info { <br>
+> &ensp;&ensp;   value: "v0" <br>
+> &ensp;&ensp;   version: 1 <br>
+> &ensp; } <br>
+> } <br>
+> item { <br>
+> &ensp; key: "key2" <br>
+> &ensp; value_info { <br>
+> &ensp;&ensp;   value: "v1" <br>
+> &ensp;&ensp;   version: 1 <br>
+> &ensp; } <br>
+>}
+
+
+### Non-Version-Based Function ###
+#### Set #####
+Set `value` to the key `key`.
+
+      kv_service_tools --config config_file --cmd set --key key --value value
+
+|  parameters   |  descriptions |
+|  ----  | ----  |
+| config  | the path of the client config which points to the db entrance |
+| cmd  | set |
+| key  | the key you want to set |
+| value | the new value |
+
+Example:
+
+      bazel-bin/service/tools/kv/api_tools/kv_service_tools --config service/tools/config/interface/service.config --cmd set --key key1 --value value1
+
+Results:
+> set key = key1, value = v1, done, ret = 0
+
+#### Get ####
+Obtain the value of `key`.
+
+      kv_service_tools --config config_file --cmd get --key key
+
+|  parameters   |  descriptions |
+|  ----  | ----  |
+| config  | the path of the client config which points to the db entrance |
+| cmd  | get |
+| key  | the key you want to obtain |
+
+Example:
+
+      bazel-bin/service/tools/kv/api_tools/kv_service_tools --config service/tools/config/interface/service.config --cmd get --key key1
+
+Results:
+> get key = key1, value = "v2"
+
+
+#### Get Key Range ####
+Obtain the values of the keys in the ranges [`key1`, `key2`]. Do not use this function in your practice code
+
+      kv_service_tools --config config_path --cmd get_key_range --min_key key1 --max_key key2
+
+|  parameters   |  descriptions |
+|  ----  | ----  |
+| config  | the path of the client config which points to the db entrance |
+| cmd  | get_key_range |
+| min_key  | the minimum key  |
+| max_key | the maximum key |
+
+Example:
+
+      bazel-bin/service/tools/kv/api_tools/kv_service_tools --config service/tools/config/interface/service.config --cmd get_key_range --min_key key1 --max_key key3
+
+Results:
+> getrange min key = key1, max key = key3 <br>
+> value = [v3,v2,v1]
 
-    client getallvalues value = [test_value]
 
 ## Deployment Script
 
@@ -143,14 +333,4 @@ We also provide access to a [deployment script](https://github.com/resilientdb/r
    docker exec -it myserver bash
    ```
 
-   Verify the functionality of the service by performing set and get operations:
-
-   - Set a test value:
-   ```shell
-   bazel-bin/service/tools/kv/api_tools/kv_service_tools service/tools/config/interface/service.config set test test_value
-   ```
-
-   - Retrieve the test value:
-   ```
-    bazel-bin/service/tools/kv/api_tools/kv_service_tools service/tools/config/interface/service.config get test
-   ```
+   Verify the functionality of the service by performing set and get operations provided above [functions](README.md#functions).

benchmark/protocols/pbft/BUILD

@@ -6,6 +6,7 @@ cc_binary(
     name = "kv_server_performance",
     srcs = ["kv_server_performance.cpp"],
     deps = [
+        "//chain/storage:memory_db",
         "//executor/kv:kv_executor",
         "//platform/config:resdb_config_utils",
         "//platform/consensus/ordering/pbft:consensus_manager_pbft",

benchmark/protocols/pbft/kv_server_performance.cpp

@@ -25,7 +25,7 @@
 
 #include <glog/logging.h>
 
-#include "chain/state/chain_state.h"
+#include "chain/storage/memory_db.h"
 #include "executor/kv/kv_executor.h"
 #include "platform/config/resdb_config_utils.h"
 #include "platform/consensus/ordering/pbft/consensus_manager_pbft.h"
@@ -34,6 +34,7 @@
 #include "proto/kv/kv.pb.h"
 
 using namespace resdb;
+using namespace resdb::storage;
 
 void ShowUsage() {
   printf("<config> <private_key> <cert_file> [logging_dir]\n");
@@ -69,7 +70,7 @@ int main(int argc, char** argv) {
   config->RunningPerformance(true);
 
   auto performance_consens = std::make_unique<ConsensusManagerPBFT>(
-      *config, std::make_unique<KVExecutor>(std::make_unique<ChainState>()));
+      *config, std::make_unique<KVExecutor>(std::make_unique<MemoryDB>()));
   performance_consens->SetupPerformanceDataFunc([]() {
     KVRequest request;
     request.set_cmd(KVRequest::SET);

chain/state/BUILD

@@ -5,8 +5,8 @@ cc_library(
     srcs = ["chain_state.cpp"],
     hdrs = ["chain_state.h"],
     deps = [
-        "//chain/storage",
         "//common:comm",
+        "//platform/proto:resdb_cc_proto",
     ],
 )
 

chain/state/chain_state.cpp

@@ -29,64 +29,22 @@
 
 namespace resdb {
 
-ChainState::ChainState(std::unique_ptr<Storage> storage)
-    : storage_(std::move(storage)) {}
+ChainState::ChainState() : max_seq_(0) {}
 
-Storage* ChainState::GetStorage() {
-  return storage_ ? storage_.get() : nullptr;
-}
-
-int ChainState::SetValue(const std::string& key, const std::string& value) {
-  if (storage_) {
-    return storage_->SetValue(key, value);
+Request* ChainState::Get(uint64_t seq) {
+  std::unique_lock<std::mutex> lk(mutex_);
+  if (data_.find(seq) == data_.end()) {
+    return nullptr;
   }
-  kv_map_[key] = value;
-  return 0;
+  return data_[seq].get();
 }
 
-std::string ChainState::GetValue(const std::string& key) {
-  if (storage_) {
-    return storage_->GetValue(key);
-  }
-  auto search = kv_map_.find(key);
-  if (search != kv_map_.end())
-    return search->second;
-  else {
-    return "";
-  }
+void ChainState::Put(std::unique_ptr<Request> request) {
+  std::unique_lock<std::mutex> lk(mutex_);
+  max_seq_ = request->seq();
+  data_[max_seq_] = std::move(request);
 }
 
-std::string ChainState::GetAllValues(void) {
-  if (storage_) {
-    return storage_->GetAllValues();
-  }
-  std::string values = "[";
-  bool first_iteration = true;
-  for (auto kv : kv_map_) {
-    if (!first_iteration) values.append(",");
-    first_iteration = false;
-    values.append(kv.second);
-  }
-  values.append("]");
-  return values;
-}
-
-std::string ChainState::GetRange(const std::string& min_key,
-                                 const std::string& max_key) {
-  if (storage_) {
-    return storage_->GetRange(min_key, max_key);
-  }
-  std::string values = "[";
-  bool first_iteration = true;
-  for (auto kv : kv_map_) {
-    if (kv.first >= min_key && kv.first <= max_key) {
-      if (!first_iteration) values.append(",");
-      first_iteration = false;
-      values.append(kv.second);
-    }
-  }
-  values.append("]");
-  return values;
-}
+uint64_t ChainState::GetMaxSeq() { return max_seq_; }
 
 }  // namespace resdb