Merge pull request #39 from resilientdb/fix_build_pybind

Fix build pybind

.bazelrc

@@ -1 +1,4 @@
-build --cxxopt='-std=c++17' --copt=-O3 --jobs=40
+build --cxxopt='-std=c++17' --copt=-O3 --jobs=40 
+build --action_env=PYTHON_BIN_PATH="/usr/bin/python3.10"
+build --action_env=PYTHON_LIB_PATH="/usr/include/python3.10"
+

INSTALL.sh

@@ -22,6 +22,7 @@ sudo apt-get install autoconf automake libtool -y
 
 # for pybind
 sudo apt-get install python3.10-dev -y
+sudo apt-get install python3-dev -y
 
 # for crow
 sudo apt-get install libboost-dev -y

README.md

@@ -79,7 +79,15 @@ You will see the following result if successful:
 
     client getvalues value = [test_value]
 
+## Enable SDK
 
+If you want to enable sdk verification, add a flag to the command
+
+    ./example/start_kv_server.sh --define enable_sdk=True
+
+And in example/kv_config.config
+
+    require_txn_validation:true,
 
 ## FAQ
 

application/kv_server_executor/kv_server_executor_test.cpp

@@ -125,7 +125,7 @@ class KVServerExecutorTest : public Test {
       txn_dict = json.loads(transaction)
       ret = txn_dict['is_valid']
     )",
-             pybind11::globals(), locals);
+                   pybind11::globals(), locals);
 
     return locals["ret"].cast<bool>();
   }

common/data_comm/data_comm.h

@@ -24,6 +24,8 @@
  */
 
 #pragma once
+#include <stdlib.h>
+
 #include <memory>
 
 namespace resdb {

example/start_kv_server.sh

@@ -4,7 +4,7 @@ SERVER_PATH=./bazel-bin/kv_server/kv_server
 SERVER_CONFIG=example/kv_config.config
 WORK_PATH=$PWD
 
-bazel build kv_server:kv_server
+bazel build kv_server:kv_server $@
 nohup $SERVER_PATH $SERVER_CONFIG $WORK_PATH/cert/node1.key.pri $WORK_PATH/cert/cert_1.cert > server0.log &
 nohup $SERVER_PATH $SERVER_CONFIG $WORK_PATH/cert/node2.key.pri $WORK_PATH/cert/cert_2.cert > server1.log &
 nohup $SERVER_PATH $SERVER_CONFIG $WORK_PATH/cert/node3.key.pri $WORK_PATH/cert/cert_3.cert > server2.log &

kv_server/BUILD

@@ -1,19 +1,59 @@
 package(default_visibility = ["//visibility:public"])
 
+load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
+
+bool_flag(
+    name = "enable_sdk",
+    build_setting_default = False,
+    visibility = ["//visibility:private"],
+)
+
+config_setting(
+    name = "custom",
+    values = {"define": "enable_sdk=True"},
+    visibility = ["//visibility:private"],
+)
+
+cc_library(
+    name = "py_verificator",
+    srcs = ["py_verificator.cpp"],
+    hdrs = ["py_verificator.h"],
+    deps = [
+        "//common:comm",
+        "@pybind11",
+        "@pybind11//:pybind11_embed",
+    ],
+)
+
+cc_test(
+    name = "py_verificator_test",
+    srcs = ["py_verificator_test.cpp"],
+    tags = ["manual"],
+    deps = [
+        ":py_verificator",
+        "//common/test:test_main",
+    ],
+)
+
 cc_library(
     name = "kv_server_executor",
     srcs = ["kv_server_executor.cpp"],
     hdrs = ["kv_server_executor.h"],
+    copts = select({
+        ":custom": ["-DENABLED_SDK"],
+        "//conditions:default": [],
+    }),
     deps = [
         "//common:comm",
         "//config:resdb_config_utils",
         "//durable_layer:leveldb_durable",
         "//durable_layer:rocksdb_durable",
         "//execution:transaction_executor_impl",
         "//proto:kv_server_cc_proto",
-        "@pybind11",
-        "@pybind11//:pybind11_embed",
-    ],
+    ] + select({
+        ":custom": [":py_verificator"],
+        "//conditions:default": [],
+    }),
 )
 
 cc_test(

kv_server/kv_server_executor.cpp

@@ -26,13 +26,9 @@
 #include "kv_server/kv_server_executor.h"
 
 #include <glog/logging.h>
-#include <pybind11/embed.h>
 
 #include "proto/kv_server.pb.h"
 
-namespace py = pybind11;
-using namespace py::literals;
-
 namespace resdb {
 
 KVServerExecutor::KVServerExecutor(const ResConfigData& config_data,
@@ -41,13 +37,13 @@ KVServerExecutor::KVServerExecutor(const ResConfigData& config_data,
       r_storage_layer_(cert_file, config_data) {
   equip_rocksdb_ = config_data.rocksdb_info().enable_rocksdb();
   equip_leveldb_ = config_data.leveldb_info().enable_leveldb();
+
+#ifdef ENABLED_SDK
   require_txn_validation_ = config_data.require_txn_validation();
   if (require_txn_validation_) {
-    py::initialize_interpreter();
-    py::exec(R"(
-      import json
-    )");
+    py_verificator_ = std::make_unique<PYVerificator>();
   }
+#endif
 }
 
 KVServerExecutor::KVServerExecutor(void) {}
@@ -80,35 +76,16 @@ std::unique_ptr<std::string> KVServerExecutor::ExecuteData(
   return resp_str;
 }
 
-// Validate transactions committed by the Python SDK
-bool KVServerExecutor::Validate(const std::string& transaction) {
-  auto locals = py::dict("transaction"_a = transaction);
-
-  py::exec(R"(
-    from sdk_validator.validator import is_valid_tx
-
-    try:
-      txn_dict = json.loads(transaction)
-      ret = is_valid_tx(txn_dict)
-      is_valid = ret[0] == 0
-    except (KeyError, AttributeError, ValueError):
-      is_valid = False
-  )",
-           py::globals(), locals);
-
-  bool is_valid = locals["is_valid"].cast<bool>();
-
-  return is_valid;
-}
-
 void KVServerExecutor::Set(const std::string& key, const std::string& value) {
+#ifdef ENABLED_SDK
   if (require_txn_validation_) {
-    bool is_valid = Validate(value);
+    bool is_valid = py_verificator_->Validate(value);
     if (!is_valid) {
       LOG(ERROR) << "Invalid transaction for " << key;
       return;
     }
   }
+#endif
 
   if (equip_rocksdb_) {
     r_storage_layer_.SetValue(key, value);

kv_server/kv_server_executor.h

@@ -34,6 +34,10 @@
 #include "durable_layer/rocksdb_durable.h"
 #include "execution/transaction_executor_impl.h"
 
+#ifdef ENABLED_SDK
+#include "kv_server/py_verificator.h"
+#endif
+
 namespace resdb {
 
 class KVServerExecutor : public TransactionExecutorImpl {
@@ -45,13 +49,16 @@ class KVServerExecutor : public TransactionExecutorImpl {
   std::unique_ptr<std::string> ExecuteData(const std::string& request) override;
 
  private:
-  bool Validate(const std::string& transaction);
   void Set(const std::string& key, const std::string& value);
   std::string Get(const std::string& key);
   std::string GetValues();
   std::string GetRange(const std::string& min_key, const std::string& max_key);
 
  private:
+#ifdef ENABLED_SDK
+  std::unique_ptr<PYVerificator> py_verificator_;
+#endif
+
   std::unordered_map<std::string, std::string> kv_map_;
   LevelDurable l_storage_layer_;
   RocksDurable r_storage_layer_;

kv_server/kv_server_executor_test.cpp

@@ -27,7 +27,6 @@
 
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
-#include <pybind11/embed.h>
 
 #include "config/resdb_config_utils.h"
 #include "proto/kv_server.pb.h"
@@ -113,23 +112,6 @@ class KVServerExecutorTest : public Test {
     return kv_response.value();
   }
 
-  // Test pybind11 is working and parse simple JSON object
-  bool PythonValidate(const std::string& transaction) {
-    using namespace pybind11::literals;
-    pybind11::scoped_interpreter guard{};
-
-    auto locals = pybind11::dict("transaction"_a = transaction);
-    pybind11::exec(R"(
-      import json
-
-      txn_dict = json.loads(transaction)
-      ret = txn_dict['is_valid']
-    )",
-             pybind11::globals(), locals);
-
-    return locals["ret"].cast<bool>();
-  }
-
  private:
   KVServerExecutor impl_;
 };
@@ -257,11 +239,6 @@ TEST_F(KVServerExecutorTestRocksDB, GetValue) {
   EXPECT_EQ(Get("test_key"), "");
 }
 
-TEST_F(KVServerExecutorTest, PythonValidate) {
-  EXPECT_EQ(PythonValidate("{\"is_valid\": true}"), true);
-  EXPECT_EQ(PythonValidate("{\"is_valid\": false}"), false);
-}
-
 }  // namespace
 
 }  // namespace resdb

kv_server/py_verificator.cpp

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2019-2022 ExpoLab, UC Davis
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+/*
+ * Copyright (c) 2019 The Pybind Development Team. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You are under no obligation whatsoever to provide any bug fixes, patches, or
+ * upgrades to the features, functionality or performance of the source code
+ * ("Enhancements") to anyone; however, if you choose to make your Enhancements
+ * available either publicly, or directly to the author of this software,
+ * without imposing a separate written license agreement for such Enhancements,
+ * then you hereby grant the following license: a non-exclusive, royalty-free
+ * perpetual license to install, use, modify, prepare derivative works,
+ * incorporate into other computer software, distribute, and sublicense such
+ * enhancements or derivative works thereof, in binary and source code form.
+ */
+
+#include "kv_server/py_verificator.h"
+
+#include <glog/logging.h>
+#include <pybind11/embed.h>
+
+namespace py = pybind11;
+using namespace py::literals;
+
+namespace resdb {
+
+PYVerificator::PYVerificator() {
+  py::initialize_interpreter();
+  py::exec(R"(
+      import json
+    )");
+}
+
+PYVerificator::~PYVerificator() { py::finalize_interpreter(); }
+
+// Validate transactions committed by the Python SDK
+bool PYVerificator::Validate(const std::string& transaction) {
+  auto locals = py::dict("transaction"_a = transaction);
+
+  py::exec(R"(
+    from sdk_validator.validator import is_valid_tx
+
+    try:
+      txn_dict = json.loads(transaction)
+      ret = is_valid_tx(txn_dict)
+      is_valid = ret[0] == 0
+    except (KeyError, AttributeError, ValueError):
+      is_valid = False
+  )",
+           py::globals(), locals);
+
+  return locals["is_valid"].cast<bool>();
+}
+
+}  // namespace resdb