enable disk cache testing

* Updated QueryRunner and DHHandlerTestHelper to allow for a --use-disk-cache flag to enable generic disk cache testing with more test suites. Tests that use QueryRunner need to be manually enabled, so UpdelStorageTest and ExecuteTest were updated to handle this option. DBHandler based tests have the option supported going forward without alterations.
yma11 · Jun 4, 2021 · 15233f7 · 15233f7
1 parent e761513
commit 15233f7
Show file tree

Hide file tree

Showing 7 changed files with 83 additions and 16 deletions.
diff --git a/DataMgr/FileMgr/CachingFileMgr.h b/DataMgr/FileMgr/CachingFileMgr.h
@@ -70,6 +70,9 @@ struct DiskCacheConfig {
     }
     return "";
   }
+  static std::string getDefaultPath(const std::string& base_path) {
+    return base_path + "/omnisci_disk_cache";
+  }
 };
 
 inline std::string get_dir_name_for_table(int db_id, int tb_id) {

diff --git a/QueryRunner/QueryRunner.cpp b/QueryRunner/QueryRunner.cpp
@@ -93,6 +93,25 @@ QueryRunner* QueryRunner::init(const char* db_path,
                            reserved_gpu_mem);
 }
 
+QueryRunner* QueryRunner::init(const File_Namespace::DiskCacheConfig* disk_cache_config,
+                               const char* db_path,
+                               const std::vector<LeafHostInfo>& string_servers,
+                               const std::vector<LeafHostInfo>& leaf_servers) {
+  return QueryRunner::init(db_path,
+                           std::string{OMNISCI_ROOT_USER},
+                           "HyperInteractive",
+                           std::string{OMNISCI_DEFAULT_DB},
+                           string_servers,
+                           leaf_servers,
+                           "",
+                           true,
+                           0,
+                           256 << 20,
+                           false,
+                           false,
+                           disk_cache_config);
+}
+
 QueryRunner* QueryRunner::init(const char* db_path,
                                const std::string& user,
                                const std::string& pass,
@@ -104,7 +123,8 @@ QueryRunner* QueryRunner::init(const char* db_path,
                                const size_t max_gpu_mem,
                                const int reserved_gpu_mem,
                                const bool create_user,
-                               const bool create_db) {
+                               const bool create_db,
+                               const File_Namespace::DiskCacheConfig* disk_cache_config) {
   // Whitelist root path for tests by default
   ddl_utils::FilePathWhitelist::clear();
   ddl_utils::FilePathWhitelist::initialize(db_path, "[\"/\"]", "[\"/\"]");
@@ -121,7 +141,8 @@ QueryRunner* QueryRunner::init(const char* db_path,
                                      max_gpu_mem,
                                      reserved_gpu_mem,
                                      create_user,
-                                     create_db));
+                                     create_db,
+                                     disk_cache_config));
   return qr_instance_.get();
 }
 
@@ -136,7 +157,8 @@ QueryRunner::QueryRunner(const char* db_path,
                          const size_t max_gpu_mem,
                          const int reserved_gpu_mem,
                          const bool create_user,
-                         const bool create_db)
+                         const bool create_db,
+                         const File_Namespace::DiskCacheConfig* cache_config)
     : dispatch_queue_(std::make_unique<QueryDispatchQueue>(1)) {
   g_serialize_temp_tables = true;
   boost::filesystem::path base_path{db_path};
@@ -146,6 +168,9 @@ QueryRunner::QueryRunner(const char* db_path,
   auto data_dir = base_path / "mapd_data";
   File_Namespace::DiskCacheConfig disk_cache_config{
       (base_path / "omnisci_disk_cache").string(), File_Namespace::DiskCacheLevel::fsi};
+  if (cache_config) {
+    disk_cache_config = *cache_config;
+  }
   Catalog_Namespace::UserMetadata user;
   Catalog_Namespace::DBMetadata db;
 

diff --git a/QueryRunner/QueryRunner.h b/QueryRunner/QueryRunner.h
@@ -63,6 +63,11 @@ class QueryRunner {
                            const size_t max_gpu_mem = 0,  // use all available mem
                            const int reserved_gpu_mem = 256 << 20);
 
+  static QueryRunner* init(const File_Namespace::DiskCacheConfig* disk_cache_config,
+                           const char* db_path,
+                           const std::vector<LeafHostInfo>& string_servers = {},
+                           const std::vector<LeafHostInfo>& leaf_servers = {});
+
   static QueryRunner* init(const char* db_path,
                            const std::vector<LeafHostInfo>& string_servers,
                            const std::vector<LeafHostInfo>& leaf_servers) {
@@ -85,7 +90,8 @@ class QueryRunner {
                            const size_t max_gpu_mem = 0,  // use all available mem
                            const int reserved_gpu_mem = 256 << 20,
                            const bool create_user = false,
-                           const bool create_db = false);
+                           const bool create_db = false,
+                           const File_Namespace::DiskCacheConfig* config = nullptr);
 
   static QueryRunner* init(std::unique_ptr<Catalog_Namespace::SessionInfo>& session) {
     qr_instance_.reset(new QueryRunner(std::move(session)));
@@ -201,8 +207,8 @@ class QueryRunner {
               const size_t max_gpu_mem,
               const int reserved_gpu_mem,
               const bool create_user,
-              const bool create_db);
-
+              const bool create_db,
+              const File_Namespace::DiskCacheConfig* disk_cache_config = nullptr);
   static std::unique_ptr<QueryRunner> qr_instance_;
 
   std::shared_ptr<Catalog_Namespace::SessionInfo> session_info_;

diff --git a/Tests/DBHandlerTestHelpers.h b/Tests/DBHandlerTestHelpers.h
@@ -186,7 +186,9 @@ class DBHandlerTestFixture : public testing::Test {
     desc.add_options()("cluster",
                        po::value<std::string>(&cluster_config_file_path_),
                        "Path to data leaves list JSON file.");
-
+    desc.add_options()("use-disk-cache",
+                       po::value<bool>(&use_disk_cache_),
+                       "Enable disk cache for all tables.");
     po::variables_map vm;
     po::store(po::command_line_parser(argc, argv).options(desc).run(), vm);
     po::notify(vm);
@@ -208,8 +210,7 @@ class DBHandlerTestFixture : public testing::Test {
 
   static void TearDownTestSuite() {}
 
-  static void createDBHandler(
-      File_Namespace::DiskCacheLevel cache_level = File_Namespace::DiskCacheLevel::fsi) {
+  static void createDBHandler() {
     if (!db_handler_) {
       setupSignalHandler();
 
@@ -239,8 +240,14 @@ class DBHandlerTestFixture : public testing::Test {
       system_parameters_.omnisci_server_port = -1;
       system_parameters_.calcite_port = 3280;
 
+      File_Namespace::DiskCacheLevel cache_level{File_Namespace::DiskCacheLevel::fsi};
+      if (use_disk_cache_) {
+        cache_level = File_Namespace::DiskCacheLevel::all;
+      }
       File_Namespace::DiskCacheConfig disk_cache_config{
-          std::string(BASE_PATH) + "/omnisci_disk_cache", cache_level};
+          File_Namespace::DiskCacheConfig::getDefaultPath(std::string(BASE_PATH)),
+          cache_level};
+
       db_handler_ = std::make_unique<DBHandler>(db_leaves_,
                                                 string_leaves_,
                                                 BASE_PATH,
@@ -579,6 +586,7 @@ class DBHandlerTestFixture : public testing::Test {
 
  public:
   static std::string cluster_config_file_path_;
+  static bool use_disk_cache_;
 };
 
 TSessionId DBHandlerTestFixture::session_id_{};
@@ -598,3 +606,4 @@ std::string DBHandlerTestFixture::cluster_config_file_path_{};
 #ifdef ENABLE_GEOS
 std::string DBHandlerTestFixture::libgeos_so_filename_{};
 #endif
+bool DBHandlerTestFixture::use_disk_cache_{false};
diff --git a/Tests/DiskCacheQueryTest.cpp b/Tests/DiskCacheQueryTest.cpp
@@ -45,7 +45,8 @@ class TableTest : public DBHandlerTestFixture {
   inline static PersistentStorageMgr* psm_;
 
   static void SetUpTestSuite() {
-    DBHandlerTestFixture::createDBHandler(File_Namespace::DiskCacheLevel::all);
+    use_disk_cache_ = true;
+    DBHandlerTestFixture::SetUpTestSuite();
     cat_ = &getCatalog();
     ASSERT_NE(cat_, nullptr);
     psm_ = cat_->getDataMgr().getPersistentStorageMgr();
@@ -54,8 +55,6 @@ class TableTest : public DBHandlerTestFixture {
     ASSERT_NE(cache_, nullptr);
   }
 
-  static void TearDownTestSuite() {}
-
   void SetUp() override {
     sqlDropTable();
     cache_->clear();
@@ -111,7 +110,7 @@ class TableTest : public DBHandlerTestFixture {
   }
 };
 
-TEST_F(TableTest, DISABLED_InsertWithCache) {
+TEST_F(TableTest, InsertWithCache) {
   sqlCreateTable("(i INTEGER) WITH (fragment_size = 1)");
   const ChunkKey key1 = getChunkKeyFromTable(*cat_, default_table_name, {1, 0});
   const ChunkKey key2 = getChunkKeyFromTable(*cat_, default_table_name, {1, 1});

diff --git a/Tests/ExecuteTest.cpp b/Tests/ExecuteTest.cpp
@@ -21346,6 +21346,8 @@ int main(int argc, char** argv) {
                          ->default_value(g_use_tbb_pool)
                          ->implicit_value(true),
                      "Use TBB thread pool implementation for query dispatch.");
+  desc.add_options()("use-disk-cache",
+                     "Use the disk cache for all tables with minimum size settings.");
 
   desc.add_options()(
       "test-help",
@@ -21381,7 +21383,14 @@ int main(int argc, char** argv) {
   g_enable_window_functions = true;
   g_enable_interop = false;
 
-  QR::init(BASE_PATH);
+  File_Namespace::DiskCacheConfig disk_cache_config{};
+  if (vm.count("use-disk-cache")) {
+    disk_cache_config = File_Namespace::DiskCacheConfig{
+        File_Namespace::DiskCacheConfig::getDefaultPath(std::string(BASE_PATH)),
+        File_Namespace::DiskCacheLevel::all};
+  }
+
+  QR::init(&disk_cache_config, BASE_PATH);
   if (vm.count("with-sharding")) {
     g_shard_count = choose_shard_count();
   }

diff --git a/Tests/UpdelStorageTest.cpp b/Tests/UpdelStorageTest.cpp
@@ -1193,7 +1193,23 @@ int main(int argc, char** argv) {
   TestHelpers::init_logger_stderr_only(argc, argv);
   testing::InitGoogleTest(&argc, argv);
 
-  QR::init(BASE_PATH);
+  namespace po = boost::program_options;
+  po::options_description desc("Options");
+  // these two are here to allow passing correctly google testing parameters
+  desc.add_options()("gtest_list_tests", "list all test");
+  desc.add_options()("gtest_filter", "filters tests, use --help for details");
+  desc.add_options()("use-disk-cache",
+                     "Use the disk cache for all tables with default size settings.");
+  po::variables_map vm;
+  po::store(po::command_line_parser(argc, argv).options(desc).run(), vm);
+  po::notify(vm);
+  File_Namespace::DiskCacheConfig disk_cache_config{};
+  if (vm.count("use-disk-cache")) {
+    disk_cache_config = File_Namespace::DiskCacheConfig{
+        File_Namespace::DiskCacheConfig::getDefaultPath(std::string(BASE_PATH)),
+        File_Namespace::DiskCacheLevel::all};
+  }
+  QR::init(&disk_cache_config, BASE_PATH);
 
   // the data files for perf tests are too big to check in, so perf tests
   // are done privately in someone's dev host. prog option seems a overkill.