From 15233f77420d168011f007ab46d04b2da0ae4d2c Mon Sep 17 00:00:00 2001 From: Misiu Godfrey <40667992+misiugodfrey@users.noreply.github.com> Date: Thu, 3 Jun 2021 12:12:26 -0700 Subject: [PATCH] enable disk cache testing * Updated QueryRunner and DHHandlerTestHelper to allow for a --use-disk-cache flag to enable generic disk cache testing with more test suites. Tests that use QueryRunner need to be manually enabled, so UpdelStorageTest and ExecuteTest were updated to handle this option. DBHandler based tests have the option supported going forward without alterations. --- DataMgr/FileMgr/CachingFileMgr.h | 3 +++ QueryRunner/QueryRunner.cpp | 31 ++++++++++++++++++++++++++++--- QueryRunner/QueryRunner.h | 12 +++++++++--- Tests/DBHandlerTestHelpers.h | 17 +++++++++++++---- Tests/DiskCacheQueryTest.cpp | 7 +++---- Tests/ExecuteTest.cpp | 11 ++++++++++- Tests/UpdelStorageTest.cpp | 18 +++++++++++++++++- 7 files changed, 83 insertions(+), 16 deletions(-) diff --git a/DataMgr/FileMgr/CachingFileMgr.h b/DataMgr/FileMgr/CachingFileMgr.h index d6ee6a19d8..b76c149507 100644 --- a/DataMgr/FileMgr/CachingFileMgr.h +++ b/DataMgr/FileMgr/CachingFileMgr.h @@ -70,6 +70,9 @@ struct DiskCacheConfig { } return ""; } + static std::string getDefaultPath(const std::string& base_path) { + return base_path + "/omnisci_disk_cache"; + } }; inline std::string get_dir_name_for_table(int db_id, int tb_id) { diff --git a/QueryRunner/QueryRunner.cpp b/QueryRunner/QueryRunner.cpp index 68c2b7c4d9..171b765148 100644 --- a/QueryRunner/QueryRunner.cpp +++ b/QueryRunner/QueryRunner.cpp @@ -93,6 +93,25 @@ QueryRunner* QueryRunner::init(const char* db_path, reserved_gpu_mem); } +QueryRunner* QueryRunner::init(const File_Namespace::DiskCacheConfig* disk_cache_config, + const char* db_path, + const std::vector& string_servers, + const std::vector& leaf_servers) { + return QueryRunner::init(db_path, + std::string{OMNISCI_ROOT_USER}, + "HyperInteractive", + std::string{OMNISCI_DEFAULT_DB}, + string_servers, + leaf_servers, + "", + true, + 0, + 256 << 20, + false, + false, + disk_cache_config); +} + QueryRunner* QueryRunner::init(const char* db_path, const std::string& user, const std::string& pass, @@ -104,7 +123,8 @@ QueryRunner* QueryRunner::init(const char* db_path, const size_t max_gpu_mem, const int reserved_gpu_mem, const bool create_user, - const bool create_db) { + const bool create_db, + const File_Namespace::DiskCacheConfig* disk_cache_config) { // Whitelist root path for tests by default ddl_utils::FilePathWhitelist::clear(); ddl_utils::FilePathWhitelist::initialize(db_path, "[\"/\"]", "[\"/\"]"); @@ -121,7 +141,8 @@ QueryRunner* QueryRunner::init(const char* db_path, max_gpu_mem, reserved_gpu_mem, create_user, - create_db)); + create_db, + disk_cache_config)); return qr_instance_.get(); } @@ -136,7 +157,8 @@ QueryRunner::QueryRunner(const char* db_path, const size_t max_gpu_mem, const int reserved_gpu_mem, const bool create_user, - const bool create_db) + const bool create_db, + const File_Namespace::DiskCacheConfig* cache_config) : dispatch_queue_(std::make_unique(1)) { g_serialize_temp_tables = true; boost::filesystem::path base_path{db_path}; @@ -146,6 +168,9 @@ QueryRunner::QueryRunner(const char* db_path, auto data_dir = base_path / "mapd_data"; File_Namespace::DiskCacheConfig disk_cache_config{ (base_path / "omnisci_disk_cache").string(), File_Namespace::DiskCacheLevel::fsi}; + if (cache_config) { + disk_cache_config = *cache_config; + } Catalog_Namespace::UserMetadata user; Catalog_Namespace::DBMetadata db; diff --git a/QueryRunner/QueryRunner.h b/QueryRunner/QueryRunner.h index 116f90cf09..6c3524606d 100644 --- a/QueryRunner/QueryRunner.h +++ b/QueryRunner/QueryRunner.h @@ -63,6 +63,11 @@ class QueryRunner { const size_t max_gpu_mem = 0, // use all available mem const int reserved_gpu_mem = 256 << 20); + static QueryRunner* init(const File_Namespace::DiskCacheConfig* disk_cache_config, + const char* db_path, + const std::vector& string_servers = {}, + const std::vector& leaf_servers = {}); + static QueryRunner* init(const char* db_path, const std::vector& string_servers, const std::vector& leaf_servers) { @@ -85,7 +90,8 @@ class QueryRunner { const size_t max_gpu_mem = 0, // use all available mem const int reserved_gpu_mem = 256 << 20, const bool create_user = false, - const bool create_db = false); + const bool create_db = false, + const File_Namespace::DiskCacheConfig* config = nullptr); static QueryRunner* init(std::unique_ptr& session) { qr_instance_.reset(new QueryRunner(std::move(session))); @@ -201,8 +207,8 @@ class QueryRunner { const size_t max_gpu_mem, const int reserved_gpu_mem, const bool create_user, - const bool create_db); - + const bool create_db, + const File_Namespace::DiskCacheConfig* disk_cache_config = nullptr); static std::unique_ptr qr_instance_; std::shared_ptr session_info_; diff --git a/Tests/DBHandlerTestHelpers.h b/Tests/DBHandlerTestHelpers.h index fab76375da..cd18f22b9b 100644 --- a/Tests/DBHandlerTestHelpers.h +++ b/Tests/DBHandlerTestHelpers.h @@ -186,7 +186,9 @@ class DBHandlerTestFixture : public testing::Test { desc.add_options()("cluster", po::value(&cluster_config_file_path_), "Path to data leaves list JSON file."); - + desc.add_options()("use-disk-cache", + po::value(&use_disk_cache_), + "Enable disk cache for all tables."); po::variables_map vm; po::store(po::command_line_parser(argc, argv).options(desc).run(), vm); po::notify(vm); @@ -208,8 +210,7 @@ class DBHandlerTestFixture : public testing::Test { static void TearDownTestSuite() {} - static void createDBHandler( - File_Namespace::DiskCacheLevel cache_level = File_Namespace::DiskCacheLevel::fsi) { + static void createDBHandler() { if (!db_handler_) { setupSignalHandler(); @@ -239,8 +240,14 @@ class DBHandlerTestFixture : public testing::Test { system_parameters_.omnisci_server_port = -1; system_parameters_.calcite_port = 3280; + File_Namespace::DiskCacheLevel cache_level{File_Namespace::DiskCacheLevel::fsi}; + if (use_disk_cache_) { + cache_level = File_Namespace::DiskCacheLevel::all; + } File_Namespace::DiskCacheConfig disk_cache_config{ - std::string(BASE_PATH) + "/omnisci_disk_cache", cache_level}; + File_Namespace::DiskCacheConfig::getDefaultPath(std::string(BASE_PATH)), + cache_level}; + db_handler_ = std::make_unique(db_leaves_, string_leaves_, BASE_PATH, @@ -579,6 +586,7 @@ class DBHandlerTestFixture : public testing::Test { public: static std::string cluster_config_file_path_; + static bool use_disk_cache_; }; TSessionId DBHandlerTestFixture::session_id_{}; @@ -598,3 +606,4 @@ std::string DBHandlerTestFixture::cluster_config_file_path_{}; #ifdef ENABLE_GEOS std::string DBHandlerTestFixture::libgeos_so_filename_{}; #endif +bool DBHandlerTestFixture::use_disk_cache_{false}; diff --git a/Tests/DiskCacheQueryTest.cpp b/Tests/DiskCacheQueryTest.cpp index 41c6626ff4..b52881cc13 100644 --- a/Tests/DiskCacheQueryTest.cpp +++ b/Tests/DiskCacheQueryTest.cpp @@ -45,7 +45,8 @@ class TableTest : public DBHandlerTestFixture { inline static PersistentStorageMgr* psm_; static void SetUpTestSuite() { - DBHandlerTestFixture::createDBHandler(File_Namespace::DiskCacheLevel::all); + use_disk_cache_ = true; + DBHandlerTestFixture::SetUpTestSuite(); cat_ = &getCatalog(); ASSERT_NE(cat_, nullptr); psm_ = cat_->getDataMgr().getPersistentStorageMgr(); @@ -54,8 +55,6 @@ class TableTest : public DBHandlerTestFixture { ASSERT_NE(cache_, nullptr); } - static void TearDownTestSuite() {} - void SetUp() override { sqlDropTable(); cache_->clear(); @@ -111,7 +110,7 @@ class TableTest : public DBHandlerTestFixture { } }; -TEST_F(TableTest, DISABLED_InsertWithCache) { +TEST_F(TableTest, InsertWithCache) { sqlCreateTable("(i INTEGER) WITH (fragment_size = 1)"); const ChunkKey key1 = getChunkKeyFromTable(*cat_, default_table_name, {1, 0}); const ChunkKey key2 = getChunkKeyFromTable(*cat_, default_table_name, {1, 1}); diff --git a/Tests/ExecuteTest.cpp b/Tests/ExecuteTest.cpp index bb86a5c661..29b1a0163f 100644 --- a/Tests/ExecuteTest.cpp +++ b/Tests/ExecuteTest.cpp @@ -21346,6 +21346,8 @@ int main(int argc, char** argv) { ->default_value(g_use_tbb_pool) ->implicit_value(true), "Use TBB thread pool implementation for query dispatch."); + desc.add_options()("use-disk-cache", + "Use the disk cache for all tables with minimum size settings."); desc.add_options()( "test-help", @@ -21381,7 +21383,14 @@ int main(int argc, char** argv) { g_enable_window_functions = true; g_enable_interop = false; - QR::init(BASE_PATH); + File_Namespace::DiskCacheConfig disk_cache_config{}; + if (vm.count("use-disk-cache")) { + disk_cache_config = File_Namespace::DiskCacheConfig{ + File_Namespace::DiskCacheConfig::getDefaultPath(std::string(BASE_PATH)), + File_Namespace::DiskCacheLevel::all}; + } + + QR::init(&disk_cache_config, BASE_PATH); if (vm.count("with-sharding")) { g_shard_count = choose_shard_count(); } diff --git a/Tests/UpdelStorageTest.cpp b/Tests/UpdelStorageTest.cpp index 463545fafe..466f212d6b 100644 --- a/Tests/UpdelStorageTest.cpp +++ b/Tests/UpdelStorageTest.cpp @@ -1193,7 +1193,23 @@ int main(int argc, char** argv) { TestHelpers::init_logger_stderr_only(argc, argv); testing::InitGoogleTest(&argc, argv); - QR::init(BASE_PATH); + namespace po = boost::program_options; + po::options_description desc("Options"); + // these two are here to allow passing correctly google testing parameters + desc.add_options()("gtest_list_tests", "list all test"); + desc.add_options()("gtest_filter", "filters tests, use --help for details"); + desc.add_options()("use-disk-cache", + "Use the disk cache for all tables with default size settings."); + po::variables_map vm; + po::store(po::command_line_parser(argc, argv).options(desc).run(), vm); + po::notify(vm); + File_Namespace::DiskCacheConfig disk_cache_config{}; + if (vm.count("use-disk-cache")) { + disk_cache_config = File_Namespace::DiskCacheConfig{ + File_Namespace::DiskCacheConfig::getDefaultPath(std::string(BASE_PATH)), + File_Namespace::DiskCacheLevel::all}; + } + QR::init(&disk_cache_config, BASE_PATH); // the data files for perf tests are too big to check in, so perf tests // are done privately in someone's dev host. prog option seems a overkill.