From 4cdb8ba541e6b82016ef09be62e7d56dc1a8a81f Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Mon, 30 Sep 2024 01:45:00 -0700 Subject: [PATCH] Make statistics extensible Signed-off-by: Yang Zhang --- db/db_test.cc | 8 +-- include/rocksdb/statistics.h | 2 + java/rocksjni/statisticsjni.h | 2 +- monitoring/statistics.cc | 108 +++++++++++++++++++++------------- monitoring/statistics_impl.h | 33 +++++------ options/customizable_test.cc | 18 ++---- 6 files changed, 95 insertions(+), 76 deletions(-) diff --git a/db/db_test.cc b/db/db_test.cc index 95a35a82b45..646e3101f50 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -36,6 +36,7 @@ #include "db/write_batch_internal.h" #include "env/mock_env.h" #include "file/filename.h" +#include "monitoring/statistics_impl.h" #include "monitoring/thread_status_util.h" #include "port/port.h" #include "port/stack_trace.h" @@ -1161,7 +1162,6 @@ class DelayFilterFactory : public CompactionFilterFactory { }; } // anonymous namespace - static std::string CompressibleString(Random* rnd, int len) { std::string r; test::CompressibleString(rnd, 0.8, len, &r); @@ -4351,7 +4351,6 @@ TEST_F(DBTest, ConcurrentMemtableNotSupported) { ASSERT_NOK(db_->CreateColumnFamily(cf_options, "name", &handle)); } - TEST_F(DBTest, SanitizeNumThreads) { for (int attempt = 0; attempt < 2; attempt++) { const size_t kTotalTasks = 8; @@ -5721,7 +5720,6 @@ TEST_F(DBTest, FileCreationRandomFailure) { } } - TEST_F(DBTest, DynamicMiscOptions) { // Test max_sequential_skip_in_iterations Options options; @@ -6101,7 +6099,7 @@ TEST_P(DBTestWithParam, FilterCompactionTimeTest) { // CPUMicros() is not supported. See WinClock::CPUMicros(). TEST_P(DBTestWithParam, CompactionTotalTimeTest) { int record_count = 0; - class TestStatistics : public StatisticsImpl { + class TestStatistics : public StatisticsImpl<> { public: explicit TestStatistics(int* record_count) : StatisticsImpl(nullptr), record_count_(record_count) {} @@ -7178,7 +7176,6 @@ TEST_F(DBTest, ReusePinnableSlice) { 1); } - TEST_F(DBTest, DeletingOldWalAfterDrop) { ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( {{"Test:AllowFlushes", "DBImpl::BGWorkFlush"}, @@ -7303,7 +7300,6 @@ TEST_F(DBTest, LargeBlockSizeTest) { ASSERT_NOK(TryReopenWithColumnFamilies({"default", "pikachu"}, options)); } - TEST_F(DBTest, CreationTimeOfOldestFile) { const int kNumKeysPerFile = 32; const int kNumLevelFiles = 2; diff --git a/include/rocksdb/statistics.h b/include/rocksdb/statistics.h index de3052a1f5b..618f438f2ec 100644 --- a/include/rocksdb/statistics.h +++ b/include/rocksdb/statistics.h @@ -794,6 +794,8 @@ class Statistics : public Customizable { }; // Create a concrete DBStatistics object +template std::shared_ptr CreateDBStatistics(); } // namespace ROCKSDB_NAMESPACE diff --git a/java/rocksjni/statisticsjni.h b/java/rocksjni/statisticsjni.h index 3262b296cf5..0440497ceab 100644 --- a/java/rocksjni/statisticsjni.h +++ b/java/rocksjni/statisticsjni.h @@ -18,7 +18,7 @@ namespace ROCKSDB_NAMESPACE { -class StatisticsJni : public StatisticsImpl { +class StatisticsJni : public StatisticsImpl<> { public: StatisticsJni(std::shared_ptr stats); StatisticsJni(std::shared_ptr stats, diff --git a/monitoring/statistics.cc b/monitoring/statistics.cc index d7c9642e8e1..95fc36ae12f 100644 --- a/monitoring/statistics.cc +++ b/monitoring/statistics.cc @@ -343,17 +343,14 @@ const std::vector> HistogramsNameMap = { "rocksdb.table.open.prefetch.tail.read.bytes"}, }; -std::shared_ptr CreateDBStatistics() { - return std::make_shared(nullptr); -} - static int RegisterBuiltinStatistics(ObjectLibrary& library, const std::string& /*arg*/) { library.AddFactory( - StatisticsImpl::kClassName(), + StatisticsImpl::kClassName(), [](const std::string& /*uri*/, std::unique_ptr* guard, std::string* /* errmsg */) { - guard->reset(new StatisticsImpl(nullptr)); + guard->reset( + new StatisticsImpl(nullptr)); return guard->get(); }); return 1; @@ -367,8 +364,10 @@ Status Statistics::CreateFromString(const ConfigOptions& config_options, RegisterBuiltinStatistics(*(ObjectLibrary::Default().get()), ""); }); Status s; - if (id == "" || id == StatisticsImpl::kClassName()) { - result->reset(new StatisticsImpl(nullptr)); + if (id == "" || + id == StatisticsImpl::kClassName()) { + result->reset( + new StatisticsImpl(nullptr)); } else if (id == kNullptrString) { result->reset(); } else { @@ -383,20 +382,27 @@ static std::unordered_map stats_type_info = { OptionTypeFlags::kCompareNever)}, }; -StatisticsImpl::StatisticsImpl(std::shared_ptr stats) +template +StatisticsImpl::StatisticsImpl( + std::shared_ptr stats) : stats_(std::move(stats)) { RegisterOptions("StatisticsOptions", &stats_, &stats_type_info); } -StatisticsImpl::~StatisticsImpl() = default; +template +StatisticsImpl::~StatisticsImpl() = default; -uint64_t StatisticsImpl::getTickerCount(uint32_t tickerType) const { +template +uint64_t StatisticsImpl::getTickerCount( + uint32_t tickerType) const { MutexLock lock(&aggregate_lock_); return getTickerCountLocked(tickerType); } -uint64_t StatisticsImpl::getTickerCountLocked(uint32_t tickerType) const { - assert(tickerType < TICKER_ENUM_MAX); +template +uint64_t StatisticsImpl::getTickerCountLocked( + uint32_t tickerType) const { + assert(tickerType < TICKER_MAX); uint64_t res = 0; for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { res += per_core_stats_.AccessAtCore(core_idx)->tickers_[tickerType]; @@ -404,15 +410,18 @@ uint64_t StatisticsImpl::getTickerCountLocked(uint32_t tickerType) const { return res; } -void StatisticsImpl::histogramData(uint32_t histogramType, - HistogramData* const data) const { +template +void StatisticsImpl::histogramData( + uint32_t histogramType, HistogramData* const data) const { MutexLock lock(&aggregate_lock_); getHistogramImplLocked(histogramType)->Data(data); } -std::unique_ptr StatisticsImpl::getHistogramImplLocked( +template +std::unique_ptr +StatisticsImpl::getHistogramImplLocked( uint32_t histogramType) const { - assert(histogramType < HISTOGRAM_ENUM_MAX); + assert(histogramType < HISTOGRAM_MAX); std::unique_ptr res_hist(new HistogramImpl()); for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { res_hist->Merge( @@ -421,23 +430,29 @@ std::unique_ptr StatisticsImpl::getHistogramImplLocked( return res_hist; } -std::string StatisticsImpl::getHistogramString(uint32_t histogramType) const { +template +std::string StatisticsImpl::getHistogramString( + uint32_t histogramType) const { MutexLock lock(&aggregate_lock_); return getHistogramImplLocked(histogramType)->ToString(); } -void StatisticsImpl::setTickerCount(uint32_t tickerType, uint64_t count) { +template +void StatisticsImpl::setTickerCount( + uint32_t tickerType, uint64_t count) { { MutexLock lock(&aggregate_lock_); setTickerCountLocked(tickerType, count); } - if (stats_ && tickerType < TICKER_ENUM_MAX) { + if (stats_ && tickerType < TICKER_MAX) { stats_->setTickerCount(tickerType, count); } } -void StatisticsImpl::setTickerCountLocked(uint32_t tickerType, uint64_t count) { - assert(tickerType < TICKER_ENUM_MAX); +template +void StatisticsImpl::setTickerCountLocked( + uint32_t tickerType, uint64_t count) { + assert(tickerType < TICKER_MAX); for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { if (core_idx == 0) { per_core_stats_.AccessAtCore(core_idx)->tickers_[tickerType] = count; @@ -447,28 +462,32 @@ void StatisticsImpl::setTickerCountLocked(uint32_t tickerType, uint64_t count) { } } -uint64_t StatisticsImpl::getAndResetTickerCount(uint32_t tickerType) { +template +uint64_t StatisticsImpl::getAndResetTickerCount( + uint32_t tickerType) { uint64_t sum = 0; { MutexLock lock(&aggregate_lock_); - assert(tickerType < TICKER_ENUM_MAX); + assert(tickerType < TICKER_MAX); for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { sum += per_core_stats_.AccessAtCore(core_idx)->tickers_[tickerType].exchange( 0, std::memory_order_relaxed); } } - if (stats_ && tickerType < TICKER_ENUM_MAX) { + if (stats_ && tickerType < TICKER_MAX) { stats_->setTickerCount(tickerType, 0); } return sum; } -void StatisticsImpl::recordTick(uint32_t tickerType, uint64_t count) { +template +void StatisticsImpl::recordTick(uint32_t tickerType, + uint64_t count) { if (get_stats_level() <= StatsLevel::kExceptTickers) { return; } - if (tickerType < TICKER_ENUM_MAX) { + if (tickerType < TICKER_MAX) { per_core_stats_.Access()->tickers_[tickerType].fetch_add( count, std::memory_order_relaxed); if (stats_) { @@ -479,23 +498,26 @@ void StatisticsImpl::recordTick(uint32_t tickerType, uint64_t count) { } } -void StatisticsImpl::recordInHistogram(uint32_t histogramType, uint64_t value) { - assert(histogramType < HISTOGRAM_ENUM_MAX); +template +void StatisticsImpl::recordInHistogram( + uint32_t histogramType, uint64_t value) { + assert(histogramType < HISTOGRAM_MAX); if (get_stats_level() <= StatsLevel::kExceptHistogramOrTimers) { return; } per_core_stats_.Access()->histograms_[histogramType].Add(value); - if (stats_ && histogramType < HISTOGRAM_ENUM_MAX) { + if (stats_ && histogramType < HISTOGRAM_MAX) { stats_->recordInHistogram(histogramType, value); } } -Status StatisticsImpl::Reset() { +template +Status StatisticsImpl::Reset() { MutexLock lock(&aggregate_lock_); - for (uint32_t i = 0; i < TICKER_ENUM_MAX; ++i) { + for (uint32_t i = 0; i < TICKER_MAX; ++i) { setTickerCountLocked(i, 0); } - for (uint32_t i = 0; i < HISTOGRAM_ENUM_MAX; ++i) { + for (uint32_t i = 0; i < HISTOGRAM_MAX; ++i) { for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { per_core_stats_.AccessAtCore(core_idx)->histograms_[i].Clear(); } @@ -510,19 +532,20 @@ const int kTmpStrBufferSize = 200; } // namespace -std::string StatisticsImpl::ToString() const { +template +std::string StatisticsImpl::ToString() const { MutexLock lock(&aggregate_lock_); std::string res; res.reserve(20000); for (const auto& t : TickersNameMap) { - assert(t.first < TICKER_ENUM_MAX); + assert(t.first < TICKER_MAX); char buffer[kTmpStrBufferSize]; snprintf(buffer, kTmpStrBufferSize, "%s COUNT : %" PRIu64 "\n", t.second.c_str(), getTickerCountLocked(t.first)); res.append(buffer); } for (const auto& h : HistogramsNameMap) { - assert(h.first < HISTOGRAM_ENUM_MAX); + assert(h.first < HISTOGRAM_MAX); char buffer[kTmpStrBufferSize]; HistogramData hData; getHistogramImplLocked(h.first)->Data(&hData); @@ -544,7 +567,8 @@ std::string StatisticsImpl::ToString() const { return res; } -bool StatisticsImpl::getTickerMap( +template +bool StatisticsImpl::getTickerMap( std::map* stats_map) const { assert(stats_map); if (!stats_map) { @@ -553,14 +577,18 @@ bool StatisticsImpl::getTickerMap( stats_map->clear(); MutexLock lock(&aggregate_lock_); for (const auto& t : TickersNameMap) { - assert(t.first < TICKER_ENUM_MAX); + assert(t.first < TICKER_MAX); (*stats_map)[t.second.c_str()] = getTickerCountLocked(t.first); } return true; } -bool StatisticsImpl::HistEnabledForType(uint32_t type) const { - return type < HISTOGRAM_ENUM_MAX; +template +bool StatisticsImpl::HistEnabledForType( + uint32_t type) const { + return type < HISTOGRAM_MAX; } +template class StatisticsImpl; + } // namespace ROCKSDB_NAMESPACE diff --git a/monitoring/statistics_impl.h b/monitoring/statistics_impl.h index e0dc29d2846..754db76e927 100644 --- a/monitoring/statistics_impl.h +++ b/monitoring/statistics_impl.h @@ -29,16 +29,8 @@ namespace ROCKSDB_NAMESPACE { -enum TickersInternal : uint32_t { - INTERNAL_TICKER_ENUM_START = TICKER_ENUM_MAX, - INTERNAL_TICKER_ENUM_MAX -}; - -enum HistogramsInternal : uint32_t { - INTERNAL_HISTOGRAM_START = HISTOGRAM_ENUM_MAX, - INTERNAL_HISTOGRAM_ENUM_MAX -}; - +template class StatisticsImpl : public Statistics { public: StatisticsImpl(std::shared_ptr stats); @@ -84,14 +76,13 @@ class StatisticsImpl : public Statistics { // // Alignment attributes expand to nothing depending on the platform struct ALIGN_AS(CACHE_LINE_SIZE) StatisticsData { - std::atomic_uint_fast64_t tickers_[INTERNAL_TICKER_ENUM_MAX] = {{0}}; - HistogramImpl histograms_[INTERNAL_HISTOGRAM_ENUM_MAX]; + std::atomic_uint_fast64_t tickers_[TICKER_MAX] = {{0}}; + HistogramImpl histograms_[HISTOGRAM_MAX]; #ifndef HAVE_ALIGNED_NEW - char - padding[(CACHE_LINE_SIZE - - (INTERNAL_TICKER_ENUM_MAX * sizeof(std::atomic_uint_fast64_t) + - INTERNAL_HISTOGRAM_ENUM_MAX * sizeof(HistogramImpl)) % - CACHE_LINE_SIZE)] ROCKSDB_FIELD_UNUSED; + char padding[(CACHE_LINE_SIZE - + (TICKER_MAX * sizeof(std::atomic_uint_fast64_t) + + HISTOGRAM_MAX * sizeof(HistogramImpl)) % + CACHE_LINE_SIZE)] ROCKSDB_FIELD_UNUSED; #endif void* operator new(size_t s) { return port::cacheline_aligned_alloc(s); } void* operator new[](size_t s) { return port::cacheline_aligned_alloc(s); } @@ -141,4 +132,12 @@ inline void SetTickerCount(Statistics* statistics, uint32_t ticker_type, } } +template +std::shared_ptr CreateDBStatistics() { + return std::make_shared>(nullptr); +} + +template std::shared_ptr +CreateDBStatistics(); + } // namespace ROCKSDB_NAMESPACE diff --git a/options/customizable_test.cc b/options/customizable_test.cc index 696f1b25edf..099f696ba0a 100644 --- a/options/customizable_test.cc +++ b/options/customizable_test.cc @@ -18,6 +18,7 @@ #include "db/db_test_util.h" #include "memory/jemalloc_nodump_allocator.h" #include "memory/memkind_kmem_allocator.h" +#include "monitoring/statistics_impl.h" #include "options/options_helper.h" #include "options/options_parser.h" #include "port/stack_trace.h" @@ -78,9 +79,7 @@ class TestCustomizable : public Customizable { public: TestCustomizable(const std::string& name) : name_(name) {} // Method to allow CheckedCast to work for this class - static const char* kClassName() { - return "TestCustomizable"; - } + static const char* kClassName() { return "TestCustomizable"; } const char* Name() const override { return name_.c_str(); } static const char* Type() { return "test.custom"; } @@ -611,10 +610,9 @@ TEST_F(CustomizableTest, PrepareOptionsTest) { namespace { static std::unordered_map inner_option_info = { - {"inner", - OptionTypeInfo::AsCustomSharedPtr( - 0, OptionVerificationType::kNormal, OptionTypeFlags::kStringNameOnly)} -}; + {"inner", OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kNormal, + OptionTypeFlags::kStringNameOnly)}}; struct InnerOptions { static const char* kName() { return "InnerOptions"; } @@ -939,7 +937,6 @@ TEST_F(CustomizableTest, NoNameTest) { ASSERT_EQ(copts->cu, nullptr); } - TEST_F(CustomizableTest, IgnoreUnknownObjects) { ConfigOptions ignore = config_options_; std::shared_ptr shared; @@ -1223,7 +1220,6 @@ TEST_F(CustomizableTest, CreateManagedObjects) { ASSERT_EQ(mc1, obj); } - namespace { class TestSecondaryCache : public SecondaryCache { public: @@ -1257,7 +1253,7 @@ class TestSecondaryCache : public SecondaryCache { std::string GetPrintableOptions() const override { return ""; } }; -class TestStatistics : public StatisticsImpl { +class TestStatistics : public StatisticsImpl<> { public: TestStatistics() : StatisticsImpl(nullptr) {} const char* Name() const override { return kClassName(); } @@ -1347,8 +1343,6 @@ class DummyFileSystem : public FileSystemWrapper { const char* Name() const override { return kClassName(); } }; - - class MockTablePropertiesCollectorFactory : public TablePropertiesCollectorFactory { private: