From bd08b3b2d3bb69d89e05cd60c596200d3ad3535d Mon Sep 17 00:00:00 2001 From: tinswzy Date: Wed, 4 Dec 2024 20:24:40 +0800 Subject: [PATCH] enhance: Add mmap file usage metric (#38211) issue: #38156 cherry-pick from https://github.com/milvus-io/milvus/pull/38193 Signed-off-by: tinswzy --- internal/core/src/mmap/Column.h | 4 ++++ internal/core/src/monitor/prometheus_client.cpp | 12 ++++++++++++ internal/core/src/monitor/prometheus_client.h | 3 +++ internal/core/src/storage/MmapChunkManager.cpp | 2 ++ 4 files changed, 21 insertions(+) diff --git a/internal/core/src/mmap/Column.h b/internal/core/src/mmap/Column.h index fb7a41bf6f704..2e9e71f31f780 100644 --- a/internal/core/src/mmap/Column.h +++ b/internal/core/src/mmap/Column.h @@ -388,11 +388,13 @@ class ColumnBase { mapped_size); milvus::monitor::internal_mmap_in_used_space_bytes_anon.Increment( mapped_size); + milvus::monitor::internal_mmap_in_used_count_anon.Increment(); } else if (mapping_type_ == MappingType::MAP_WITH_FILE) { milvus::monitor::internal_mmap_allocated_space_bytes_file.Observe( mapped_size); milvus::monitor::internal_mmap_in_used_space_bytes_file.Increment( mapped_size); + milvus::monitor::internal_mmap_in_used_count_file.Increment(); } // else: does not update metric for MAP_WITH_MANAGER, MmapChunkManagerPtr // will update metric itself. @@ -403,9 +405,11 @@ class ColumnBase { if (mapping_type_ == MappingType::MAP_WITH_ANONYMOUS) { milvus::monitor::internal_mmap_in_used_space_bytes_anon.Decrement( mapped_size); + milvus::monitor::internal_mmap_in_used_count_anon.Decrement(); } else if (mapping_type_ == MappingType::MAP_WITH_FILE) { milvus::monitor::internal_mmap_in_used_space_bytes_file.Decrement( mapped_size); + milvus::monitor::internal_mmap_in_used_count_file.Decrement(); } // else: does not update metric for MAP_WITH_MANAGER, MmapChunkManagerPtr // will update metric itself. diff --git a/internal/core/src/monitor/prometheus_client.cpp b/internal/core/src/monitor/prometheus_client.cpp index 50ca5550559a2..9b5cf5603ed83 100644 --- a/internal/core/src/monitor/prometheus_client.cpp +++ b/internal/core/src/monitor/prometheus_client.cpp @@ -196,6 +196,10 @@ std::map mmapAllocatedSpaceAnonLabel = { {"type", "anon"}}; std::map mmapAllocatedSpaceFileLabel = { {"type", "file"}}; +std::map mmapAllocatedCountAnonLabel = { + {"type", "anon"}}; +std::map mmapAllocatedCountFileLabel = { + {"type", "file"}}; DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(internal_mmap_allocated_space_bytes, "[cpp]mmap allocated space stats") @@ -218,4 +222,12 @@ DEFINE_PROMETHEUS_GAUGE(internal_mmap_in_used_space_bytes_anon, DEFINE_PROMETHEUS_GAUGE(internal_mmap_in_used_space_bytes_file, internal_mmap_in_used_space_bytes, mmapAllocatedSpaceFileLabel) +DEFINE_PROMETHEUS_GAUGE_FAMILY(internal_mmap_in_used_count, + "[cpp]mmap in used count stats") +DEFINE_PROMETHEUS_GAUGE(internal_mmap_in_used_count_anon, + internal_mmap_in_used_count, + mmapAllocatedCountAnonLabel) +DEFINE_PROMETHEUS_GAUGE(internal_mmap_in_used_count_file, + internal_mmap_in_used_count, + mmapAllocatedCountFileLabel) } // namespace milvus::monitor diff --git a/internal/core/src/monitor/prometheus_client.h b/internal/core/src/monitor/prometheus_client.h index 29b625b5e9fc2..23b4f58a87e1b 100644 --- a/internal/core/src/monitor/prometheus_client.h +++ b/internal/core/src/monitor/prometheus_client.h @@ -127,6 +127,9 @@ DECLARE_PROMETHEUS_HISTOGRAM(internal_mmap_allocated_space_bytes_file); DECLARE_PROMETHEUS_GAUGE_FAMILY(internal_mmap_in_used_space_bytes); DECLARE_PROMETHEUS_GAUGE(internal_mmap_in_used_space_bytes_anon); DECLARE_PROMETHEUS_GAUGE(internal_mmap_in_used_space_bytes_file); +DECLARE_PROMETHEUS_GAUGE_FAMILY(internal_mmap_in_used_count); +DECLARE_PROMETHEUS_GAUGE(internal_mmap_in_used_count_anon); +DECLARE_PROMETHEUS_GAUGE(internal_mmap_in_used_count_file); // search metrics DECLARE_PROMETHEUS_HISTOGRAM_FAMILY(internal_core_search_latency); diff --git a/internal/core/src/storage/MmapChunkManager.cpp b/internal/core/src/storage/MmapChunkManager.cpp index fffd1ea7d09e1..f2acc0adebfc0 100644 --- a/internal/core/src/storage/MmapChunkManager.cpp +++ b/internal/core/src/storage/MmapChunkManager.cpp @@ -73,6 +73,7 @@ MmapBlock::Init() { file_size_); milvus::monitor::internal_mmap_in_used_space_bytes_file.Increment( file_size_); + milvus::monitor::internal_mmap_in_used_count_file.Increment(); is_valid_ = true; allocated_size_.fetch_add(file_size_); } @@ -98,6 +99,7 @@ MmapBlock::Close() { allocated_size_.fetch_sub(file_size_); milvus::monitor::internal_mmap_in_used_space_bytes_file.Decrement( file_size_); + milvus::monitor::internal_mmap_in_used_count_file.Decrement(); is_valid_ = false; }