From 8370caa4a677d7d851fa92d73293249ab6c9f595 Mon Sep 17 00:00:00 2001 From: aoiasd <45024769+aoiasd@users.noreply.github.com> Date: Tue, 29 Oct 2024 17:38:22 +0800 Subject: [PATCH] enhance: [Cherry-pick]Add collection name label for some metric (#36951) (#37159) pr: https://github.com/milvus-io/milvus/pull/36951 Signed-off-by: aoiasd --- internal/datacoord/index_meta.go | 2 +- internal/datacoord/meta.go | 2 +- internal/querynodev2/metrics_info.go | 10 ++-- internal/querynodev2/segments/collection.go | 11 +++++ .../segments/mock_collection_manager.go | 47 +++++++++++++++++++ pkg/metrics/datacoord_metrics.go | 4 +- pkg/metrics/querynode_metrics.go | 1 + 7 files changed, 70 insertions(+), 7 deletions(-) diff --git a/internal/datacoord/index_meta.go b/internal/datacoord/index_meta.go index a3c488bff5408..ec2e7c9c7e118 100644 --- a/internal/datacoord/index_meta.go +++ b/internal/datacoord/index_meta.go @@ -800,7 +800,7 @@ func (m *indexMeta) SetStoredIndexFileSizeMetric(collections map[UniqueID]*colle for _, segmentIdx := range m.buildID2SegmentIndex { coll, ok := collections[segmentIdx.CollectionID] if ok { - metrics.DataCoordStoredIndexFilesSize.WithLabelValues(coll.DatabaseName, + metrics.DataCoordStoredIndexFilesSize.WithLabelValues(coll.DatabaseName, coll.Schema.GetName(), fmt.Sprint(segmentIdx.CollectionID), fmt.Sprint(segmentIdx.SegmentID)).Set(float64(segmentIdx.IndexSize)) total += segmentIdx.IndexSize } diff --git a/internal/datacoord/meta.go b/internal/datacoord/meta.go index b940a9ceb667a..b1138c8b5f354 100644 --- a/internal/datacoord/meta.go +++ b/internal/datacoord/meta.go @@ -437,7 +437,7 @@ func (m *meta) GetQuotaInfo() *metricsinfo.DataCoordQuotaMetrics { coll, ok := m.collections[collectionID] if ok { for state, rows := range statesRows { - metrics.DataCoordNumStoredRows.WithLabelValues(coll.DatabaseName, fmt.Sprint(collectionID), state.String()).Set(float64(rows)) + metrics.DataCoordNumStoredRows.WithLabelValues(coll.DatabaseName, fmt.Sprint(collectionID), coll.Schema.GetName(), state.String()).Set(float64(rows)) } } } diff --git a/internal/querynodev2/metrics_info.go b/internal/querynodev2/metrics_info.go index f60a07e036f9c..151b4e03fba31 100644 --- a/internal/querynodev2/metrics_info.go +++ b/internal/querynodev2/metrics_info.go @@ -59,7 +59,7 @@ func getQuotaMetrics(node *QueryNode) (*metricsinfo.QueryNodeQuotaMetrics, error } minTsafeChannel, minTsafe := node.tSafeManager.Min() - collections := node.manager.Collection.List() + collections := node.manager.Collection.ListWithName() nodeID := fmt.Sprint(node.GetNodeID()) metrics.QueryNodeNumEntities.Reset() @@ -70,7 +70,7 @@ func getQuotaMetrics(node *QueryNode) (*metricsinfo.QueryNodeQuotaMetrics, error growingGroupByCollection := lo.GroupBy(growingSegments, func(seg segments.Segment) int64 { return seg.Collection() }) - for _, collection := range collections { + for collection := range collections { segs := growingGroupByCollection[collection] size := lo.SumBy(segs, func(seg segments.Segment) int64 { return seg.MemSize() @@ -90,6 +90,7 @@ func getQuotaMetrics(node *QueryNode) (*metricsinfo.QueryNodeQuotaMetrics, error segment := segs[0] metrics.QueryNodeNumEntities.WithLabelValues( segment.DatabaseName(), + collections[segment.Collection()], nodeID, fmt.Sprint(segment.Collection()), fmt.Sprint(segment.Partition()), @@ -101,7 +102,7 @@ func getQuotaMetrics(node *QueryNode) (*metricsinfo.QueryNodeQuotaMetrics, error sealedGroupByCollection := lo.GroupBy(sealedSegments, func(seg segments.Segment) int64 { return seg.Collection() }) - for _, collection := range collections { + for collection := range collections { segs := sealedGroupByCollection[collection] size := lo.SumBy(segs, func(seg segments.Segment) int64 { return seg.MemSize() @@ -119,6 +120,7 @@ func getQuotaMetrics(node *QueryNode) (*metricsinfo.QueryNodeQuotaMetrics, error segment := segs[0] metrics.QueryNodeNumEntities.WithLabelValues( segment.DatabaseName(), + collections[segment.Collection()], nodeID, fmt.Sprint(segment.Collection()), fmt.Sprint(segment.Partition()), @@ -148,7 +150,7 @@ func getQuotaMetrics(node *QueryNode) (*metricsinfo.QueryNodeQuotaMetrics, error GrowingSegmentsSize: totalGrowingSize, Effect: metricsinfo.NodeEffect{ NodeID: node.GetNodeID(), - CollectionIDs: collections, + CollectionIDs: lo.Keys(collections), }, DeleteBufferInfo: metricsinfo.DeleteBufferInfo{ CollectionDeleteBufferNum: deleteBufferNum, diff --git a/internal/querynodev2/segments/collection.go b/internal/querynodev2/segments/collection.go index 11becda6d16fa..51e6bab37326e 100644 --- a/internal/querynodev2/segments/collection.go +++ b/internal/querynodev2/segments/collection.go @@ -48,6 +48,7 @@ import ( type CollectionManager interface { List() []int64 + ListWithName() map[int64]string Get(collectionID int64) *Collection PutOrRef(collectionID int64, schema *schemapb.CollectionSchema, meta *segcorepb.CollectionIndexMeta, loadMeta *querypb.LoadMetaInfo) Ref(collectionID int64, count uint32) bool @@ -75,6 +76,16 @@ func (m *collectionManager) List() []int64 { return lo.Keys(m.collections) } +// return all collections by map id --> name +func (m *collectionManager) ListWithName() map[int64]string { + m.mut.RLock() + defer m.mut.RUnlock() + + return lo.MapValues(m.collections, func(coll *Collection, _ int64) string { + return coll.Schema().GetName() + }) +} + func (m *collectionManager) Get(collectionID int64) *Collection { m.mut.RLock() defer m.mut.RUnlock() diff --git a/internal/querynodev2/segments/mock_collection_manager.go b/internal/querynodev2/segments/mock_collection_manager.go index 049b326c714b1..c2e381bb9ea77 100644 --- a/internal/querynodev2/segments/mock_collection_manager.go +++ b/internal/querynodev2/segments/mock_collection_manager.go @@ -110,6 +110,53 @@ func (_c *MockCollectionManager_List_Call) RunAndReturn(run func() []int64) *Moc return _c } +// ListWithName provides a mock function with given fields: +func (_m *MockCollectionManager) ListWithName() map[int64]string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ListWithName") + } + + var r0 map[int64]string + if rf, ok := ret.Get(0).(func() map[int64]string); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[int64]string) + } + } + + return r0 +} + +// MockCollectionManager_ListWithName_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListWithName' +type MockCollectionManager_ListWithName_Call struct { + *mock.Call +} + +// ListWithName is a helper method to define mock.On call +func (_e *MockCollectionManager_Expecter) ListWithName() *MockCollectionManager_ListWithName_Call { + return &MockCollectionManager_ListWithName_Call{Call: _e.mock.On("ListWithName")} +} + +func (_c *MockCollectionManager_ListWithName_Call) Run(run func()) *MockCollectionManager_ListWithName_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockCollectionManager_ListWithName_Call) Return(_a0 map[int64]string) *MockCollectionManager_ListWithName_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockCollectionManager_ListWithName_Call) RunAndReturn(run func() map[int64]string) *MockCollectionManager_ListWithName_Call { + _c.Call.Return(run) + return _c +} + // PutOrRef provides a mock function with given fields: collectionID, schema, meta, loadMeta func (_m *MockCollectionManager) PutOrRef(collectionID int64, schema *schemapb.CollectionSchema, meta *segcorepb.CollectionIndexMeta, loadMeta *querypb.LoadMetaInfo) { _m.Called(collectionID, schema, meta, loadMeta) diff --git a/pkg/metrics/datacoord_metrics.go b/pkg/metrics/datacoord_metrics.go index 30b78dcad31c9..92e851c77a577 100644 --- a/pkg/metrics/datacoord_metrics.go +++ b/pkg/metrics/datacoord_metrics.go @@ -95,6 +95,7 @@ var ( }, []string{ databaseLabelName, collectionIDLabelName, + collectionName, segmentStateLabelName, }) @@ -162,6 +163,7 @@ var ( Help: "index files size of the segments", }, []string{ databaseLabelName, + collectionName, collectionIDLabelName, segmentIDLabelName, }) @@ -390,7 +392,7 @@ func CleanupDataCoordSegmentMetrics(dbName string, collectionID int64, segmentID collectionIDLabelName: fmt.Sprint(collectionID), segmentIDLabelName: fmt.Sprint(segmentID), }) - DataCoordStoredIndexFilesSize.Delete(prometheus.Labels{ + DataCoordStoredIndexFilesSize.DeletePartialMatch(prometheus.Labels{ databaseLabelName: dbName, collectionIDLabelName: fmt.Sprint(collectionID), segmentIDLabelName: fmt.Sprint(segmentID), diff --git a/pkg/metrics/querynode_metrics.go b/pkg/metrics/querynode_metrics.go index 41f7c2873e569..f11ab8f6c2eff 100644 --- a/pkg/metrics/querynode_metrics.go +++ b/pkg/metrics/querynode_metrics.go @@ -439,6 +439,7 @@ var ( Help: "number of entities which can be searched/queried, clustered by collection, partition and state", }, []string{ databaseLabelName, + collectionName, nodeIDLabelName, collectionIDLabelName, partitionIDLabelName,