diff --git a/deployments/monitor/grafana/milvus-dashboard.json b/deployments/monitor/grafana/milvus-dashboard.json index d15750edf0c68..3397707e2c2c3 100644 --- a/deployments/monitor/grafana/milvus-dashboard.json +++ b/deployments/monitor/grafana/milvus-dashboard.json @@ -3,7 +3,10 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -18,1700 +21,2623 @@ } ] }, + "description": "Recommend grafana with v8.5.20", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 34, - "iteration": 1667987821492, + "id": 110, "links": [], "liveNow": false, "panels": [ { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 123248, - "panels": [], - "title": "Proxy", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "collapsed": true, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "per-second increasing rate of searching vectors.", - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 6, - "w": 8, + "h": 1, + "w": 24, "x": 0, - "y": 1 - }, - "hiddenSeries": false, - "id": 123250, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true + "y": 0 }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "id": 123399, + "panels": [ { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_search_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Search Vector Count Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3414", - "format": "short", - "logBase": 1, - "min": "0", - "show": true + "description": "Slow query count in 1 minute", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 123450, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "increase(milvus_proxy_slow_query_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Slow Query (in 1m)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:536", + "decimals": 0, + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:537", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { - "$$hashKey": "object:3415", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "per-second increasing rate of inserting vectors.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 1 - }, - "hiddenSeries": false, - "id": 123315, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_insert_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Insert Vector Count Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3414", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3415", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of search request over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 1 - }, - "hiddenSeries": false, - "id": 123264, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "description": "", + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 1 + }, + "hiddenSeries": false, + "id": 123397, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(milvus_proxy_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"success\"}[1m])) by(function_name) ", + "interval": "", + "legendFormat": "{{function_name}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Successful Requests (in 1m)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:319", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:320", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "hide": false, - "interval": "", - "legendFormat": "p99-{{query_type}}-{{pod}}-{{node_id}}", - "refId": "A" + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 1 + }, + "hiddenSeries": false, + "id": 123411, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(milvus_proxy_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status!~\"success|total\"}[1m])) by(function_name) ", + "interval": "", + "legendFormat": "{{function_name}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Failed Requests (in 1m)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:319", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:320", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_sq_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{query_type}}-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Search Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3538", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3539", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of search request for a specified collection over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 7 - }, - "hiddenSeries": false, - "id": 123377, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "description": "The 99th percentile and average latency of mutation request over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 123405, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{msg_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(milvus_proxy_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{msg_type}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "hide": false, + "legendFormat": "max-{{pod}}-{{node-id}}-{{msg_type}}", + "range": true, + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Mutation Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3538", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3539", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_collection_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])))", - "hide": false, - "interval": "", - "legendFormat": "p99-{{query_type}}-{{pod}}-{{node_id}}", - "refId": "A" + "description": "The 99th percentile and average latency of search request over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 9 + }, + "hiddenSeries": false, + "id": 123404, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le,pod, node_id) (rate(milvus_proxy_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", query_type=\"search\"}[2m])))", + "hide": false, + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(milvus_proxy_sq_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", query_type=\"search\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_sq_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", query_type=\"search\"}[2m])) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum by (le,pod, node_id) (rate(milvus_proxy_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", query_type=\"search\"}[2m])))", + "hide": false, + "legendFormat": "max-{{pod}}-{{node-id}}", + "range": true, + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Search Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3538", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3539", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_collection_sq_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_collection_sq_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, query_type)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{query_type}}-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Collection Search Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3538", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3539", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of mutation request over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 7 - }, - "hiddenSeries": false, - "id": 123320, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "description": "The 99th percentile and average latency of query request over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 9 + }, + "hiddenSeries": false, + "id": 123410, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", query_type=\"query\"}[2m])))", + "hide": false, + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(milvus_proxy_sq_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", query_type=\"query\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_sq_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", query_type=\"query\"}[2m])) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum by (le, pod, node_id) (rate(milvus_proxy_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", query_type=\"query\"}[2m])))", + "hide": false, + "legendFormat": "max-{{pod}}-{{node_id}}", + "range": true, + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Query Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3538", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3539", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{msg_type}}-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 123402, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(milvus_rootcoord_quota_states{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (quota_states)", + "interval": "", + "legendFormat": "{{quota_states}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "RootCoord Quota State", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:376", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:377", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{msg_type}}-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Mutation Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3538", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3539", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of mutation request for a specified collection over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 7 - }, - "hiddenSeries": false, - "id": 123378, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "description": "Rate limit request per second by status", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 17 + }, + "hiddenSeries": false, + "id": 123447, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(milvus_proxy_rate_limit_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"fail\"}[1m])) by (msg_type)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{msg_type}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "RateLimit Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:376", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:377", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_collection_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}-{{msg_type}}", - "queryType": "randomWalk", - "refId": "A" + "description": "the channel cp lag, unit seconds", + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 17 + }, + "hiddenSeries": false, + "id": 123434, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(timestamp(milvus_datacoord_channel_checkpoint_unix_seconds{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})-milvus_datacoord_channel_checkpoint_unix_seconds{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (channel_name)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{channel_name}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Channel Checkpoint Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:484", + "format": "s", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:485", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_collection_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_collection_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, msg_type)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}-{{msg_type}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Collection Mutation Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3538", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3539", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of wait search result over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 13 - }, - "hiddenSeries": false, - "id": 123348, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "description": "The max time tick delay of flow graphs. unit ms", + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 123407, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(milvus_rootcoord_time_tick_delay{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (role_name, node_id)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{role_name}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Time Tick Delay", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:484", + "format": "ms", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:485", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_wait_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "hide": false, - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", - "refId": "A" + "description": "", + "fieldConfig": { + "defaults": { + "unit": "decmbytes" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 25 + }, + "hiddenSeries": false, + "id": 123412, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(milvus_querynode_disk_used_size{namespace=\"$namespace\",pod=~\"$instance-milvus.*\"} ) by(pod) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:484", + "format": "decmbytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:485", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_sq_wait_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_wait_result_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Wait Search Result Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3538", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3539", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of reduce search result over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 13 - }, - "hiddenSeries": false, - "id": 123316, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" + "description": "", + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_reduce_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", - "queryType": "randomWalk", - "refId": "A" + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 25 + }, + "hiddenSeries": false, + "id": 123408, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(process_resident_memory_bytes{namespace=\"$namespace\",pod=~\"$instance-milvus.*\"} ) by(pod) / sum(container_spec_memory_limit_bytes{namespace=\"$namespace\",pod=~\"$instance-milvus.*\",image!=\"\",container!=\"\"} ) by(pod)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Usage Ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:484", + "format": "percentunit", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:485", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_sq_reduce_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_reduce_result_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Reduce Search Result Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3538", - "format": "short", - "logBase": 1, - "min": "0", - "show": true + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 33 + }, + "hiddenSeries": false, + "id": 123409, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\",pod=~\"$instance-milvus.*\",image!=\"\",container!=\"\"}[2m])) by (pod, namespace) / (sum(container_spec_cpu_quota{namespace=\"$namespace\",pod=~\"$instance-milvus.*\",image!=\"\",container!=\"\"}/100000) by (pod, namespace)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU Usage Ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:484", + "format": "percentunit", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:485", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { - "$$hashKey": "object:3539", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of decode search result over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 13 - }, - "hiddenSeries": false, - "id": 123317, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_decode_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "hide": false, - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", - "queryType": "randomWalk", - "refId": "A" + "description": "The 99th percentile and average latency of search request over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 33 + }, + "hiddenSeries": false, + "id": 123468, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(milvus_proxy_restful_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[$__rate_interval])) by (phase) / sum(increase(milvus_proxy_restful_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[$__rate_interval])) by (phase)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{phase}}", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Restful Search Latency Avg", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3538", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3539", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_sq_decode_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_decode_resultlatency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Decode Search Result Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3538", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3539", - "format": "short", - "logBase": 1, - "show": true + "description": "The 99th percentile and average latency of search request over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 33 + }, + "hiddenSeries": false, + "id": 123469, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, phase) (rate(milvus_proxy_restful_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[$__rate_interval])))", + "hide": false, + "interval": "", + "legendFormat": "p99-{{phase}}", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Restful Search Latency P99", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3538", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3539", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } } ], - "yaxis": { - "align": false - } + "title": "Service Quality", + "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Average, maximum and minimum values of the number of msgstream objects created on all physical topics.", - "fill": 1, - "fillGradient": 0, + "collapsed": true, "gridPos": { - "h": 6, - "w": 8, + "h": 1, + "w": 24, "x": 0, - "y": 19 - }, - "hiddenSeries": false, - "id": 123319, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true + "y": 1 }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "id": 123455, + "panels": [ { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "avg(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}-avg", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" + "description": "The 99th percentile of latency for meta operations", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 2 }, - "exemplar": true, - "expr": "max(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-max", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" + "hiddenSeries": false, + "id": 123453, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "exemplar": true, - "expr": "min(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-min", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Msg Stream Object Num", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "format": "short", - "logBase": 1, - "min": "0", - "show": true + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, meta_op_type) (rate(milvus_meta_request_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "{{meta_op_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Meta Operate Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of send mutation over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 19 - }, - "hiddenSeries": false, - "id": 123321, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_send_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{msg_type}}-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" + "description": "The 99th percentile of latency for storage operations", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 2 + }, + "hiddenSeries": false, + "id": 123456, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, persistent_data_op_type) (rate(milvus_storage_request_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "{{persistent_data_op_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Storage Operate Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{msg_type}}-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Mutation Send Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3538", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3539", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Average cache hits per minute of cache operations.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 19 - }, - "hiddenSeries": false, - "id": 123322, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" + "description": "The 99th percentile of latency for storage operations", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 2 }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", cache_state=\"hit\"}[2m])/120) by(cache_name, pod, node_id) / sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by(cache_name, pod, node_id)", - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-{{cache_name}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Cache Hit rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3414", - "format": "short", - "logBase": 1, - "max": "1", - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3415", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of update cache over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 25 - }, - "hiddenSeries": false, - "id": 123323, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" + "hiddenSeries": false, + "id": 123457, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_cache_update_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, message_op_type) (rate(milvus_msgstream_request_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "{{message_op_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Message Operate Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_cache_update_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_cache_update_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Cache Update Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3538", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3539", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "description": "Average, maximum and minimum values of the timestamps for time tick behind.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 25 - }, - "hiddenSeries": false, - "id": 123325, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" + "description": "Meta request rate", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 10 }, - "exemplar": true, - "expr": "avg(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}-avg", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" + "hiddenSeries": false, + "id": 123458, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "exemplar": true, - "expr": "max(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-max", - "refId": "B" + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(milvus_meta_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by (meta_op_type, status)", + "interval": "", + "legendFormat": "{{meta_op_type}}-{{status}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Meta Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "min(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-min", - "refId": "C" + "description": "Storage request rate", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 10 + }, + "hiddenSeries": false, + "id": 123459, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(milvus_storage_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by (persistent_data_op_type, status)", + "interval": "", + "legendFormat": "{{persistent_data_op_type}}-{{status}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Storage Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "", - "hide": false, - "interval": "", - "legendFormat": "", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Produced Timetick Lag Behind Now", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "decimals": 0, - "format": "ms", - "logBase": 1, - "show": true + "description": "Message request rate", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 10 + }, + "hiddenSeries": false, + "id": 123460, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(milvus_msgstream_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by (message_op_type, status)", + "interval": "", + "legendFormat": "{{message_op_type}}-{{status}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Message Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile of latency for applying PK over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 25 - }, - "hiddenSeries": false, - "id": 123326, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_pk_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" + "description": "Meta operation throughput", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 123461, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(milvus_meta_kv_size_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by (meta_op_type)", + "interval": "", + "legendFormat": "{{meta_op_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Meta Operation Throughput", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "Bps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_apply_pk_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_pk_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Apply PK Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3538", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3539", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of apply timestamp over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 31 - }, - "hiddenSeries": false, - "id": 123327, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "description": "Storage operation throughput", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 18 + }, + "hiddenSeries": false, + "id": 123462, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(milvus_storage_kv_size_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by (persistent_data_op_type)", + "interval": "", + "legendFormat": "{{persistent_data_op_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Storage Operation Throughput", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "Bps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_timestamp_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" + "description": "The 99th percentile of latency for storage operations", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 18 + }, + "hiddenSeries": false, + "id": 123463, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, persistent_data_op_type) (rate(internal_storage_request_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "{{persistent_data_op_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Storage Operate Latency[cpp]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_apply_timestamp_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_timestamp_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Apply Timestamp Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3538", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3539", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "per-second increasing rate of success requests.", + "description": "Storage request rate in cpp", + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 123464, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(internal_storage_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (persistent_data_op_type, status)", + "interval": "", + "legendFormat": "{{persistent_data_op_type}}-{{status}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Storage Request Rate[cpp]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Storage operation throughput in cpp", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 26 + }, + "hiddenSeries": false, + "id": 123465, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(internal_storage_kv_size_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by (persistent_data_op_type)", + "interval": "", + "legendFormat": "{{persistent_data_op_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Storage Operation Throughput[cpp]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "Bps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Dependencies", + "type": "row" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 123248, + "panels": [], + "title": "Proxy", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of searching vectors.", + "fieldConfig": { + "defaults": { + "unit": "cps" + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 31 + "x": 0, + "y": 3 }, "hiddenSeries": false, - "id": 123329, + "id": 123250, "legend": { "avg": false, "current": false, @@ -1728,7 +2654,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -1740,19 +2666,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_proxy_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"success\"}[2m])/120) by(function_name, pod, node_id)", + "expr": "sum(increase(milvus_proxy_search_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", "interval": "", - "legendFormat": "{{function_name}}-{{pod}}-{{node_id}}", + "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(milvus_proxy_search_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m]))", + "hide": false, + "legendFormat": "total", + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Success Request Rate", + "title": "Search Vector Count Rate", "tooltip": { "shared": true, "sort": 0, @@ -1767,7 +2705,7 @@ "yaxes": [ { "$$hashKey": "object:3414", - "format": "short", + "format": "cps", "logBase": 1, "min": "0", "show": true @@ -1790,19 +2728,25 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" + }, + "description": "per-second increasing rate of inserting vectors.", + "fieldConfig": { + "defaults": { + "unit": "cps" + }, + "overrides": [] }, - "description": "per-second increasing rate of faild requests.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 31 + "x": 8, + "y": 3 }, "hiddenSeries": false, - "id": 123379, + "id": 123315, "legend": { "avg": false, "current": false, @@ -1819,7 +2763,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -1831,19 +2775,19 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_proxy_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"fail\"}[2m])/120) by(function_name, pod, node_id)", + "expr": "sum(increase(milvus_proxy_insert_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", "interval": "", - "legendFormat": "{{function_name}}-{{pod}}-{{node_id}}", + "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Faild Request Rate", + "title": "Insert Vector Count Rate", "tooltip": { "shared": true, "sort": 0, @@ -1858,7 +2802,7 @@ "yaxes": [ { "$$hashKey": "object:3414", - "format": "short", + "format": "cps", "logBase": 1, "min": "0", "show": true @@ -1881,19 +2825,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of request over the last 2 minutes.", + "description": "The 99th percentile and average latency of search request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 37 + "x": 16, + "y": 3 }, "hiddenSeries": false, - "id": 123381, + "id": 123264, "legend": { "avg": false, "current": false, @@ -1910,7 +2854,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -1922,31 +2866,35 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, function_name) (rate(milvus_proxy_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "hide": false, "interval": "", - "legendFormat": "p99-{{function_name}}-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" + "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", + "range": true, + "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "sum(increase(milvus_proxy_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, function_name) / sum(increase(milvus_proxy_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, function_name)", + "expr": "sum(increase(milvus_proxy_sq_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)", "hide": false, "interval": "", - "legendFormat": "avg-{{function_name}}-{{pod}}-{{node_id}}", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", + "range": true, "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Request Latency", + "title": "Search Latency", "tooltip": { "shared": true, "sort": 0, @@ -1961,7 +2909,7 @@ "yaxes": [ { "$$hashKey": "object:3538", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -1984,19 +2932,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "per-second increasing rate of bytes received in proxy", + "description": "The 99th percentile and average latency of search request for a specified collection over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 37 + "x": 0, + "y": 11 }, "hiddenSeries": false, - "id": 123368, + "id": 123377, "legend": { "avg": false, "current": false, @@ -2013,7 +2961,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2025,19 +2973,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_proxy_receive_bytes_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by(pod, node_id)", + "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_collection_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])))", + "hide": false, "interval": "", - "legendFormat": "{{pod}}-{{node_id}}", - "queryType": "randomWalk", + "legendFormat": "p99-{{query_type}}-{{pod}}-{{node_id}}", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_proxy_collection_sq_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_collection_sq_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, query_type)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{query_type}}-{{pod}}-{{node_id}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Received Byte Rate", + "title": "Collection Search Latency", "tooltip": { "shared": true, "sort": 0, @@ -2051,14 +3011,14 @@ }, "yaxes": [ { - "$$hashKey": "object:3414", - "format": "short", + "$$hashKey": "object:3538", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:3415", + "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true @@ -2075,19 +3035,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "per-second increasing rate of bytes sent back to the client by the Proxy in response to a Search or Query request.", + "description": "The 99th percentile and average latency of mutation request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 37 + "x": 8, + "y": 11 }, "hiddenSeries": false, - "id": 123369, + "id": 123320, "legend": { "avg": false, "current": false, @@ -2104,7 +3064,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2116,19 +3076,35 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "sum(increase(milvus_proxy_send_bytes_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by(pod, node_id)", + "expr": "histogram_quantile(0.95, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{msg_type}}", "queryType": "randomWalk", + "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(milvus_proxy_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{msg_type}}", + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Send Byte Rate", + "title": "Mutation Latency", "tooltip": { "shared": true, "sort": 0, @@ -2142,14 +3118,14 @@ }, "yaxes": [ { - "$$hashKey": "object:3414", - "format": "short", + "$$hashKey": "object:3538", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:3415", + "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true @@ -2159,19 +3135,6 @@ "align": false } }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 123155, - "panels": [], - "title": "Root Coordinator", - "type": "row" - }, { "aliasColors": {}, "bars": false, @@ -2179,19 +3142,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Number of proxy nodes which has register with etcd", + "description": "The 99th percentile and average latency of mutation request for a specified collection over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 44 + "x": 16, + "y": 11 }, "hiddenSeries": false, - "id": 123141, + "id": 123378, "legend": { "avg": false, "current": false, @@ -2208,7 +3171,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2220,20 +3183,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_rootcoord_proxy_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", + "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_collection_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "num", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{msg_type}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_proxy_collection_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_collection_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, msg_type)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{msg_type}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Proxy Node Num", + "title": "Collection Mutation Latency", "tooltip": { "shared": true, "sort": 0, @@ -2247,14 +3221,14 @@ }, "yaxes": [ { - "$$hashKey": "object:4353", - "format": "short", + "$$hashKey": "object:3538", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:4354", + "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true @@ -2271,19 +3245,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${datasource}" }, - "description": "Average, maximum and minimum values of the timestamps for time tick behind.", + "description": "The 99th percentile and average search latency in step", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 44 + "x": 0, + "y": 19 }, "hiddenSeries": false, - "id": 123384, + "id": 123348, "legend": { "avg": false, "current": false, @@ -2300,7 +3274,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2312,45 +3286,83 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "avg(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_wait_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "hide": false, "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}-avg", - "queryType": "randomWalk", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}-wait-result", + "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "max(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "sum(increase(milvus_proxy_sq_wait_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_wait_result_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)", "hide": false, "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-max", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}-wait-result", + "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "min(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_reduce_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-min", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}-reduce", + "range": true, "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(milvus_proxy_sq_reduce_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_reduce_result_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)", + "hide": false, + "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}-reduce", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_decode_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))\n", + "hide": false, + "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}-decode", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(milvus_proxy_sq_decode_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_decode_resultlatency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)", + "hide": false, + "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}-decode", + "range": true, + "refId": "F" } ], "thresholds": [], "timeRegions": [], - "title": "Produced Timetick Lag Behind Now", + "title": "Search Latency In Step", "tooltip": { "shared": true, "sort": 0, @@ -2364,14 +3376,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", - "decimals": 0, + "$$hashKey": "object:3538", "format": "ms", "logBase": 1, + "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true @@ -2388,19 +3400,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency for rootcoord to finish synchronizing timestamp messages to all pchanels.", + "description": "The 99th percentile and average latency of send mutation over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 44 + "x": 8, + "y": 19 }, "hiddenSeries": false, - "id": 123338, + "id": 123321, "legend": { "avg": false, "current": false, @@ -2417,7 +3429,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2429,31 +3441,35 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_rootcoord_sync_timetick_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_send_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "legendFormat": "p99-latency", + "legendFormat": "p99{{pod}}-{{node_id}}-{{msg_type}}", "queryType": "randomWalk", + "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "sum(increase(milvus_rootcoord_sync_timetick_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_rootcoord_sync_timetick_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))", + "expr": "sum(increase(milvus_proxy_mutation_send_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_send_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type)", "hide": false, "interval": "", - "legendFormat": "avg-latency", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{msg_type}}", + "range": true, "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Produced Timetick Time Taken", + "title": "Mutation Send Latency", "tooltip": { "shared": true, "sort": 0, @@ -2467,14 +3483,14 @@ }, "yaxes": [ { - "$$hashKey": "object:161", + "$$hashKey": "object:3538", "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:162", + "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true @@ -2491,19 +3507,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of DDL request over the last 2 minutes.", + "description": "The 99th percentile and average latency of mq mutation over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 50 + "x": 16, + "y": 19 }, "hiddenSeries": false, - "id": 123337, + "id": 123448, "legend": { "avg": false, "current": false, @@ -2520,7 +3536,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2532,31 +3548,35 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, function_name) (rate(milvus_rootcoord_ddl_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "histogram_quantile(0.99, sum by (le, pod) (rate(milvus_msgstream_request_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", message_op_type=\"produce\"}[2m])))", "interval": "", - "legendFormat": "p99-{{function_name}}", + "legendFormat": "p99-{{pod}}", "queryType": "randomWalk", + "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "sum(increase(milvus_rootcoord_ddl_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (function_name) / sum(increase(milvus_rootcoord_ddl_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (function_name)", + "expr": "sum(increase(milvus_msgstream_request_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", message_op_type=\"produce\"}[2m])) by (pod) / sum(increase(milvus_msgstream_request_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", message_op_type=\"produce\"}[2m])) by (pod)", "hide": false, "interval": "", - "legendFormat": "avg-{{function_name}}", + "legendFormat": "avg-{{pod}}", + "range": true, "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "DDL Request Latency", + "title": "MQ Send Latency", "tooltip": { "shared": true, "sort": 0, @@ -2570,14 +3590,14 @@ }, "yaxes": [ { - "$$hashKey": "object:161", - "format": "short", + "$$hashKey": "object:3538", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:162", + "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true @@ -2594,19 +3614,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "RootCoord stores pre-assigned timestamps in the metastore", + "description": "The 99th percentile and average latency of apply timestamp over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 50 + "x": 0, + "y": 27 }, "hiddenSeries": false, - "id": 123340, + "id": 123327, "legend": { "avg": false, "current": false, @@ -2623,7 +3643,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2635,20 +3655,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "milvus_rootcoord_timestamp_saved{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_timestamp_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "tiestamp", + "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_proxy_apply_timestamp_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_timestamp_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Timestamp Saved", + "title": "Apply Timestamp Latency", "tooltip": { "shared": true, "sort": 0, @@ -2662,15 +3693,15 @@ }, "yaxes": [ { - "$$hashKey": "object:4353", - "format": "short", + "$$hashKey": "object:3538", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:4354", - "format": "short", + "$$hashKey": "object:3539", + "format": "ms", "logBase": 1, "show": true } @@ -2686,19 +3717,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "per-second increasing rate of DDL request", + "description": "The 99th percentile of latency for applying PK over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 50 + "x": 8, + "y": 27 }, "hiddenSeries": false, - "id": 123347, + "id": 123326, "legend": { "avg": false, "current": false, @@ -2715,7 +3746,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2727,19 +3758,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_rootcoord_ddl_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, function_name)", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_pk_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "legendFormat": "{{function_name}}-{{status}}", + "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_proxy_apply_pk_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_pk_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "DDL Request Rate", + "title": "Apply PK Latency", "tooltip": { "shared": true, "sort": 0, @@ -2753,14 +3796,14 @@ }, "yaxes": [ { - "$$hashKey": "object:1456", - "format": "short", + "$$hashKey": "object:3538", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:1457", + "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true @@ -2777,19 +3820,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "RoootCoord current latest timestamp", + "description": "The 99th percentile and average latency of rpc in queue over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 56 + "x": 16, + "y": 27 }, "hiddenSeries": false, - "id": 123339, + "id": 123470, "legend": { "avg": false, "current": false, @@ -2806,7 +3849,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2818,20 +3861,35 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "milvus_rootcoord_timestamp{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", + "expr": "histogram_quantile(0.99, sum by (le, pod, function_name) (rate(milvus_proxy_req_in_queue_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "tiestamp", + "legendFormat": "p99-{{pod}}-{{function_name}}", "queryType": "randomWalk", + "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(milvus_proxy_req_in_queue_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, function_name) / sum(increase(milvus_proxy_req_in_queue_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, function_name)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{function_name}}", + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Timestamp", + "title": "In Queue Latency", "tooltip": { "shared": true, "sort": 0, @@ -2845,14 +3903,14 @@ }, "yaxes": [ { - "$$hashKey": "object:4353", - "format": "short", + "$$hashKey": "object:3538", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:4354", + "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true @@ -2869,19 +3927,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The number of DML channels", + "description": "Average, maximum and minimum values of the number of msgstream objects created on all physical topics.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 56 + "x": 0, + "y": 35 }, "hiddenSeries": false, - "id": 123345, + "id": 123319, "legend": { "avg": false, "current": false, @@ -2898,7 +3956,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2910,20 +3968,44 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_rootcoord_dml_channel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", + "expr": "avg(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, - "legendFormat": "num", + "legendFormat": "{{pod}}-{{node_id}}-avg", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "max(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-max", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "min(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-min", + "refId": "C" } ], "thresholds": [], "timeRegions": [], - "title": "DML Channel Num", + "title": "Msg Stream Object Num", "tooltip": { "shared": true, "sort": 0, @@ -2937,14 +4019,14 @@ }, "yaxes": [ { - "$$hashKey": "object:4353", + "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:4354", + "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true @@ -2961,19 +4043,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "per-second increasing rate of IDs assigned by RootCoord", + "description": "Average, maximum and minimum values of the timestamps for time tick behind.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 56 + "x": 8, + "y": 35 }, "hiddenSeries": false, - "id": 123221, + "id": 123325, "legend": { "avg": false, "current": false, @@ -2990,7 +4072,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -3002,19 +4084,57 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_rootcoord_id_alloc_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120)", + "expr": "avg(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "hide": false, "interval": "", - "legendFormat": "total", + "intervalFactor": 2, + "legendFormat": "{{pod}}-{{node_id}}-avg", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "max(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-max", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "min(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-min", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "D" } ], "thresholds": [], "timeRegions": [], - "title": "ID Alloc Rate", + "title": "Produced Timetick Lag Behind Now", "tooltip": { "shared": true, "sort": 0, @@ -3028,14 +4148,14 @@ }, "yaxes": [ { - "$$hashKey": "object:1456", - "format": "short", + "$$hashKey": "object:536", + "decimals": 0, + "format": "ms", "logBase": 1, - "min": "0", "show": true }, { - "$$hashKey": "object:1457", + "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true @@ -3052,19 +4172,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The number of partitions.", + "description": "The 99th percentile of latency for assigning segment id over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 62 + "x": 16, + "y": 35 }, "hiddenSeries": false, - "id": 123344, + "id": 123449, "legend": { "avg": false, "current": false, @@ -3081,7 +4201,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -3093,20 +4213,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_rootcoord_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_assign_segmentID_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "num", + "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_proxy_assign_segmentID_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_assign_segmentID_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Partition Num", + "title": "Assign Segment ID Latency", "tooltip": { "shared": true, "sort": 0, @@ -3120,14 +4251,14 @@ }, "yaxes": [ { - "$$hashKey": "object:4353", - "format": "short", + "$$hashKey": "object:3538", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:4354", + "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true @@ -3144,19 +4275,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The max time tick delay of flow graphs. unit ms", + "description": "The 99th percentile and average latency of request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 62 + "x": 0, + "y": 43 }, "hiddenSeries": false, - "id": 123383, + "id": 123381, "legend": { "avg": false, "current": false, @@ -3173,7 +4304,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -3185,19 +4316,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_rootcoord_time_tick_delay{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (role_name, node_id)", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{role_name}}-{{node_id}}", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, function_name) (rate(milvus_proxy_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "p99-{{function_name}}-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Time Tick Delay", + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_proxy_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, function_name) / sum(increase(milvus_proxy_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, function_name)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{function_name}}-{{pod}}-{{node_id}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Request Latency", "tooltip": { "shared": true, "sort": 0, @@ -3211,14 +4354,105 @@ }, "yaxes": [ { - "$$hashKey": "object:4353", + "$$hashKey": "object:3538", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3539", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of success requests.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 43 + }, + "hiddenSeries": false, + "id": 123329, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_proxy_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"success\"}[2m])/120) by(function_name, pod, node_id)", + "interval": "", + "legendFormat": "{{function_name}}-{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Success Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:4354", + "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true @@ -3235,19 +4469,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The number of collections.", + "description": "The 99th percentile and average latency of update cache over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 62 + "y": 43 }, "hiddenSeries": false, - "id": 123342, + "id": 123323, "legend": { "avg": false, "current": false, @@ -3264,7 +4498,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -3276,20 +4510,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_rootcoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_cache_update_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "num", + "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_proxy_cache_update_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_cache_update_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Collection Num", + "title": "Cache Update Latency", "tooltip": { "shared": true, "sort": 0, @@ -3303,14 +4548,14 @@ }, "yaxes": [ { - "$$hashKey": "object:4353", - "format": "short", + "$$hashKey": "object:3538", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:4354", + "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true @@ -3327,19 +4572,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The number of credential", + "description": "per-second increasing rate of bytes sent back to the client by the Proxy in response to a Search or Query request.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 68 + "y": 51 }, "hiddenSeries": false, - "id": 123343, + "id": 123369, "legend": { "avg": false, "current": false, @@ -3356,7 +4601,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -3368,20 +4613,19 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_rootcoord_credential_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", + "expr": "sum(rate(milvus_proxy_send_bytes_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[30s])) by(pod, node_id)", "interval": "", - "intervalFactor": 2, - "legendFormat": "num", + "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Credential Num", + "title": "Send Byte Rate", "tooltip": { "shared": true, "sort": 0, @@ -3395,14 +4639,14 @@ }, "yaxes": [ { - "$$hashKey": "object:4353", - "format": "short", + "$$hashKey": "object:3414", + "format": "binBps", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:4354", + "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true @@ -3419,19 +4663,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The number of Msgstream objects.", + "description": "per-second increasing rate of bytes received in proxy", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 68 + "y": 51 }, "hiddenSeries": false, - "id": 123346, + "id": 123368, "legend": { "avg": false, "current": false, @@ -3448,7 +4692,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -3460,20 +4704,19 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_rootcoord_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", + "expr": "sum(rate(milvus_proxy_receive_bytes_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by(pod, node_id)", "interval": "", - "intervalFactor": 2, - "legendFormat": "num", + "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Msgstream Num", + "title": "Received Byte Rate", "tooltip": { "shared": true, "sort": 0, @@ -3487,14 +4730,14 @@ }, "yaxes": [ { - "$$hashKey": "object:4353", - "format": "short", + "$$hashKey": "object:3414", + "format": "binBps", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:4354", + "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true @@ -3505,982 +4748,764 @@ } }, { - "collapsed": true, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of faild requests.", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 74 + "h": 8, + "w": 8, + "x": 16, + "y": 51 }, - "id": 123246, - "panels": [ + "hiddenSeries": false, + "id": 123379, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "Total number of loaded collections.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 3 - }, - "hiddenSeries": false, - "id": 123288, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "uid": "${datasource}" }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_querycoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "num", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Collection Loaded Num", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "sum(increase(milvus_proxy_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"fail\"}[2m])/120) by(function_name, pod, node_id)", + "interval": "", + "legendFormat": "{{function_name}}-{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Faild Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3414", + "format": "short", + "logBase": 1, + "min": "0", + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "$$hashKey": "object:3415", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Average, maximum and minimum values of the timestamps for time tick behind.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 59 + }, + "hiddenSeries": false, + "id": 123436, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "Total number of loaded entities.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 3 - }, - "hiddenSeries": false, - "id": 123289, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_querycoord_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "num", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Entity Loaded Num", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "uid": "${datasource}" }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "editorMode": "code", + "exemplar": true, + "expr": "avg(milvus_proxy_limiter_rate{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id, msg_type)", + "hide": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}-avg", + "queryType": "randomWalk", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "yaxes": [ - { - "$$hashKey": "object:536", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "editorMode": "code", + "exemplar": true, + "expr": "max(milvus_proxy_limiter_rate{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id, msg_type)", + "hide": true, + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}-max", + "range": true, + "refId": "B" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "per-second increasing rate of load requests.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 3 + "editorMode": "code", + "exemplar": true, + "expr": "min(milvus_proxy_limiter_rate{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id, msg_type)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}-min", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "hiddenSeries": false, - "id": 123291, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querycoord_load_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status)", - "interval": "", - "legendFormat": "{{status}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Load Request Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:101", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:102", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "D" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Rate Limit", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:536", + "decimals": 0, + "format": "bytes", + "logBase": 1, + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "per-second increasing rate of relaase requests.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 9 - }, - "hiddenSeries": false, - "id": 123292, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true + "$$hashKey": "object:537", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Average cache hits per minute of cache operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querycoord_release_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status)", - "interval": "", - "legendFormat": "{{status}}", - "queryType": "randomWalk", - "refId": "A" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - ], - "thresholds": [], - "timeRegions": [], - "title": "Release Request Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, - "yaxes": [ - { - "$$hashKey": "object:101", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:102", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 59 + }, + "id": 123322, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.3.3", + "targets": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of load request over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 9 + "uid": "${datasource}" }, - "hiddenSeries": false, - "id": 123294, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_load_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-latency", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querycoord_load_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_querycoord_load_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg-latency", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Load Request Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "exemplar": true, + "expr": "sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", cache_state=\"hit\"}[2m])/120) by(cache_name, pod, node_id) / sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by(cache_name, pod, node_id)", + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-{{cache_name}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Cache Hit rate", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 67 + }, + "id": 123155, + "panels": [], + "title": "Root Coordinator", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of proxy nodes which has register with etcd", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 68 + }, + "hiddenSeries": false, + "id": 123141, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "exemplar": true, + "expr": "sum(milvus_rootcoord_proxy_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "num", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Proxy Node Num", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:4353", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:4354", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Average, maximum and minimum values of the timestamps for time tick behind.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 68 + }, + "hiddenSeries": false, + "id": 123384, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "yaxes": [ - { - "$$hashKey": "object:161", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:162", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "avg(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}-{{node_id}}-avg", + "queryType": "randomWalk", + "refId": "A" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of release request over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 9 - }, - "hiddenSeries": false, - "id": 123313, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_release_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-latency", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querycoord_release_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_querycoord_release_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg-latency", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Release Request Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:161", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:162", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "max(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-max", + "refId": "B" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "Total number of sub-load task.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 15 - }, - "hiddenSeries": false, - "id": 123295, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_querycoord_child_task_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "num", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Sub-Load Task", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "uid": "${datasource}" }, - "yaxes": [ - { - "$$hashKey": "object:536", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "min(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-min", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Produced Timetick Lag Behind Now", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:536", + "decimals": 0, + "format": "ms", + "logBase": 1, + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "$$hashKey": "object:537", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency for rootcoord to finish synchronizing timestamp messages to all pchanels.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 68 + }, + "hiddenSeries": false, + "id": 123338, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "Total number pf parent loading task.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 15 + "uid": "${datasource}" }, - "hiddenSeries": false, - "id": 123296, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_querycoord_parent_task_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "num", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Parent Load Task", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_rootcoord_sync_timetick_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "p99-latency", + "queryType": "randomWalk", + "refId": "A" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of sub-load task request over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 15 - }, - "hiddenSeries": false, - "id": 123298, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_child_task_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-latency", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querycoord_child_task_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_querycoord_child_task_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg-latency", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Sub-Load Task Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "uid": "${datasource}" }, - "yaxes": [ - { - "$$hashKey": "object:161", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:162", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "sum(increase(milvus_rootcoord_sync_timetick_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_rootcoord_sync_timetick_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))", + "hide": false, + "interval": "", + "legendFormat": "avg-latency", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Produced Timetick Time Taken", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of DDL request over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 76 + }, + "hiddenSeries": false, + "id": 123337, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "Number of Query nodes which has register with etcd.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 21 + "uid": "${datasource}" }, - "hiddenSeries": false, - "id": 123297, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_querycoord_querynode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "num", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Query Node Num", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, function_name) (rate(milvus_rootcoord_ddl_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "p99-{{function_name}}", + "queryType": "randomWalk", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "yaxes": [ - { - "$$hashKey": "object:536", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "sum(increase(milvus_rootcoord_ddl_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (function_name) / sum(increase(milvus_rootcoord_ddl_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (function_name)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{function_name}}", + "refId": "B" } ], - "title": "Query Coordinator", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 75 + "thresholds": [], + "timeRegions": [], + "title": "DDL Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "id": 123244, - "panels": [], - "title": "Query Node", - "type": "row" + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { "aliasColors": {}, @@ -4489,19 +5514,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Total number of the loaded collections in QueryNode.", + "description": "RootCoord stores pre-assigned timestamps in the metastore", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, + "x": 8, "y": 76 }, "hiddenSeries": false, - "id": 123299, + "id": 123340, "legend": { "avg": false, "current": false, @@ -4518,7 +5543,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -4530,20 +5555,22 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "sum(milvus_querynode_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "sum(milvus_rootcoord_timestamp_saved{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", + "intervalFactor": 1, + "legendFormat": "timestamp", "queryType": "randomWalk", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Collection Loaded Num", + "title": "Timestamp Saved", "tooltip": { "shared": true, "sort": 0, @@ -4557,15 +5584,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", - "decimals": 0, + "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true @@ -4582,19 +5608,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Total number of the loaded partitions in QueryNode.", + "description": "per-second increasing rate of DDL request", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, + "x": 16, "y": 76 }, "hiddenSeries": false, - "id": 123303, + "id": 123347, "legend": { "avg": false, "current": false, @@ -4611,7 +5637,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -4623,20 +5649,19 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_querynode_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "sum(increase(milvus_rootcoord_ddl_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, function_name)", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "{{function_name}}-{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Partition Loaded Num", + "title": "DDL Request Rate", "tooltip": { "shared": true, "sort": 0, @@ -4650,15 +5675,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", - "decimals": 0, + "$$hashKey": "object:1456", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:1457", "format": "short", "logBase": 1, "show": true @@ -4675,19 +5699,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Total number of the loaded segment in QueryNode.", + "description": "RoootCoord current latest timestamp", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 76 + "x": 0, + "y": 84 }, "hiddenSeries": false, - "id": 123305, + "id": 123339, "legend": { "avg": false, "current": false, @@ -4704,32 +5728,34 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "sum(milvus_querynode_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id, segment_state)", + "expr": "sum(milvus_rootcoord_timestamp{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}-{{segment_state}}", + "intervalFactor": 1, + "legendFormat": "timestamp", "queryType": "randomWalk", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Segment Loaded Num", + "title": "Timestamp", "tooltip": { "shared": true, "sort": 0, @@ -4743,15 +5769,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", - "decimals": 0, + "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true @@ -4768,19 +5793,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${datasource}" }, - "description": "Average, maximum and minimum values of the timestamps for time tick behind.", + "description": "The number of DML channels", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 82 + "x": 8, + "y": 84 }, "hiddenSeries": false, - "id": 123385, + "id": 123345, "legend": { "avg": false, "current": false, @@ -4797,7 +5822,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -4809,45 +5834,20 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "avg(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)", - "hide": false, + "expr": "sum(milvus_rootcoord_dml_channel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}-avg", + "legendFormat": "num", "queryType": "randomWalk", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "max(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-max", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "min(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-min", - "refId": "C" } ], "thresholds": [], "timeRegions": [], - "title": "TimeTick Lag Behind Now (Consumed Insert)", + "title": "DML Channel Num", "tooltip": { "shared": true, "sort": 0, @@ -4861,14 +5861,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", - "decimals": 0, - "format": "ms", + "$$hashKey": "object:4353", + "format": "short", "logBase": 1, + "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true @@ -4885,19 +5885,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${datasource}" }, - "description": "Average, maximum and minimum values of the timestamps for time tick behind.", + "description": "per-second increasing rate of IDs assigned by RootCoord", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 82 + "x": 16, + "y": 84 }, "hiddenSeries": false, - "id": 123386, + "id": 123221, "legend": { "avg": false, "current": false, @@ -4914,7 +5914,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -4926,45 +5926,19 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "avg(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)", - "hide": false, + "expr": "sum(increase(milvus_rootcoord_id_alloc_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120)", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}-avg", + "legendFormat": "total", "queryType": "randomWalk", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "max(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-max", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "min(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-min", - "refId": "C" } ], "thresholds": [], "timeRegions": [], - "title": "TimeTick Lag Behind Now (Consumed Delete)", + "title": "ID Alloc Rate", "tooltip": { "shared": true, "sort": 0, @@ -4978,14 +5952,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", - "decimals": 0, - "format": "ms", + "$$hashKey": "object:1456", + "format": "short", "logBase": 1, + "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:1457", "format": "short", "logBase": 1, "show": true @@ -5002,19 +5976,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "per-second increasing rate of consuming message", + "description": "The number of partitions.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 82 + "x": 0, + "y": 92 }, "hiddenSeries": false, - "id": 123387, + "id": 123344, "legend": { "avg": false, "current": false, @@ -5031,7 +6005,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -5043,32 +6017,20 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_querynode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id, msg_type)", - "hide": false, + "expr": "sum(milvus_rootcoord_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", + "intervalFactor": 2, + "legendFormat": "num", "queryType": "randomWalk", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-all", - "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Consumed Message Rate", + "title": "Partition Num", "tooltip": { "shared": true, "sort": 0, @@ -5082,14 +6044,14 @@ }, "yaxes": [ { - "$$hashKey": "object:3414", - "format": "cps", + "$$hashKey": "object:4353", + "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:3415", + "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true @@ -5106,19 +6068,25 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" + }, + "description": "The max time tick delay of flow graphs. unit ms", + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] }, - "description": "Total number of the queryable entities in QueryNode.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 88 + "x": 8, + "y": 92 }, "hiddenSeries": false, - "id": 123365, + "id": 123383, "legend": { "avg": false, "current": false, @@ -5135,7 +6103,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -5147,20 +6115,20 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_querynode_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "sum(milvus_rootcoord_time_tick_delay{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (role_name, node_id)", "interval": "", "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "{{role_name}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Queryable Entity Num", + "title": "Time Tick Delay", "tooltip": { "shared": true, "sort": 0, @@ -5174,15 +6142,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", - "decimals": 0, - "format": "short", + "$$hashKey": "object:4353", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true @@ -5199,19 +6166,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Total number of dml virtual channels for QueryNode watch", + "description": "The number of collections.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 88 + "x": 16, + "y": 92 }, "hiddenSeries": false, - "id": 123304, + "id": 123342, "legend": { "avg": false, "current": false, @@ -5228,7 +6195,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -5240,20 +6207,20 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_querynode_dml_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "sum(milvus_rootcoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "DML Virtual Channel", + "title": "Collection Num", "tooltip": { "shared": true, "sort": 0, @@ -5267,14 +6234,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", + "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true @@ -5291,19 +6258,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Total number of delta virtual channels for QueryNode watch", + "description": "The number of credential", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 88 + "x": 0, + "y": 100 }, "hiddenSeries": false, - "id": 123306, + "id": 123343, "legend": { "avg": false, "current": false, @@ -5320,7 +6287,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -5332,21 +6299,20 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_querynode_delta_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "hide": false, + "expr": "sum(milvus_rootcoord_credential_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Delta Virtual Channel", + "title": "Credential Num", "tooltip": { "shared": true, "sort": 0, @@ -5360,14 +6326,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", + "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true @@ -5384,19 +6350,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Total number of consumers in the QueryNode.", + "description": "The number of Msgstream objects.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 94 + "x": 8, + "y": 100 }, "hiddenSeries": false, - "id": 123307, + "id": 123346, "legend": { "avg": false, "current": false, @@ -5413,7 +6379,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -5425,21 +6391,20 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_querynode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "hide": false, + "expr": "sum(milvus_rootcoord_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Consumer Num", + "title": "Msgstream Num", "tooltip": { "shared": true, "sort": 0, @@ -5453,14 +6418,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", + "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true @@ -5470,6 +6435,23 @@ "align": false } }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 108 + }, + "id": 123246, + "panels": [], + "title": "Query Coordinator", + "type": "row" + }, { "aliasColors": {}, "bars": false, @@ -5477,19 +6459,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "per-second increasing rate of searching requests.", + "description": "Total number of loaded collections.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 94 + "x": 0, + "y": 109 }, "hiddenSeries": false, - "id": 123350, + "id": 123288, "legend": { "avg": false, "current": false, @@ -5506,7 +6488,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -5518,19 +6500,20 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_querynode_sq_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (query_type, status, pod, node_id)", + "expr": "sum(milvus_querycoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-{{query_type}}-{{status}}", + "intervalFactor": 2, + "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Search Request Rate", + "title": "Collection Loaded Num", "tooltip": { "shared": true, "sort": 0, @@ -5544,14 +6527,14 @@ }, "yaxes": [ { - "$$hashKey": "object:3414", + "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:3415", + "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true @@ -5568,19 +6551,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of search and query request over the last 2 minutes.", + "description": "Total number of loaded collections.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 94 + "x": 8, + "y": 109 }, "hiddenSeries": false, - "id": 123366, + "id": 123403, "legend": { "avg": false, "current": false, @@ -5597,7 +6580,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -5609,31 +6592,22 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "sum(milvus_querycoord_task_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance,querycoord_task_type)", "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", + "intervalFactor": 2, + "legendFormat": "{{querycoord_task_type}}", "queryType": "randomWalk", + "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_sq_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", - "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Search Request Latency", + "title": "Task Num", "tooltip": { "shared": true, "sort": 0, @@ -5647,14 +6621,14 @@ }, "yaxes": [ { - "$$hashKey": "object:161", + "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:162", + "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true @@ -5671,19 +6645,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of search or query in queue over the last 2 minutes.", + "description": "per-second increasing rate of load requests.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 100 + "x": 16, + "y": 109 }, "hiddenSeries": false, - "id": 123372, + "id": 123291, "legend": { "avg": false, "current": false, @@ -5700,7 +6674,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -5712,31 +6686,19 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_queue_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "sum(increase(milvus_querycoord_load_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status)", "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", + "legendFormat": "{{status}}", "queryType": "randomWalk", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_sq_queue_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_queue_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", - "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Search in Queue Latency", + "title": "Load Request Rate", "tooltip": { "shared": true, "sort": 0, @@ -5750,14 +6712,14 @@ }, "yaxes": [ { - "$$hashKey": "object:161", + "$$hashKey": "object:101", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:162", + "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true @@ -5774,19 +6736,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of search segment over the last 2 minutes.", + "description": "Number of Query nodes which has register with etcd.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 100 + "x": 0, + "y": 117 }, "hiddenSeries": false, - "id": 123374, + "id": 123297, "legend": { "avg": false, "current": false, @@ -5803,7 +6765,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -5815,31 +6777,20 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type, segment_state) (rate(milvus_querynode_sq_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "exemplar": true, + "expr": "sum(milvus_querycoord_querynode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}-{{segment_state}}", + "intervalFactor": 2, + "legendFormat": "num", "queryType": "randomWalk", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_sq_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type, segment_state) / sum(increase(milvus_querynode_sq_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type, segment_state)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}_{{segment_state}}", - "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Search Segment Latency", + "title": "Query Node Num", "tooltip": { "shared": true, "sort": 0, @@ -5853,14 +6804,14 @@ }, "yaxes": [ { - "$$hashKey": "object:161", + "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:162", + "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true @@ -5877,19 +6828,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of search at the segcore step over the last 2 minutes.", + "description": "Segment Num On Stopping Node", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 100 + "x": 8, + "y": 117 }, "hiddenSeries": false, - "id": 123310, + "id": 123439, "legend": { "avg": false, "current": false, @@ -5906,7 +6857,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -5918,31 +6869,21 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_querynode_sq_core_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "sum(milvus_querynode_stopping_balance_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", + "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", + "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_sq_core_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_core_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", - "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Segcore Request Latency", + "title": "Segment Num On Stopping Node", "tooltip": { "shared": true, "sort": 0, @@ -5980,19 +6921,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of search or query reduce over the last 2 minutes.", + "description": "Channel Num On Stopping Node", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 106 + "x": 16, + "y": 117 }, "hiddenSeries": false, - "id": 123367, + "id": 123444, "legend": { "avg": false, "current": false, @@ -6009,7 +6950,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6021,31 +6962,21 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_reduce_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "sum(milvus_querynode_stopping_balance_channel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", + "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", + "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_sq_reduce_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_reduce_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", - "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Search Reduce Latency", + "title": "Channel Num On Stopping Node", "tooltip": { "shared": true, "sort": 0, @@ -6076,6 +7007,23 @@ "align": false } }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 125 + }, + "id": 123244, + "panels": [], + "title": "Query Node", + "type": "row" + }, { "aliasColors": {}, "bars": false, @@ -6083,19 +7031,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of load segment over the last 2 minutes.", + "description": "Total number of the loaded collections in QueryNode.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 106 + "x": 0, + "y": 126 }, "hiddenSeries": false, - "id": 123311, + "id": 123299, "legend": { "avg": false, "current": false, @@ -6112,7 +7060,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6124,31 +7072,20 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_load_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "sum(milvus_querynode_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", + "intervalFactor": 2, + "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_load_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_load_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", - "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Load Segment Latency", + "title": "Collection Loaded Num", "tooltip": { "shared": true, "sort": 0, @@ -6162,14 +7099,15 @@ }, "yaxes": [ { - "$$hashKey": "object:161", + "$$hashKey": "object:536", + "decimals": 0, "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:162", + "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true @@ -6186,19 +7124,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Total number of flowgraph", + "description": "Total number of the loaded partitions in QueryNode.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 106 + "x": 8, + "y": 126 }, "hiddenSeries": false, - "id": 123312, + "id": 123303, "legend": { "avg": false, "current": false, @@ -6215,7 +7153,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6227,10 +7165,10 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_querynode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "sum(milvus_querynode_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", @@ -6240,7 +7178,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "Flowgraph Num", + "title": "Partition Loaded Num", "tooltip": { "shared": true, "sort": 0, @@ -6255,6 +7193,7 @@ "yaxes": [ { "$$hashKey": "object:536", + "decimals": 0, "format": "short", "logBase": 1, "min": "0", @@ -6278,19 +7217,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The length of the task queue for unsolved read requests", + "description": "Total number of the loaded segment in QueryNode.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 112 + "x": 16, + "y": 126 }, "hiddenSeries": false, - "id": 123351, + "id": 123305, "legend": { "avg": false, "current": false, @@ -6307,7 +7246,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6319,20 +7258,20 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_querynode_read_task_unsolved_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "sum(milvus_querynode_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id, segment_state)", "interval": "", "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "{{pod}}-{{node_id}}-{{segment_state}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Unsolved Read Task Length", + "title": "Segment Loaded Num", "tooltip": { "shared": true, "sort": 0, @@ -6347,6 +7286,7 @@ "yaxes": [ { "$$hashKey": "object:536", + "decimals": 0, "format": "short", "logBase": 1, "min": "0", @@ -6370,19 +7310,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The length of the task queue of read requests to be executed", + "description": "Average, maximum and minimum values of the timestamps for time tick behind.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 112 + "x": 0, + "y": 134 }, "hiddenSeries": false, - "id": 123356, + "id": 123385, "legend": { "avg": false, "current": false, @@ -6399,7 +7339,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6411,20 +7351,45 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_querynode_read_task_ready_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "avg(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)", + "hide": false, "interval": "", "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "{{pod}}-{{node_id}}-avg", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "max(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-max", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "min(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-min", + "refId": "C" } ], "thresholds": [], "timeRegions": [], - "title": "Ready Read Task Length", + "title": "TimeTick Lag Behind Now (Consumed Insert)", "tooltip": { "shared": true, "sort": 0, @@ -6439,9 +7404,9 @@ "yaxes": [ { "$$hashKey": "object:536", - "format": "short", + "decimals": 0, + "format": "ms", "logBase": 1, - "min": "0", "show": true }, { @@ -6462,19 +7427,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Number of read requests currently being executed in parallel", + "description": "Total number of dml virtual channels for QueryNode watch", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 112 + "x": 8, + "y": 134 }, "hiddenSeries": false, - "id": 123357, + "id": 123304, "legend": { "avg": false, "current": false, @@ -6491,7 +7456,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6503,10 +7468,10 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_querynode_read_task_concurrency{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "sum(milvus_querynode_dml_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", @@ -6516,7 +7481,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "Parallel Read Task Num", + "title": "DML Virtual Channel", "tooltip": { "shared": true, "sort": 0, @@ -6554,19 +7519,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "cpu utilization under scheduler evaluation", + "description": "per-second increasing rate of consuming message", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 118 + "x": 16, + "y": 134 }, "hiddenSeries": false, - "id": 123358, + "id": 123387, "legend": { "avg": false, "current": false, @@ -6583,7 +7548,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6595,20 +7560,32 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_querynode_estimate_cpu_usage{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "sum(increase(milvus_querynode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id, msg_type)", + "hide": false, "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querynode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-all", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Estimate CPU Usage", + "title": "Consumed Message Rate", "tooltip": { "shared": true, "sort": 0, @@ -6622,14 +7599,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", - "format": "short", + "$$hashKey": "object:3414", + "format": "cps", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true @@ -6646,19 +7623,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The number of original tasks contained in the merged search task", + "description": "The 99th percentile and average latency of delegator process insert/delete over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 118 + "x": 0, + "y": 142 }, "hiddenSeries": false, - "id": 123352, + "id": 123438, "legend": { "avg": false, "current": false, @@ -6675,7 +7652,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6687,31 +7664,35 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, msg_type) (rate(milvus_querynode_process_insert_or_delete_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{msg_type}}", "queryType": "randomWalk", + "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "sum(increase(milvus_querynode_search_group_size_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_size_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "expr": "sum(increase(milvus_querynode_process_insert_or_delete_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, msg_type) / sum(increase(milvus_querynode_process_insert_or_delete_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, msg_type)", "hide": false, "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{msg_type}}", + "range": true, "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Search Group Size", + "title": "Processing Insert/Delete Latency", "tooltip": { "shared": true, "sort": 0, @@ -6726,7 +7707,7 @@ "yaxes": [ { "$$hashKey": "object:161", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -6749,19 +7730,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Number of queries for search requests", + "description": "Total number of the queryable entities in QueryNode.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 118 + "x": 8, + "y": 142 }, "hiddenSeries": false, - "id": 123361, + "id": 123365, "legend": { "avg": false, "current": false, @@ -6778,7 +7759,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6790,31 +7771,34 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "sum(milvus_querynode_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id, collection_id, segment_state)", "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", + "intervalFactor": 2, + "legendFormat": "{{pod}}-{{node_id}}-{{collection_id}}-{{segment_state}}", "queryType": "randomWalk", + "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_search_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "editorMode": "code", + "expr": "sum(milvus_querynode_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", + "legendFormat": "Total", + "range": true, "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Search NQ", + "title": "Queryable Entity Num", "tooltip": { "shared": true, "sort": 0, @@ -6828,14 +7812,15 @@ }, "yaxes": [ { - "$$hashKey": "object:161", + "$$hashKey": "object:536", + "decimals": 0, "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:162", + "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true @@ -6852,19 +7837,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Number of queries for the merged search requests", + "description": "per-second increasing rate of searching requests.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 124 + "x": 16, + "y": 142 }, "hiddenSeries": false, - "id": 123360, + "id": 123350, "legend": { "avg": false, "current": false, @@ -6881,7 +7866,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6893,31 +7878,19 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "sum(increase(milvus_querynode_sq_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (query_type, status, pod, node_id)", "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", + "legendFormat": "{{pod}}-{{node_id}}-{{query_type}}-{{status}}", "queryType": "randomWalk", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_search_group_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "avg-milvus_querynode_search_group_nq", - "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Search Group NQ", + "title": "Search Request Rate", "tooltip": { "shared": true, "sort": 0, @@ -6931,14 +7904,14 @@ }, "yaxes": [ { - "$$hashKey": "object:161", + "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:162", + "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true @@ -6955,19 +7928,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Top_K for search requests", + "description": "search latency by phase", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 124 + "x": 0, + "y": 150 }, "hiddenSeries": false, - "id": 123359, + "id": 123466, "legend": { "avg": false, "current": false, @@ -6984,7 +7957,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6996,31 +7969,81 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "histogram_quantile(0.99, sum by (le, query_type) (rate(milvus_querynode_sq_queue_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", + "legendFormat": "p99-{{query_type}}-InQueue", "queryType": "randomWalk", + "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_search_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "editorMode": "code", + "expr": "sum(increase(milvus_querynode_sq_queue_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (query_type) / sum(increase(milvus_querynode_sq_queue_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (query_type) ", "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", + "legendFormat": "avg-{{query_type}}-InQueue", + "range": true, "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, query_type, segment_state) (rate(milvus_querynode_sq_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "hide": false, + "legendFormat": "p99-{{query_type}}-{{segment_state}}-Segment", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(milvus_querynode_sq_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (query_type, segment_state) / sum(increase(milvus_querynode_sq_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (query_type, segment_state)", + "hide": false, + "legendFormat": "avg-{{query_type}}-{{segment_state}}-Segment", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, query_type) (rate(milvus_querynode_sq_reduce_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "hide": false, + "legendFormat": "p99-{{query_type}}-Reduce", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(sum(increase(milvus_querynode_sq_reduce_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (query_type) / sum(increase(milvus_querynode_sq_reduce_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (query_type)) ", + "hide": false, + "legendFormat": "avg-{{query_type}}-Reduce", + "range": true, + "refId": "F" } ], "thresholds": [], "timeRegions": [], - "title": "Search Top_K", + "title": "Search Latency By Phase", "tooltip": { "shared": true, "sort": 0, @@ -7034,14 +8057,14 @@ }, "yaxes": [ { - "$$hashKey": "object:161", - "format": "short", + "$$hashKey": "object:3414", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:162", + "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true @@ -7058,19 +8081,25 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of search segment over the last 2 minutes.", + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] }, - "description": "Top_K for the merged search requests.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 124 + "x": 8, + "y": 150 }, "hiddenSeries": false, - "id": 123362, + "id": 123374, "legend": { "avg": false, "current": false, @@ -7087,7 +8116,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -7099,31 +8128,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "exemplar": false, + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type, segment_state) (rate(milvus_querynode_sq_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}-{{segment_state}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_querynode_search_group_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "expr": "sum(increase(milvus_querynode_sq_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type, segment_state) / sum(increase(milvus_querynode_sq_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type, segment_state)", "hide": false, "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}_{{segment_state}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Search Group Top_K", + "title": "Search Segment Latency", "tooltip": { "shared": true, "sort": 0, @@ -7138,7 +8167,7 @@ "yaxes": [ { "$$hashKey": "object:161", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -7161,19 +8190,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "per-second increasing rate of evicted read requests.", + "description": "The 99th percentile and average latency of search and query request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 130 + "x": 16, + "y": 150 }, "hiddenSeries": false, - "id": 123364, + "id": 123366, "legend": { "avg": false, "current": false, @@ -7190,7 +8219,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -7202,19 +8231,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_querynode_read_evicted_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querynode_sq_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Evicted Read Requests Rate", + "title": "Search Request Latency", "tooltip": { "shared": true, "sort": 0, @@ -7228,14 +8269,14 @@ }, "yaxes": [ { - "$$hashKey": "object:3414", - "format": "short", + "$$hashKey": "object:161", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:3415", + "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true @@ -7246,22 +8287,25 @@ } }, { - "id": 123397, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of search or query in queue over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 130 - }, - "type": "graph", - "title": "Knowhere Search Top_K", - "datasource": { - "uid": "$datasource", - "type": "prometheus" + "x": 0, + "y": 158 }, - "thresholds": [], - "pluginVersion": "8.5.20", - "description": "Top_K for knowhere search requests", + "hiddenSeries": false, + "id": 123372, "legend": { "avg": false, "current": false, @@ -7271,62 +8315,65 @@ "total": false, "values": false }, - "aliasColors": {}, - "dashLength": 10, - "fill": 1, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", "pointradius": 2, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(knowhere_search_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_queue_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(knowhere_search_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(knowhere_search_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "expr": "sum(increase(milvus_querynode_sq_queue_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_queue_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", "hide": false, "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", "refId": "B" } ], + "thresholds": [], "timeRegions": [], + "title": "Search in Queue Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, + "type": "graph", "xaxis": { "mode": "time", "show": true, - "values": [], - "name": null, - "buckets": null + "values": [] }, "yaxes": [ { "$$hashKey": "object:161", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -7340,35 +8387,28 @@ ], "yaxis": { "align": false - }, + } + }, + { + "aliasColors": {}, "bars": false, + "dashLength": 10, "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of load segment over the last 2 minutes.", + "fill": 1, "fillGradient": 0, - "hiddenSeries": false, - "percentage": false, - "points": false, - "stack": false, - "steppedLine": false, - "timeFrom": null, - "timeShift": null - }, - { - "id": 123395, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 130 + "x": 8, + "y": 158 }, - "type": "graph", - "title": "Knowhere Search Count", - "datasource": { - "uid": "$datasource", - "type": "prometheus" - }, - "thresholds": [], - "pluginVersion": "8.5.20", - "description": "Total number of knowhere search cnt.", + "hiddenSeries": false, + "id": 123311, "legend": { "avg": false, "current": false, @@ -7378,58 +8418,71 @@ "total": false, "values": false }, - "aliasColors": {}, - "dashLength": 10, - "fill": 1, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", "pointradius": 2, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(knowhere_search_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_load_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querynode_load_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_load_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" } ], + "thresholds": [], "timeRegions": [], + "title": "Load Segment Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, + "type": "graph", "xaxis": { "mode": "time", "show": true, - "values": [], - "name": null, - "buckets": null + "values": [] }, "yaxes": [ { - "$$hashKey": "object:536", - "decimals": 0, - "format": "short", + "$$hashKey": "object:161", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true @@ -7437,35 +8490,28 @@ ], "yaxis": { "align": false - }, + } + }, + { + "aliasColors": {}, "bars": false, + "dashLength": 10, "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of search at the segcore step over the last 2 minutes.", + "fill": 1, "fillGradient": 0, - "hiddenSeries": false, - "percentage": false, - "points": false, - "stack": false, - "steppedLine": false, - "timeFrom": null, - "timeShift": null - }, - { - "id": 123396, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 136 - }, - "type": "graph", - "title": "Knowhere Range Search Count", - "datasource": { - "uid": "$datasource", - "type": "prometheus" + "x": 16, + "y": 158 }, - "thresholds": [], - "pluginVersion": "8.5.20", - "description": "Total number of knowhere range search cnt.", + "hiddenSeries": false, + "id": 123310, "legend": { "avg": false, "current": false, @@ -7475,58 +8521,71 @@ "total": false, "values": false }, - "aliasColors": {}, - "dashLength": 10, - "fill": 1, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", "pointradius": 2, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(knowhere_range_search_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_querynode_sq_core_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querynode_sq_core_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_core_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", + "refId": "B" } ], + "thresholds": [], "timeRegions": [], + "title": "Segcore Request Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, + "type": "graph", "xaxis": { "mode": "time", "show": true, - "values": [], - "name": null, - "buckets": null + "values": [] }, "yaxes": [ { - "$$hashKey": "object:536", - "decimals": 0, - "format": "short", + "$$hashKey": "object:161", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true @@ -7534,30 +8593,7 @@ ], "yaxis": { "align": false - }, - "bars": false, - "dashes": false, - "fillGradient": 0, - "hiddenSeries": false, - "percentage": false, - "points": false, - "stack": false, - "steppedLine": false, - "timeFrom": null, - "timeShift": null - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 136 - }, - "id": 123172, - "panels": [], - "title": "Data Coordinator", - "type": "row" + } }, { "aliasColors": {}, @@ -7566,19 +8602,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Number of data nodes which has register with etcd.", + "description": "The 99th percentile and average latency of search or query reduce over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 137 + "y": 166 }, "hiddenSeries": false, - "id": 123207, + "id": 123367, "legend": { "avg": false, "current": false, @@ -7595,7 +8631,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -7607,20 +8643,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_datacoord_datanode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_reduce_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "total", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querynode_sq_reduce_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_reduce_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Data Node Num", + "title": "Search Reduce Latency", "tooltip": { "shared": true, "sort": 0, @@ -7634,14 +8681,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", - "format": "short", + "$$hashKey": "object:161", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true @@ -7658,19 +8705,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${datasource}" }, - "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now from datanode.", + "description": "The length of the task queue of read requests to be executed", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 137 + "y": 166 }, "hiddenSeries": false, - "id": 123390, + "id": 123356, "legend": { "avg": false, "current": false, @@ -7687,7 +8734,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -7699,45 +8746,20 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "avg(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "hide": false, + "expr": "sum(milvus_querynode_read_task_ready_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}-avg", + "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "max(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-max", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "min(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-min", - "refId": "C" } ], "thresholds": [], "timeRegions": [], - "title": "TimeTick Lag Behind Now From DataNode", + "title": "Ready Read Task Length", "tooltip": { "shared": true, "sort": 0, @@ -7752,9 +8774,9 @@ "yaxes": [ { "$$hashKey": "object:536", - "decimals": 0, - "format": "ms", + "format": "short", "logBase": 1, + "min": "0", "show": true }, { @@ -7775,19 +8797,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The number of rows of valid data accumulated in DataCoord that flushed.", + "description": "The 99th percentile and average latency of search at the segcore step over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 137 + "y": 166 }, "hiddenSeries": false, - "id": 123269, + "id": 123435, "legend": { "avg": false, "current": false, @@ -7804,7 +8826,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -7816,20 +8838,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_datacoord_stored_rows_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_wait_tsafe_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "row", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querynode_sq_wait_tsafe_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_wait_tsafe_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Stored Rows", + "title": "Wait tSafe Latency", "tooltip": { "shared": true, "sort": 0, @@ -7843,14 +8876,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", - "format": "short", + "$$hashKey": "object:161", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true @@ -7867,19 +8900,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "per-second increasing rate of data flushed.", + "description": "The length of the task queue for unsolved read requests", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 143 + "y": 174 }, "hiddenSeries": false, - "id": 123371, + "id": 123351, "legend": { "avg": false, "current": false, @@ -7896,7 +8929,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -7908,11 +8941,12 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_datacoord_stored_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", + "expr": "sum(milvus_querynode_read_task_unsolved_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", + "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" @@ -7920,7 +8954,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "Stored Rows Rate", + "title": "Unsolved Read Task Length", "tooltip": { "shared": true, "sort": 0, @@ -7934,14 +8968,14 @@ }, "yaxes": [ { - "$$hashKey": "object:3414", + "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:3415", + "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true @@ -7958,19 +8992,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Number of segments with different states in the Meta of DataCoord", + "description": "Number of queries for search requests", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 143 + "y": 174 }, "hiddenSeries": false, - "id": 123267, + "id": 123361, "legend": { "avg": false, "current": false, @@ -7987,7 +9021,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -7999,20 +9033,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_datacoord_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (segment_state)", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{segment_state}}", + "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querynode_search_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Segment Num", + "title": "Search NQ", "tooltip": { "shared": true, "sort": 0, @@ -8026,14 +9071,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", + "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true @@ -8050,19 +9095,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "binlog size of all collections/segments, unit byte", + "description": "Total number of flowgraph", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 143 + "y": 174 }, "hiddenSeries": false, - "id": 123382, + "id": 123312, "legend": { "avg": false, "current": false, @@ -8079,7 +9124,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -8091,20 +9136,20 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_datacoord_stored_binlog_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", + "expr": "sum(milvus_querynode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, - "legendFormat": "size", + "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Stored Binlog Size", + "title": "Flowgraph Num", "tooltip": { "shared": true, "sort": 0, @@ -8119,7 +9164,7 @@ "yaxes": [ { "$$hashKey": "object:536", - "format": "bytes", + "format": "short", "logBase": 1, "min": "0", "show": true @@ -8135,19 +9180,6 @@ "align": false } }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 149 - }, - "id": 123242, - "panels": [], - "title": "Data Node", - "type": "row" - }, { "aliasColors": {}, "bars": false, @@ -8155,19 +9187,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Total number of flowgraph", + "description": "Number of queries for the merged search requests", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 150 + "y": 182 }, "hiddenSeries": false, - "id": 123272, + "id": 123360, "legend": { "avg": false, "current": false, @@ -8184,7 +9216,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -8196,20 +9228,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_datanode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querynode_search_group_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-milvus_querynode_search_group_nq", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Flowgraph Num", + "title": "Search Group NQ", "tooltip": { "shared": true, "sort": 0, @@ -8223,14 +9266,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", + "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true @@ -8247,19 +9290,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Total number of prodecer created on the DataNode", + "description": "The number of original tasks contained in the merged search task", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 150 + "y": 182 }, "hiddenSeries": false, - "id": 123277, + "id": 123352, "legend": { "avg": false, "current": false, @@ -8276,7 +9319,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -8288,20 +9331,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_datanode_producer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querynode_search_group_size_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_size_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Producer Num", + "title": "Search Group Size", "tooltip": { "shared": true, "sort": 0, @@ -8315,14 +9369,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", + "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true @@ -8339,19 +9393,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${datasource}" }, - "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now.", + "description": "Number of read requests currently being executed in parallel", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 150 + "y": 182 }, "hiddenSeries": false, - "id": 123393, + "id": 123357, "legend": { "avg": false, "current": false, @@ -8368,7 +9422,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -8380,45 +9434,20 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "avg(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)", - "hide": false, + "expr": "sum(milvus_querynode_read_task_concurrency{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "interval": "", "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}-avg", + "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "max(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-max", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "min(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-min", - "refId": "C" } ], "thresholds": [], "timeRegions": [], - "title": "TimeTick Lag Behind Now (Consumed All)", + "title": "Parallel Read Task Num", "tooltip": { "shared": true, "sort": 0, @@ -8433,9 +9462,9 @@ "yaxes": [ { "$$hashKey": "object:536", - "decimals": 0, - "format": "ms", + "format": "short", "logBase": 1, + "min": "0", "show": true }, { @@ -8456,19 +9485,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${datasource}" }, - "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now, which send to DataCoord", + "description": "per-second increasing rate of evicted read requests.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 156 + "y": 190 }, "hiddenSeries": false, - "id": 123388, + "id": 123364, "legend": { "avg": false, "current": false, @@ -8485,7 +9514,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -8497,45 +9526,19 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "avg(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "hide": false, + "expr": "sum(increase(milvus_querynode_read_evicted_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}-avg", + "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "max(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-max", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "min(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-min", - "refId": "C" } ], "thresholds": [], "timeRegions": [], - "title": "Produced TimeTick Lag Behind Now For DC", + "title": "Evicted Read Requests Rate", "tooltip": { "shared": true, "sort": 0, @@ -8549,14 +9552,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", - "decimals": 0, - "format": "ms", + "$$hashKey": "object:3414", + "format": "short", "logBase": 1, + "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true @@ -8573,19 +9576,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "per-second increasing rate of consuming message", + "description": "Top_K for search requests", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 156 + "y": 190 }, "hiddenSeries": false, - "id": 123389, + "id": 123359, "legend": { "avg": false, "current": false, @@ -8602,7 +9605,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -8614,19 +9617,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_datanode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id, msg_type)", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", + "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querynode_search_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Consumed Message Rate", + "title": "Search Top_K", "tooltip": { "shared": true, "sort": 0, @@ -8640,14 +9655,14 @@ }, "yaxes": [ { - "$$hashKey": "object:3414", - "format": "cps", + "$$hashKey": "object:161", + "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:3415", + "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true @@ -8664,19 +9679,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "forward delete and timetick message to delta channel latency", + "description": "Top_K for the merged search requests.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 156 + "y": 190 }, "hiddenSeries": false, - "id": 123394, + "id": 123362, "legend": { "avg": false, "current": false, @@ -8693,7 +9708,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -8705,31 +9720,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_forward_delete_msg_time_taken_ms_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "hide": false, + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", - "refId": "B" + "queryType": "randomWalk", + "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_datanode_forward_delete_msg_time_taken_ms_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_forward_delete_msg_time_taken_ms_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "expr": "sum(increase(milvus_querynode_search_group_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", - "refId": "C" + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Forward Delete&Timetick Message latency", + "title": "Search Group Top_K", "tooltip": { "shared": true, "sort": 0, @@ -8743,14 +9758,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", - "decimals": 0, - "format": "ms", + "$$hashKey": "object:161", + "format": "short", "logBase": 1, + "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true @@ -8761,842 +9776,1154 @@ } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Total number of consumers created on the DataNode", - "fill": 1, - "fillGradient": 0, + "collapsed": false, "gridPos": { - "h": 6, - "w": 8, + "h": 1, + "w": 24, "x": 0, - "y": 162 + "y": 198 }, - "hiddenSeries": false, - "id": 123276, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "id": 123419, + "panels": [], + "title": "Knowhere", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 199 + }, + "id": 123420, "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(milvus_datanode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", - "queryType": "randomWalk", + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le,pod, module) (rate(search_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "legendFormat": "p99-{{module}}-{{pod}}", + "range": true, "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Consumer Num", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "format": "short", - "logBase": 1, - "min": "0", - "show": true }, { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(search_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module) / sum(increase(search_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module)", + "hide": false, + "legendFormat": "avg-{{module}}-{{pod}}", + "range": true, + "refId": "B" } ], - "yaxis": { - "align": false - } + "title": "search latency", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] }, - "description": "The 99th percentile and average latency of encode the data in the buffer over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 162 - }, - "hiddenSeries": false, - "id": 123282, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "y": 199 }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "id": 123421, "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_encode_buffer_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", - "queryType": "randomWalk", + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le,pod, module) (rate(range_search_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "legendFormat": "p99-{{module}}-{{pod}}", + "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_datanode_encode_buffer_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_encode_buffer_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "editorMode": "code", + "expr": "sum(increase(range_search_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module) / sum(increase(range_search_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module)", "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", + "legendFormat": "avg-{{module}}-{{pod}}", + "range": true, "refId": "B" } ], - "thresholds": [], - "timeRegions": [], - "title": "Encode Buffer Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:161", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:162", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "title": "range search latency", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] }, - "description": "per-second increasing rate of messages consumed for insert and delete operation.", - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 162 - }, - "hiddenSeries": false, - "id": 123274, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "y": 199 }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "id": 123428, "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_datanode_msg_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)", - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", - "queryType": "randomWalk", + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le,pod, module) (rate(build_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "legendFormat": "p99-{{module}}-{{pod}}", + "range": true, "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Msg Rows Consumed Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:101", - "format": "cps", - "logBase": 1, - "min": "0", - "show": true }, { - "$$hashKey": "object:102", - "format": "short", - "logBase": 1, - "show": true + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(build_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module) / sum(increase(knowhere_build_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module)", + "hide": false, + "legendFormat": "avg-{{module}}-{{pod}}", + "range": true, + "refId": "B" } ], - "yaxis": { - "align": false - } + "title": "build index latency", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] }, - "description": "Total number of segment that has been not flushed.", - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 168 - }, - "hiddenSeries": false, - "id": 123280, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "y": 207 }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "id": 123425, "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(milvus_datanode_unflushed_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", - "queryType": "randomWalk", + "editorMode": "code", + "expr": "sum(increase(search_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, module)", + "legendFormat": "{{module}}-{{pod}}", + "range": true, "refId": "A" } ], - "thresholds": [], - "timeRegions": [], - "title": "Unflushed Segment Num", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "title": "search rate", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] }, - "description": "per-second increasing rate of auto flush operate.", - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 168 - }, - "hiddenSeries": false, - "id": 123285, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "y": 207 }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "id": 123423, "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_datanode_autoflush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)", - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-{{status}}", - "queryType": "randomWalk", + "editorMode": "code", + "expr": "sum(increase(range_search_latencycount{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, module)", + "legendFormat": "{{module}}-{{pod}}", + "range": true, "refId": "A" } ], - "thresholds": [], - "timeRegions": [], - "title": "Autoflush Operate Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:101", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:102", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "title": "range search rate", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] }, - "description": "per-second increasing rate of each message that has been flushed.", - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 168 + "y": 207 }, - "hiddenSeries": false, - "id": 123275, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "id": 123417, "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_datanode_flushed_data_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)", - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", - "queryType": "randomWalk", + "editorMode": "code", + "expr": "sum(increase(build_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, module)", + "legendFormat": "{{module}}-{{pod}}", + "range": true, "refId": "A" } ], - "thresholds": [], - "timeRegions": [], - "title": "Flush Data Size Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:101", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:102", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "title": "build index rate", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] }, - "description": "per-second increasing rate of flush operete.", - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 174 - }, - "hiddenSeries": false, - "id": 123284, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "y": 215 }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "id": 123443, "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_datanode_flush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)", - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-{{status}}", - "queryType": "randomWalk", + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, module, pod) (rate(load_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "instant": false, + "legendFormat": "p99-{{module}}-{{pod}}", + "range": true, "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Flush Operate Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:101", - "format": "short", - "logBase": 1, - "min": "0", - "show": true }, { - "$$hashKey": "object:102", - "format": "short", - "logBase": 1, - "show": true + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(load_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (module, pod) / sum(increase(load_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (module, pod)", + "hide": false, + "instant": false, + "legendFormat": "avg-{{module}}-{{pod}}", + "range": true, + "refId": "B" } ], - "yaxis": { - "align": false - } + "title": "Load Latency", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "per-second increasing rate of flush requests.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 174 - }, - "hiddenSeries": false, - "id": 123286, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true + "uid": "${datasource}" }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_datanode_flush_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)", - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-{{status}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Flush Request Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 215 }, - "yaxes": [ + "id": 123427, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { - "$$hashKey": "object:101", - "format": "short", - "logBase": 1, - "min": "0", - "show": true + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le,pod, module) (rate(ann_iterator_init_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "legendFormat": "p99-{{module}}-{{pod}}", + "range": true, + "refId": "A" }, { - "$$hashKey": "object:102", - "format": "short", - "logBase": 1, - "show": true + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(ann_iterator_init_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module) / sum(increase(ann_iterator_init_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module)", + "hide": false, + "legendFormat": "avg-{{module}}-{{pod}}", + "range": true, + "refId": "B" } ], - "yaxis": { - "align": false - } + "title": "ann iterator init latency", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] }, - "description": "The 99th percentile and average latency of writte the data in buffer to storage over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 174 - }, - "hiddenSeries": false, - "id": 123283, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "y": 215 }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "id": 123429, "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_save_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", - "queryType": "randomWalk", + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le,pod) (rate(knowhere_index_version_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "legendFormat": "p99-{{pod}}", + "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_datanode_save_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_save_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "editorMode": "code", + "expr": "sum(increase(knowhere_index_version_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod) / sum(increase(knowhere_index_version_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod)", "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", + "legendFormat": "avg-{{pod}}", + "range": true, "refId": "B" } ], - "thresholds": [], - "timeRegions": [], - "title": "Save Data Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "title": "knowhere create version index latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "yaxes": [ - { - "$$hashKey": "object:161", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:162", - "format": "short", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 223 + }, + "id": 123433, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le,pod, module) (rate(search_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "legendFormat": "p99-{{module}}-{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(search_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module) / sum(increase(search_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module)", + "hide": false, + "legendFormat": "avg-{{module}}-{{pod}}", + "range": true, + "refId": "B" + } + ], + "title": "search topk ", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 231 + }, + "id": 123172, + "panels": [], + "title": "Data Coordinator", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of data nodes which has register with etcd.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 232 + }, + "hiddenSeries": false, + "id": 123207, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(milvus_datacoord_datanode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "total", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Data Node Num", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:536", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:537", + "format": "short", "logBase": 1, "show": true } @@ -9612,19 +10939,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of compaction over the last 2 minutes.", + "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now from datanode.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 180 + "x": 8, + "y": 232 }, "hiddenSeries": false, - "id": 123314, + "id": 123390, "legend": { "avg": false, "current": false, @@ -9641,7 +10968,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -9653,32 +10980,45 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_compaction_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "avg(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", + "intervalFactor": 2, + "legendFormat": "{{pod}}-{{node_id}}-avg", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_datanode_compaction_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_compaction_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "expr": "max(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", "hide": false, "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", + "legendFormat": "{{pod}}-{{node_id}}-max", "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "min(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-min", + "refId": "C" } ], "thresholds": [], "timeRegions": [], - "title": "Compaction Latency", + "title": "TimeTick Lag Behind Now From DataNode", "tooltip": { "shared": true, "sort": 0, @@ -9692,14 +11032,14 @@ }, "yaxes": [ { - "$$hashKey": "object:161", - "format": "short", + "$$hashKey": "object:536", + "decimals": 0, + "format": "ms", "logBase": 1, - "min": "0", "show": true }, { - "$$hashKey": "object:162", + "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true @@ -9710,1716 +11050,1765 @@ } }, { - "collapsed": true, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of rows of valid data accumulated in DataCoord that flushed.", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 186 + "h": 8, + "w": 8, + "x": 16, + "y": 232 }, - "id": 123223, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { + "hiddenSeries": false, + "id": 123269, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "per-second increasing rate of building index requests recived by IndexCoord.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 7 - }, - "hiddenSeries": false, - "id": 123225, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_indexcoord_index_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status)", - "interval": "", - "legendFormat": "{{status}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Index Request Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "uid": "${datasource}" }, - "yaxes": [ - { - "$$hashKey": "object:372", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:373", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "sum(milvus_datacoord_stored_rows_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "row", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Stored Rows", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:536", + "format": "short", + "logBase": 1, + "min": "0", + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "$$hashKey": "object:537", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of data flushed.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 240 + }, + "hiddenSeries": false, + "id": 123371, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "Number of indexing tasks with different states in IndexCoord's meta", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 7 - }, - "hiddenSeries": false, - "id": 123227, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_indexcoord_index_task_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (index_task_status)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{index_task_status}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Index Task Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "uid": "${datasource}" }, - "yaxes": [ - { - "$$hashKey": "object:401", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:402", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "sum(increase(milvus_datacoord_stored_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Stored Rows Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3414", + "format": "short", + "logBase": 1, + "min": "0", + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "$$hashKey": "object:3415", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of segments with different states in the Meta of DataCoord", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 240 + }, + "hiddenSeries": false, + "id": 123267, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "Number of index nodes which has register with etcd", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 7 - }, - "hiddenSeries": false, - "id": 123229, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_indexcoord_index_node_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "total", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Index Node Num", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "uid": "${datasource}" }, - "yaxes": [ - { - "$$hashKey": "object:479", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:480", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "sum(milvus_datacoord_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (segment_state)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{segment_state}}", + "queryType": "randomWalk", + "refId": "A" } ], - "title": "Index Coordinator", - "type": "row" + "thresholds": [], + "timeRegions": [], + "title": "Segment Num", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:536", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:537", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { - "collapsed": true, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "binlog size of all collections/segments, unit byte", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 187 + "h": 8, + "w": 8, + "x": 16, + "y": 240 }, - "id": 123231, - "panels": [ + "hiddenSeries": false, + "id": 123382, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "per-second increasing rate of index tasks recevied by IndexNode", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 8 - }, - "hiddenSeries": false, - "id": 123233, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_indexnode_index_task_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)", - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-{{status}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Index Task Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:101", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:102", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "sum(milvus_datacoord_stored_binlog_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "size", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Stored Binlog Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:536", + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "$$hashKey": "object:537", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now from datanode.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 248 + }, + "hiddenSeries": false, + "id": 123437, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency ofr load the FieldData over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 8 + "exemplar": true, + "expr": "topK(10, timestamp(milvus_datacoord_channel_checkpoint_unix_seconds{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) - milvus_datacoord_channel_checkpoint_unix_seconds{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, channel_name)", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}-{{node_id}}-avg", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "channel cp lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:536", + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:537", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of building index requests recived by IndexCoord.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 248 + }, + "hiddenSeries": false, + "id": 123225, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "hiddenSeries": false, - "id": 123235, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_load_field_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_indexnode_load_field_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_load_field_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Load Field Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:161", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:162", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, + "exemplar": true, + "expr": "sum(rate(milvus_datacoord_index_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by (status)", + "interval": "", + "legendFormat": "{{status}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Index Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of decod the FieldData over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 8 - }, - "hiddenSeries": false, - "id": 123238, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_decode_field_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_indexnode_decode_field_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_decode_field_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Decode Field Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:190", - "format": "short", - "label": "", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:191", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "$$hashKey": "object:372", + "format": "short", + "logBase": 1, + "min": "0", + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "$$hashKey": "object:373", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of building index requests recived by IndexCoord.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 248 + }, + "hiddenSeries": false, + "id": 123451, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of build index over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 14 - }, - "hiddenSeries": false, - "id": 123237, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_build_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_indexnode_build_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_build_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Build Index Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:246", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:247", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(milvus_datacoord_index_task_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by (index_task_status)", + "interval": "", + "legendFormat": "{{index_task_status}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Index Task Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of encode index over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 14 - }, - "hiddenSeries": false, - "id": 123239, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_encode_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_indexnode_encode_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_encode_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Encode Index Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:275", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:276", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "$$hashKey": "object:372", + "format": "short", + "logBase": 1, + "min": "0", + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "$$hashKey": "object:373", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 256 + }, + "id": 123242, + "panels": [], + "title": "Data Node", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of flowgraph", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 257 + }, + "hiddenSeries": false, + "id": 123272, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile of latency for save index over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 14 - }, - "hiddenSeries": false, - "id": 123240, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_save_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_indexnode_save_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_save_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Save Index Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "uid": "${datasource}" }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "sum(milvus_datanode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Flowgraph Num", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:536", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:537", + "format": "short", + "logBase": 1, + "show": true } ], - "title": "Index Node", - "type": "row" + "yaxis": { + "align": false + } }, { - "collapsed": true, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of consuming message", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 188 + "h": 8, + "w": 8, + "x": 8, + "y": 257 }, - "id": 123157, - "panels": [ + "hiddenSeries": false, + "id": 123389, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "Total cpu usage of all milvus components.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 183 - }, - "hiddenSeries": false, - "id": 123202, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m]))", - "interval": "", - "intervalFactor": 2, - "legendFormat": "cpu usage", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU Usage Total", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "uid": "${datasource}" }, - "yaxes": [ - { - "$$hashKey": "object:8411", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:8412", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "sum(increase(milvus_datanode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id, msg_type)", + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Consumed Message Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3414", + "format": "cps", + "logBase": 1, + "min": "0", + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 183 - }, - "hiddenSeries": false, - "id": 123204, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})", - "interval": "", - "intervalFactor": 2, - "legendFormat": "memory", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory Total", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "$$hashKey": "object:3415", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 257 + }, + "hiddenSeries": false, + "id": 123393, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "yaxes": [ - { - "$$hashKey": "object:1340", - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:1341", - "format": "bytes", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "avg(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}-{{node_id}}-avg", + "queryType": "randomWalk", + "refId": "A" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "$datasource" - }, - "description": "Number of goroutines that currently exist.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 183 - }, - "hiddenSeries": false, - "id": 123206, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "type": "prometheus", + "uid": "${datasource}" }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true + "exemplar": true, + "expr": "max(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-max", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})", - "interval": "", - "intervalFactor": 2, - "legendFormat": "goroutines", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Goroutines Total", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "exemplar": true, + "expr": "min(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-min", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "TimeTick Lag Behind Now (Consumed All)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:536", + "decimals": 0, + "format": "ms", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:537", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of flush operete.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 265 + }, + "hiddenSeries": false, + "id": 123284, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "exemplar": true, + "expr": "sum(increase(milvus_datanode_flush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)", + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-{{status}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Flush Operate Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:101", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:102", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of encode the data in the buffer over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 265 + }, + "hiddenSeries": false, + "id": 123282, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "yaxes": [ - { - "$$hashKey": "object:1281", - "format": "short", - "label": "", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:1282", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_encode_buffer_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "$datasource" - }, - "description": "process cpu usage", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 189 - }, - "hiddenSeries": false, - "id": 123135, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "type": "prometheus", + "uid": "${datasource}" }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "exemplar": true, + "expr": "sum(increase(milvus_datanode_encode_buffer_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_encode_buffer_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Encode Buffer Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of messages consumed for insert and delete operation.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 265 + }, + "hiddenSeries": false, + "id": 123274, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "sum(increase(milvus_datanode_msg_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)", + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Msg Rows Consumed Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:101", + "format": "cps", + "logBase": 1, + "min": "0", + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "$$hashKey": "object:102", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of compaction over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 273 + }, + "hiddenSeries": false, + "id": 123314, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "Process memory of milvus pod.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 189 - }, - "hiddenSeries": false, - "id": 123133, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": false, - "expr": "process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "uid": "${datasource}" }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "bytes", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_compaction_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "hide": false, + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { - "uid": "$datasource" - }, - "description": "Number of goroutines that currently exist.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 189 - }, - "hiddenSeries": false, - "id": 123125, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Goroutines", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "type": "prometheus", + "uid": "${datasource}" }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "exemplar": true, + "expr": "sum(increase(milvus_datanode_compaction_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_compaction_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Compaction Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of auto flush operate.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 273 + }, + "hiddenSeries": false, + "id": 123285, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "sum(increase(milvus_datanode_autoflush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)", + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-{{status}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Autoflush Operate Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:101", + "format": "short", + "logBase": 1, + "min": "0", + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "$$hashKey": "object:102", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of each message that has been flushed.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 273 + }, + "hiddenSeries": false, + "id": 123275, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { "datasource": { - "uid": "$datasource" - }, - "description": "Number of bytes allocated and still in use.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 195 - }, - "hiddenSeries": false, - "id": 123137, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "go_memstats_alloc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Go Allocated Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "type": "prometheus", + "uid": "${datasource}" }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "bytes", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "sum(increase(milvus_datanode_flushed_data_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)", + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Flush Data Size Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:101", + "format": "binBps", + "logBase": 1, + "min": "0", + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "$$hashKey": "object:102", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of flush requests.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 281 + }, + "hiddenSeries": false, + "id": 123286, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { "datasource": { - "uid": "$datasource" - }, - "description": "GC Max duration seconds.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 195 - }, - "hiddenSeries": false, - "id": 123131, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "go_gc_duration_seconds{quantile=\"1\", app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", - "interval": "", - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "GC Max duration seconds", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "type": "prometheus", + "uid": "${datasource}" }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "sum(increase(milvus_datanode_flush_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)", + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-{{status}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Flush Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:101", + "format": "short", + "logBase": 1, + "min": "0", + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "$$hashKey": "object:102", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of writte the data in buffer to storage over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 281 + }, + "hiddenSeries": false, + "id": 123283, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { "datasource": { - "uid": "$datasource" - }, - "description": "Number of OS threads created.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 195 - }, - "hiddenSeries": false, - "id": 123127, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "go_threads{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "OS Threads", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "type": "prometheus", + "uid": "${datasource}" }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_save_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": true, + "expr": "sum(increase(milvus_datanode_save_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_save_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Save Data Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 289 + }, + "id": 123231, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "type": "prometheus", + "uid": "${datasource}" }, - "description": "Next GC Bytes.", + "description": "per-second increasing rate of index tasks recevied by IndexNode", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 201 + "y": 306 }, "hiddenSeries": false, - "id": 123129, + "id": 123233, "legend": { "avg": false, "current": false, @@ -11436,7 +12825,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -11446,18 +12835,21 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "go_memstats_next_gc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", + "expr": "sum(increase(milvus_indexnode_index_task_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", + "legendFormat": "{{pod}}-{{node_id}}-{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Next GC Bytes", + "title": "Index Task Rate", "tooltip": { "shared": true, "sort": 0, @@ -11471,12 +12863,14 @@ }, "yaxes": [ { - "format": "bytes", + "$$hashKey": "object:101", + "format": "short", "logBase": 1, "min": "0", "show": true }, { + "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true @@ -11492,19 +12886,26 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency ofr load the FieldData over the last 2 minutes.", + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] }, - "description": "Number of process opened fds.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 201 + "y": 306 }, "hiddenSeries": false, - "id": 123211, + "id": 123235, "legend": { "avg": false, "current": false, @@ -11521,7 +12922,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -11531,103 +12932,33 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "process_open_fds{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_load_field_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", + "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Process Opened Fds", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true }, { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "description": "The fraction of this program's available CPU time used by the GC since the program started.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 201 - }, - "hiddenSeries": false, - "id": 123212, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "go_memstats_gc_cpu_fraction{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "expr": "sum(increase(milvus_indexnode_load_field_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_load_field_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "GC CPU time", + "title": "Load Field Latency", "tooltip": { "shared": true, "sort": 0, @@ -11641,12 +12972,13 @@ }, "yaxes": [ { - "format": "short", + "$$hashKey": "object:161", + "format": "ms", "logBase": 1, - "min": "0", "show": true }, { + "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true @@ -11662,19 +12994,20 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "type": "prometheus", + "uid": "${datasource}" }, - "description": "Rate of memory frees.", + "description": "The 99th percentile and average latency of decod the FieldData over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 207 + "x": 16, + "y": 306 }, "hiddenSeries": false, - "id": 123213, + "id": 123238, "legend": { "avg": false, "current": false, @@ -11691,7 +13024,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -11701,103 +13034,33 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(go_memstats_frees_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_decode_field_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", + "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory Free Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true }, { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "description": "Rate of go memory mallocs.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 207 - }, - "hiddenSeries": false, - "id": 123215, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "rate(go_memstats_mallocs_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[2m])", + "expr": "sum(increase(milvus_indexnode_decode_field_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_decode_field_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Go Memory mallocs rate.", + "title": "Decode Field Latency", "tooltip": { "shared": true, "sort": 0, @@ -11811,12 +13074,14 @@ }, "yaxes": [ { + "$$hashKey": "object:190", "format": "short", + "label": "", "logBase": 1, - "min": "0", "show": true }, { + "$$hashKey": "object:191", "format": "short", "logBase": 1, "show": true @@ -11832,19 +13097,26 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of build index over the last 2 minutes.", + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] }, - "description": "Number of allocated heap objects.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 207 + "x": 0, + "y": 314 }, "hiddenSeries": false, - "id": 123216, + "id": 123237, "legend": { "avg": false, "current": false, @@ -11861,7 +13133,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -11871,18 +13143,33 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "go_memstats_heap_objects{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_build_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", + "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_indexnode_build_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_build_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Allocated Heap Objects", + "title": "Build Index Latency", "tooltip": { "shared": true, "sort": 0, @@ -11896,12 +13183,14 @@ }, "yaxes": [ { - "format": "short", + "$$hashKey": "object:246", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { + "$$hashKey": "object:247", "format": "short", "logBase": 1, "show": true @@ -11918,19 +13207,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Rate of bytes allocated, even if freed.", + "description": "The 99th percentile and average latency of encode index over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 213 + "x": 8, + "y": 314 }, "hiddenSeries": false, - "id": 123217, + "id": 123239, "legend": { "avg": false, "current": false, @@ -11947,7 +13236,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -11959,20 +13248,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "rate(go_memstats_alloc_bytes_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_encode_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", + "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_indexnode_encode_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_encode_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Memory Allocated Rate", + "title": "Encode Index Latency", "tooltip": { "shared": true, "sort": 0, @@ -11986,14 +13286,14 @@ }, "yaxes": [ { - "$$hashKey": "object:8539", + "$$hashKey": "object:275", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:8540", + "$$hashKey": "object:276", "format": "short", "logBase": 1, "show": true @@ -12009,19 +13309,20 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "type": "prometheus", + "uid": "${datasource}" }, - "description": "Number of heap bytes released to OS.", + "description": "The 99th percentile of latency for save index over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 213 + "x": 16, + "y": 314 }, "hiddenSeries": false, - "id": 123218, + "id": 123240, "legend": { "avg": false, "current": false, @@ -12038,7 +13339,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -12048,18 +13349,33 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "exemplar": true, - "expr": "go_memstats_heap_released_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_save_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", + "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_indexnode_save_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_save_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Go Heap released", + "title": "Save Index Latency", "tooltip": { "shared": true, "sort": 0, @@ -12073,12 +13389,13 @@ }, "yaxes": [ { - "format": "bytes", + "$$hashKey": "object:331", + "format": "short", "logBase": 1, - "min": "0", "show": true }, { + "$$hashKey": "object:332", "format": "short", "logBase": 1, "show": true @@ -12087,1742 +13404,1725 @@ "yaxis": { "align": false } + } + ], + "title": "Index Node", + "type": "row" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 290 + }, + "id": 123157, + "panels": [], + "title": "Runtime", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total cpu usage of all milvus components.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 291 + }, + "hiddenSeries": false, + "id": 123202, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "cpu usage", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU Usage Total", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:8411", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:8412", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 291 + }, + "hiddenSeries": false, + "id": 123204, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "memory", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Total", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1340", + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "description": "Number of heap bytes waiting to be used.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 213 - }, - "hiddenSeries": false, - "id": 123219, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "go_memstats_heap_idle_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Go Heap idle", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "$$hashKey": "object:1341", + "format": "bytes", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "description": "Number of goroutines that currently exist.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 291 + }, + "hiddenSeries": false, + "id": 123206, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "goroutines", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Goroutines Total", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1281", + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "$$hashKey": "object:1282", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "process cpu usage", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 299 + }, + "hiddenSeries": false, + "id": 123135, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "Number of heap bytes that be used.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 219 - }, - "hiddenSeries": false, - "id": 123375, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "go_memstats_heap_inuse_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Go Heap in Use", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "uid": "${datasource}" }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "editorMode": "code", + "exemplar": true, + "expr": "rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[1m])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:544", + "format": "short", + "logBase": 1, + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "$$hashKey": "object:545", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Process memory of milvus pod.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 299 + }, + "hiddenSeries": false, + "id": 123133, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "Number of container threads created.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 219 - }, - "hiddenSeries": false, - "id": 123376, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "container_threads{image=\"\", namespace=\"$namespace\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Container Threads", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "uid": "${datasource}" }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "exemplar": false, + "expr": "process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:598", + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:599", + "format": "bytes", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "description": "Number of goroutines that currently exist.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 299 + }, + "hiddenSeries": false, + "id": 123125, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true } ], - "title": "Runtime", - "type": "row" + "yaxis": { + "align": false + } }, { - "collapsed": false, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "description": "Number of bytes allocated and still in use.", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 1, - "w": 24, + "h": 8, + "w": 8, "x": 0, - "y": 4 + "y": 307 }, - "id": 123402, - "panels": [], - "title": "storage", - "type": "row" + "hiddenSeries": false, + "id": 123137, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "go_memstats_alloc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Go Allocated Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "bytes", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { - "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, + "description": "GC Max duration seconds.", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 10, - "w": 14, - "x": 5, - "y": 5 + "h": 8, + "w": 8, + "x": 8, + "y": 307 + }, + "hiddenSeries": false, + "id": 123131, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "id": 123406, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "go_gc_duration_seconds{quantile=\"1\", app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", + "interval": "", + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "GC Max duration seconds", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true }, - "tooltip": { - "mode": "single", - "sort": "none" + { + "format": "short", + "logBase": 1, + "show": true } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "description": "Number of OS threads created.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 307 + }, + "hiddenSeries": false, + "id": 123127, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.90, milvus_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", - "legendFormat": "90% {{persistent_data_op_type}}", - "range": true, + "exemplar": true, + "expr": "go_threads{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "refId": "A" - }, + } + ], + "thresholds": [], + "timeRegions": [], + "title": "OS Threads", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, milvus_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", - "hide": false, - "legendFormat": "75% {{persistent_data_op_type}}", - "range": true, - "refId": "B" + "format": "short", + "logBase": 1, + "min": "0", + "show": true }, { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, milvus_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", - "hide": false, - "legendFormat": "50% {{persistent_data_op_type}}", - "range": true, - "refId": "C" + "format": "short", + "logBase": 1, + "show": true } ], - "title": "[go]kv size", - "type": "timeseries" + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { - "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, + "description": "Next GC Bytes.", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 10, - "w": 14, - "x": 5, - "y": 15 + "h": 8, + "w": 8, + "x": 0, + "y": 315 + }, + "hiddenSeries": false, + "id": 123129, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "id": 123446, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Name", - "sortDesc": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "rate(milvus_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", persistent_data_op_type=\"get\"}[2m])", - "legendFormat": "{{le}}", - "range": true, + "exemplar": true, + "expr": "go_memstats_next_gc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "[go]get", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "thresholds": [], + "timeRegions": [], + "title": "Next GC Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true }, - "overrides": [] + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" }, + "description": "Number of process opened fds.", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 10, - "w": 14, - "x": 5, - "y": 25 + "h": 8, + "w": 8, + "x": 8, + "y": 315 + }, + "hiddenSeries": false, + "id": 123211, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "id": 123447, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "rate(milvus_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", persistent_data_op_type=\"put\"}[2m])", - "legendFormat": "{{le}}", - "range": true, + "exemplar": true, + "expr": "process_open_fds{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "[go]put", - "type": "timeseries" + "thresholds": [], + "timeRegions": [], + "title": "Process Opened Fds", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { - "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "list::success", - "remove::total" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, + "description": "The fraction of this program's available CPU time used by the GC since the program started.", + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 35 + "w": 8, + "x": 16, + "y": 315 + }, + "hiddenSeries": false, + "id": 123212, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "id": 123404, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(increase(milvus_storage_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status!=\"fail\"}[2m])/120) by (status, pod, persistent_data_op_type)", - "legendFormat": "{{persistent_data_op_type}}::{{status}}", - "range": true, + "exemplar": true, + "expr": "go_memstats_gc_cpu_fraction{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "[go]total/success request", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] + "thresholds": [], + "timeRegions": [], + "title": "GC CPU time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 35 + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] }, - "id": 123444, - "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le, pod, persistent_data_op_type) (rate(milvus_storage_request_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "legendFormat": "{{persistent_data_op_type}}", - "range": true, - "refId": "A" + "format": "short", + "logBase": 1, + "show": true } ], - "title": "[go]pct99", - "type": "timeseries" + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { - "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, + "description": "Rate of memory frees.", + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 43 + "y": 323 + }, + "hiddenSeries": false, + "id": 123213, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "id": 123405, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(increase(milvus_storage_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"fail\"}[2m])/120) by (status, pod, persistent_data_op_type)", - "legendFormat": "{{persistent_data_op_type}}::{{status}}", - "range": true, + "exemplar": true, + "expr": "rate(go_memstats_frees_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "[go]failed request", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] + "thresholds": [], + "timeRegions": [], + "title": "Memory Free Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 43 + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] }, - "id": 123407, - "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(increase(milvus_storage_request_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, persistent_data_op_type) / sum(increase(milvus_storage_request_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, persistent_data_op_type)", - "legendFormat": "{{persistent_data_op_type}}", - "range": true, - "refId": "A" + "format": "short", + "logBase": 1, + "show": true } ], - "title": "[go]avg", - "type": "timeseries" - },{ + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { - "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, + "description": "Rate of go memory mallocs.", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 10, - "w": 14, - "x": 5, - "y": 5 + "h": 8, + "w": 8, + "x": 8, + "y": 323 + }, + "hiddenSeries": false, + "id": 123215, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "id": 123506, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.90, internal_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", - "legendFormat": "90% {{persistent_data_op_type}}", - "range": true, + "exemplar": true, + "expr": "rate(go_memstats_mallocs_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[2m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "refId": "A" - }, + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Go Memory mallocs rate.", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, internal_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", - "hide": false, - "legendFormat": "75% {{persistent_data_op_type}}", - "range": true, - "refId": "B" + "format": "short", + "logBase": 1, + "min": "0", + "show": true }, { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, internal_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", - "hide": false, - "legendFormat": "50% {{persistent_data_op_type}}", - "range": true, - "refId": "C" + "format": "short", + "logBase": 1, + "show": true } ], - "title": "[cpp]kv size", - "type": "timeseries" + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { - "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, + "description": "Number of allocated heap objects.", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 10, - "w": 14, - "x": 5, - "y": 15 + "h": 8, + "w": 8, + "x": 16, + "y": 323 + }, + "hiddenSeries": false, + "id": 123216, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "id": 123546, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Name", - "sortDesc": false - }, - "tooltip": { - "mode": "single", - "sort": "none" + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "go_memstats_heap_objects{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "refId": "A" } + ], + "thresholds": [], + "timeRegions": [], + "title": "Allocated Heap Objects", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] }, - "targets": [ + "yaxes": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "rate(internal_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", persistent_data_op_type=\"get\"}[2m])", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true } ], - "title": "[cpp]get", - "type": "timeseries" + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, + "description": "Rate of bytes allocated, even if freed.", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 10, - "w": 14, - "x": 5, - "y": 25 + "h": 8, + "w": 8, + "x": 0, + "y": 331 + }, + "hiddenSeries": false, + "id": 123217, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "id": 123547, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "code", - "expr": "rate(internal_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", persistent_data_op_type=\"put\"}[2m])", - "legendFormat": "{{le}}", - "range": true, + "exemplar": true, + "expr": "rate(go_memstats_alloc_bytes_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "[cpp]put", - "type": "timeseries" + "thresholds": [], + "timeRegions": [], + "title": "Memory Allocated Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:8539", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:8540", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { - "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "list::success", - "remove::total" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "description": "Number of heap bytes released to OS.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 331 + }, + "hiddenSeries": false, + "id": 123218, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "go_memstats_heap_released_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Go Heap released", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 35 + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] }, - "id": 123504, - "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(increase(internal_storage_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status!=\"fail\"}[2m])/120) by (status, pod, persistent_data_op_type)", - "legendFormat": "{{persistent_data_op_type}}::{{status}}", - "range": true, - "refId": "A" + "format": "short", + "logBase": 1, + "show": true } ], - "title": "[cpp]total/success request", - "type": "timeseries" + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { - "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, + "description": "Number of heap bytes waiting to be used.", + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 35 + "w": 8, + "x": 16, + "y": 331 + }, + "hiddenSeries": false, + "id": 123219, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "id": 123544, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le, pod, persistent_data_op_type) (rate(internal_storage_request_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "legendFormat": "{{persistent_data_op_type}}", - "range": true, + "exemplar": true, + "expr": "go_memstats_heap_idle_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "[cpp]pct99", - "type": "timeseries" + "thresholds": [], + "timeRegions": [], + "title": "Go Heap idle", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, + "description": "Number of heap bytes that be used.", + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 43 + "y": 339 }, - "id": 123505, - "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "hiddenSeries": false, + "id": 123375, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "code", - "expr": "sum(increase(internal_storage_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"fail\"}[2m])/120) by (status, pod, persistent_data_op_type)", - "legendFormat": "{{persistent_data_op_type}}::{{status}}", - "range": true, + "exemplar": true, + "expr": "go_memstats_heap_inuse_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "[cpp]failed request", - "type": "timeseries" + "thresholds": [], + "timeRegions": [], + "title": "Go Heap in Use", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, + "description": "Number of container threads created.", + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 43 + "w": 8, + "x": 8, + "y": 339 + }, + "hiddenSeries": false, + "id": 123376, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "id": 123507, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { "datasource": { @@ -13830,30 +15130,59 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(internal_storage_request_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, persistent_data_op_type) / sum(increase(internal_storage_request_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, persistent_data_op_type)", - "legendFormat": "{{persistent_data_op_type}}", + "exemplar": true, + "expr": "container_threads{image=\"\", namespace=\"$namespace\", pod=~\"$instance-.*\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "range": true, "refId": "A" } ], - "title": "[cpp]avg", - "type": "timeseries" + "thresholds": [], + "timeRegions": [], + "title": "Container Threads", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:420", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:421", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } } ], - "refresh": false, - "schemaVersion": 34, - "style": "dark", + "refresh": "", + "schemaVersion": 38, "tags": [ - "milvus2.0" + "milvus2.4", + "milvus2.3" ], "templating": { "list": [ { - "current": { - "selected": false, - "text": "prometheus", - "value": "prometheus" - }, "hide": 0, "includeAll": false, "multi": false, @@ -13867,42 +15196,20 @@ "type": "datasource" }, { - "current": { - "selected": true, - "text": "milvus", - "value": "milvus" - }, - "hide": 2, - "name": "app_name", - "options": [ - { - "selected": true, - "text": "milvus", - "value": "milvus" - } - ], - "query": "milvus", - "skipUrlSync": false, - "type": "textbox" - }, - { - "current": { - "selected": true, - "text": "chaos-testing", - "value": "chaos-testing" - }, "datasource": { - "uid": "$datasource" + "type": "prometheus", + "uid": "${datasource}" }, - "definition": "label_values(kube_pod_info, namespace)", + "definition": "label_values(go_info,namespace)", "hide": 0, "includeAll": false, "multi": false, "name": "namespace", "options": [], "query": { - "query": "label_values(kube_pod_info, namespace)", - "refId": "prometheus-namespace-Variable-Query" + "qryType": 1, + "query": "label_values(go_info,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", @@ -13914,23 +15221,20 @@ "useTags": false }, { - "current": { - "selected": false, - "text": "bulk-insert-test", - "value": "bulk-insert-test" - }, "datasource": { - "uid": "$datasource" + "type": "prometheus", + "uid": "${datasource}" }, - "definition": "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}, app_kubernetes_io_instance)", + "definition": "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"},app_kubernetes_io_instance)", "hide": 0, "includeAll": false, "multi": false, "name": "instance", "options": [], "query": { - "query": "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}, app_kubernetes_io_instance)", - "refId": "StandardVariableQuery" + "qryType": 1, + "query": "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"},app_kubernetes_io_instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "regex": "", @@ -13950,17 +15254,18 @@ }, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "definition": "label_values(milvus_proxy_collection_mutation_latency_sum{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"}, collection_name)", + "definition": "label_values(milvus_proxy_collection_mutation_latency_sum{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\"},collection_name)", "hide": 0, "includeAll": true, "multi": false, "name": "collection", "options": [], "query": { - "query": "label_values(milvus_proxy_collection_mutation_latency_sum{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"}, collection_name)", - "refId": "StandardVariableQuery" + "qryType": 1, + "query": "label_values(milvus_proxy_collection_mutation_latency_sum{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\"},collection_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", @@ -13970,60 +15275,22 @@ }, { "current": { - "selected": false, - "text": "bulk-insert-test-milvus-standalone-55968cfc55-cxnps", - "value": "bulk-insert-test-milvus-standalone-55968cfc55-cxnps" - }, - "datasource": { - "uid": "$datasource" - }, - "definition": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , pod)", - "hide": 2, - "includeAll": false, - "multi": true, - "name": "pod", - "options": [], - "query": { - "query": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , pod)", - "refId": "prometheus-pod-Variable-Query" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": { - "uid": "$datasource" + "selected": true, + "text": "milvus", + "value": "milvus" }, - "definition": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , app_kubernetes_io_component)", "hide": 2, - "includeAll": false, - "multi": true, - "name": "component", - "options": [], - "query": { - "query": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , app_kubernetes_io_component)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", + "name": "app_name", + "options": [ + { + "selected": true, + "text": "milvus", + "value": "milvus" + } + ], + "query": "milvus", "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "textbox" } ] }, @@ -14060,7 +15327,7 @@ }, "timezone": "browser", "title": "Milvus2.0", - "uid": "uLf5cJ3Gz", - "version": 5, + "uid": "uLf5cJ3Ga", + "version": 3, "weekStart": "" }