Adding support for badnwidth and iops collection in MINIO REST collector

dss_metrics/README.md

@@ -4,8 +4,11 @@
 ## Overview
 The DSS Metrics Agent runs on a per node level. It can collect data from MINIO, target, and other sources. The data is then processed
 into metrics that can be stored into a Promotheus database or exposed at an endpoint. The metrics are associated with tags such as 
-cluster_id, susbsystem_id, target_id, etc.. in order to be able to correlate the metric with a given point/layer in the system. It 
-is assumed that all captured metrics will be time series data.
+cluster_id, susbsystem_id, target_id, etc.. in order to be able to correlate the metric with a given point/layer in the system. It is assumed that all captured metrics will be time series data.
+
+## Dependencies
+- install required python packages listed in requirements.txt
+- set environment variable MINIO_REPORT_METRICS=1, will also need to add to MINIO startup script before running MINIO
 
 ## Execution
 Until the agent is deployed along with the deploy DSS playbook, it will need to be manually run. Currently, the agent supports

dss_metrics/metrics_scope.json

@@ -2,5 +2,7 @@
     "target.subsystem\\d+.kvio.(puts|gets|dels|putBandwidth|getBandwidth)": "target",
     "minio_upstream.outstanding.s3_get_req": "minio cluster",
     "minio_disk_storage_used_bytes": "minio cluster",
-    "minio_disk_storage_total_capacity_bytes": "minio cluster"
+    "minio_disk_storage_total_capacity_bytes": "minio cluster",
+    "minio_metrics_(get|put)_bw": "minio endpoint",
+    "minio_metrics_(del|get|put|)_iops": "minio endpoint"
 }

dss_metrics/minio_rest_collector.py

@@ -50,8 +50,13 @@ def __init__(self, configs, metrics_scopes, whitelist_patterns,
         self.metrics_scopes = metrics_scopes
         self.whitelist_patterns = whitelist_patterns
         self.filter = filter
-        self.minio_metrics = {'minio_disk_storage_used_bytes',
-                              'minio_disk_storage_total_capacity_bytes'}
+        # self.minio_metrics = {'minio_disk_storage_used_bytes',
+        #                       'minio_disk_storage_total_capacity_bytes'}
+        self.ignored_minio_metrics = {
+            'minio_http_requests_duration_seconds_bucket',
+            'minio_http_requests_duration_seconds_sum',
+            'minio_http_requests_duration_seconds_count'}
+
         self.url_prefix = "http://"
         self.cluster_id_url_suffix = "/minio/cluster_id"
         self.metrics_url_suffix = "/minio/prometheus/metrics"
@@ -138,7 +143,9 @@ def get_minio_metrics_from_endpoint(self, endpoint):
         r = requests.get(url)
         metrics_data = []
         for line in r.text.splitlines():
-            if (any(line.startswith(metric) for metric in self.minio_metrics)):
+            if (line.startswith("minio")
+                and not (any(line.startswith(x)
+                             for x in self.ignored_minio_metrics))):
                 key, val = line.split(" ")
                 metrics_data.append((key, float(val)))
         return metrics_data

dss_metrics/whitelist.txt

@@ -1,4 +1,6 @@
 target.subsystem\d+.kvio.(puts|gets|dels|putBandwidth|getBandwidth)
 minio_upstream.outstanding.s3_get_req
 minio_disk_storage_used_bytes
-minio_disk_storage_total_capacity_bytes
+minio_disk_storage_total_capacity_bytes
+minio_metrics_(del|get|put|)_iops
+minio_metrics_(get|put)_bw