From 43a3fe62dd9455ef886f55e8dcb0b04f8b29af8f Mon Sep 17 00:00:00 2001 From: Bruno Soares Date: Fri, 4 Nov 2016 11:16:08 +0000 Subject: [PATCH 1/2] Added couchbase support - Metrics from statsmetrics plugin (https://github.com/brunopsoares/statsmetrics) - API request supports authentication if environment variables are defined (COUCHBASE_USERNAME, COUCHBASE_PASSWORD) --- README.rst | 8 +- docker/base/newrelic-plugin-agent.cfg | 5 + etc/newrelic/newrelic-plugin-agent.cfg | 5 + newrelic_plugin_agent/plugins/__init__.py | 1 + newrelic_plugin_agent/plugins/couchbase.py | 123 +++++++++++++++++++++ 5 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 newrelic_plugin_agent/plugins/couchbase.py diff --git a/README.rst b/README.rst index 917bf8f..920383f 100644 --- a/README.rst +++ b/README.rst @@ -6,6 +6,7 @@ NewRelic platform. Currently supported backend systems are: - Alternative PHP Cache - Apache HTTP Server +- Couchbase - CouchDB - Elasticsearch - HAProxy @@ -254,7 +255,7 @@ UWSGI Installation Notes ------------------------ The UWSGI plugin can communicate either over UNIX domain sockets using the path configuration variable or TCP/IP using the host and port variables. Do not include both. -Make sure you have `enabled stats server +Make sure you have `enabled stats server `_ in your uwsgi config. Configuration Example @@ -284,6 +285,11 @@ Configuration Example path: /server-status #verify_ssl_cert: true + couchbase: + name: localhost + host: localhost + port: 8091 + couchdb: - name: localhost host: localhost diff --git a/docker/base/newrelic-plugin-agent.cfg b/docker/base/newrelic-plugin-agent.cfg index f957d04..8f6ebe3 100644 --- a/docker/base/newrelic-plugin-agent.cfg +++ b/docker/base/newrelic-plugin-agent.cfg @@ -14,6 +14,11 @@ Application: # port: 80 # path: /server-status + #couchbase: + # name: localhost + # host: localhost + # port: 8091 + #couchdb: # name: localhost # host: localhost diff --git a/etc/newrelic/newrelic-plugin-agent.cfg b/etc/newrelic/newrelic-plugin-agent.cfg index f957d04..8f6ebe3 100644 --- a/etc/newrelic/newrelic-plugin-agent.cfg +++ b/etc/newrelic/newrelic-plugin-agent.cfg @@ -14,6 +14,11 @@ Application: # port: 80 # path: /server-status + #couchbase: + # name: localhost + # host: localhost + # port: 8091 + #couchdb: # name: localhost # host: localhost diff --git a/newrelic_plugin_agent/plugins/__init__.py b/newrelic_plugin_agent/plugins/__init__.py index 98c5cb9..990998c 100644 --- a/newrelic_plugin_agent/plugins/__init__.py +++ b/newrelic_plugin_agent/plugins/__init__.py @@ -5,6 +5,7 @@ """ available = { 'apache_httpd': 'newrelic_plugin_agent.plugins.apache_httpd.ApacheHTTPD', + 'couchbase': 'newrelic_plugin_agent.plugins.couchbase.Couchbase', 'couchdb': 'newrelic_plugin_agent.plugins.couchdb.CouchDB', 'edgecast': 'newrelic_plugin_agent.plugins.edgecast.Edgecast', 'elasticsearch': diff --git a/newrelic_plugin_agent/plugins/couchbase.py b/newrelic_plugin_agent/plugins/couchbase.py new file mode 100644 index 0000000..be6002f --- /dev/null +++ b/newrelic_plugin_agent/plugins/couchbase.py @@ -0,0 +1,123 @@ +""" +Couchbase +""" +import os, requests + +from requests.auth import HTTPBasicAuth +from newrelic_plugin_agent.plugins import base +from statsmetrics import couchbase as couchbasemetrics + +class Couchbase(base.JSONStatsPlugin): + + GUID = 'com.meetme.Couchbase' + + # Get metrics from statsmetrics plugin and transform to a list of dicts + def _get_metrics(self): + self.metrics = [] + for key,values in couchbasemetrics.get_metrics().items(): + for metric in values['metrics']: + obj = { + 'type': key, + 'label': metric['id'], + 'suffix': metric['suffix'] + } + self.metrics.append(obj) + if 'bucket_stats' in values: + for metric in values['bucket_stats']: + obj = { + 'type': 'bucket_stats', + 'label': metric['id'], + 'suffix': metric['suffix'] + } + self.metrics.append(obj) + + def add_datapoints(self, data): + """Add data points for all couchbase nodes. + :param dict stats: stats for all nodes + """ + self._get_metrics() + # fetch metrics for each metric type (cluster, nodes, buckets) + for typ, stats in data.iteritems(): + # set items to fetch stats from, + # and set item key name, where the item's name will be fetched from + if typ == 'nodes': + items = stats['nodes'] + name_key = 'hostname' + else: + items = stats + name_key = 'name' + + for metric in [m for m in self.metrics if m['type'] == typ]: + # add gauge value for current metric. + # cluster metrics are not repeated + # nodes and bucket metrics are repeated, + # bucket_stats metrics are a list of metrics + # in the last minute, so we calculate average value + if typ == 'cluster': + self._add_gauge_value( + metric, typ, items[name_key], items) + elif typ == 'bucket_stats': + for item in items: + self._add_gauge_value( + metric, typ, item, items[item]["op"]["samples"]) + else: + for item in items: + self._add_gauge_value( + metric, typ, item[name_key], item) + + + def _add_gauge_value(self, metric, typ, name, items): + """Adds a gauge value for a nested metric. + Some stats are missing from memcached bucket types, + thus we use dict.get() + :param dict m: metric as defined at the top of this class. + :param str typ: metric type: cluster, nodes, buckets, bucket_stats. + :param str name: cluster, node, or bucket name. + :param dict items: stats to lookup the metric in. + """ + label = metric['label'] + value = items + for key in label.split('.'): + if typ == 'bucket_stats': + valueList = value[key] + value = sum(valueList) / float(len(valueList)) + else: + value = value.get(key, 0) + self.add_gauge_value('%s/%s/%s' % (typ, name, label), + metric['suffix'], value) + + def fetch_data(self): + """Fetch data from multiple couchbase stats URLs. + Returns a dictionary with three keys: cluster, nodes and buckets. + Each key holds the JSON response from the API request. + If checking bucket, there is another API request to get bucket stats. + :rtype: dict + """ + data = {} + for path, typ in [('pools/default', 'cluster'), + ('pools/nodes', 'nodes'), + ('pools/default/buckets', 'buckets')]: + res = self.api_request(path) + res = res and res.json() or {} + data[typ] = res + # Check if there are bucket stats + if typ == 'buckets': + data_bucket = {} + for bucket in res: + if 'stats' in bucket: + res_bucket = self.api_request(bucket['stats']['uri']) + res_bucket = res_bucket and res_bucket.json() or {} + data_bucket[bucket['name']] = res_bucket + data["bucket_stats"] = data_bucket + return data + + def api_request(self, path): + """Request data using Couchbase API + Allows basic authentication with ENV variables + Returns a Response object + """ + if set(["COUCHBASE_USERNAME","COUCHBASE_PASSWORD"]).issubset(os.environ): + res = requests.get(self.stats_url + path, auth=HTTPBasicAuth(os.environ["COUCHBASE_USERNAME"], os.environ["COUCHBASE_PASSWORD"])) + else: + res = requests.get(self.stats_url + path) + return res From 45e4778a749edf7cfabcb0ee77296aff55bea66a Mon Sep 17 00:00:00 2001 From: Bruno Soares Date: Fri, 4 Nov 2016 12:14:00 +0000 Subject: [PATCH 2/2] Updated README file - Couchbase installation notes --- README.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.rst b/README.rst index 920383f..d1ebef9 100644 --- a/README.rst +++ b/README.rst @@ -143,6 +143,17 @@ The agent requires the extended information to parse metrics. If you are not see If you are monitoring Apache HTTPd via a HTTPS connection you can use the ``verify_ssl_cert`` configuration value in the httpd configuration section to disable SSL certificate verification. +Couchbase Installation Notes +---------------------------- +In order to get all the couchbase metrics, couchbase plugin uses an external plugin: StatsMetrics_. + +.. _StatsMetrics: https://github.com/brunopsoares/statsmetrics + +:: + + $ pip install statsmetrics + + Memcached Installation Notes ---------------------------- The memcached plugin can communicate either over UNIX domain sockets using the path configuration variable or TCP/IP using the host and port variables. Do not include both.