Skip to content

Commit

Permalink
complete rewrite of couchbase plugin, supports many more metrics
Browse files Browse the repository at this point in the history 
- add memcached support to couchbase plugin; added logging + pep8 + pep257
- also much easier to add/remove additional metrics in the future
- metrics are according to api reference v3.0 or v3.1
  • Loading branch information
@oryband
oryband committed Oct 28, 2015
1 parent 8160951 commit 33a1a57
Show file tree
Hide file tree
Showing 4 changed files with 149 additions and 54 deletions.
1 change: 0 additions & 1 deletion README.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,6 @@ Configuration Example
name: localhost
host: localhost
port: 8091
path: pools/default/buckets

elasticsearch:
name: clustername
Expand Down
1 change: 0 additions & 1 deletion docker/base/newrelic-plugin-agent.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ Application:
# name: localhost
# host: localhost
# port: 8091
# path: pools/default/buckets

#elasticsearch:
# name: Clustername
Expand Down
1 change: 0 additions & 1 deletion etc/newrelic/newrelic-plugin-agent.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ Application:
# name: localhost
# host: localhost
# port: 8091
# path: pools/default/buckets

#elasticsearch:
# name: Clustername
Expand Down
200 changes: 149 additions & 51 deletions newrelic_plugin_agent/plugins/couchbase.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,59 +2,157 @@
couchbase
"""

from newrelic_plugin_agent.plugins import base


class Couchbase(base.JSONStatsPlugin):

GUID = 'com.meetme.couchbase'

def add_datapoints(self, stats):
for bucket_data in stats:
bucket_name = bucket_data['name']
bucket_stats = bucket_data['basicStats']
if bucket_stats:
""" Example JSON
"basicStats":{
"quotaPercentUsed":7.548735046386719,
"opsPerSec":0,
"diskFetches":0,
"itemCount":2222,
"diskUsed":229366539,
"dataUsed":203511808,
"memUsed":79154224
}
"""
self.add_gauge_bucket_metric(
bucket_name, 'quotaPercentUsed', 'percent', bucket_stats['quotaPercentUsed']
)
self.add_gauge_bucket_metric(
bucket_name, 'opsPerSec', 'ops', bucket_stats['opsPerSec']
)
self.add_gauge_bucket_metric(
bucket_name, 'diskFetches', 'count', bucket_stats['diskFetches']
)
self.add_gauge_bucket_metric(
bucket_name, 'itemCount', 'count', bucket_stats['itemCount']
)
self.add_gauge_bucket_metric(
bucket_name, 'diskUsed', 'Byte', bucket_stats['diskUsed']
)
self.add_gauge_bucket_metric(
bucket_name, 'dataUsed', 'Byte', bucket_stats['dataUsed']
)
self.add_gauge_bucket_metric(
bucket_name, 'memUsed', 'Byte', bucket_stats['memUsed']
)

# Summary metrics
self.add_gauge_value('Summary/%s/quotaPercentUsed' % bucket_name, 'percent',
bucket_stats['quotaPercentUsed'],
min_val=0, max_val=0)
self.add_gauge_value('Summary/%s/diskUsed' % bucket_name, 'byte',
bucket_stats['diskUsed'])

def add_gauge_bucket_metric(self, bucket_name, metric_name, units, metric_value):
if metric_value:
self.add_gauge_value('%s/%s' % (bucket_name, metric_name), units, metric_value)
GUID = 'com.meetme.newrelic_couchbase_agent'

# metrics are according to api reference v3.0 or v3.1
METRICS = [
{'type': 'cluster', 'label': 'storageTotals.ram.total', 'suffix': 'bytes'},
{'type': 'cluster', 'label': 'storageTotals.ram.used', 'suffix': 'bytes'},
{'type': 'cluster', 'label': 'storageTotals.ram.usedByData', 'suffix': 'bytes'},
{'type': 'cluster', 'label': 'storageTotals.ram.quotaTotal', 'suffix': 'bytes'},
{'type': 'cluster', 'label': 'storageTotals.ram.quotaUsed', 'suffix': 'bytes'},

{'type': 'cluster', 'label': 'storageTotals.hdd.total', 'suffix': 'bytes'},
{'type': 'cluster', 'label': 'storageTotals.hdd.used', 'suffix': 'bytes'},
{'type': 'cluster', 'label': 'storageTotals.hdd.usedByData', 'suffix': 'bytes'},
{'type': 'cluster', 'label': 'storageTotals.hdd.quotaTotal', 'suffix': 'bytes'},
{'type': 'cluster', 'label': 'storageTotals.hdd.free', 'suffix': 'bytes'},

{'type': 'cluster', 'label': 'counters.rebalance_success', 'suffix': 'count'},
{'type': 'cluster', 'label': 'counters.rebalance_start', 'suffix': 'count'},
{'type': 'cluster', 'label': 'counters.rebalance_fail', 'suffix': 'count'},
{'type': 'cluster', 'label': 'counters.rebalance_node', 'suffix': 'count'},

{'type': 'nodes', 'label': 'systemStats.cpu_utilization_rate', 'suffix': 'count'},
{'type': 'nodes', 'label': 'systemStats.swap_total', 'suffix': 'byte'},
{'type': 'nodes', 'label': 'systemStats.swap_used', 'suffix': 'byte'},
{'type': 'nodes', 'label': 'systemStats.mem_total', 'suffix': 'byte'},
{'type': 'nodes', 'label': 'systemStats.mem_free', 'suffix': 'byte'},

{'type': 'nodes', 'label': 'interestingStats.couch_docs_actual_disk_size', 'suffix': 'byte'},
{'type': 'nodes', 'label': 'interestingStats.couch_docs_data_size', 'suffix': 'byte'},
{'type': 'nodes', 'label': 'interestingStats.couch_views_actual_disk_size', 'suffix': 'byte'},
{'type': 'nodes', 'label': 'interestingStats.couch_views_data_size', 'suffix': 'byte'},
{'type': 'nodes', 'label': 'interestingStats.mem_used', 'suffix': 'byte'},
{'type': 'nodes', 'label': 'interestingStats.ops', 'suffix': 'count'},
{'type': 'nodes', 'label': 'interestingStats.curr_items', 'suffix': 'count'},
{'type': 'nodes', 'label': 'interestingStats.curr_items_tot', 'suffix': 'count'},
{'type': 'nodes', 'label': 'interestingStats.vb_replica_curr_items', 'suffix': 'count'},

{'type': 'nodes', 'scoreboard': True, 'score_value': 'active', 'label': 'clusterMembership', 'suffix': 'count'},
{'type': 'nodes', 'scoreboard': True, 'score_value': 'healthy', 'label': 'status', 'suffix': 'count'},

{'type': 'buckets', 'label': 'basicStats.quotaPercentUsed', 'suffix': 'percent'},
{'type': 'buckets', 'label': 'basicStats.opsPerSec', 'suffix': 'ops'},
{'type': 'buckets', 'label': 'basicStats.diskFetches', 'suffix': 'percent'},
{'type': 'buckets', 'label': 'basicStats.itemCount', 'suffix': 'percent'},
{'type': 'buckets', 'label': 'basicStats.diskUsed', 'suffix': 'byte'},
{'type': 'buckets', 'label': 'basicStats.dataUsed', 'suffix': 'byte'},
{'type': 'buckets', 'label': 'basicStats.memUsed', 'suffix': 'byte'},
]

def add_datapoints(self, data):
"""Add data points for all couchbase nodes.
:param dict stats: stats for all nodes
"""
# fetch metrics for each metric type (cluster, nodes, buckets)
for typ, stats in data.iteritems():
for metric in [m for m in self.METRICS if m['type'] == typ]:
if typ == 'cluster':
items = stats
name_key = 'name'
elif typ == 'nodes':
items = stats['nodes']
name_key = 'hostname'
elif typ == 'buckets':
items = stats
name_key = 'name'

# count score for scoreboard metrics,
# otherwise just add the gauge value from the API repsonse
if metric.get('scoreboard', False):
self.add_gauge_value(
'%s/_scoreboard/%s' % (typ, metric['label']),
metric['suffix'],
self._get_metric_score(metric, items))
else:
# add gauge value for current metric.
# NOTE cluster metrics are not repeated,
# as there is a single cluster.
# nodes and bucket metrics are repeated,
# as there are (usually) several of them
if typ == 'cluster':
self._add_gauge_value(
metric, typ, items[name_key], items)
else:
for item in items:
self._add_gauge_value(
metric, typ, item[name_key], item)

def _add_gauge_value(self, metric, typ, name, items):
"""Adds a gauge value for a nested metric.
Some stats are missing from memcached bucket types,
thus we use dict.get()
:param dict m: metric as defined at the top of this class.
:param str typ: metric type: cluster, nodes, buckets.
:param str name: cluster, node, or bucket name.
:param dict items: stats to lookup the metric in.
"""
label = metric['label']
value = items
for key in label.split('.'):
value = value.get(key, 0)
self.add_gauge_value('%s/%s/%s' % (typ, name, label),
metric['suffix'], value)

def _get_metric_score(self, metric, items):
"""Calculate metric score for scoreboard type metrics.
A "scoreboard" metric is a counter of a specific value for a field.
E.g How many times the pair {"status": "healthy"} is found in a
list of object.
NOTE that scoreboard metrics are meant to be used on lists of objects,
i.e. a list of node objects.
:param dict metric: metric to calculate score for.
:param items metric: stats to calculate score from.
:rtype: int
"""
score = 0
label = metric['label']
score_value = metric['score_value']
for d in items:
score += 1 if d[label] == score_value else 0
return score

def fetch_data(self):
"""Fetch data from multiple couchbase stats URLs.
Returns a dictionary with three keys: cluster, nodes, buckets.
Each key holds the JSON response from the API request.
:rtype: dict
"""
data = {}
for path, typ in [('pools/default', 'cluster'),
('pools/nodes', 'nodes'),
('pools/default/buckets', 'buckets')]:
res = self.http_get(self.stats_url + path)
if res:
data[typ] = res.json() if res else {}

return data

0 comments on commit 33a1a57

Please sign in to comment.