Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow filtering out metrics for pools with no connections #58

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,12 @@ pgbouncers:
extra_labels:
pool_id: 1

# Omit metrics for pools for which PgBouncer has no client or server
# connections open. Recommended if you have a lot of different database
# users, because PgBouncer never forgets about users it has seen.
# If omitted, no filtering is done.
filter_empty_pools: true

- dsn: postgresql://pgbouncer:$(PGBOUNCER_PASS)@localhost:6432/pgbouncer
exclude_databases:
- pgbouncer
Expand Down
29 changes: 25 additions & 4 deletions prometheus_pgbouncer_exporter/collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,9 @@ def collect(self):
return metrics.values()

def _instanceMetric(self, data):
if data["type"] is "counter":
if data["type"] == "counter":
return CounterMetricFamily(data["name"], data["help"], labels=data["labels"].keys())
elif data["type"] is "gauge":
elif data["type"] == "gauge":
return GaugeMetricFamily(data["name"], data["help"], labels=data["labels"].keys())
else:
raise Exception("Unsupported metric type: {type}".format(type=data['type']))
Expand Down Expand Up @@ -80,6 +80,8 @@ def collect(self):
if results:
results = self._filterMetricsByIncludeDatabases(results, self.config.getIncludeDatabases())
results = self._filterMetricsByExcludeDatabases(results, self.config.getExcludeDatabases())
if self.config.getFilterEmptyPools():
results = self._filterEmptyPools(results)
metrics += self._exportMetrics(results, "pgbouncer_pools_", [
{"type": "gauge", "column": "cl_active", "metric": "client_active_connections", "help": "Client connections that are linked to server connection and can process queries"},
{"type": "gauge", "column": "cl_waiting", "metric": "client_waiting_connections", "help": "Client connections have sent queries but have not yet got a server connection"},
Expand Down Expand Up @@ -185,14 +187,33 @@ def _filterMetricsByIncludeDatabases(self, results, databases):
if not databases:
return results

return list(filter(lambda item: item["database"] in databases, results))
return filter(lambda item: item["database"] in databases, results)

def _filterMetricsByExcludeDatabases(self, results, databases):
# No filtering if empty
if not databases:
return results

return list(filter(lambda item: item["database"] not in databases, results))
return filter(lambda item: item["database"] not in databases, results)

def _filterEmptyPools(self, results):
# Filter out pools that have no client or server connections.
return filter(
lambda item: sum(
item[x]
for x in [
"cl_active",
"cl_waiting",
"sv_active",
"sv_idle",
"sv_used",
"sv_tested",
"sv_login",
]
)
> 0,
results,
)

def _fetchMetrics(self, conn, query):
cursor = False
Expand Down
3 changes: 3 additions & 0 deletions prometheus_pgbouncer_exporter/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,9 @@ def getIncludeDatabases(self):
def getExcludeDatabases(self):
return self.config["exclude_databases"] if "exclude_databases" in self.config else []

def getFilterEmptyPools(self):
return self.config.get("filter_empty_pools", False)

def getExtraLabels(self):
# Lazy instance extra labels
if self.labels is False:
Expand Down
42 changes: 42 additions & 0 deletions tests/test_collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,17 @@ def fetchMetricsPartialFailureFromPgBouncer17Mock(conn, query):
def fetchMetricsFailureMock(conn, query):
raise Exception("Error while fetching metrics")

def fetchMetricsWithEmptyPool(conn, query):
if query == "SHOW POOLS":
return [
{"database": "test", "user": "marco", "cl_active": 1, "cl_waiting": 2, "sv_active": 3, "sv_idle": 4, "sv_used": 5, "sv_tested": 6, "sv_login": 7, "maxwait": 8, "maxwait_us": 100000 },
{"database": "prod", "user": "marco", "cl_active": 8, "cl_waiting": 7, "sv_active": 6, "sv_idle": 5, "sv_used": 4, "sv_tested": 3, "sv_login": 2, "maxwait": 1, "maxwait_us": 200000 },
{"database": "test", "user": "empty", "cl_active": 0, "cl_waiting": 0, "sv_active": 0, "sv_idle": 0, "sv_used": 0, "sv_tested": 0, "sv_login": 0, "maxwait": 0, "maxwait_us": 0 },
{"database": "prod", "user": "empty", "cl_active": 0, "cl_waiting": 0, "sv_active": 0, "sv_idle": 0, "sv_used": 0, "sv_tested": 0, "sv_login": 0, "maxwait": 0, "maxwait_us": 0 },
]

return []

#
# Tests
#
Expand Down Expand Up @@ -429,6 +440,37 @@ def testShouldReturnScrapedMetricsOnPartialFailure(self):
metrics = getMetricsByName(collector.collect(), "pgbouncer_pools_client_active_connections")
self.assertEqual(len(metrics), 0)

#
# Empty pool filtering
#

def testEmptyPoolFiltering(self):
# By default no filtering should be done
config_default = PgbouncerConfig({})
collector = PgbouncerMetricsCollector(config_default)
collector._createConnection = MagicMock(return_value=False)
collector._fetchMetrics = MagicMock(side_effect=fetchMetricsWithEmptyPool)

metrics = getMetricsByName(collector.collect(), "pgbouncer_pools_client_active_connections")
self.assertEqual(len(metrics), 4)
self.assertEqual(list(x["labels"]["user"] for x in metrics), ["marco", "marco", "empty", "empty"])

# Same when explicitly disabled:
config_no_filter = PgbouncerConfig({"filter_empty_pools": False})
collector.config = config_no_filter

metrics = getMetricsByName(collector.collect(), "pgbouncer_pools_client_active_connections")
self.assertEqual(len(metrics), 4)
self.assertEqual(list(x["labels"]["user"] for x in metrics), ["marco", "marco", "empty", "empty"])

# When filtering is on only the non-empty pools should remain:
config_yes_filter = PgbouncerConfig({"filter_empty_pools": True})
collector.config = config_yes_filter

metrics = getMetricsByName(collector.collect(), "pgbouncer_pools_client_active_connections")
self.assertEqual(len(metrics), 2)
self.assertEqual(list(x["labels"]["user"] for x in metrics), ["marco", "marco"])


if __name__ == '__main__':
unittest.main()