From d8d3ad479c60b7b3def6ddc77dac7829dd7158ed Mon Sep 17 00:00:00 2001 From: Zhengda Lu Date: Wed, 3 Jul 2024 11:31:33 -0400 Subject: [PATCH] [mongo] Support auto-discover available databases for the monitored mongodb instance (#17959) * autodiscover mongodb databases * Add database autodiscovery support * remove print * only list authorized collections not views * ignore collections from config when database autodiscovery is enabled * add changelog * update changelog * fix license header * update include list with deprecated dbnames * fix test * update comments * update changelog * return databases and count * update readme * update config description to dbnames --- mongo/README.md | 89 ++- mongo/assets/configuration/spec.yaml | 73 +- mongo/changelog.d/17959.added | 4 + mongo/changelog.d/17959.deprecated | 2 + mongo/datadog_checks/mongo/api.py | 17 + .../mongo/collectors/coll_stats.py | 13 +- .../mongo/collectors/index_stats.py | 10 +- mongo/datadog_checks/mongo/config.py | 35 + .../mongo/config_models/instance.py | 13 + .../mongo/data/conf.yaml.example | 60 +- .../mongo/dbm/operation_samples.py | 19 +- mongo/datadog_checks/mongo/discovery.py | 53 ++ mongo/datadog_checks/mongo/mongo.py | 72 +- mongo/tests/compose/scripts/init-router.js | 2 +- mongo/tests/conftest.py | 18 + mongo/tests/fixtures/$currentOp-mongos | 10 +- mongo/tests/fixtures/list_collection_names | 1 + mongo/tests/fixtures/list_database_names | 2 +- mongo/tests/mocked_api.py | 4 + .../metrics-collection-autodiscover.json | 730 ++++++++++++++++++ mongo/tests/results/metrics-count-dbs.json | 2 +- .../metrics-indexes-stats-autodiscover.json | 90 +++ .../tests/results/metrics-indexes-stats.json | 6 +- .../results/opeartion-activities-mongos.json | 10 +- mongo/tests/test_e2e.py | 8 +- mongo/tests/test_integration.py | 41 + mongo/tests/test_unit.py | 1 + mongo/tests/test_unit_config.py | 26 + 28 files changed, 1331 insertions(+), 80 deletions(-) create mode 100644 mongo/changelog.d/17959.added create mode 100644 mongo/changelog.d/17959.deprecated create mode 100644 mongo/datadog_checks/mongo/discovery.py create mode 100644 mongo/tests/fixtures/list_collection_names create mode 100644 mongo/tests/results/metrics-collection-autodiscover.json create mode 100644 mongo/tests/results/metrics-indexes-stats-autodiscover.json diff --git a/mongo/README.md b/mongo/README.md index ca12e2e901953..d03c0d4af2f12 100644 --- a/mongo/README.md +++ b/mongo/README.md @@ -44,7 +44,12 @@ db.createUser({ "roles": [ { role: "read", db: "admin" }, { role: "clusterMonitor", db: "admin" }, - { role: "read", db: "local" } + { role: "read", db: "local" }, + # Grant additional read-only access to the database you want to collect collection/index statistics from. + { role: "read", db: "mydb" }, + { role: "read", db: "myanotherdb" }, + # Alternatively, grant read-only access to all databases. + { role: "readAnyDatabase", db: "admin" } ] }) ``` @@ -72,7 +77,12 @@ db.createUser({ "roles": [ { role: "read", db: "admin" }, { role: "clusterMonitor", db: "admin" }, - { role: "read", db: "local" } + { role: "read", db: "local" }, + # Grant additional read-only access to the database you want to collect collection/index statistics from. + { role: "read", db: "mydb" }, + { role: "read", db: "myanotherdb" }, + # Alternatively, grant read-only access to all databases. + { role: "readAnyDatabase", db: "admin" } ] }) ``` @@ -200,6 +210,81 @@ To configure this check for an Agent running on a host: 2. [Restart the Agent][6]. +##### Database Autodiscovery + +Starting from Datadog Agent v7.56, you can enable database autodiscovery to automatically collect metrics from all your databases on the MongoDB instance. +Please note that database autodiscovery is disabled by default. Read access to the autodiscovered databases is required to collect metrics from them. +To enable it, add the following configuration to your `mongo.d/conf.yaml` file: + +```yaml + init_config: + + instances: + ## @param hosts - list of strings - required + ## Hosts to collect metrics from, as is appropriate for your deployment topology. + ## E.g. for a standalone deployment, specify the hostname and port of the mongod instance. + ## For replica sets or sharded clusters, see instructions in the sample conf.yaml. + ## Only specify multiple hosts when connecting through mongos + # + - hosts: + - : + + ## @param username - string - optional + ## The username to use for authentication. + # + username: datadog + + ## @param password - string - optional + ## The password to use for authentication. + # + password: + + ## @param options - mapping - optional + ## Connection options. For a complete list, see: + ## https://docs.mongodb.com/manual/reference/connection-string/#connections-connection-options + # + options: + authSource: admin + + ## @param database_autodiscovery - mapping - optional + ## Enable database autodiscovery to automatically collect metrics from all your MongoDB databases. + # + database_autodiscovery: + ## @param enabled - boolean - required + ## Enable database autodiscovery. + # + enabled: true + + ## @param include - list of strings - optional + ## List of databases to include in the autodiscovery. Use regular expressions to match multiple databases. + ## For example, to include all databases starting with "mydb", use "^mydb.*". + ## By default, include is set to ".*" and all databases are included. + # + include: + - "^mydb.*" + + ## @param exclude - list of strings - optional + ## List of databases to exclude from the autodiscovery. Use regular expressions to match multiple databases. + ## For example, to exclude all databases starting with "mydb", use "^mydb.*". + ## When the exclude list conflicts with include list, the exclude list takes precedence. + # + exclude: + - "^mydb2.*" + - "admin$" + + ## @param max_databases - integer - optional + ## Maximum number of databases to collect metrics from. The default value is 100. + # + max_databases: 100 + + ## @param refresh_interval - integer - optional + ## Interval in seconds to refresh the list of databases. The default value is 600 seconds. + # + refresh_interval: 600 + ``` + +2. [Restart the Agent][6]. + ##### Trace collection Datadog APM integrates with Mongo to see the traces across your distributed system. Trace collection is enabled by default in the Datadog Agent v6+. To start collecting traces: diff --git a/mongo/assets/configuration/spec.yaml b/mongo/assets/configuration/spec.yaml index cafffa3693d2a..b33c39d407e81 100644 --- a/mongo/assets/configuration/spec.yaml +++ b/mongo/assets/configuration/spec.yaml @@ -80,15 +80,81 @@ files: type: object properties: [] - name: dbnames + deprecation: + Agent version: "7.56.0" + Migration: | + dbnames is deprecated. Set database_autodiscovery.enabled to true to enable database autodiscovery. + Use database_autodiscovery.include or database_autodiscovery.exclude to include or exclude + specific databases to collect metrics from. description: | - Set a list of the names of all databases to collect metrics from. - If this key does not exist, all metrics from all databases on the server will be collected. + Set a list of the names of all databases to collect dbstats metrics from. + If this key does not exist, all dbstats metrics from all databases on the server will be collected. value: type: array items: type: string example: [ one_database, other_database ] + - name: database_autodiscovery + description: | + Define the configuration for database autodiscovery. + Complete this section if you want to auto-discover databases on this MongoDB instance. + options: + - name: enabled + description: Enable database autodiscovery. + value: + type: boolean + example: false + display_default: false + - name: max_databases + description: The maximum number of databases this host should monitor. + value: + type: integer + example: 100 + display_default: 100 + - name: include + description: | + Regular expression for database names to include as part of + database autodiscovery. + Will report metrics for databases that are found in this instance, + ignores databases listed but not found. + Character casing is ignored. The regular expressions start matching from + the beginning, so to match anywhere, prepend `.*`. For exact matches append `$`. + Defaults to `.*` to include everything. + value: + type: array + items: + type: string + example: + - "mydatabase$" + - "orders.*" + display_default: + - ".*" + - name: exclude + description: | + Regular expression for database names to exclude as part of `database_autodiscovery`. + Character casing is ignored. The regular expressions start matching from the beginning, + so to match anywhere, prepend `.*`. For exact matches append `$`. + In case of conflicts, database exclusion via `exclude` takes precedence over + those found via `include` + value: + type: array + items: + type: string + example: + - "admin$" + - "config$" + - "local$" + display_default: + - "admin$" + - "config$" + - "local$" + - name: refresh_interval + description: Frequency in seconds of scans for new databases. Defaults to 10 minutes. + value: + type: integer + example: 600 + display_default: 600 - name: dbm description: | Set to `true` enable Database Monitoring. @@ -218,6 +284,9 @@ files: * `db:` e.g. `db:` * `collection:` e.g. `collection:` Each collection generates many metrics, up to 8 + the number of indices on the collection for each collection. + + NOTE: This option is ignored when database_autodiscovery is enabled. + Metrics are collected for all authorized collections on autodiscovered databases. value: type: array items: diff --git a/mongo/changelog.d/17959.added b/mongo/changelog.d/17959.added new file mode 100644 index 0000000000000..163a9d54c9eec --- /dev/null +++ b/mongo/changelog.d/17959.added @@ -0,0 +1,4 @@ +Support auto-discover available databases (up to 100 databases) for the monitored mongodb instance. +By default, database autodiscovery is disabled. Set `database_autodiscovery.enabled` to true to enable database autodiscovery. +When enabled, the integration will automatically discover the databases available in the monitored mongodb instance and refresh the list of databases every 10 minutes. +Use `database_autodiscovery.include` and `database_autodiscovery.exclude` to filter the list of databases to monitor. \ No newline at end of file diff --git a/mongo/changelog.d/17959.deprecated b/mongo/changelog.d/17959.deprecated new file mode 100644 index 0000000000000..2ea42d88e7c79 --- /dev/null +++ b/mongo/changelog.d/17959.deprecated @@ -0,0 +1,2 @@ +Configuration option `dbnames` is deprecated and will be removed in a future release. +To monitor multiple databases, set `database_autodiscovery.enabled` to true and configure `database_autodiscovery.include` and `database_autodiscovery.exclude` filters instead. diff --git a/mongo/datadog_checks/mongo/api.py b/mongo/datadog_checks/mongo/api.py index 587588b9de9c7..d7edfa5576837 100644 --- a/mongo/datadog_checks/mongo/api.py +++ b/mongo/datadog_checks/mongo/api.py @@ -187,6 +187,23 @@ def refresh_shards(self): def server_status(self): return self['admin'].command('serverStatus') + def list_authorized_collections(self, db_name): + try: + return self[db_name].list_collection_names( + filter={"type": "collection"}, # Only return collections, not views + authorizedCollections=True, + ) + except OperationFailure: + # The user is not authorized to run listCollections on this database. + # This is NOT a critical error, so we log it as a warning. + self._log.warning( + "Not authorized to run 'listCollections' on db %s, " + "please make sure the user has read access on the database or " + "add the database to the `database_autodiscovery.exclude` list in the configuration file", + db_name, + ) + return [] + @property def hostname(self): try: diff --git a/mongo/datadog_checks/mongo/collectors/coll_stats.py b/mongo/datadog_checks/mongo/collectors/coll_stats.py index ae9804c0e6bdc..97f106d6dde74 100644 --- a/mongo/datadog_checks/mongo/collectors/coll_stats.py +++ b/mongo/datadog_checks/mongo/collectors/coll_stats.py @@ -12,9 +12,6 @@ class CollStatsCollector(MongoCollector): """Collects metrics from the 'collstats' command. Note: Collecting those metrics requires that 'collection' is set in the 'additional_metrics' section of the config. - Also, the collections to be monitored have to be explicitly listed in the config as well. - Finally, it is currently not possible to monitor collections from multiple databases using a single check instance. - The check will always use the main database name defined in the configuration (or 'admin' by default). """ def __init__(self, check, db_name, tags, coll_names=None): @@ -26,11 +23,17 @@ def compatible_with(self, deployment): # Can only be run once per cluster. return deployment.is_principal() + def _get_collections(self, api): + if self.coll_names: + return self.coll_names + return api.list_authorized_collections(self.db_name) + def collect(self, api): # Ensure that you're on the right db db = api[self.db_name] - # Loop through the collections - for coll_name in self.coll_names: + coll_names = self._get_collections(api) + + for coll_name in coll_names: # Grab the stats from the collection payload = {'collection': db.command("collstats", coll_name)} additional_tags = ["db:%s" % self.db_name, "collection:%s" % coll_name] diff --git a/mongo/datadog_checks/mongo/collectors/index_stats.py b/mongo/datadog_checks/mongo/collectors/index_stats.py index 219c685d80ee5..78e1f73210cae 100644 --- a/mongo/datadog_checks/mongo/collectors/index_stats.py +++ b/mongo/datadog_checks/mongo/collectors/index_stats.py @@ -17,14 +17,22 @@ def compatible_with(self, deployment): # Can only be run once per cluster. return deployment.is_principal() + def _get_collections(self, api): + if self.coll_names: + return self.coll_names + return api.list_authorized_collections(self.db_name) + def collect(self, api): db = api[self.db_name] - for coll_name in self.coll_names: + coll_names = self._get_collections(api) + + for coll_name in coll_names: try: for stats in db[coll_name].aggregate([{"$indexStats": {}}], cursor={}): idx_tags = self.base_tags + [ "name:{0}".format(stats.get('name', 'unknown')), "collection:{0}".format(coll_name), + "db:{0}".format(self.db_name), ] val = int(stats.get('accesses', {}).get('ops', 0)) self.gauge('mongodb.collection.indexes.accesses.ops', val, idx_tags) diff --git a/mongo/datadog_checks/mongo/config.py b/mongo/datadog_checks/mongo/config.py index 01ef54aceb743..2857593cf20ed 100644 --- a/mongo/datadog_checks/mongo/config.py +++ b/mongo/datadog_checks/mongo/config.py @@ -110,7 +110,11 @@ def __init__(self, instance, log): # MongoDB instance hostname override self.reported_database_hostname = instance.get('reported_database_hostname', None) + # MongoDB database auto-discovery, disabled by default + self.database_autodiscovery_config = self._get_database_autodiscovery_config(instance) + # Generate tags for service checks and metrics + # TODO: service check and metric tags should be updated to be dynamic with auto-discovered databases self.service_check_tags = self._compute_service_check_tags() self.metric_tags = self._compute_metric_tags() @@ -167,3 +171,34 @@ def operation_samples(self): self._operation_samples_config.get('explained_operations_per_hour_per_query', 10) ), } + + def _get_database_autodiscovery_config(self, instance): + database_autodiscovery_config = instance.get('database_autodiscovery', {"enabled": False}) + if database_autodiscovery_config['enabled']: + if self.db_name != 'admin': + # If database_autodiscovery is enabled, the `database` parameter should not be set + # because we want to monitor all databases. Unless the `database` parameter is set to 'admin'. + self.log.warning( + "The `database` parameter should not be set when `database_autodiscovery` is enabled. " + "The `database` parameter will be ignored." + ) + if self.coll_names: + self.log.warning( + "The `collections` parameter should not be set when `database_autodiscovery` is enabled. " + "The `collections` parameter will be ignored." + ) + if self.db_names: + # dbnames is deprecated and will be removed in a future version + self.log.warning( + "The `dbnames` parameter is deprecated and will be removed in a future version. " + "To monitor more databases, enable `database_autodiscovery` and use " + "`database_autodiscovery.include` instead." + ) + include_list = [f"{db}$" for db in self.db_names] # Append $ to each db name for exact match + if not database_autodiscovery_config['enabled']: + # if database_autodiscovery is not enabled, we should enable it + database_autodiscovery_config['enabled'] = True + if not database_autodiscovery_config.get('include'): + # if database_autodiscovery is enabled but include list is not set, set the include list + database_autodiscovery_config['include'] = include_list + return database_autodiscovery_config diff --git a/mongo/datadog_checks/mongo/config_models/instance.py b/mongo/datadog_checks/mongo/config_models/instance.py index cdfa146bf5ed4..fa990a8fb47fd 100644 --- a/mongo/datadog_checks/mongo/config_models/instance.py +++ b/mongo/datadog_checks/mongo/config_models/instance.py @@ -42,6 +42,18 @@ class CustomQuery(BaseModel): tags: Optional[tuple[str, ...]] = None +class DatabaseAutodiscovery(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = None + exclude: Optional[tuple[str, ...]] = None + include: Optional[tuple[str, ...]] = None + max_databases: Optional[int] = None + refresh_interval: Optional[int] = None + + class MetricPatterns(BaseModel): model_config = ConfigDict( arbitrary_types_allowed=True, @@ -74,6 +86,7 @@ class InstanceConfig(BaseModel): connection_scheme: Optional[str] = None custom_queries: Optional[tuple[CustomQuery, ...]] = None database: Optional[str] = None + database_autodiscovery: Optional[DatabaseAutodiscovery] = None database_instance_collection_interval: Optional[float] = None dbm: Optional[bool] = None dbnames: Optional[tuple[str, ...]] = None diff --git a/mongo/datadog_checks/mongo/data/conf.yaml.example b/mongo/datadog_checks/mongo/data/conf.yaml.example index b175488fdb4db..d52325a6dbf14 100644 --- a/mongo/datadog_checks/mongo/data/conf.yaml.example +++ b/mongo/datadog_checks/mongo/data/conf.yaml.example @@ -77,13 +77,66 @@ instances: # options: {} ## @param dbnames - list of strings - optional - ## Set a list of the names of all databases to collect metrics from. - ## If this key does not exist, all metrics from all databases on the server will be collected. + ## Set a list of the names of all databases to collect dbstats metrics from. + ## If this key does not exist, all dbstats metrics from all databases on the server will be collected. + ## + ## + ## <<< DEPRECATED >>> + ## + ## Agent version: 7.56.0 + ## Migration: dbnames is deprecated. Set database_autodiscovery.enabled to true to enable database autodiscovery. + ## Use database_autodiscovery.include or database_autodiscovery.exclude to include or exclude + ## specific databases to collect metrics from. # # dbnames: # - one_database # - other_database + ## Define the configuration for database autodiscovery. + ## Complete this section if you want to auto-discover databases on this MongoDB instance. + # + # database_autodiscovery: + + ## @param enabled - boolean - optional - default: false + ## Enable database autodiscovery. + # + # enabled: false + + ## @param max_databases - integer - optional - default: 100 + ## The maximum number of databases this host should monitor. + # + # max_databases: 100 + + ## @param include - list of strings - optional - default: ['.*'] + ## Regular expression for database names to include as part of + ## database autodiscovery. + ## Will report metrics for databases that are found in this instance, + ## ignores databases listed but not found. + ## Character casing is ignored. The regular expressions start matching from + ## the beginning, so to match anywhere, prepend `.*`. For exact matches append `$`. + ## Defaults to `.*` to include everything. + # + # include: + # - mydatabase$ + # - orders.* + + ## @param exclude - list of strings - optional - default: ['admin$', 'config$', 'local$'] + ## Regular expression for database names to exclude as part of `database_autodiscovery`. + ## Character casing is ignored. The regular expressions start matching from the beginning, + ## so to match anywhere, prepend `.*`. For exact matches append `$`. + ## In case of conflicts, database exclusion via `exclude` takes precedence over + ## those found via `include` + # + # exclude: + # - admin$ + # - config$ + # - local$ + + ## @param refresh_interval - integer - optional - default: 600 + ## Frequency in seconds of scans for new databases. Defaults to 10 minutes. + # + # refresh_interval: 600 + ## @param dbm - boolean - optional - default: false ## Set to `true` enable Database Monitoring. ## @@ -195,6 +248,9 @@ instances: ## * `db:` e.g. `db:` ## * `collection:` e.g. `collection:` ## Each collection generates many metrics, up to 8 + the number of indices on the collection for each collection. + ## + ## NOTE: This option is ignored when database_autodiscovery is enabled. + ## Metrics are collected for all authorized collections on autodiscovered databases. # # collections: # - diff --git a/mongo/datadog_checks/mongo/dbm/operation_samples.py b/mongo/datadog_checks/mongo/dbm/operation_samples.py index 74d4dee70bb8c..889538eae6d47 100644 --- a/mongo/datadog_checks/mongo/dbm/operation_samples.py +++ b/mongo/datadog_checks/mongo/dbm/operation_samples.py @@ -96,7 +96,9 @@ def collect_operation_samples(self): activities = [] - for activity, sample in self._get_operation_samples(now): + for activity, sample in self._get_operation_samples( + now, databases_monitored=self._check._database_autodiscovery.databases + ): if sample: self._check.log.debug("Sending operation sample: %s", sample) self._check.database_monitoring_query_sample(json_util.dumps(sample)) @@ -115,10 +117,10 @@ def _should_collect_operation_samples(self) -> bool: return False return True - def _get_operation_samples(self, now): + def _get_operation_samples(self, now, databases_monitored: List[str]): for operation in self._get_current_op(): try: - if not self._should_include_operation(operation): + if not self._should_include_operation(operation, databases_monitored): continue command = operation.get("command") @@ -155,7 +157,7 @@ def _get_current_op(self): self._check.log.debug("Found operation: %s", operation) yield operation - def _should_include_operation(self, operation: dict) -> bool: + def _should_include_operation(self, operation: dict, databases_monitored: List[str]) -> bool: # Skip operations from db that are not configured to be monitored namespace = operation.get("ns") if not namespace: @@ -163,12 +165,9 @@ def _should_include_operation(self, operation: dict) -> bool: return False db, _ = namespace.split(".", 1) - if self._check._config.db_names is not None: - if db not in self._check._config.db_names: - self._check.log.debug( - "Skipping operation for database %s because it is not configured to be monitored", db - ) - return False + if db not in databases_monitored: + self._check.log.debug("Skipping operation for database %s because it is not configured to be monitored", db) + return False if db in SYSTEM_DATABASES: self._check.log.debug("Skipping operation for system database %s", db) diff --git a/mongo/datadog_checks/mongo/discovery.py b/mongo/datadog_checks/mongo/discovery.py new file mode 100644 index 0000000000000..9ffae77883f1b --- /dev/null +++ b/mongo/datadog_checks/mongo/discovery.py @@ -0,0 +1,53 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + + +from datadog_checks.base.utils.discovery import Discovery +from datadog_checks.mongo.common import ReplicaSetDeployment + +DEFAULT_MAX_DATABASES = 100 +DEFAULT_REFRESH_INTERVAL = 600 + + +class MongoDBDatabaseAutodiscovery(Discovery): + def __init__(self, check): + self._autodiscovery_config = check._config.database_autodiscovery_config + self.autodiscovery_enabled = self._autodiscovery_config.get("enabled", False) + + super(MongoDBDatabaseAutodiscovery, self).__init__( + self._list_databases, + include={db: 0 for db in self._autodiscovery_config.get("include", [".*"])}, + exclude=self._autodiscovery_config.get("exclude"), + interval=self._autodiscovery_config.get('refresh_interval', DEFAULT_REFRESH_INTERVAL), + ) + self._check = check + self._log = self._check.log + self._max_databases = self._autodiscovery_config.get("max_databases", DEFAULT_MAX_DATABASES) + + self.database_count = 0 # total number of databases on the server + + def _list_databases(self): + deployment = self._check.api_client.deployment_type + + databases = [] + if isinstance(deployment, ReplicaSetDeployment) and deployment.is_arbiter: + self._log.debug("Replicaset and arbiter deployment, no databases will be checked") + elif isinstance(deployment, ReplicaSetDeployment) and deployment.replset_state == 3: + self._log.debug("Replicaset is in recovering state, will skip reading database names") + else: + databases = self._check.api_client.list_database_names() + self.database_count = len(databases) + return databases + + @property + def databases(self): + ''' + The databases property returns a list of database names to monitor, capped at the max_databases limit. + ''' + dbnames = [database[1] for database in self.get_items()] + dbnames = dbnames[: self._max_databases] # limit the number of databases to monitor + return dbnames + + def get_databases_and_count(self): + return self.databases, self.database_count diff --git a/mongo/datadog_checks/mongo/mongo.py b/mongo/datadog_checks/mongo/mongo.py index 95553650a755c..fdeb6d7850131 100644 --- a/mongo/datadog_checks/mongo/mongo.py +++ b/mongo/datadog_checks/mongo/mongo.py @@ -32,6 +32,7 @@ from datadog_checks.mongo.common import SERVICE_CHECK_NAME, MongosDeployment, ReplicaSetDeployment from datadog_checks.mongo.config import MongoConfig from datadog_checks.mongo.dbm.operation_samples import MongoOperationSamples +from datadog_checks.mongo.discovery import MongoDBDatabaseAutodiscovery from . import metrics @@ -98,6 +99,9 @@ def __init__(self, name, init_config, instances=None): # DBM self._operation_samples = MongoOperationSamples(check=self) + # Database autodiscovery + self._database_autodiscovery = MongoDBDatabaseAutodiscovery(check=self) + @cached_property def api_client(self): # This needs to be a property for our unit test mocks to work. @@ -109,7 +113,6 @@ def refresh_collectors(self, deployment_type, all_dbs, tags): ConnPoolStatsCollector(self, tags), ReplicationOpLogCollector(self, tags), FsyncLockCollector(self, tags), - CollStatsCollector(self, self._config.db_name, tags, coll_names=self._config.coll_names), ServerStatusCollector(self, self._config.db_name, tags, tcmalloc=collect_tcmalloc_metrics), ] if self._config.replica_check: @@ -121,21 +124,32 @@ def refresh_collectors(self, deployment_type, all_dbs, tags): assert self._mongo_version is not None, "No MongoDB version is set, make sure you refreshed the metadata." if self._mongo_version_parsed >= Version("3.6"): potential_collectors.append(SessionStatsCollector(self, tags)) - if self._config.collections_indexes_stats: - if self._mongo_version_parsed >= Version("3.2"): - potential_collectors.append( - IndexStatsCollector(self, self._config.db_name, tags, self._config.coll_names) - ) - else: - self.log.debug( - "'collections_indexes_stats' is only available starting from mongo 3.2: " - "your mongo version is %s", - self._mongo_version, - ) + dbstats_tag_dbname = self._config.dbstats_tag_dbname for db_name in all_dbs: + # DbStatCollector is always collected on all monitored databases (filtered by db_names config option) + # For backward compatibility, we keep collecting from all monitored databases + # regardless of the auto-discovery settings. potential_collectors.append(DbStatCollector(self, db_name, dbstats_tag_dbname, tags)) + monitored_dbs = all_dbs if self._database_autodiscovery.autodiscovery_enabled else [self._config.db_name] + # When autodiscovery is enabled, we collect collstats and indexstats for all auto-discovered databases + # Otherwise, we collect collstats and indexstats for the database specified in the configuration + for db_name in monitored_dbs: + # For backward compatibility, coll_names is ONLY applied when autodiscovery is not enabled + # Otherwise, we collect collstats & indexstats for all auto-discovered databases and authorized collections + coll_names = None if self._database_autodiscovery.autodiscovery_enabled else self._config.coll_names + potential_collectors.append(CollStatsCollector(self, db_name, tags, coll_names=coll_names)) + if self._config.collections_indexes_stats: + if self._mongo_version_parsed >= Version("3.2"): + potential_collectors.append(IndexStatsCollector(self, db_name, tags, coll_names=coll_names)) + else: + self.log.debug( + "'collections_indexes_stats' is only available starting from mongo 3.2: " + "your mongo version is %s", + self._mongo_version, + ) + # Custom queries are always collected except if the node is a secondary or an arbiter in a replica set. # It is possible to collect custom queries from secondary nodes as well but this has to be explicitly # stated in the configuration of the query. @@ -260,7 +274,7 @@ def _collect_metrics(self): deployment = self.api_client.deployment_type tags = self._get_tags(include_deployment_tags=True, include_internal_resource_tags=True) - dbnames = self._get_db_names(self.api_client, deployment, tags) + dbnames = self._get_db_names(tags) self.refresh_collectors(deployment, dbnames, tags) for collector in self.collectors: try: @@ -275,34 +289,10 @@ def _collect_metrics(self): "Unable to collect logs from collector %s. Some metrics will be missing.", collector, exc_info=True ) - def _get_db_names(self, api, deployment, tags): - if isinstance(deployment, ReplicaSetDeployment) and deployment.is_arbiter: - self.log.debug("Replicaset and arbiter deployment, no databases will be checked") - dbnames = [] - elif isinstance(deployment, ReplicaSetDeployment) and deployment.replset_state == 3: - self.log.debug("Replicaset is in recovering state, will skip reading database names") - dbnames = [] - else: - server_databases = api.list_database_names() - self.gauge('mongodb.dbs', len(server_databases), tags=tags) - if self._config.db_names is None: - self.log.debug("No databases configured. Retrieving list of databases from the mongo server") - dbnames = server_databases - else: - self.log.debug("Collecting only from the configured databases: %s", self._config.db_names) - dbnames = [] - self.log.debug("Checking the configured databases that exist on the mongo server") - for config_dbname in self._config.db_names: - if config_dbname in server_databases: - self.log.debug("'%s' database found on the mongo server", config_dbname) - dbnames.append(config_dbname) - else: - self.log.warning( - "'%s' database not found on the mongo server" - ", will not append to list of databases to check", - config_dbname, - ) - self.log.debug("List of databases to check: %s", dbnames) + def _get_db_names(self, tags): + dbnames, database_count = self._database_autodiscovery.get_databases_and_count() + if database_count: + self.gauge('mongodb.dbs', database_count, tags=tags) return dbnames def _diagnose_tls(self): diff --git a/mongo/tests/compose/scripts/init-router.js b/mongo/tests/compose/scripts/init-router.js index f2a1eb43d5c3a..c4d0ebcf85e8a 100644 --- a/mongo/tests/compose/scripts/init-router.js +++ b/mongo/tests/compose/scripts/init-router.js @@ -6,4 +6,4 @@ sh.addShard("shard02/shard02a:27019") sh.addShard("shard02/shard02b:27019") sh.addShard("shard03/shard03a:27020") -sh.addShard("shard03/shard03b:27020") \ No newline at end of file +sh.addShard("shard03/shard03b:27020") diff --git a/mongo/tests/conftest.py b/mongo/tests/conftest.py index b64fee5b00c0f..b0c2740c76ee2 100644 --- a/mongo/tests/conftest.py +++ b/mongo/tests/conftest.py @@ -126,6 +126,15 @@ def instance_integration(instance_custom_queries): return instance +@pytest.fixture +def instance_integration_autodiscovery(instance_integration): + instance = copy.deepcopy(instance_integration) + instance["database_autodiscovery"] = { + "enabled": True, + } + return instance + + @pytest.fixture def instance_integration_cluster(instance_integration): instance = copy.deepcopy(instance_integration) @@ -133,6 +142,15 @@ def instance_integration_cluster(instance_integration): return instance +@pytest.fixture +def instance_integration_cluster_autodiscovery(instance_integration_cluster): + instance = copy.deepcopy(instance_integration_cluster) + instance["database_autodiscovery"] = { + "enabled": True, + } + return instance + + @pytest.fixture(scope='session', autouse=True) def mock_local_tls_dns(): with mock_local(HOSTNAME_TO_PORT_MAPPING): diff --git a/mongo/tests/fixtures/$currentOp-mongos b/mongo/tests/fixtures/$currentOp-mongos index 2d68aadaf67d4..a5acd5992c3e9 100644 --- a/mongo/tests/fixtures/$currentOp-mongos +++ b/mongo/tests/fixtures/$currentOp-mongos @@ -280,7 +280,7 @@ "secs_running": 0, "microsecs_running": 26789, "op": "getmore", - "ns": "dbmorders_1.movies", + "ns": "integration.movies", "redacted": false, "command": { "getMore": 7153547462305880513, @@ -340,7 +340,7 @@ }, { "role": "readWrite", - "db": "dbmorders_1" + "db": "integration" } ] }, @@ -369,7 +369,7 @@ } }, "mayBypassWriteBlocking": true, - "$db": "dbmorders_1" + "$db": "integration" }, "queryFramework": "classic", "planSummary": "COLLSCAN", @@ -508,7 +508,7 @@ }, { "role": "readWrite", - "db": "dbmorders_1" + "db": "integration" } ] }, @@ -537,7 +537,7 @@ } }, "mayBypassWriteBlocking": true, - "$db": "dbmorders_1" + "$db": "integration" }, "planSummary": "COLLSCAN", "operationUsingCursorId": 155914227 diff --git a/mongo/tests/fixtures/list_collection_names b/mongo/tests/fixtures/list_collection_names new file mode 100644 index 0000000000000..a6ca513e1de3e --- /dev/null +++ b/mongo/tests/fixtures/list_collection_names @@ -0,0 +1 @@ +["foo", "bar"] \ No newline at end of file diff --git a/mongo/tests/fixtures/list_database_names b/mongo/tests/fixtures/list_database_names index 841642dabfe37..0086cf0583516 100644 --- a/mongo/tests/fixtures/list_database_names +++ b/mongo/tests/fixtures/list_database_names @@ -1 +1 @@ -["admin", "config", "test", "local"] \ No newline at end of file +["admin", "config", "test", "local", "integration"] \ No newline at end of file diff --git a/mongo/tests/mocked_api.py b/mongo/tests/mocked_api.py index 33b363b6460ef..280da967011e6 100644 --- a/mongo/tests/mocked_api.py +++ b/mongo/tests/mocked_api.py @@ -65,6 +65,10 @@ def command(self, command, *args, **_): with open(os.path.join(HERE, "fixtures", filename), 'r') as f: return json.load(f, object_hook=json_util.object_hook) + def list_collection_names(self, session=None, filter=None, comment=None, **kwargs): + with open(os.path.join(HERE, "fixtures", "list_collection_names"), 'r') as f: + return json.load(f) + def aggregate(self, pipeline, session=None, **kwargs): if pipeline[0] == {'$currentOp': {'allUsers': True}}: # mock the $currentOp aggregation used for operation sampling diff --git a/mongo/tests/results/metrics-collection-autodiscover.json b/mongo/tests/results/metrics-collection-autodiscover.json new file mode 100644 index 0000000000000..f13f8ef455b65 --- /dev/null +++ b/mongo/tests/results/metrics-collection-autodiscover.json @@ -0,0 +1,730 @@ +[ + { + "name": "mongodb.collection.size", + "type": 0, + "value": 5670.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.size", + "type": 0, + "value": 2600.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.avgobjsize", + "type": 0, + "value": 27.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.avgobjsize", + "type": 0, + "value": 26.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.count", + "type": 0, + "value": 210.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.count", + "type": 0, + "value": 100.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.capped", + "type": 0, + "value": 0.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.capped", + "type": 0, + "value": 0.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.max", + "type": 0, + "value": 10.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.max", + "type": 0, + "value": 10.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.maxsize", + "type": 0, + "value": 10.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.maxsize", + "type": 0, + "value": 10.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.storagesize", + "type": 0, + "value": 16384.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.storagesize", + "type": 0, + "value": 16384.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.nindexes", + "type": 0, + "value": 1.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.nindexes", + "type": 0, + "value": 1.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.indexsizes", + "type": 0, + "value": 16384.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:foo", + "index:_id_" + ] + }, + { + "name": "mongodb.collection.indexsizes", + "type": 0, + "value": 16384.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:test", + "collection:bar", + "index:_id_" + ] + }, + { + "name": "mongodb.collection.size", + "type": 0, + "value": 5670.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.size", + "type": 0, + "value": 2600.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.avgobjsize", + "type": 0, + "value": 27.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.avgobjsize", + "type": 0, + "value": 26.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.count", + "type": 0, + "value": 210.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.count", + "type": 0, + "value": 100.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.capped", + "type": 0, + "value": 0.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.capped", + "type": 0, + "value": 0.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.max", + "type": 0, + "value": 10.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.max", + "type": 0, + "value": 10.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.maxsize", + "type": 0, + "value": 10.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.maxsize", + "type": 0, + "value": 10.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.storagesize", + "type": 0, + "value": 16384.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.storagesize", + "type": 0, + "value": 16384.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.nindexes", + "type": 0, + "value": 1.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.nindexes", + "type": 0, + "value": 1.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.indexsizes", + "type": 0, + "value": 16384.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:foo", + "index:_id_" + ] + }, + { + "name": "mongodb.collection.indexsizes", + "type": 0, + "value": 16384.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:admin", + "collection:bar", + "index:_id_" + ] + }, + { + "name": "mongodb.collection.size", + "type": 0, + "value": 5670.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.size", + "type": 0, + "value": 2600.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.avgobjsize", + "type": 0, + "value": 27.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.avgobjsize", + "type": 0, + "value": 26.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.count", + "type": 0, + "value": 210.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.count", + "type": 0, + "value": 100.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.capped", + "type": 0, + "value": 0.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.capped", + "type": 0, + "value": 0.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.max", + "type": 0, + "value": 10.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.max", + "type": 0, + "value": 10.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.maxsize", + "type": 0, + "value": 10.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.maxsize", + "type": 0, + "value": 10.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.storagesize", + "type": 0, + "value": 16384.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.storagesize", + "type": 0, + "value": 16384.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.nindexes", + "type": 0, + "value": 1.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.nindexes", + "type": 0, + "value": 1.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.indexsizes", + "type": 0, + "value": 16384.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:foo", + "index:_id_" + ] + }, + { + "name": "mongodb.collection.indexsizes", + "type": 0, + "value": 16384.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:local", + "collection:bar", + "index:_id_" + ] + }, + { + "name": "mongodb.collection.size", + "type": 0, + "value": 5670.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.size", + "type": 0, + "value": 2600.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.avgobjsize", + "type": 0, + "value": 27.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.avgobjsize", + "type": 0, + "value": 26.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.count", + "type": 0, + "value": 210.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.count", + "type": 0, + "value": 100.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.capped", + "type": 0, + "value": 0.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.capped", + "type": 0, + "value": 0.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.max", + "type": 0, + "value": 10.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.max", + "type": 0, + "value": 10.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.maxsize", + "type": 0, + "value": 10.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.maxsize", + "type": 0, + "value": 10.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.storagesize", + "type": 0, + "value": 16384.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.storagesize", + "type": 0, + "value": 16384.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.nindexes", + "type": 0, + "value": 1.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:foo" + ] + }, + { + "name": "mongodb.collection.nindexes", + "type": 0, + "value": 1.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:bar" + ] + }, + { + "name": "mongodb.collection.indexsizes", + "type": 0, + "value": 16384.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:foo", + "index:_id_" + ] + }, + { + "name": "mongodb.collection.indexsizes", + "type": 0, + "value": 16384.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "db:integration", + "collection:bar", + "index:_id_" + ] + } +] \ No newline at end of file diff --git a/mongo/tests/results/metrics-count-dbs.json b/mongo/tests/results/metrics-count-dbs.json index 3dd6481eeeb38..7a656f091b746 100644 --- a/mongo/tests/results/metrics-count-dbs.json +++ b/mongo/tests/results/metrics-count-dbs.json @@ -2,7 +2,7 @@ { "name": "mongodb.dbs", "type": 0, - "value": 4.0, + "value": 5.0, "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test" ] diff --git a/mongo/tests/results/metrics-indexes-stats-autodiscover.json b/mongo/tests/results/metrics-indexes-stats-autodiscover.json new file mode 100644 index 0000000000000..e1f64ace03346 --- /dev/null +++ b/mongo/tests/results/metrics-indexes-stats-autodiscover.json @@ -0,0 +1,90 @@ +[ + { + "name": "mongodb.collection.indexes.accesses.ops", + "type": 0, + "value": 1243.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "name:_id_", + "collection:foo", + "db:test" + ] + }, + { + "name": "mongodb.collection.indexes.accesses.ops", + "type": 0, + "value": 12.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "name:_id_", + "collection:bar", + "db:test" + ] + }, + { + "name": "mongodb.collection.indexes.accesses.ops", + "type": 0, + "value": 1243.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "name:_id_", + "collection:foo", + "db:admin" + ] + }, + { + "name": "mongodb.collection.indexes.accesses.ops", + "type": 0, + "value": 12.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "name:_id_", + "collection:bar", + "db:admin" + ] + }, + { + "name": "mongodb.collection.indexes.accesses.ops", + "type": 0, + "value": 1243.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "name:_id_", + "collection:foo", + "db:config" + ] + }, + { + "name": "mongodb.collection.indexes.accesses.ops", + "type": 0, + "value": 12.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "name:_id_", + "collection:bar", + "db:config" + ] + }, + { + "name": "mongodb.collection.indexes.accesses.ops", + "type": 0, + "value": 1243.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "name:_id_", + "collection:foo", + "db:integration" + ] + }, + { + "name": "mongodb.collection.indexes.accesses.ops", + "type": 0, + "value": 12.0, + "tags": [ + "server:mongodb://testUser2:*****@localhost:27017/test", + "name:_id_", + "collection:bar", + "db:integration" + ] + } +] \ No newline at end of file diff --git a/mongo/tests/results/metrics-indexes-stats.json b/mongo/tests/results/metrics-indexes-stats.json index dea5b20e76070..4977ad2f1d274 100644 --- a/mongo/tests/results/metrics-indexes-stats.json +++ b/mongo/tests/results/metrics-indexes-stats.json @@ -6,7 +6,8 @@ "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "name:_id_", - "collection:foo" + "collection:foo", + "db:test" ] }, { @@ -16,7 +17,8 @@ "tags": [ "server:mongodb://testUser2:*****@localhost:27017/test", "name:_id_", - "collection:bar" + "collection:bar", + "db:test" ] } ] \ No newline at end of file diff --git a/mongo/tests/results/opeartion-activities-mongos.json b/mongo/tests/results/opeartion-activities-mongos.json index 43aa9d99b12ba..117e04d930298 100644 --- a/mongo/tests/results/opeartion-activities-mongos.json +++ b/mongo/tests/results/opeartion-activities-mongos.json @@ -82,12 +82,12 @@ }, { "now": 1715911398.1112723, - "query_signature": "21940c8bee157379", - "statement": "{\"getMore\": 7153547462305880513, \"collection\": \"movies\", \"lsid\": {\"id\": {\"$binary\": {\"base64\": \"zLd+DNpkQdqkWN1AZ828SA==\", \"subType\": \"04\"}}, \"uid\": {\"$binary\": {\"base64\": \"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=\", \"subType\": \"00\"}}}, \"$clusterTime\": {\"clusterTime\": {\"$timestamp\": {\"t\": 1718311810, \"i\": 6}}, \"signature\": {\"hash\": {\"$binary\": {\"base64\": \"AAAAAAAAAAAAAAAAAAAAAAAAAAA=\", \"subType\": \"00\"}}, \"keyId\": 0}}, \"$configTime\": {\"$timestamp\": {\"t\": 1718311809, \"i\": 1}}, \"$topologyTime\": {\"$timestamp\": {\"t\": 1716428552, \"i\": 3}}, \"$audit\": {\"$impersonatedUser\": {\"user\": \"shopper_4\", \"db\": \"admin\"}, \"$impersonatedRoles\": [{\"role\": \"readWrite\", \"db\": \"dbmorders\"}, {\"role\": \"readWrite\", \"db\": \"dbmorders_1\"}]}, \"$client\": {\"application\": {\"name\": \"orders-mongo\"}, \"driver\": {\"name\": \"mongo-go-driver\", \"version\": \"v1.15.0\"}, \"os\": {\"type\": \"linux\", \"architecture\": \"amd64\"}, \"platform\": \"go1.22.1\", \"env\": {\"container\": {\"runtime\": \"docker\"}}, \"mongos\": {\"host\": \"ip-172-29-203-167:27017\", \"client\": \"127.0.0.1:50388\", \"version\": \"7.0.9\"}}, \"mayBypassWriteBlocking\": true, \"$db\": \"dbmorders_1\"}", + "query_signature": "c9f4db9be9cc6b96", + "statement": "{\"getMore\": 7153547462305880513, \"collection\": \"movies\", \"lsid\": {\"id\": {\"$binary\": {\"base64\": \"zLd+DNpkQdqkWN1AZ828SA==\", \"subType\": \"04\"}}, \"uid\": {\"$binary\": {\"base64\": \"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=\", \"subType\": \"00\"}}}, \"$clusterTime\": {\"clusterTime\": {\"$timestamp\": {\"t\": 1718311810, \"i\": 6}}, \"signature\": {\"hash\": {\"$binary\": {\"base64\": \"AAAAAAAAAAAAAAAAAAAAAAAAAAA=\", \"subType\": \"00\"}}, \"keyId\": 0}}, \"$configTime\": {\"$timestamp\": {\"t\": 1718311809, \"i\": 1}}, \"$topologyTime\": {\"$timestamp\": {\"t\": 1716428552, \"i\": 3}}, \"$audit\": {\"$impersonatedUser\": {\"user\": \"shopper_4\", \"db\": \"admin\"}, \"$impersonatedRoles\": [{\"role\": \"readWrite\", \"db\": \"dbmorders\"}, {\"role\": \"readWrite\", \"db\": \"integration\"}]}, \"$client\": {\"application\": {\"name\": \"orders-mongo\"}, \"driver\": {\"name\": \"mongo-go-driver\", \"version\": \"v1.15.0\"}, \"os\": {\"type\": \"linux\", \"architecture\": \"amd64\"}, \"platform\": \"go1.22.1\", \"env\": {\"container\": {\"runtime\": \"docker\"}}, \"mongos\": {\"host\": \"ip-172-29-203-167:27017\", \"client\": \"127.0.0.1:50388\", \"version\": \"7.0.9\"}}, \"mayBypassWriteBlocking\": true, \"$db\": \"integration\"}", "active": true, "desc": "conn703087", "opid": "shard2:155914227", - "ns": "dbmorders_1.movies", + "ns": "integration.movies", "plan_summary": "COLLSCAN", "current_op_time": "2024-06-13T20:50:10.834+00:00", "microsecs_running": 26789, @@ -129,14 +129,14 @@ "no_cursor_timeout": false, "tailable": false, "await_data": false, - "originating_command": "{\"aggregate\": \"movies\", \"pipeline\": [{\"$lookup\": {\"from\": \"comments\", \"as\": \"comments\", \"localField\": \"_id\", \"foreignField\": \"movie_id\"}}], \"cursor\": {\"batchSize\": 0}, \"needsMerge\": true, \"let\": {\"NOW\": {\"$literal\": {\"$date\": \"2024-06-13T20:50:10.805Z\"}}, \"CLUSTER_TIME\": {\"$literal\": {\"$timestamp\": {\"t\": 1718311810, \"i\": 6}}}}, \"fromMongos\": true, \"collation\": {\"locale\": \"simple\"}, \"readConcern\": {\"level\": \"local\", \"provenance\": \"implicitDefault\"}, \"writeConcern\": {\"w\": \"majority\", \"wtimeout\": 0, \"provenance\": \"implicitDefault\"}, \"shardVersion\": {\"e\": {\"$oid\": \"666b52b852fa5a96da6d59ae\"}, \"t\": {\"$timestamp\": {\"t\": 1718309560, \"i\": 188}}, \"v\": {\"$timestamp\": {\"t\": 1, \"i\": 5}}}, \"clientOperationKey\": {\"$binary\": {\"base64\": \"zc+70kUAQ5m1zVRdJjwYxw==\", \"subType\": \"04\"}}, \"lsid\": {\"id\": {\"$binary\": {\"base64\": \"zLd+DNpkQdqkWN1AZ828SA==\", \"subType\": \"04\"}}, \"uid\": {\"$binary\": {\"base64\": \"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=\", \"subType\": \"00\"}}}, \"$clusterTime\": {\"clusterTime\": {\"$timestamp\": {\"t\": 1718311810, \"i\": 6}}, \"signature\": {\"hash\": {\"$binary\": {\"base64\": \"AAAAAAAAAAAAAAAAAAAAAAAAAAA=\", \"subType\": \"00\"}}, \"keyId\": 0}}, \"$configTime\": {\"$timestamp\": {\"t\": 1718311809, \"i\": 1}}, \"$topologyTime\": {\"$timestamp\": {\"t\": 1716428552, \"i\": 3}}, \"$audit\": {\"$impersonatedUser\": {\"user\": \"shopper_4\", \"db\": \"admin\"}, \"$impersonatedRoles\": [{\"role\": \"readWrite\", \"db\": \"dbmorders\"}, {\"role\": \"readWrite\", \"db\": \"dbmorders_1\"}]}, \"$client\": {\"application\": {\"name\": \"orders-mongo\"}, \"driver\": {\"name\": \"mongo-go-driver\", \"version\": \"v1.15.0\"}, \"os\": {\"type\": \"linux\", \"architecture\": \"amd64\"}, \"platform\": \"go1.22.1\", \"env\": {\"container\": {\"runtime\": \"docker\"}}, \"mongos\": {\"host\": \"ip-172-29-203-167:27017\", \"client\": \"127.0.0.1:50388\", \"version\": \"7.0.9\"}}, \"mayBypassWriteBlocking\": true, \"$db\": \"dbmorders_1\"}", + "originating_command": "{\"aggregate\": \"movies\", \"pipeline\": [{\"$lookup\": {\"from\": \"comments\", \"as\": \"comments\", \"localField\": \"_id\", \"foreignField\": \"movie_id\"}}], \"cursor\": {\"batchSize\": 0}, \"needsMerge\": true, \"let\": {\"NOW\": {\"$literal\": {\"$date\": \"2024-06-13T20:50:10.805Z\"}}, \"CLUSTER_TIME\": {\"$literal\": {\"$timestamp\": {\"t\": 1718311810, \"i\": 6}}}}, \"fromMongos\": true, \"collation\": {\"locale\": \"simple\"}, \"readConcern\": {\"level\": \"local\", \"provenance\": \"implicitDefault\"}, \"writeConcern\": {\"w\": \"majority\", \"wtimeout\": 0, \"provenance\": \"implicitDefault\"}, \"shardVersion\": {\"e\": {\"$oid\": \"666b52b852fa5a96da6d59ae\"}, \"t\": {\"$timestamp\": {\"t\": 1718309560, \"i\": 188}}, \"v\": {\"$timestamp\": {\"t\": 1, \"i\": 5}}}, \"clientOperationKey\": {\"$binary\": {\"base64\": \"zc+70kUAQ5m1zVRdJjwYxw==\", \"subType\": \"04\"}}, \"lsid\": {\"id\": {\"$binary\": {\"base64\": \"zLd+DNpkQdqkWN1AZ828SA==\", \"subType\": \"04\"}}, \"uid\": {\"$binary\": {\"base64\": \"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=\", \"subType\": \"00\"}}}, \"$clusterTime\": {\"clusterTime\": {\"$timestamp\": {\"t\": 1718311810, \"i\": 6}}, \"signature\": {\"hash\": {\"$binary\": {\"base64\": \"AAAAAAAAAAAAAAAAAAAAAAAAAAA=\", \"subType\": \"00\"}}, \"keyId\": 0}}, \"$configTime\": {\"$timestamp\": {\"t\": 1718311809, \"i\": 1}}, \"$topologyTime\": {\"$timestamp\": {\"t\": 1716428552, \"i\": 3}}, \"$audit\": {\"$impersonatedUser\": {\"user\": \"shopper_4\", \"db\": \"admin\"}, \"$impersonatedRoles\": [{\"role\": \"readWrite\", \"db\": \"dbmorders\"}, {\"role\": \"readWrite\", \"db\": \"integration\"}]}, \"$client\": {\"application\": {\"name\": \"orders-mongo\"}, \"driver\": {\"name\": \"mongo-go-driver\", \"version\": \"v1.15.0\"}, \"os\": {\"type\": \"linux\", \"architecture\": \"amd64\"}, \"platform\": \"go1.22.1\", \"env\": {\"container\": {\"runtime\": \"docker\"}}, \"mongos\": {\"host\": \"ip-172-29-203-167:27017\", \"client\": \"127.0.0.1:50388\", \"version\": \"7.0.9\"}}, \"mayBypassWriteBlocking\": true, \"$db\": \"integration\"}", "plan_summary": "COLLSCAN", "operation_using_cursor_id": 155914227 }, "type": "op", "op": "getmore", "shard": null, - "dbname": "dbmorders_1", + "dbname": "integration", "application": "orders-mongo", "collection": "movies", "comment": null, diff --git a/mongo/tests/test_e2e.py b/mongo/tests/test_e2e.py index 4af0c5a705bed..b1f6ab97dbe53 100644 --- a/mongo/tests/test_e2e.py +++ b/mongo/tests/test_e2e.py @@ -64,7 +64,9 @@ @standalone @pytest.mark.e2e -def test_e2e_mongo_standalone(dd_agent_check, instance_user): +@pytest.mark.parametrize('database_autodiscovery_enabled', [True, False]) +def test_e2e_mongo_standalone(dd_agent_check, instance_user, database_autodiscovery_enabled): + instance_user['database_autodiscovery'] = {'enabled': database_autodiscovery_enabled} aggregator = dd_agent_check(instance_user, rate=True) for metric in MONGOD_METRICS: aggregator.assert_metric(metric) @@ -73,7 +75,9 @@ def test_e2e_mongo_standalone(dd_agent_check, instance_user): @shard @pytest.mark.e2e -def test_e2e_mongo_shard(dd_agent_check, instance_authdb): +@pytest.mark.parametrize('database_autodiscovery_enabled', [True, False]) +def test_e2e_mongo_shard(dd_agent_check, instance_authdb, database_autodiscovery_enabled): + instance_authdb['database_autodiscovery'] = {'enabled': database_autodiscovery_enabled} aggregator = dd_agent_check(instance_authdb, rate=True) for metric in MONGOS_METRICS: diff --git a/mongo/tests/test_integration.py b/mongo/tests/test_integration.py index 7ccdb742a0dd6..a46b2a48141ce 100644 --- a/mongo/tests/test_integration.py +++ b/mongo/tests/test_integration.py @@ -1082,3 +1082,44 @@ def test_integration_reemit_mongodb_instance_on_deployment_change( mongo_check.api_client.deployment_type.replset_state = SECONDARY_STATE_ID dd_run_check(mongo_check) assert _get_mongodb_instance_event(aggregator) is not None + + +def test_integration_database_autodiscovery(instance_integration_autodiscovery, aggregator, check, dd_run_check): + mongo_check = check(instance_integration_autodiscovery) + + with mock_pymongo("replica-primary"): + dd_run_check(mongo_check) + + replica_tags = [ + 'replset_name:replset', + 'replset_state:primary', + 'replset_me:replset-data-0.mongo.default.svc.cluster.local:27017', + ] + metrics_categories = [ + 'count-dbs', + 'serverStatus', + 'custom-queries', + 'oplog', + 'replset-primary', + 'top', + 'dbstats-local', + 'fsynclock', + 'dbstats', + 'indexes-stats-autodiscover', + 'collection-autodiscover', + ] + _assert_metrics(mongo_check, aggregator, metrics_categories, replica_tags) + # Lag metrics are tagged with the state of the member and not with the current one. + _assert_metrics(mongo_check, aggregator, ['replset-lag-from-primary']) + + aggregator.assert_all_metrics_covered() + aggregator.assert_metrics_using_metadata( + get_metadata_metrics(), + exclude=[ + 'dd.custom.mongo.aggregate.total', + 'dd.custom.mongo.count', + 'dd.custom.mongo.query_a.amount', + 'dd.custom.mongo.query_a.el', + ], + check_submission_type=True, + ) diff --git a/mongo/tests/test_unit.py b/mongo/tests/test_unit.py index 0cfe918f9a13a..022d6614f5d4b 100644 --- a/mongo/tests/test_unit.py +++ b/mongo/tests/test_unit.py @@ -635,6 +635,7 @@ def test_when_version_lower_than_3_6_then_no_session_metrics_reported(aggregator @pytest.mark.parametrize("error_cls", CRITICAL_FAILURE) def test_service_check_critical_when_connection_dies(error_cls, aggregator, check, instance, dd_run_check): + instance['database_autodiscovery'] = {'enabled': True, 'refresh_interval': 0} # force refresh on every run check = check(instance) with mock_pymongo('standalone') as mocked_client: dd_run_check(check) diff --git a/mongo/tests/test_unit_config.py b/mongo/tests/test_unit_config.py index 2cbda301afa2b..e9b68386f8899 100644 --- a/mongo/tests/test_unit_config.py +++ b/mongo/tests/test_unit_config.py @@ -123,3 +123,29 @@ def test_mongo_operation_samples_enabled( mongo_check = check(instance_integration_cluster) assert mongo_check._config.operation_samples.get('enabled') == operation_samples_enabled + + +def test_database_autodiscovery_disabled(instance_user): + config = MongoConfig(instance_user, mock.Mock()) + assert config.database_autodiscovery_config is not None + assert config.database_autodiscovery_config['enabled'] is False + + +def test_database_autodiscovery_enabled(instance_user): + instance_user['database_autodiscovery'] = {'enabled': True, 'include': ['test.*'], 'exclude': ['admin']} + config = MongoConfig(instance_user, mock.Mock()) + assert config.database_autodiscovery_config is not None + assert config.database_autodiscovery_config['enabled'] is True + assert config.database_autodiscovery_config['include'] == ['test.*'] + assert config.database_autodiscovery_config['exclude'] == ['admin'] + + +def test_database_autodiscovery_dbnames_deprecation(instance_user): + # dbnames is deprecated in favor of database_autodiscovery + # for backwards compatibility, we implicitly enable database_autodiscovery if dbnames is set + # and set the include list to the dbnames list + instance_user['dbnames'] = ['test', 'integration'] + config = MongoConfig(instance_user, mock.Mock()) + assert config.database_autodiscovery_config is not None + assert config.database_autodiscovery_config['enabled'] is True + assert config.database_autodiscovery_config['include'] == ['test$', 'integration$']