diff --git a/README.md b/README.md index 9e14360..24debc0 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Get VMware vCenter information: Alternatively, if you don't wish to install the package, run it using `$ vmware_exporter/vmware_exporter.py` or use the following docker command: ``` -docker run -it --rm -p 9272:9272 -e VSPHERE_USER=${VSPHERE_USERNAME} -e VSPHERE_PASSWORD=${VSPHERE_PASSWORD} -e VSPHERE_HOST=${VSPHERE_HOST} -e VSPHERE_IGNORE_SSL=True --name vmware_exporter pryorda/vmware_exporter +docker run -it --rm -p 9272:9272 -e VSPHERE_USER=${VSPHERE_USERNAME} -e VSPHERE_PASSWORD=${VSPHERE_PASSWORD} -e VSPHERE_HOST=${VSPHERE_HOST} -e VSPHERE_IGNORE_SSL=True -e VSPHERE_SPECS_SIZE=2000 --name vmware_exporter pryorda/vmware_exporter ``` ### Configuration and limiting data collection @@ -54,6 +54,7 @@ default: vsphere_user: "user" vsphere_password: "password" ignore_ssl: False + specs_size: 5000 collect_only: vms: True vmguests: True @@ -66,6 +67,7 @@ esx: vsphere_user: 'root' vsphere_password: 'password' ignore_ssl: True + specs_size: 5000 collect_only: vms: False vmguests: True @@ -78,6 +80,7 @@ limited: vsphere_user: 'administrator@vsphere.local' vsphere_password: 'password' ignore_ssl: True + specs_size: 5000 collect_only: vms: False vmguests: False @@ -94,6 +97,7 @@ Switching sections can be done by adding ?section=limited to the URL. | `VSPHERE_HOST` | config, env, get_param | n/a | vsphere server to connect to | | `VSPHERE_USER` | config, env | n/a | User for connecting to vsphere | | `VSPHERE_PASSWORD` | config, env | n/a | Password for connecting to vsphere | +| `VSPHERE_SPECS_SIZE` | config, env | 5000 | Size of specs list for query stats function | | `VSPHERE_IGNORE_SSL` | config, env | False | Ignore the ssl cert on the connection to vsphere host | | `VSPHERE_COLLECT_HOSTS` | config, env | True | Set to false to disable collection of host metrics | | `VSPHERE_COLLECT_DATASTORES` | config, env | True | Set to false to disable collection of datastore metrics | @@ -108,6 +112,7 @@ You can create new sections as well, with very similiar variables. For example, | `VSPHERE_LIMITED_HOST` | config, env, get_param | n/a | vsphere server to connect to | | `VSPHERE_LIMITED_USER` | config, env | n/a | User for connecting to vsphere | | `VSPHERE_LIMITED_PASSWORD` | config, env | n/a | Password for connecting to vsphere | +| `VSPHERE_LIMITED_SPECS_SIZE` | config, env | 5000 | Size of specs list for query stats function | | `VSPHERE_LIMITED_IGNORE_SSL` | config, env | False | Ignore the ssl cert on the connection to vsphere host | | `VSPHERE_LIMITED_COLLECT_HOSTS` | config, env | True | Set to false to disable collection of host metrics | | `VSPHERE_LIMITED_COLLECT_DATASTORES` | config, env | True | Set to false to disable collection of datastore metrics | diff --git a/tests/unit/test_vmware_exporter.py b/tests/unit/test_vmware_exporter.py index 093d167..11ff806 100644 --- a/tests/unit/test_vmware_exporter.py +++ b/tests/unit/test_vmware_exporter.py @@ -71,6 +71,7 @@ def test_collect_vms(): 'root', 'password', collect_only, + 5000, ) collector.content = _succeed(mock.Mock()) @@ -103,6 +104,7 @@ def test_collect_vms(): 'root', 'password', collect_only, + 5000, ) collector.content = _succeed(mock.Mock()) @@ -144,6 +146,7 @@ def test_collect_vms(): 'root', 'password', collect_only, + 5000, ) collector.content = _succeed(mock.Mock()) @@ -340,6 +343,7 @@ def test_metrics_without_hostaccess(): 'root', 'password', collect_only, + 5000, ) metrics = collector._create_metric_containers() collector.content = _succeed(mock.Mock()) @@ -398,6 +402,7 @@ def test_no_error_onempty_vms(): 'root', 'password', collect_only, + 5000, ignore_ssl=True, ) @@ -479,6 +484,7 @@ def test_collect_vm_perf(): 'root', 'password', collect_only, + 5000, ) metrics = collector._create_metric_containers() @@ -595,6 +601,7 @@ def test_collect_hosts(): 'root', 'password', collect_only, + 5000, ) collector.content = _succeed(mock.Mock()) @@ -679,6 +686,7 @@ def test_collect_host_perf(): 'root', 'password', collect_only, + 5000, ) metrics = collector._create_metric_containers() @@ -792,6 +800,7 @@ def test_collect_datastore(): 'root', 'password', collect_only, + 5000, ) collector.content = _succeed(mock.Mock()) @@ -854,6 +863,7 @@ def test_collect(): 'root', 'password', collect_only, + 5000, ignore_ssl=True, ) collector.content = _succeed(mock.Mock()) @@ -889,6 +899,7 @@ def test_collect_deferred_error_works(): 'root', 'password', collect_only, + 5000, ignore_ssl=True, ) collector.content = _succeed(mock.Mock()) @@ -990,6 +1001,7 @@ def test_vmware_get_inventory(): 'root', 'password', collect_only, + 5000, ignore_ssl=True, ) collector.content = content @@ -1033,6 +1045,7 @@ def test_vmware_connect(): 'root', 'password', collect_only, + 5000, ignore_ssl=True, ) @@ -1059,6 +1072,7 @@ def test_vmware_disconnect(): '127.0.0.1', 'root', 'password', + 5000, collect_only, ) @@ -1093,6 +1107,7 @@ def test_counter_ids(): '127.0.0.1', 'root', 'password', + 5000, collect_only, ) collector.content = content @@ -1233,6 +1248,7 @@ def test_vmware_resource_async_render_GET_section(): 'vsphere_host': '127.0.0.10', 'vsphere_user': 'username1', 'vsphere_password': 'password1', + 'specs_size': 5000, 'collect_only': { 'datastores': True, 'hosts': True, @@ -1246,6 +1262,7 @@ def test_vmware_resource_async_render_GET_section(): 'vsphere_host': '127.0.0.11', 'vsphere_user': 'username2', 'vsphere_password': 'password2', + 'specs_size': 5000, 'collect_only': { 'datastores': True, 'hosts': True, @@ -1265,6 +1282,7 @@ def test_vmware_resource_async_render_GET_section(): 'username2', 'password2', resource.config['mysection']['collect_only'], + 5000, 'On' ) @@ -1278,9 +1296,11 @@ def test_config_env_multiple_sections(): 'VSPHERE_HOST': '127.0.0.10', 'VSPHERE_USER': 'username1', 'VSPHERE_PASSWORD': 'password1', + 'VSPHERE_SPECS_SIZE': 5000, 'VSPHERE_MYSECTION_HOST': '127.0.0.11', 'VSPHERE_MYSECTION_USER': 'username2', 'VSPHERE_MYSECTION_PASSWORD': 'password2', + 'VSPHERE_MYSECTION_SPECS_SIZE': 5000, } args = mock.Mock() @@ -1295,6 +1315,7 @@ def test_config_env_multiple_sections(): 'vsphere_host': '127.0.0.10', 'vsphere_user': 'username1', 'vsphere_password': 'password1', + 'specs_size': 5000, 'collect_only': { 'datastores': True, 'hosts': True, @@ -1308,6 +1329,7 @@ def test_config_env_multiple_sections(): 'vsphere_host': '127.0.0.11', 'vsphere_user': 'username2', 'vsphere_password': 'password2', + 'specs_size': 5000, 'collect_only': { 'datastores': True, 'hosts': True, diff --git a/vmware_exporter/vmware_exporter.py b/vmware_exporter/vmware_exporter.py index d8fdce8..8ba1a98 100755 --- a/vmware_exporter/vmware_exporter.py +++ b/vmware_exporter/vmware_exporter.py @@ -38,12 +38,13 @@ class VmwareCollector(): - def __init__(self, host, username, password, collect_only, ignore_ssl=False): + def __init__(self, host, username, password, collect_only, specs_size, ignore_ssl=False): self.host = host self.username = username self.password = password self.ignore_ssl = ignore_ssl self.collect_only = collect_only + self.specs_size = specs_size def _create_metric_containers(self): metric_list = {} @@ -636,17 +637,19 @@ def _vmware_get_vm_perf_manager_metrics(self, vm_metrics): content = yield self.content if len(specs) > 0: - results, labels = yield parallelize( - threads.deferToThread(content.perfManager.QueryStats, querySpec=specs), - self.vm_labels, - ) + chunks = [specs[x:x+self.specs_size] for x in range(0, len(specs), self.specs_size)] + for list_specs in chunks: + results, labels = yield parallelize( + threads.deferToThread(content.perfManager.QueryStats, querySpec=list_specs), + self.vm_labels, + ) - for ent in results: - for metric in ent.value: - vm_metrics[metric_names[metric.id.counterId]].add_metric( - labels[ent.entity._moId], - float(sum(metric.value)), - ) + for ent in results: + for metric in ent.value: + vm_metrics[metric_names[metric.id.counterId]].add_metric( + labels[ent.entity._moId], + float(sum(metric.value)), + ) logging.info('FIN: _vmware_get_vm_perf_manager_metrics') @@ -949,6 +952,7 @@ def configure(self, args): 'vsphere_user': os.environ.get('VSPHERE_USER'), 'vsphere_password': os.environ.get('VSPHERE_PASSWORD'), 'ignore_ssl': get_bool_env('VSPHERE_IGNORE_SSL', False), + 'specs_size': os.environ.get('VSPHERE_SPECS_SIZE', 5000), 'collect_only': { 'vms': get_bool_env('VSPHERE_COLLECT_VMS', True), 'vmguests': get_bool_env('VSPHERE_COLLECT_VMGUESTS', True), @@ -972,6 +976,7 @@ def configure(self, args): 'vsphere_user': os.environ.get('VSPHERE_{}_USER'.format(section)), 'vsphere_password': os.environ.get('VSPHERE_{}_PASSWORD'.format(section)), 'ignore_ssl': get_bool_env('VSPHERE_{}_IGNORE_SSL'.format(section), False), + 'specs_size': os.environ.get('VSPHERE_{}_SPECS_SIZE'.format(section), 5000), 'collect_only': { 'vms': get_bool_env('VSPHERE_{}_COLLECT_VMS'.format(section), True), 'vmguests': get_bool_env('VSPHERE_{}_COLLECT_VMGUESTS'.format(section), True), @@ -1026,6 +1031,7 @@ def generate_latest_metrics(self, request): self.config[section]['vsphere_user'], self.config[section]['vsphere_password'], self.config[section]['collect_only'], + self.config[section]['specs_size'], self.config[section]['ignore_ssl'], ) metrics = yield collector.collect()