Skip to content

Commit

Permalink
feat(improve_performance): split specs in multiple chunks of 5000 spe…
Browse files Browse the repository at this point in the history 
…cs (#158)

* split specs in multiple chunks of 5000 specs

* put specs_size in config

* fix unit tests

* fix unit tests

* fix unit tests

* fix readme

* fix len(specs) part

* fix len(specs) part
  • Loading branch information
@alecorps @pryorda
alecorps authored and pryorda committed Dec 1, 2019
1 parent bbbc8fb commit bfa941d
Show file tree
Hide file tree
Showing 3 changed files with 45 additions and 12 deletions.
7 changes: 6 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ Get VMware vCenter information:
Alternatively, if you don't wish to install the package, run it using `$ vmware_exporter/vmware_exporter.py` or use the following docker command:

```
docker run -it --rm -p 9272:9272 -e VSPHERE_USER=${VSPHERE_USERNAME} -e VSPHERE_PASSWORD=${VSPHERE_PASSWORD} -e VSPHERE_HOST=${VSPHERE_HOST} -e VSPHERE_IGNORE_SSL=True --name vmware_exporter pryorda/vmware_exporter
docker run -it --rm -p 9272:9272 -e VSPHERE_USER=${VSPHERE_USERNAME} -e VSPHERE_PASSWORD=${VSPHERE_PASSWORD} -e VSPHERE_HOST=${VSPHERE_HOST} -e VSPHERE_IGNORE_SSL=True -e VSPHERE_SPECS_SIZE=2000 --name vmware_exporter pryorda/vmware_exporter
```

### Configuration and limiting data collection
Expand All @@ -54,6 +54,7 @@ default:
vsphere_user: "user"
vsphere_password: "password"
ignore_ssl: False
specs_size: 5000
collect_only:
vms: True
vmguests: True
Expand All @@ -66,6 +67,7 @@ esx:
vsphere_user: 'root'
vsphere_password: 'password'
ignore_ssl: True
specs_size: 5000
collect_only:
vms: False
vmguests: True
Expand All @@ -78,6 +80,7 @@ limited:
vsphere_user: '[email protected]'
vsphere_password: 'password'
ignore_ssl: True
specs_size: 5000
collect_only:
vms: False
vmguests: False
Expand All @@ -94,6 +97,7 @@ Switching sections can be done by adding ?section=limited to the URL.
| `VSPHERE_HOST` | config, env, get_param | n/a | vsphere server to connect to |
| `VSPHERE_USER` | config, env | n/a | User for connecting to vsphere |
| `VSPHERE_PASSWORD` | config, env | n/a | Password for connecting to vsphere |
| `VSPHERE_SPECS_SIZE` | config, env | 5000 | Size of specs list for query stats function |
| `VSPHERE_IGNORE_SSL` | config, env | False | Ignore the ssl cert on the connection to vsphere host |
| `VSPHERE_COLLECT_HOSTS` | config, env | True | Set to false to disable collection of host metrics |
| `VSPHERE_COLLECT_DATASTORES` | config, env | True | Set to false to disable collection of datastore metrics |
Expand All @@ -108,6 +112,7 @@ You can create new sections as well, with very similiar variables. For example,
| `VSPHERE_LIMITED_HOST` | config, env, get_param | n/a | vsphere server to connect to |
| `VSPHERE_LIMITED_USER` | config, env | n/a | User for connecting to vsphere |
| `VSPHERE_LIMITED_PASSWORD` | config, env | n/a | Password for connecting to vsphere |
| `VSPHERE_LIMITED_SPECS_SIZE` | config, env | 5000 | Size of specs list for query stats function |
| `VSPHERE_LIMITED_IGNORE_SSL` | config, env | False | Ignore the ssl cert on the connection to vsphere host |
| `VSPHERE_LIMITED_COLLECT_HOSTS` | config, env | True | Set to false to disable collection of host metrics |
| `VSPHERE_LIMITED_COLLECT_DATASTORES` | config, env | True | Set to false to disable collection of datastore metrics |
Expand Down
22 changes: 22 additions & 0 deletions tests/unit/test_vmware_exporter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ def test_collect_vms():
'root',
'password',
collect_only,
5000,
)
collector.content = _succeed(mock.Mock())

Expand Down Expand Up @@ -103,6 +104,7 @@ def test_collect_vms():
'root',
'password',
collect_only,
5000,
)
collector.content = _succeed(mock.Mock())

Expand Down Expand Up @@ -144,6 +146,7 @@ def test_collect_vms():
'root',
'password',
collect_only,
5000,
)
collector.content = _succeed(mock.Mock())

Expand Down Expand Up @@ -340,6 +343,7 @@ def test_metrics_without_hostaccess():
'root',
'password',
collect_only,
5000,
)
metrics = collector._create_metric_containers()
collector.content = _succeed(mock.Mock())
Expand Down Expand Up @@ -398,6 +402,7 @@ def test_no_error_onempty_vms():
'root',
'password',
collect_only,
5000,
ignore_ssl=True,
)

Expand Down Expand Up @@ -479,6 +484,7 @@ def test_collect_vm_perf():
'root',
'password',
collect_only,
5000,
)

metrics = collector._create_metric_containers()
Expand Down Expand Up @@ -595,6 +601,7 @@ def test_collect_hosts():
'root',
'password',
collect_only,
5000,
)
collector.content = _succeed(mock.Mock())

Expand Down Expand Up @@ -679,6 +686,7 @@ def test_collect_host_perf():
'root',
'password',
collect_only,
5000,
)

metrics = collector._create_metric_containers()
Expand Down Expand Up @@ -792,6 +800,7 @@ def test_collect_datastore():
'root',
'password',
collect_only,
5000,
)
collector.content = _succeed(mock.Mock())

Expand Down Expand Up @@ -854,6 +863,7 @@ def test_collect():
'root',
'password',
collect_only,
5000,
ignore_ssl=True,
)
collector.content = _succeed(mock.Mock())
Expand Down Expand Up @@ -889,6 +899,7 @@ def test_collect_deferred_error_works():
'root',
'password',
collect_only,
5000,
ignore_ssl=True,
)
collector.content = _succeed(mock.Mock())
Expand Down Expand Up @@ -990,6 +1001,7 @@ def test_vmware_get_inventory():
'root',
'password',
collect_only,
5000,
ignore_ssl=True,
)
collector.content = content
Expand Down Expand Up @@ -1033,6 +1045,7 @@ def test_vmware_connect():
'root',
'password',
collect_only,
5000,
ignore_ssl=True,
)

Expand All @@ -1059,6 +1072,7 @@ def test_vmware_disconnect():
'127.0.0.1',
'root',
'password',
5000,
collect_only,
)

Expand Down Expand Up @@ -1093,6 +1107,7 @@ def test_counter_ids():
'127.0.0.1',
'root',
'password',
5000,
collect_only,
)
collector.content = content
Expand Down Expand Up @@ -1233,6 +1248,7 @@ def test_vmware_resource_async_render_GET_section():
'vsphere_host': '127.0.0.10',
'vsphere_user': 'username1',
'vsphere_password': 'password1',
'specs_size': 5000,
'collect_only': {
'datastores': True,
'hosts': True,
Expand All @@ -1246,6 +1262,7 @@ def test_vmware_resource_async_render_GET_section():
'vsphere_host': '127.0.0.11',
'vsphere_user': 'username2',
'vsphere_password': 'password2',
'specs_size': 5000,
'collect_only': {
'datastores': True,
'hosts': True,
Expand All @@ -1265,6 +1282,7 @@ def test_vmware_resource_async_render_GET_section():
'username2',
'password2',
resource.config['mysection']['collect_only'],
5000,
'On'
)

Expand All @@ -1278,9 +1296,11 @@ def test_config_env_multiple_sections():
'VSPHERE_HOST': '127.0.0.10',
'VSPHERE_USER': 'username1',
'VSPHERE_PASSWORD': 'password1',
'VSPHERE_SPECS_SIZE': 5000,
'VSPHERE_MYSECTION_HOST': '127.0.0.11',
'VSPHERE_MYSECTION_USER': 'username2',
'VSPHERE_MYSECTION_PASSWORD': 'password2',
'VSPHERE_MYSECTION_SPECS_SIZE': 5000,
}

args = mock.Mock()
Expand All @@ -1295,6 +1315,7 @@ def test_config_env_multiple_sections():
'vsphere_host': '127.0.0.10',
'vsphere_user': 'username1',
'vsphere_password': 'password1',
'specs_size': 5000,
'collect_only': {
'datastores': True,
'hosts': True,
Expand All @@ -1308,6 +1329,7 @@ def test_config_env_multiple_sections():
'vsphere_host': '127.0.0.11',
'vsphere_user': 'username2',
'vsphere_password': 'password2',
'specs_size': 5000,
'collect_only': {
'datastores': True,
'hosts': True,
Expand Down
28 changes: 17 additions & 11 deletions vmware_exporter/vmware_exporter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,13 @@

class VmwareCollector():

def __init__(self, host, username, password, collect_only, ignore_ssl=False):
def __init__(self, host, username, password, collect_only, specs_size, ignore_ssl=False):
self.host = host
self.username = username
self.password = password
self.ignore_ssl = ignore_ssl
self.collect_only = collect_only
self.specs_size = specs_size

def _create_metric_containers(self):
metric_list = {}
Expand Down Expand Up @@ -636,17 +637,19 @@ def _vmware_get_vm_perf_manager_metrics(self, vm_metrics):
content = yield self.content

if len(specs) > 0:
results, labels = yield parallelize(
threads.deferToThread(content.perfManager.QueryStats, querySpec=specs),
self.vm_labels,
)
chunks = [specs[x:x+self.specs_size] for x in range(0, len(specs), self.specs_size)]
for list_specs in chunks:
results, labels = yield parallelize(
threads.deferToThread(content.perfManager.QueryStats, querySpec=list_specs),
self.vm_labels,
)

for ent in results:
for metric in ent.value:
vm_metrics[metric_names[metric.id.counterId]].add_metric(
labels[ent.entity._moId],
float(sum(metric.value)),
)
for ent in results:
for metric in ent.value:
vm_metrics[metric_names[metric.id.counterId]].add_metric(
labels[ent.entity._moId],
float(sum(metric.value)),
)

logging.info('FIN: _vmware_get_vm_perf_manager_metrics')

Expand Down Expand Up @@ -949,6 +952,7 @@ def configure(self, args):
'vsphere_user': os.environ.get('VSPHERE_USER'),
'vsphere_password': os.environ.get('VSPHERE_PASSWORD'),
'ignore_ssl': get_bool_env('VSPHERE_IGNORE_SSL', False),
'specs_size': os.environ.get('VSPHERE_SPECS_SIZE', 5000),
'collect_only': {
'vms': get_bool_env('VSPHERE_COLLECT_VMS', True),
'vmguests': get_bool_env('VSPHERE_COLLECT_VMGUESTS', True),
Expand All @@ -972,6 +976,7 @@ def configure(self, args):
'vsphere_user': os.environ.get('VSPHERE_{}_USER'.format(section)),
'vsphere_password': os.environ.get('VSPHERE_{}_PASSWORD'.format(section)),
'ignore_ssl': get_bool_env('VSPHERE_{}_IGNORE_SSL'.format(section), False),
'specs_size': os.environ.get('VSPHERE_{}_SPECS_SIZE'.format(section), 5000),
'collect_only': {
'vms': get_bool_env('VSPHERE_{}_COLLECT_VMS'.format(section), True),
'vmguests': get_bool_env('VSPHERE_{}_COLLECT_VMGUESTS'.format(section), True),
Expand Down Expand Up @@ -1026,6 +1031,7 @@ def generate_latest_metrics(self, request):
self.config[section]['vsphere_user'],
self.config[section]['vsphere_password'],
self.config[section]['collect_only'],
self.config[section]['specs_size'],
self.config[section]['ignore_ssl'],
)
metrics = yield collector.collect()
Expand Down

0 comments on commit bfa941d

Please sign in to comment.