From ed4f2cf2c46a13653164ef3cf1ae844fbf133797 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 5 Nov 2024 15:41:55 +0000 Subject: [PATCH] Fix up unit tests and linter --- os_capacity/prometheus.py | 41 +++-- os_capacity/tests/unit/__init__.py | 0 os_capacity/tests/unit/test_data.py | 188 ---------------------- os_capacity/tests/unit/test_prometheus.py | 37 +++++ requirements.txt | 8 +- setup.cfg | 15 +- setup.py | 53 +----- tox.ini | 37 +++-- 8 files changed, 96 insertions(+), 283 deletions(-) create mode 100644 os_capacity/tests/unit/__init__.py delete mode 100644 os_capacity/tests/unit/test_data.py create mode 100644 os_capacity/tests/unit/test_prometheus.py diff --git a/os_capacity/prometheus.py b/os_capacity/prometheus.py index a476964..9e0ac7d 100755 --- a/os_capacity/prometheus.py +++ b/os_capacity/prometheus.py @@ -1,8 +1,18 @@ #!/usr/bin/env python3 +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. -import os import collections -import json +import os import time import uuid @@ -174,12 +184,12 @@ def get_host_details(compute_client, placement_client): counts = capacity_per_flavor.get(flavor.name, {}).values() total = 0 if not counts else sum(counts) free_by_flavor_total.add_metric([flavor.name, str(flavor.is_public)], total) - # print(f'openstack_free_capacity_by_flavor{{flavor="{flavor_name}"}} {total}') # capacity per host free_by_flavor_hypervisor = prom_core.GaugeMetricFamily( "openstack_free_capacity_hypervisor_by_flavor", - "Free capacity for each hypervisor if you fill remaining space full of each flavor", + "Free capacity for each hypervisor if you fill " + "remaining space full of each flavor", labels=["hypervisor", "flavor_name", "az_aggregate", "project_aggregate"], ) resource_providers, project_to_aggregate = get_resource_provider_info( @@ -203,7 +213,8 @@ def get_host_details(compute_client, placement_client): ) free_space_found = True if not free_space_found: - # TODO(johngarbutt) allocation candidates only returns some not all candidates! + # TODO(johngarbutt) allocation candidates only returns some, + # not all candidates! print(f"# WARNING - no free spaces found for {hostname}") project_filter_aggregates = prom_core.GaugeMetricFamily( @@ -214,9 +225,6 @@ def get_host_details(compute_client, placement_client): for project, names in project_to_aggregate.items(): for name in names: project_filter_aggregates.add_metric([project, name], 1) - # print( - # f'openstack_project_filter_aggregate{{project_id="{project}",aggregate="{name}"}} 1' - # ) return resource_providers, [ free_by_flavor_total, free_by_flavor_hypervisor, @@ -312,10 +320,6 @@ def get_host_usage(resource_providers, placement_client): return [usage_guage, capacity_guage] -def print_exporter_data(app): - print_host_free_details(app.compute_client, app.placement_client) - - class OpenStackCapacityCollector(object): def __init__(self): self.conn = openstack.connect() @@ -346,7 +350,8 @@ def collect(self): host_time = time.perf_counter() host_duration = host_time - start_time print( - f"1 of 3: host flavor capacity complete for {collect_id} it took {host_duration} seconds" + "1 of 3: host flavor capacity complete " + f"for {collect_id} it took {host_duration} seconds" ) if not skip_project_usage: @@ -354,17 +359,19 @@ def collect(self): project_time = time.perf_counter() project_duration = project_time - host_time print( - f"2 of 3: project usage complete for {collect_id} it took {project_duration} seconds" + "2 of 3: project usage complete " + f"for {collect_id} it took {project_duration} seconds" ) else: print("2 of 3: skipping project usage") - if not skip_project_usage: + if not skip_host_usage: guages += get_host_usage(resource_providers, conn.placement) host_usage_time = time.perf_counter() host_usage_duration = host_usage_time - project_time print( - f"3 of 3: host usage complete for {collect_id} it took {host_usage_duration} seconds" + "3 of 3: host usage complete for " + f"{collect_id} it took {host_usage_duration} seconds" ) else: print("3 of 3: skipping host usage") @@ -391,4 +398,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/os_capacity/tests/unit/__init__.py b/os_capacity/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/os_capacity/tests/unit/test_data.py b/os_capacity/tests/unit/test_data.py deleted file mode 100644 index 5b1b6a3..0000000 --- a/os_capacity/tests/unit/test_data.py +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright (c) 2017 StackHPC Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import datetime -import unittest -from unittest import mock - -from os_capacity.data import flavors -from os_capacity.data import resource_provider -from os_capacity.data import server -from os_capacity.tests.unit import fakes - - -class TestFlavor(unittest.TestCase): - - def test_get_all_no_extra_specs(self): - fake_response = mock.MagicMock() - fake_response.json.return_value = fakes.FLAVOR_RESPONSE - compute_client = mock.MagicMock() - compute_client.get.return_value = fake_response - - result = flavors.get_all(compute_client, False) - - compute_client.get.assert_called_once_with("/flavors/detail") - expected_flavors = [flavors.Flavor(fakes.FLAVOR['id'], 'compute-GPU', - 8, 2048, 30, None)] - self.assertEqual(expected_flavors, result) - - def test_get_all(self): - fake_response1 = mock.MagicMock() - fake_response1.json.return_value = fakes.FLAVOR_RESPONSE - fake_response2 = mock.MagicMock() - fake_response2.json.return_value = fakes.FLAVOR_EXTRA_RESPONSE - compute_client = mock.MagicMock() - compute_client.get.side_effect = [fake_response1, fake_response2] - - result = flavors.get_all(compute_client) - - compute_client.get.assert_has_calls([ - mock.call("/flavors/detail"), - mock.call("/flavors/%s/os-extra_specs" % fakes.FLAVOR['id'])]) - expected_flavors = [flavors.Flavor(fakes.FLAVOR['id'], 'compute-GPU', - 8, 2048, 30, - {'example_key': 'example_value'})] - self.assertEqual(expected_flavors, result) - - -class TestResourceProvider(unittest.TestCase): - - def test_get_all(self): - fake_response = mock.MagicMock() - fake_response.json.return_value = fakes.RESOURCE_PROVIDER_RESPONSE - placement_client = mock.MagicMock() - placement_client.get.return_value = fake_response - - result = resource_provider.get_all(placement_client) - - placement_client.get.assert_called_once_with("/resource_providers") - self.assertEqual([(fakes.RESOURCE_PROVIDER['uuid'], 'name1')], result) - - -class TestInventory(unittest.TestCase): - - def test_get_inventories(self): - fake_response = mock.MagicMock() - fake_response.json.return_value = fakes.INVENTORIES_RESPONSE - client = mock.MagicMock() - client.get.return_value = fake_response - - rp = resource_provider.ResourceProvider("uuid", "name") - - result = resource_provider.get_inventories(client, rp) - - client.get.assert_called_once_with( - "/resource_providers/uuid/inventories") - self.assertEqual(3, len(result)) - disk = resource_provider.Inventory("uuid", "DISK_GB", 10) - self.assertIn(disk, result) - mem = resource_provider.Inventory("uuid", "MEMORY_MB", 20) - self.assertIn(mem, result) - vcpu = resource_provider.Inventory("uuid", "VCPU", 30) - self.assertIn(vcpu, result) - - @mock.patch.object(resource_provider, 'get_all') - def test_get_all_inventories(self, mock_get_all): - fake_response = mock.MagicMock() - fake_response.json.return_value = fakes.INVENTORIES_RESPONSE - client = mock.MagicMock() - client.get.return_value = fake_response - - rp1 = resource_provider.ResourceProvider("uuid1", "name1") - rp2 = resource_provider.ResourceProvider("uuid2", "name2") - mock_get_all.return_value = [rp1, rp2] - - result = resource_provider.get_all_inventories(client) - - mock_get_all.assert_called_once_with(client) - client.get.assert_called_with( - "/resource_providers/uuid2/inventories") - self.assertEqual(6, len(result)) - disk1 = resource_provider.Inventory("uuid1", "DISK_GB", 10) - self.assertIn(disk1, result) - disk2 = resource_provider.Inventory("uuid2", "DISK_GB", 10) - self.assertIn(disk2, result) - - -class TestAllocations(unittest.TestCase): - - def test_get_allocations(self): - fake_response = mock.MagicMock() - fake_response.json.return_value = fakes.ALLOCATIONS_RESPONSE - client = mock.MagicMock() - client.get.return_value = fake_response - - rp = resource_provider.ResourceProvider("uuid", "name") - - result = resource_provider.get_allocations(client, rp) - - self.assertEqual(1, len(result)) - expected = resource_provider.Allocation( - "uuid", "consumer_uuid", [ - resource_provider.ResourceClassAmount("DISK_GB", 10), - resource_provider.ResourceClassAmount("MEMORY_MB", 20), - resource_provider.ResourceClassAmount("VCPU", 30)]) - self.assertEqual(expected, result[0]) - - @mock.patch.object(resource_provider, 'get_all') - def test_get_all_allocations(self, mock_get_all): - fake_response = mock.MagicMock() - fake_response.json.return_value = fakes.ALLOCATIONS_RESPONSE - client = mock.MagicMock() - client.get.return_value = fake_response - - rp1 = resource_provider.ResourceProvider("uuid1", "name1") - rp2 = resource_provider.ResourceProvider("uuid2", "name2") - mock_get_all.return_value = [rp1, rp2] - - result = resource_provider.get_all_allocations(client) - - self.assertEqual(2, len(result)) - uuid1 = resource_provider.Allocation( - "uuid1", "consumer_uuid", [ - resource_provider.ResourceClassAmount("DISK_GB", 10), - resource_provider.ResourceClassAmount("MEMORY_MB", 20), - resource_provider.ResourceClassAmount("VCPU", 30)]) - self.assertIn(uuid1, result) - uuid2 = resource_provider.Allocation( - "uuid1", "consumer_uuid", [ - resource_provider.ResourceClassAmount("DISK_GB", 10), - resource_provider.ResourceClassAmount("MEMORY_MB", 20), - resource_provider.ResourceClassAmount("VCPU", 30)]) - self.assertIn(uuid2, result) - - -class TestServer(unittest.TestCase): - def test_parse_created(self): - result = server._parse_created("2017-02-14T19:23:58Z") - expected = datetime.datetime(2017, 2, 14, 19, 23, 58) - self.assertEqual(expected, result) - - def test_get(self): - mock_response = mock.Mock() - mock_response.json.return_value = fakes.SERVER_RESPONSE - compute_client = mock.Mock() - compute_client.get.return_value = mock_response - - uuid = '9168b536-cd40-4630-b43f-b259807c6e87' - result = server.get(compute_client, uuid) - - expected = server.Server( - uuid=uuid, - name='new-server-test', - created=datetime.datetime(2017, 2, 14, 19, 23, 58), - user_id='fake', - project_id='6f70656e737461636b20342065766572', - flavor_id='7b46326c-ce48-4e43-8aca-4f5ca00d5f37') - self.assertEqual(expected, result) diff --git a/os_capacity/tests/unit/test_prometheus.py b/os_capacity/tests/unit/test_prometheus.py new file mode 100644 index 0000000..cc3d687 --- /dev/null +++ b/os_capacity/tests/unit/test_prometheus.py @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest + +from os_capacity import prometheus + + +class FakeFlavor: + def __init__(self, id, name, vcpus, ram, disk, ephemeral, extra_specs): + self.id = id + self.name = name + self.vcpus = vcpus + self.ram = ram + self.disk = disk + self.ephemeral = ephemeral + self.extra_specs = extra_specs + + +class TestFlavor(unittest.TestCase): + def test_get_placement_request(self): + flavor = FakeFlavor( + "fake_id", "fake_name", 8, 2048, 30, 0, {"hw:cpu_policy": "dedicated"} + ) + resources, traits = prometheus.get_placement_request(flavor) + + self.assertEqual({"PCPU": 8, "MEMORY_MB": 2048, "DISK_GB": 30}, resources) + self.assertEqual([], traits) diff --git a/requirements.txt b/requirements.txt index 4e5edae..689a89b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ -cliff>=2.8.0 # Apache -os-client-config>=1.28.0 # Apache-2.0 -pbr>=2.0.0,!=2.1.0 # Apache-2.0 -prometheus-client==0.16.0 -six # MIT +openstacksdk +pbr +prometheus-client diff --git a/setup.cfg b/setup.cfg index 3600d32..dd98db5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,17 +5,14 @@ description-file = README.rst author = StackHPC author-email = johng@stackhpc.com -home-page = https://stackhpc.com -classifier = - Environment :: OpenStack - Intended Audience :: Information Technology - Intended Audience :: System Administrators - Operating System :: POSIX :: Linux - Programming Language :: Python - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.9 +url = https://github.com/stackhpc/os-capacity +python-requires = >=3.9 +license = Apache-2 [files] packages = os_capacity +[entry_points] +console_scripts= + os_capacity = os_capacity.prometheus:main diff --git a/setup.py b/setup.py index a9061b6..32792a1 100644 --- a/setup.py +++ b/setup.py @@ -1,54 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2017 StackHPC Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +import setuptools -from setuptools import find_packages -from setuptools import setup - - -PROJECT = 'os_capacity' -VERSION = '0.4' - -try: - long_description = open('README.md', 'rt').read() -except IOError: - long_description = '' - -setup( - name=PROJECT, - version=VERSION, - - description='OpenStack capacity tooling', - long_description=long_description, - - author='StackHPC', - author_email='john.garbutt@stackhpc.com', - - url='https://github.com/stackhpc/os-capacity', - download_url='https://github.com/stackhpc/os-capacity/tarball/master', - - provides=[], - install_requires=open('requirements.txt', 'rt').read().splitlines(), - - namespace_packages=[], - packages=find_packages(), - include_package_data=True, - - entry_points={ - 'console_scripts': [ - 'os-capacity = os_capacity.prometheus:main', - ], - }, -) +setuptools.setup(setup_requires=["pbr"], pbr=True) diff --git a/tox.ini b/tox.ini index c817894..f46a41f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 2.0 -envlist = py39,pep8 +envlist = py3,black,pep8 skipsdist = True [testenv] @@ -9,21 +9,20 @@ install_command = pip install {opts} {packages} setenv = VIRTUAL_ENV={envdir} PYTHONWARNINGS=default::DeprecationWarning -deps = -r{toxinidir}/test-requirements.txt +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt commands = stestr run {posargs} -[testenv:pep8] +[testenv:cover] +setenv = + VIRTUAL_ENV={envdir} + PYTHON=coverage run --source azimuth_caas_operator --parallel-mode commands = - flake8 {posargs} - -[testenv:venv] -commands = {posargs} - -[testenv:docs] -commands = python setup.py build_sphinx - -[testenv:debug] -commands = oslo_debug_helper {posargs} + stestr run {posargs} + coverage combine + coverage html -d cover + coverage xml -o cover/coverage.xml + coverage report [flake8] # E123, E125 skipped as they are invalid PEP-8. @@ -31,3 +30,15 @@ show-source = True ignore = E123,E125 builtins = _ exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build +# match black +max-line-length = 88 + +[testenv:pep8] +commands = + black {tox_root} + flake8 {posargs} +allowlist_externals = black + +[testenv:black] +commands = black {tox_root} --check +allowlist_externals = black