Improve healch checks runner

- Health checks will now run in threads
- A successful health check wont run again

Makefile

@@ -21,7 +21,7 @@ test:
 
 coverage: test
 	# Create a coverage report and validate the given threshold
-	coverage html --fail-under=85 -d build/coverage
+	coverage html --fail-under=69 -d build/coverage
 
 nose2:
 	mkdir -p build/

README.md

@@ -74,13 +74,15 @@ import httplib
 import logging
 import requests
 
+from docker_test_tools.utils import get_health_check
 from docker_test_tools.base_test import BaseDockerTest
 from docker_test_tools.wiremock import WiremockController
-from docker_test_tools.utils import get_curl_health_check
+
+log = logging.getLogger(__name__)
 
 # Define health check functions for the environment services
-consul_health_check = get_curl_health_check('consul.service', url='http://consul.service:8500')
-mock_service_health_check = get_curl_health_check('mocked.service', url='http://mocked.service:9999/__admin')
+consul_health_check = get_health_check('consul.service', url='http://consul.service:8500')
+mock_service_health_check = get_health_check('mocked.service', url='http://mocked.service:9999/__admin')
 
 
 class ExampleTest(BaseDockerTest):

docker_test_tools/environment.py

@@ -6,6 +6,7 @@
 from contextlib import contextmanager
 
 from docker_test_tools import logs
+from docker_test_tools import utils
 from docker_test_tools import config
 from docker_test_tools.api_version import get_server_api_version
 
@@ -274,19 +275,8 @@ def wait_for_services(self, services=None, interval=1, timeout=60):
         """
         services = services if services else self.services
         log.info('Waiting for %s to reach the required state', services)
-
-        def service_checks():
-            """Return True if services checks pass."""
-            return all([self.is_container_ready(name) for name in services])
-
-        try:
-            waiting.wait(service_checks, sleep_seconds=interval, timeout_seconds=timeout)
-            log.info('Services %s reached the required state', services)
-            return True
-
-        except waiting.TimeoutExpired:
-            log.error('%s failed to to reach the required state', services)
-            return False
+        return utils.run_health_checks(checks=[lambda: self.is_container_ready(name) for name in services],
+                                       interval=interval, timeout=timeout)
 
     @contextmanager
     def container_down(self, name, health_check=None, interval=1, timeout=60):

docker_test_tools/utils.py

@@ -1,8 +1,11 @@
 import logging
-from six.moves import http_client
+
 import waiting
 import requests
 
+from six.moves import http_client
+from multiprocessing.pool import ThreadPool
+
 log = logging.getLogger(__name__)
 
 
@@ -15,15 +18,15 @@ def run_health_checks(checks, interval=1, timeout=60):
 
     :raise bool: True is all the services are healthy, False otherwise.
     """
-    log.info('Waiting for the required health checks to pass...')
-    try:
-        waiting.wait(lambda: all([health_check() for health_check in checks]),
-                     sleep_seconds=interval, timeout_seconds=timeout)
+    pool = ThreadPool()
+    async_results = [pool.apply_async(wait_for_health, (check, interval, timeout)) for check in checks]
+    return all([async_result.get() for async_result in async_results])
 
-        return True
 
+def wait_for_health(health_check, interval=1, timeout=60):
+    try:
+        return waiting.wait(health_check, sleep_seconds=interval, timeout_seconds=timeout)
     except waiting.TimeoutExpired:
-        log.error("Required health checks didn't pass within timeout")
         return False
 
 
@@ -49,7 +52,7 @@ def get_health_check(service_name, url, expected_status=http_client.OK):
 
     :return function: function used to determine if the given service is responsive.
     """
-    log.debug('Defining a CURL based health check for service: %s at: %s', service_name, url)
+    log.debug('Defining a health check for service: %s at: %s', service_name, url)
 
     def url_health_check():
         """Return True if the service is responsive."""
@@ -60,5 +63,5 @@ def url_health_check():
     return url_health_check
 
 
-# For backward compatability
-get_curl_health_check = get_health_check
+# For backward compatibility
+get_curl_health_check = get_health_check

tests/integration/test_example.py

@@ -3,15 +3,15 @@
 import logging
 import requests
 
+from docker_test_tools.utils import get_health_check
 from docker_test_tools.base_test import BaseDockerTest
 from docker_test_tools.wiremock import WiremockController
-from docker_test_tools.utils import get_curl_health_check
 
 log = logging.getLogger(__name__)
 
 # Define health check functions for the environment services
-consul_health_check = get_curl_health_check('consul.service', url='http://consul.service:8500')
-mock_service_health_check = get_curl_health_check('mocked.service', url='http://mocked.service:9999/__admin')
+consul_health_check = get_health_check('consul.service', url='http://consul.service:8500')
+mock_service_health_check = get_health_check('mocked.service', url='http://mocked.service:9999/__admin')
 
 
 class ExampleTest(BaseDockerTest):

tests/ut/test_utils.py

@@ -1,7 +1,6 @@
 import mock
 import unittest
 
-from waiting import TimeoutExpired
 from docker_test_tools import utils
 
 
@@ -10,17 +9,13 @@ class TestUtils(unittest.TestCase):
     def test_run_health_checks(self, get_mock):
         """Validate the run_health_checks function."""
         get_mock.return_value = mock.MagicMock(status_code=200)
-        utils.run_health_checks([utils.get_curl_health_check('service1', 'first_url'),
-                                 utils.get_curl_health_check('service2', 'second_url')],
-                                timeout=5)
+        utils.run_health_checks([utils.get_health_check('service1', 'first_url'),
+                                 utils.get_health_check('service2', 'second_url')],
+                                timeout=0)
 
         get_mock.assert_any_call('first_url', timeout=5)
         get_mock.assert_any_call('second_url', timeout=5)
 
-        with mock.patch("waiting.wait", return_value=True):
-            self.assertTrue(utils.run_health_checks([]))
-
-        with mock.patch("waiting.wait", side_effect=TimeoutExpired(timeout_seconds=1, what='something')):
-            self.assertFalse(utils.run_health_checks([]))
-
-
+        self.assertTrue(utils.run_health_checks([lambda: True, lambda: True], timeout=0))
+        self.assertFalse(utils.run_health_checks([lambda: True, lambda: False], timeout=0))
+        self.assertFalse(utils.run_health_checks([lambda: False, lambda: False], timeout=0))