From 42e00939051165e1866a61f4657199a6ce234ff9 Mon Sep 17 00:00:00 2001 From: sgoral-splunk <138458044+sgoral-splunk@users.noreply.github.com> Date: Mon, 30 Sep 2024 12:02:38 +0200 Subject: [PATCH 1/3] feat: add retry functionality to the splunk_rest_client (#392) **Issue number:[ADDON-75327](https://splunk.atlassian.net/browse/ADDON-75327)** ## Summary Added functionality to retries requests. ### Changes Added "max_retries" to the "HTTPAdapter" and set it to 5. ### User experience No impact / potential reliability improvement ## Checklist If your change doesn't seem to apply, please leave them unchecked. * [x] I have performed a self-review of this change * [x] Changes have been tested * [ ] Changes are documented * [x] PR title follows [conventional commit semantics](https://www.conventionalcommits.org/en/v1.0.0/) --- solnlib/splunk_rest_client.py | 10 +++++++++ tests/unit/test_splunk_rest_client.py | 29 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/solnlib/splunk_rest_client.py b/solnlib/splunk_rest_client.py index c83c2545..419c4177 100644 --- a/solnlib/splunk_rest_client.py +++ b/solnlib/splunk_rest_client.py @@ -26,6 +26,7 @@ import traceback from io import BytesIO from urllib.parse import quote +from urllib3.util.retry import Retry from splunklib import binding, client @@ -33,6 +34,7 @@ from .splunkenv import get_splunkd_access_info __all__ = ["SplunkRestClient"] +MAX_REQUEST_RETRIES = 5 def _get_proxy_info(context): @@ -98,10 +100,18 @@ def _request_handler(context): else: cert = None + retries = Retry( + total=MAX_REQUEST_RETRIES, + backoff_factor=0.3, + status_forcelist=[500, 502, 503, 504], + allowed_methods=["GET", "POST", "PUT", "DELETE"], + raise_on_status=False, + ) if context.get("pool_connections", 0): logging.info("Use HTTP connection pooling") session = requests.Session() adapter = requests.adapters.HTTPAdapter( + max_retries=retries, pool_connections=context.get("pool_connections", 10), pool_maxsize=context.get("pool_maxsize", 10), ) diff --git a/tests/unit/test_splunk_rest_client.py b/tests/unit/test_splunk_rest_client.py index 4a13e3cc..de43dbdd 100644 --- a/tests/unit/test_splunk_rest_client.py +++ b/tests/unit/test_splunk_rest_client.py @@ -17,8 +17,11 @@ from unittest import mock import pytest +from solnlib.splunk_rest_client import MAX_REQUEST_RETRIES +from requests.exceptions import ConnectionError from solnlib import splunk_rest_client +from solnlib.splunk_rest_client import SplunkRestClient @mock.patch.dict(os.environ, {"SPLUNK_HOME": "/opt/splunk"}, clear=True) @@ -80,3 +83,29 @@ def test_init_with_invalid_port(): host="localhost", port=99999, ) + + +@mock.patch.dict(os.environ, {"SPLUNK_HOME": "/opt/splunk"}, clear=True) +@mock.patch("solnlib.splunk_rest_client.get_splunkd_access_info") +@mock.patch("http.client.HTTPResponse") +@mock.patch("urllib3.HTTPConnectionPool._make_request") +def test_request_retry(http_conn_pool, http_resp, mock_get_splunkd_access_info): + mock_get_splunkd_access_info.return_value = "https", "localhost", 8089 + session_key = "123" + context = {"pool_connections": 5} + rest_client = SplunkRestClient("msg_name_1", session_key, "_", **context) + + mock_resp = http_resp() + mock_resp.status = 200 + mock_resp.reason = "TEST OK" + + side_effects = [ConnectionError(), ConnectionError(), ConnectionError(), mock_resp] + http_conn_pool.side_effect = side_effects + res = rest_client.get("test") + assert http_conn_pool.call_count == len(side_effects) + assert res.reason == mock_resp.reason + + side_effects = [ConnectionError()] * (MAX_REQUEST_RETRIES + 1) + [mock_resp] + http_conn_pool.side_effect = side_effects + with pytest.raises(ConnectionError): + rest_client.get("test") From 74ce097c441cebea9756cde18a2cfc134654d1c1 Mon Sep 17 00:00:00 2001 From: srv-rr-github-token <94607705+srv-rr-github-token@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:08:21 +0000 Subject: [PATCH 2/3] chore(release): 5.2.0-beta.2 # [5.2.0-beta.2](https://github.com/splunk/addonfactory-solutions-library-python/compare/v5.2.0-beta.1...v5.2.0-beta.2) (2024-09-30) ### Features * add retry functionality to the splunk_rest_client ([#392](https://github.com/splunk/addonfactory-solutions-library-python/issues/392)) ([42e0093](https://github.com/splunk/addonfactory-solutions-library-python/commit/42e00939051165e1866a61f4657199a6ce234ff9)) --- pyproject.toml | 2 +- solnlib/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 76add752..5e9ceb5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ [tool.poetry] name = "solnlib" -version = "5.2.0-beta.1" +version = "5.2.0-beta.2" description = "The Splunk Software Development Kit for Splunk Solutions" authors = ["Splunk "] license = "Apache-2.0" diff --git a/solnlib/__init__.py b/solnlib/__init__.py index 425e6531..1a66bc3a 100644 --- a/solnlib/__init__.py +++ b/solnlib/__init__.py @@ -56,4 +56,4 @@ "utils", ] -__version__ = "5.2.0-beta.1" +__version__ = "5.2.0-beta.2" From 470fef6365e03f67f501b90811886dd84c215fc2 Mon Sep 17 00:00:00 2001 From: srv-rr-github-token <94607705+srv-rr-github-token@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:57:14 +0000 Subject: [PATCH 3/3] chore(release): 5.3.0-beta.1 # [5.3.0-beta.1](https://github.com/splunk/addonfactory-solutions-library-python/compare/v5.2.0...v5.3.0-beta.1) (2024-09-30) ### Features * add retry functionality to the splunk_rest_client ([#392](https://github.com/splunk/addonfactory-solutions-library-python/issues/392)) ([42e0093](https://github.com/splunk/addonfactory-solutions-library-python/commit/42e00939051165e1866a61f4657199a6ce234ff9)) --- pyproject.toml | 2 +- solnlib/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 166b18d8..47abfd62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ [tool.poetry] name = "solnlib" -version = "5.2.0" +version = "5.3.0-beta.1" description = "The Splunk Software Development Kit for Splunk Solutions" authors = ["Splunk "] license = "Apache-2.0" diff --git a/solnlib/__init__.py b/solnlib/__init__.py index e462f786..1a2c73c7 100644 --- a/solnlib/__init__.py +++ b/solnlib/__init__.py @@ -56,4 +56,4 @@ "utils", ] -__version__ = "5.2.0" +__version__ = "5.3.0-beta.1"