From 5d16a030c8f8b7521c10ff20568fe9f9146d74a6 Mon Sep 17 00:00:00 2001 From: fatimarahman Date: Thu, 25 Jul 2024 16:14:32 -0500 Subject: [PATCH 1/6] Added retry functionality --- CHANGELOG.md | 5 ++++- src/nypl_py_utils/classes/avro_client.py | 11 ++++++++++- tests/test_avro_client.py | 14 +++++++------- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d080e37..13d2ce1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Changelog +## v1.2.1 7/25/24 +- Add retry for fetching Avro schemas + ## v1.2.0 7/17/24 -- Generalized Avro functions and separated encoding/decoding behavior. +- Generalized Avro functions and separated encoding/decoding behavior ## v1.1.6 7/12/24 - Add put functionality to Oauth2 Client diff --git a/src/nypl_py_utils/classes/avro_client.py b/src/nypl_py_utils/classes/avro_client.py index dd49f26..046d488 100644 --- a/src/nypl_py_utils/classes/avro_client.py +++ b/src/nypl_py_utils/classes/avro_client.py @@ -5,6 +5,7 @@ from avro.io import BinaryDecoder, BinaryEncoder, DatumReader, DatumWriter from io import BytesIO from nypl_py_utils.functions.log_helper import create_log +from requests.adapters import HTTPAdapter, Retry from requests.exceptions import JSONDecodeError, RequestException @@ -16,6 +17,12 @@ class AvroClient: def __init__(self, platform_schema_url): self.logger = create_log("avro_encoder") + retry_policy = Retry(total=2, backoff_factor=4, + status_forcelist=[500, 502, 503, 504], + allowed_methods=frozenset(['GET'])) + self.session = requests.Session() + self.session.mount("https://", + HTTPAdapter(max_retries=retry_policy)) self.schema = avro.schema.parse( self.get_json_schema(platform_schema_url)) @@ -27,7 +34,9 @@ def get_json_schema(self, platform_schema_url): self.logger.info( "Fetching Avro schema from {}".format(platform_schema_url)) try: - response = requests.get(platform_schema_url) + + response = self.session.get(url=platform_schema_url, + timeout=300) response.raise_for_status() except RequestException as e: self.logger.error( diff --git a/tests/test_avro_client.py b/tests/test_avro_client.py index af9e87c..9623bbe 100644 --- a/tests/test_avro_client.py +++ b/tests/test_avro_client.py @@ -2,7 +2,7 @@ import pytest from nypl_py_utils.classes.avro_client import ( - AvroClientError, AvroDecoder, AvroEncoder) + AvroClient, AvroClientError, AvroDecoder, AvroEncoder) from requests.exceptions import ConnectTimeout _TEST_SCHEMA = {'data': {'schema': json.dumps({ @@ -25,17 +25,17 @@ class TestAvroClient: @pytest.fixture def test_avro_encoder_instance(self, requests_mock): requests_mock.get( - 'https://test_schema_url', text=json.dumps(_TEST_SCHEMA)) - return AvroEncoder('https://test_schema_url') + "https://test_schema_url", text=json.dumps(_TEST_SCHEMA)) + return AvroEncoder("https://test_schema_url") @pytest.fixture def test_avro_decoder_instance(self, requests_mock): requests_mock.get( - 'https://test_schema_url', text=json.dumps(_TEST_SCHEMA)) - return AvroDecoder('https://test_schema_url') + "https://test_schema_url", text=json.dumps(_TEST_SCHEMA)) + return AvroDecoder("https://test_schema_url") - def test_get_json_schema(self, test_avro_encoder_instance, - test_avro_decoder_instance): + def test_get_json_schema_success(self, test_avro_encoder_instance, + test_avro_decoder_instance): assert test_avro_encoder_instance.schema == _TEST_SCHEMA['data'][ 'schema'] assert test_avro_decoder_instance.schema == _TEST_SCHEMA['data'][ From a042e6e3365e6d17cf7aaf5afd35d30545e22984 Mon Sep 17 00:00:00 2001 From: fatimarahman Date: Thu, 25 Jul 2024 16:20:26 -0500 Subject: [PATCH 2/6] Remove unused import --- tests/test_avro_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_avro_client.py b/tests/test_avro_client.py index 9623bbe..6798018 100644 --- a/tests/test_avro_client.py +++ b/tests/test_avro_client.py @@ -2,7 +2,7 @@ import pytest from nypl_py_utils.classes.avro_client import ( - AvroClient, AvroClientError, AvroDecoder, AvroEncoder) + AvroClientError, AvroDecoder, AvroEncoder) from requests.exceptions import ConnectTimeout _TEST_SCHEMA = {'data': {'schema': json.dumps({ From 5cec18d64817782695f03c0ab610d05128d0539c Mon Sep 17 00:00:00 2001 From: fatimarahman Date: Tue, 30 Jul 2024 12:15:00 -0500 Subject: [PATCH 3/6] Added retry --- src/nypl_py_utils/classes/avro_client.py | 9 ++++---- tests/test_avro_client.py | 26 ++++++++++++++++-------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/nypl_py_utils/classes/avro_client.py b/src/nypl_py_utils/classes/avro_client.py index 046d488..b9576db 100644 --- a/src/nypl_py_utils/classes/avro_client.py +++ b/src/nypl_py_utils/classes/avro_client.py @@ -1,3 +1,4 @@ +import time import avro.schema import requests @@ -16,8 +17,8 @@ class AvroClient: """ def __init__(self, platform_schema_url): - self.logger = create_log("avro_encoder") - retry_policy = Retry(total=2, backoff_factor=4, + self.logger = create_log("avro_client") + retry_policy = Retry(total=3, backoff_factor=45, status_forcelist=[500, 502, 503, 504], allowed_methods=frozenset(['GET'])) self.session = requests.Session() @@ -38,7 +39,7 @@ def get_json_schema(self, platform_schema_url): response = self.session.get(url=platform_schema_url, timeout=300) response.raise_for_status() - except RequestException as e: + except Exception as e: self.logger.error( "Failed to retrieve schema from {url}: {error}".format( url=platform_schema_url, error=e @@ -48,7 +49,7 @@ def get_json_schema(self, platform_schema_url): "Failed to retrieve schema from {url}: {error}".format( url=platform_schema_url, error=e ) - ) from None + ) try: json_response = response.json() diff --git a/tests/test_avro_client.py b/tests/test_avro_client.py index 6798018..aca9257 100644 --- a/tests/test_avro_client.py +++ b/tests/test_avro_client.py @@ -1,8 +1,10 @@ import json import pytest +from unittest.mock import patch from nypl_py_utils.classes.avro_client import ( - AvroClientError, AvroDecoder, AvroEncoder) + AvroClient, AvroClientError, AvroDecoder, AvroEncoder) +from requests import session from requests.exceptions import ConnectTimeout _TEST_SCHEMA = {'data': {'schema': json.dumps({ @@ -36,15 +38,23 @@ def test_avro_decoder_instance(self, requests_mock): def test_get_json_schema_success(self, test_avro_encoder_instance, test_avro_decoder_instance): - assert test_avro_encoder_instance.schema == _TEST_SCHEMA['data'][ - 'schema'] - assert test_avro_decoder_instance.schema == _TEST_SCHEMA['data'][ - 'schema'] + assert test_avro_encoder_instance.schema == _TEST_SCHEMA["data"][ + "schema"] + assert test_avro_decoder_instance.schema == _TEST_SCHEMA["data"][ + "schema"] - def test_request_error(self, requests_mock): - requests_mock.get('https://test_schema_url', exc=ConnectTimeout) + def test_get_json_schema_error(self, requests_mock): + requests_mock.get("https://test_schema_url", exc=ConnectTimeout) with pytest.raises(AvroClientError): - AvroEncoder('https://test_schema_url') + AvroEncoder("https://test_schema_url") + + def test_get_json_schema_success_on_retry(self, requests_mock): + requests_mock.get("https://test_schema_url", + [{"exc": ConnectionError}, + {"text": str(_TEST_SCHEMA), "status_code": 200}]) + + test_avro_client = AvroClient("https://test_schema_url") + assert test_avro_client.get_json_schema == _TEST_SCHEMA["data"]["schema"] def test_bad_json_error(self, requests_mock): requests_mock.get( From 907cd3ad1b0cf43acd94e16c7dad3b03c6478599 Mon Sep 17 00:00:00 2001 From: fatimarahman Date: Tue, 30 Jul 2024 12:17:22 -0500 Subject: [PATCH 4/6] Update timeout --- src/nypl_py_utils/classes/avro_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nypl_py_utils/classes/avro_client.py b/src/nypl_py_utils/classes/avro_client.py index b9576db..9eb9a19 100644 --- a/src/nypl_py_utils/classes/avro_client.py +++ b/src/nypl_py_utils/classes/avro_client.py @@ -37,7 +37,7 @@ def get_json_schema(self, platform_schema_url): try: response = self.session.get(url=platform_schema_url, - timeout=300) + timeout=60) response.raise_for_status() except Exception as e: self.logger.error( From 0fc7adcdf9bd6541f220153111cb12e895f0d0aa Mon Sep 17 00:00:00 2001 From: fatimarahman Date: Tue, 30 Jul 2024 13:04:59 -0500 Subject: [PATCH 5/6] Removed retry test --- tests/test_avro_client.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/test_avro_client.py b/tests/test_avro_client.py index aca9257..f773ab0 100644 --- a/tests/test_avro_client.py +++ b/tests/test_avro_client.py @@ -2,6 +2,8 @@ import pytest from unittest.mock import patch +import requests + from nypl_py_utils.classes.avro_client import ( AvroClient, AvroClientError, AvroDecoder, AvroEncoder) from requests import session @@ -48,14 +50,6 @@ def test_get_json_schema_error(self, requests_mock): with pytest.raises(AvroClientError): AvroEncoder("https://test_schema_url") - def test_get_json_schema_success_on_retry(self, requests_mock): - requests_mock.get("https://test_schema_url", - [{"exc": ConnectionError}, - {"text": str(_TEST_SCHEMA), "status_code": 200}]) - - test_avro_client = AvroClient("https://test_schema_url") - assert test_avro_client.get_json_schema == _TEST_SCHEMA["data"]["schema"] - def test_bad_json_error(self, requests_mock): requests_mock.get( 'https://test_schema_url', text='bad json') From 2036f54df87ee9143b6d0cfc8b47c1b7e30d403b Mon Sep 17 00:00:00 2001 From: fatimarahman Date: Tue, 30 Jul 2024 13:18:21 -0500 Subject: [PATCH 6/6] Fix linter --- src/nypl_py_utils/classes/avro_client.py | 3 +-- tests/test_avro_client.py | 6 +----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/nypl_py_utils/classes/avro_client.py b/src/nypl_py_utils/classes/avro_client.py index 9eb9a19..2dcaae2 100644 --- a/src/nypl_py_utils/classes/avro_client.py +++ b/src/nypl_py_utils/classes/avro_client.py @@ -1,4 +1,3 @@ -import time import avro.schema import requests @@ -7,7 +6,7 @@ from io import BytesIO from nypl_py_utils.functions.log_helper import create_log from requests.adapters import HTTPAdapter, Retry -from requests.exceptions import JSONDecodeError, RequestException +from requests.exceptions import JSONDecodeError class AvroClient: diff --git a/tests/test_avro_client.py b/tests/test_avro_client.py index f773ab0..4af11b3 100644 --- a/tests/test_avro_client.py +++ b/tests/test_avro_client.py @@ -1,12 +1,8 @@ import json import pytest -from unittest.mock import patch - -import requests from nypl_py_utils.classes.avro_client import ( - AvroClient, AvroClientError, AvroDecoder, AvroEncoder) -from requests import session + AvroClientError, AvroDecoder, AvroEncoder) from requests.exceptions import ConnectTimeout _TEST_SCHEMA = {'data': {'schema': json.dumps({