From 57f1d8fb7c817cc813afee6046308835aed1dfb6 Mon Sep 17 00:00:00 2001 From: KShivendu Date: Tue, 5 Sep 2023 16:33:26 +0530 Subject: [PATCH 1/7] fix: Async lib not found error --- supertokens_python/async_to_sync_wrapper.py | 11 +-- supertokens_python/querier.py | 82 ++++++++++++++------- supertokens_python/supertokens.py | 5 ++ supertokens_python/utils.py | 5 +- 4 files changed, 69 insertions(+), 34 deletions(-) diff --git a/supertokens_python/async_to_sync_wrapper.py b/supertokens_python/async_to_sync_wrapper.py index 9c623bf51..0e3d27486 100644 --- a/supertokens_python/async_to_sync_wrapper.py +++ b/supertokens_python/async_to_sync_wrapper.py @@ -19,17 +19,18 @@ _T = TypeVar("_T") -def check_event_loop(): +def create_or_get_event_loop() -> asyncio.AbstractEventLoop: try: - asyncio.get_event_loop() - except RuntimeError as ex: + return asyncio.get_event_loop() + except Exception as ex: if "There is no current event loop in thread" in str(ex): loop = asyncio.new_event_loop() nest_asyncio.apply(loop) # type: ignore asyncio.set_event_loop(loop) + return loop + raise ex def sync(co: Coroutine[Any, Any, _T]) -> _T: - check_event_loop() - loop = asyncio.get_event_loop() + loop = create_or_get_event_loop() return loop.run_until_complete(co) diff --git a/supertokens_python/querier.py b/supertokens_python/querier.py index 79f6ae3dc..f2bb4e331 100644 --- a/supertokens_python/querier.py +++ b/supertokens_python/querier.py @@ -39,6 +39,8 @@ from .exceptions import raise_general_exception from .process_state import AllowedProcessStates, ProcessState from .utils import find_max_version, is_4xx_error, is_5xx_error +from supertokens_python.async_to_sync_wrapper import create_or_get_event_loop +from sniffio import AsyncLibraryNotFoundError class Querier: @@ -71,6 +73,32 @@ def get_hosts_alive_for_testing(): raise_general_exception("calling testing function in non testing env") return Querier.__hosts_alive_for_testing + async def api_request( + self, + url: str, + method: str, # ["POST", "GET"] + headers: Optional[Dict[str, Any]] = None, + params: Optional[Dict[str, Any]] = None, + json: Optional[Dict[str, Any]] = None, + ) -> Response: + try: + async with AsyncClient() as client: + if method == "GET": + return await client.get(url, headers=headers, params=params, json=json) # type: ignore + if method == "POST": + return await client.post(url, headers=headers, params=params, json=json) # type: ignore + if method == "PUT": + return await client.put(url, headers=headers, params=params, json=json) # type: ignore + if method == "DELETE": + return await client.delete(url, headers=headers, params=params, json=json) # type: ignore + raise Exception("Shouldn't come here") + except AsyncLibraryNotFoundError: + # Try one more time + loop = create_or_get_event_loop() + return loop.run_until_complete( + self.api_request(url, method, headers, params, json) + ) + async def get_api_version(self): if Querier.api_version is not None: return Querier.api_version @@ -79,12 +107,11 @@ async def get_api_version(self): AllowedProcessStates.CALLING_SERVICE_IN_GET_API_VERSION ) - async def f(url: str) -> Response: + async def f(url: str, method: str) -> Response: headers = {} if Querier.__api_key is not None: headers = {API_KEY_HEADER: Querier.__api_key} - async with AsyncClient() as client: - return await client.get(url, headers=headers) # type:ignore + return await self.api_request(url, method, headers=headers) response = await self.__send_request_helper( NormalisedURLPath(API_VERSION), "GET", f, len(self.__hosts) @@ -134,13 +161,13 @@ async def send_get_request( if params is None: params = {} - async def f(url: str) -> Response: - async with AsyncClient() as client: - return await client.get( # type:ignore - url, - params=params, - headers=await self.__get_headers_with_api_version(path), - ) + async def f(url: str, method: str) -> Response: + return await self.api_request( + url, + method, + headers=await self.__get_headers_with_api_version(path), + params=params, + ) return await self.__send_request_helper(path, "GET", f, len(self.__hosts)) @@ -163,9 +190,13 @@ async def send_post_request( headers = await self.__get_headers_with_api_version(path) headers["content-type"] = "application/json; charset=utf-8" - async def f(url: str) -> Response: - async with AsyncClient() as client: - return await client.post(url, json=data, headers=headers) # type: ignore + async def f(url: str, method: str) -> Response: + return await self.api_request( + url, + method, + headers=await self.__get_headers_with_api_version(path), + json=data, + ) return await self.__send_request_helper(path, "POST", f, len(self.__hosts)) @@ -175,13 +206,13 @@ async def send_delete_request( if params is None: params = {} - async def f(url: str) -> Response: - async with AsyncClient() as client: - return await client.delete( # type:ignore - url, - params=params, - headers=await self.__get_headers_with_api_version(path), - ) + async def f(url: str, method: str) -> Response: + return await self.api_request( + url, + method, + headers=await self.__get_headers_with_api_version(path), + params=params, + ) return await self.__send_request_helper(path, "DELETE", f, len(self.__hosts)) @@ -194,9 +225,8 @@ async def send_put_request( headers = await self.__get_headers_with_api_version(path) headers["content-type"] = "application/json; charset=utf-8" - async def f(url: str) -> Response: - async with AsyncClient() as client: - return await client.put(url, json=data, headers=headers) # type: ignore + async def f(url: str, method: str) -> Response: + return await self.api_request(url, method, headers=headers, json=data) return await self.__send_request_helper(path, "PUT", f, len(self.__hosts)) @@ -223,7 +253,7 @@ async def __send_request_helper( self, path: NormalisedURLPath, method: str, - http_function: Callable[[str], Awaitable[Response]], + http_function: Callable[[str, str], Awaitable[Response]], no_of_tries: int, retry_info_map: Optional[Dict[str, int]] = None, ) -> Any: @@ -253,7 +283,7 @@ async def __send_request_helper( ProcessState.get_instance().add_state( AllowedProcessStates.CALLING_SERVICE_IN_REQUEST_HELPER ) - response = await http_function(url) + response = await http_function(url, method) if ("SUPERTOKENS_ENV" in environ) and ( environ["SUPERTOKENS_ENV"] == "testing" ): @@ -289,8 +319,8 @@ async def __send_request_helper( return response.json() except JSONDecodeError: return response.text - except (ConnectionError, NetworkError, ConnectTimeout) as _: + print("Retryingg!") return await self.__send_request_helper( path, method, http_function, no_of_tries - 1, retry_info_map ) diff --git a/supertokens_python/supertokens.py b/supertokens_python/supertokens.py index 5e9814ff4..2259c4da1 100644 --- a/supertokens_python/supertokens.py +++ b/supertokens_python/supertokens.py @@ -529,6 +529,11 @@ async def middleware( # pylint: disable=no-self-use async def handle_supertokens_error( self, request: BaseRequest, err: Exception, response: BaseResponse ): + import asyncio + + await asyncio.sleep(1) + # return send_non_200_response_with_message("err", 400, response) + log_debug_message("errorHandler: Started") log_debug_message( "errorHandler: Error is from SuperTokens recipe. Message: %s", str(err) diff --git a/supertokens_python/utils.py b/supertokens_python/utils.py index d3ca32b37..1472085ca 100644 --- a/supertokens_python/utils.py +++ b/supertokens_python/utils.py @@ -39,7 +39,7 @@ from httpx import HTTPStatusError, Response from tldextract import extract # type: ignore -from supertokens_python.async_to_sync_wrapper import check_event_loop +from supertokens_python.async_to_sync_wrapper import create_or_get_event_loop from supertokens_python.framework.django.framework import DjangoFramework from supertokens_python.framework.fastapi.framework import FastapiFramework from supertokens_python.framework.flask.framework import FlaskFramework @@ -212,8 +212,7 @@ def execute_async(mode: str, func: Callable[[], Coroutine[Any, Any, None]]): if real_mode == "wsgi": asyncio.run(func()) else: - check_event_loop() - loop = asyncio.get_event_loop() + loop = create_or_get_event_loop() loop.create_task(func()) From 129f327bc69b0127b783a7517b9558b3461b4711 Mon Sep 17 00:00:00 2001 From: KShivendu Date: Tue, 5 Sep 2023 16:54:00 +0530 Subject: [PATCH 2/7] fix: Types in async client api requests --- supertokens_python/querier.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/supertokens_python/querier.py b/supertokens_python/querier.py index f2bb4e331..60b701a4b 100644 --- a/supertokens_python/querier.py +++ b/supertokens_python/querier.py @@ -77,26 +77,25 @@ async def api_request( self, url: str, method: str, # ["POST", "GET"] - headers: Optional[Dict[str, Any]] = None, - params: Optional[Dict[str, Any]] = None, - json: Optional[Dict[str, Any]] = None, + *args: Any, + **kwargs: Any, ) -> Response: try: async with AsyncClient() as client: if method == "GET": - return await client.get(url, headers=headers, params=params, json=json) # type: ignore + return await client.get(url, *args, **kwargs) # type: ignore if method == "POST": - return await client.post(url, headers=headers, params=params, json=json) # type: ignore + return await client.post(url, *args, **kwargs) # type: ignore if method == "PUT": - return await client.put(url, headers=headers, params=params, json=json) # type: ignore + return await client.put(url, *args, **kwargs) # type: ignore if method == "DELETE": - return await client.delete(url, headers=headers, params=params, json=json) # type: ignore + return await client.delete(url, *args, **kwargs) # type: ignore raise Exception("Shouldn't come here") except AsyncLibraryNotFoundError: # Try one more time loop = create_or_get_event_loop() return loop.run_until_complete( - self.api_request(url, method, headers, params, json) + self.api_request(url, method, *args, **kwargs) ) async def get_api_version(self): From 5746a05b89c3409738566e9decc42130dc570620 Mon Sep 17 00:00:00 2001 From: KShivendu Date: Tue, 5 Sep 2023 16:59:54 +0530 Subject: [PATCH 3/7] fix: Retry exactly once --- supertokens_python/querier.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/supertokens_python/querier.py b/supertokens_python/querier.py index 60b701a4b..8c8153c8d 100644 --- a/supertokens_python/querier.py +++ b/supertokens_python/querier.py @@ -76,7 +76,8 @@ def get_hosts_alive_for_testing(): async def api_request( self, url: str, - method: str, # ["POST", "GET"] + method: str, + retry: bool, *args: Any, **kwargs: Any, ) -> Response: @@ -95,7 +96,7 @@ async def api_request( # Try one more time loop = create_or_get_event_loop() return loop.run_until_complete( - self.api_request(url, method, *args, **kwargs) + self.api_request(url, method, False, *args, **kwargs) ) async def get_api_version(self): @@ -110,7 +111,7 @@ async def f(url: str, method: str) -> Response: headers = {} if Querier.__api_key is not None: headers = {API_KEY_HEADER: Querier.__api_key} - return await self.api_request(url, method, headers=headers) + return await self.api_request(url, method, True, headers=headers) response = await self.__send_request_helper( NormalisedURLPath(API_VERSION), "GET", f, len(self.__hosts) @@ -164,6 +165,7 @@ async def f(url: str, method: str) -> Response: return await self.api_request( url, method, + True, headers=await self.__get_headers_with_api_version(path), params=params, ) @@ -193,6 +195,7 @@ async def f(url: str, method: str) -> Response: return await self.api_request( url, method, + True, headers=await self.__get_headers_with_api_version(path), json=data, ) @@ -209,6 +212,7 @@ async def f(url: str, method: str) -> Response: return await self.api_request( url, method, + True, headers=await self.__get_headers_with_api_version(path), params=params, ) @@ -225,7 +229,7 @@ async def send_put_request( headers["content-type"] = "application/json; charset=utf-8" async def f(url: str, method: str) -> Response: - return await self.api_request(url, method, headers=headers, json=data) + return await self.api_request(url, method, True, headers=headers, json=data) return await self.__send_request_helper(path, "PUT", f, len(self.__hosts)) From d12e76ffa31d0ba3ed4f0b10d0336843e298cf62 Mon Sep 17 00:00:00 2001 From: KShivendu Date: Wed, 6 Sep 2023 10:14:02 +0530 Subject: [PATCH 4/7] chores: Update CHANGELOG to mention AsyncLibraryNotFoundError retry --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ef325495..2d937a0fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] +- Retry Querier request on `AsyncLibraryNotFoundError` + ## [0.14.10] - 2023-09-31 -- Uses nest_asyncio patch in event loop - sync to async +- Uses `nest_asyncio` patch in event loop - sync to async ## [0.14.9] - 2023-09-28 From 7f4240cc09b99b3aaa52ac4fff1ea26ab767d2a0 Mon Sep 17 00:00:00 2001 From: KShivendu Date: Wed, 6 Sep 2023 10:18:22 +0530 Subject: [PATCH 5/7] refactor: Use retry_count int instead of boolean --- supertokens_python/querier.py | 19 +++++++++++-------- supertokens_python/supertokens.py | 5 ----- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/supertokens_python/querier.py b/supertokens_python/querier.py index 8c8153c8d..99292bf09 100644 --- a/supertokens_python/querier.py +++ b/supertokens_python/querier.py @@ -77,10 +77,13 @@ async def api_request( self, url: str, method: str, - retry: bool, + retry_count: int, *args: Any, **kwargs: Any, ) -> Response: + if retry_count == 0: + raise_general_exception("Retry request failed") + try: async with AsyncClient() as client: if method == "GET": @@ -96,9 +99,10 @@ async def api_request( # Try one more time loop = create_or_get_event_loop() return loop.run_until_complete( - self.api_request(url, method, False, *args, **kwargs) + self.api_request(url, method, retry_count - 1, *args, **kwargs) ) + async def get_api_version(self): if Querier.api_version is not None: return Querier.api_version @@ -111,7 +115,7 @@ async def f(url: str, method: str) -> Response: headers = {} if Querier.__api_key is not None: headers = {API_KEY_HEADER: Querier.__api_key} - return await self.api_request(url, method, True, headers=headers) + return await self.api_request(url, method, 1, headers=headers) response = await self.__send_request_helper( NormalisedURLPath(API_VERSION), "GET", f, len(self.__hosts) @@ -165,7 +169,7 @@ async def f(url: str, method: str) -> Response: return await self.api_request( url, method, - True, + 1, headers=await self.__get_headers_with_api_version(path), params=params, ) @@ -195,7 +199,7 @@ async def f(url: str, method: str) -> Response: return await self.api_request( url, method, - True, + 1, headers=await self.__get_headers_with_api_version(path), json=data, ) @@ -212,7 +216,7 @@ async def f(url: str, method: str) -> Response: return await self.api_request( url, method, - True, + 1, headers=await self.__get_headers_with_api_version(path), params=params, ) @@ -229,7 +233,7 @@ async def send_put_request( headers["content-type"] = "application/json; charset=utf-8" async def f(url: str, method: str) -> Response: - return await self.api_request(url, method, True, headers=headers, json=data) + return await self.api_request(url, method, 1, headers=headers, json=data) return await self.__send_request_helper(path, "PUT", f, len(self.__hosts)) @@ -323,7 +327,6 @@ async def __send_request_helper( except JSONDecodeError: return response.text except (ConnectionError, NetworkError, ConnectTimeout) as _: - print("Retryingg!") return await self.__send_request_helper( path, method, http_function, no_of_tries - 1, retry_info_map ) diff --git a/supertokens_python/supertokens.py b/supertokens_python/supertokens.py index 2259c4da1..5e9814ff4 100644 --- a/supertokens_python/supertokens.py +++ b/supertokens_python/supertokens.py @@ -529,11 +529,6 @@ async def middleware( # pylint: disable=no-self-use async def handle_supertokens_error( self, request: BaseRequest, err: Exception, response: BaseResponse ): - import asyncio - - await asyncio.sleep(1) - # return send_non_200_response_with_message("err", 400, response) - log_debug_message("errorHandler: Started") log_debug_message( "errorHandler: Error is from SuperTokens recipe. Message: %s", str(err) From 47e22523eeb2f3678d66f48e88915b3d6794010b Mon Sep 17 00:00:00 2001 From: KShivendu Date: Wed, 6 Sep 2023 10:18:59 +0530 Subject: [PATCH 6/7] chores: Format --- supertokens_python/querier.py | 1 - 1 file changed, 1 deletion(-) diff --git a/supertokens_python/querier.py b/supertokens_python/querier.py index 99292bf09..076a71b33 100644 --- a/supertokens_python/querier.py +++ b/supertokens_python/querier.py @@ -102,7 +102,6 @@ async def api_request( self.api_request(url, method, retry_count - 1, *args, **kwargs) ) - async def get_api_version(self): if Querier.api_version is not None: return Querier.api_version From d76e8fb045a33186fedbb45d764de8cf1d711efb Mon Sep 17 00:00:00 2001 From: KShivendu Date: Wed, 6 Sep 2023 10:22:10 +0530 Subject: [PATCH 7/7] fix: Rename retry_count to attempts_remaining --- supertokens_python/querier.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/supertokens_python/querier.py b/supertokens_python/querier.py index 076a71b33..db520f11a 100644 --- a/supertokens_python/querier.py +++ b/supertokens_python/querier.py @@ -77,11 +77,11 @@ async def api_request( self, url: str, method: str, - retry_count: int, + attempts_remaining: int, *args: Any, **kwargs: Any, ) -> Response: - if retry_count == 0: + if attempts_remaining == 0: raise_general_exception("Retry request failed") try: @@ -96,10 +96,10 @@ async def api_request( return await client.delete(url, *args, **kwargs) # type: ignore raise Exception("Shouldn't come here") except AsyncLibraryNotFoundError: - # Try one more time + # Retry loop = create_or_get_event_loop() return loop.run_until_complete( - self.api_request(url, method, retry_count - 1, *args, **kwargs) + self.api_request(url, method, attempts_remaining - 1, *args, **kwargs) ) async def get_api_version(self): @@ -114,7 +114,7 @@ async def f(url: str, method: str) -> Response: headers = {} if Querier.__api_key is not None: headers = {API_KEY_HEADER: Querier.__api_key} - return await self.api_request(url, method, 1, headers=headers) + return await self.api_request(url, method, 2, headers=headers) response = await self.__send_request_helper( NormalisedURLPath(API_VERSION), "GET", f, len(self.__hosts) @@ -168,7 +168,7 @@ async def f(url: str, method: str) -> Response: return await self.api_request( url, method, - 1, + 2, headers=await self.__get_headers_with_api_version(path), params=params, ) @@ -198,7 +198,7 @@ async def f(url: str, method: str) -> Response: return await self.api_request( url, method, - 1, + 2, headers=await self.__get_headers_with_api_version(path), json=data, ) @@ -215,7 +215,7 @@ async def f(url: str, method: str) -> Response: return await self.api_request( url, method, - 1, + 2, headers=await self.__get_headers_with_api_version(path), params=params, ) @@ -232,7 +232,7 @@ async def send_put_request( headers["content-type"] = "application/json; charset=utf-8" async def f(url: str, method: str) -> Response: - return await self.api_request(url, method, 1, headers=headers, json=data) + return await self.api_request(url, method, 2, headers=headers, json=data) return await self.__send_request_helper(path, "PUT", f, len(self.__hosts))