From 9933359fa49da28febf0559e5227590145048922 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 5 May 2024 20:13:53 +0200 Subject: [PATCH 1/7] Add traceback for error handling in LLM Apps service --- .../agenta_backend/services/llm_apps_service.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/agenta-backend/agenta_backend/services/llm_apps_service.py b/agenta-backend/agenta_backend/services/llm_apps_service.py index ea7373596c..e09eed69b1 100644 --- a/agenta-backend/agenta_backend/services/llm_apps_service.py +++ b/agenta-backend/agenta_backend/services/llm_apps_service.py @@ -1,6 +1,7 @@ import json import logging import asyncio +import traceback import aiohttp from typing import Any, Dict, List @@ -31,7 +32,8 @@ async def make_payload( for param in openapi_parameters: if param["type"] == "input": payload[param["name"]] = datapoint.get(param["name"], "") - elif param["type"] == "dict": # in case of dynamic inputs (as in our templates) + # in case of dynamic inputs (as in our templates) + elif param["type"] == "dict": # let's get the list of the dynamic inputs if ( param["name"] in parameters @@ -109,7 +111,9 @@ async def invoke_app( type="error", error=Error( message=f"{e.code}: {error_message}", - stacktrace=str(e), + stacktrace="".join( + traceback.format_exception(None, e, e.__traceback__) + ), ), ) ) @@ -122,7 +126,9 @@ async def invoke_app( type="error", error=Error( message="Unexpected error while invoking the LLM App", - stacktrace=str(e), + stacktrace="".join( + traceback.format_exception(None, e, e.__traceback__) + ), ), ) ) From e7ffce588c2ca98fdf07c37d25098ee57d85a5f3 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 5 May 2024 20:21:31 +0200 Subject: [PATCH 2/7] Refactor error handling in invoke_app function --- .../services/llm_apps_service.py | 55 ++++++++----------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/agenta-backend/agenta_backend/services/llm_apps_service.py b/agenta-backend/agenta_backend/services/llm_apps_service.py index e09eed69b1..7fd3e9931f 100644 --- a/agenta-backend/agenta_backend/services/llm_apps_service.py +++ b/agenta-backend/agenta_backend/services/llm_apps_service.py @@ -97,41 +97,32 @@ async def invoke_app( ) except aiohttp.ClientResponseError as e: - # Parse error details from the API response - error_message = "Error in invoking the LLM App:" - try: - error_message = e.message - except ValueError: - # Fallback if the error response is not JSON or doesn't have the expected structure - logger.error(f"Failed to parse error response: {e}") - - logger.error(f"Error occurred during request: {error_message}") - return InvokationResult( - result=Result( - type="error", - error=Error( - message=f"{e.code}: {error_message}", - stacktrace="".join( - traceback.format_exception(None, e, e.__traceback__) - ), - ), - ) - ) - + error_message = f"HTTP error {e.status}: {e.message}" + logger.error(f"HTTP error occurred during request: {error_message}") + except aiohttp.ClientConnectionError as e: + error_message = f"Connection error: {str(e)}" + logger.error(error_message) + except aiohttp.TimeoutError as e: + error_message = "Request timed out" + logger.error(error_message) + except json.JSONDecodeError as e: + error_message = "Failed to decode JSON from response" + logger.error(error_message) except Exception as e: - # Catch-all for any other unexpected errors - logger.error(f"Unexpected error: {e}") - return InvokationResult( - result=Result( - type="error", - error=Error( - message="Unexpected error while invoking the LLM App", - stacktrace="".join( - traceback.format_exception(None, e, e.__traceback__) - ), + error_message = f"Unexpected error: {str(e)}" + logger.error(error_message) + + return InvokationResult( + result=Result( + type="error", + error=Error( + message=error_message, + stacktrace="".join( + traceback.format_exception(None, e, e.__traceback__) ), - ) + ), ) + ) async def run_with_retry( From 95abbfe5752d7b213a180d870d2b2ae51e33af3d Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 5 May 2024 21:03:34 +0200 Subject: [PATCH 3/7] added sentry capture errors --- .../agenta_backend/services/llm_apps_service.py | 11 ++++++++++- agenta-backend/agenta_backend/utils/common.py | 6 ++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/agenta-backend/agenta_backend/services/llm_apps_service.py b/agenta-backend/agenta_backend/services/llm_apps_service.py index 7fd3e9931f..cb1854b134 100644 --- a/agenta-backend/agenta_backend/services/llm_apps_service.py +++ b/agenta-backend/agenta_backend/services/llm_apps_service.py @@ -7,6 +7,7 @@ from agenta_backend.models.db_models import InvokationResult, Result, Error +from agenta_backend.utils import common # Set logger logger = logging.getLogger(__name__) @@ -99,18 +100,23 @@ async def invoke_app( except aiohttp.ClientResponseError as e: error_message = f"HTTP error {e.status}: {e.message}" logger.error(f"HTTP error occurred during request: {error_message}") + common.capture_exception_in_sentry(e) except aiohttp.ClientConnectionError as e: error_message = f"Connection error: {str(e)}" logger.error(error_message) + common.capture_exception_in_sentry(e) except aiohttp.TimeoutError as e: error_message = "Request timed out" logger.error(error_message) + common.capture_exception_in_sentry(e) except json.JSONDecodeError as e: error_message = "Failed to decode JSON from response" logger.error(error_message) + common.capture_exception_in_sentry(e) except Exception as e: error_message = f"Unexpected error: {str(e)}" logger.error(error_message) + common.capture_exception_in_sentry(e) return InvokationResult( result=Result( @@ -161,10 +167,13 @@ async def run_with_retry( retries += 1 except Exception as e: last_exception = e - logger.info(f"Error processing datapoint: {input_data}") + logger.info(f"Error processing datapoint: {input_data}. {str(e)}") + logger.info(traceback.format_exc()) + common.capture_exception_in_sentry(e) # If max retries is reached or an exception that isn't in the second block, # update & return the last exception + logging.info("Max retries reached") exception_message = ( "Max retries reached" if retries == max_retry_count diff --git a/agenta-backend/agenta_backend/utils/common.py b/agenta-backend/agenta_backend/utils/common.py index 984160b5d6..f9b40127b9 100644 --- a/agenta-backend/agenta_backend/utils/common.py +++ b/agenta-backend/agenta_backend/utils/common.py @@ -4,6 +4,7 @@ from fastapi.types import DecoratedCallable from fastapi import APIRouter as FastAPIRouter +from sentry_sdk import capture_exception logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -76,3 +77,8 @@ def isCloudDev(): def isOss(): return os.environ["FEATURE_FLAG"] == "oss" + + +def capture_exception_in_sentry(e: Exception): + if isCloudProd(): + capture_exception(e) From f5cba9c34c8fa7ebf2a311999a35ddd53eb03cc7 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 5 May 2024 21:16:18 +0200 Subject: [PATCH 4/7] fixed error handling --- .../agenta_backend/services/llm_apps_service.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/agenta-backend/agenta_backend/services/llm_apps_service.py b/agenta-backend/agenta_backend/services/llm_apps_service.py index cb1854b134..ef59ac2d1a 100644 --- a/agenta-backend/agenta_backend/services/llm_apps_service.py +++ b/agenta-backend/agenta_backend/services/llm_apps_service.py @@ -8,7 +8,6 @@ from agenta_backend.models.db_models import InvokationResult, Result, Error from agenta_backend.utils import common - # Set logger logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -99,13 +98,14 @@ async def invoke_app( except aiohttp.ClientResponseError as e: error_message = f"HTTP error {e.status}: {e.message}" - logger.error(f"HTTP error occurred during request: {error_message}") + logger.error( + f"HTTP error occurred during request: {error_message}") common.capture_exception_in_sentry(e) except aiohttp.ClientConnectionError as e: error_message = f"Connection error: {str(e)}" logger.error(error_message) common.capture_exception_in_sentry(e) - except aiohttp.TimeoutError as e: + except aiohttp.ServerTimeoutError as e: error_message = "Request timed out" logger.error(error_message) common.capture_exception_in_sentry(e) @@ -162,7 +162,8 @@ async def run_with_retry( return result except aiohttp.ClientError as e: last_exception = e - print(f"Error in evaluation. Retrying in {retry_delay} seconds:", e) + print( + f"Error in evaluation. Retrying in {retry_delay} seconds:", e) await asyncio.sleep(retry_delay) retries += 1 except Exception as e: From bfbb4678bf77be202baec6e1385ec8a663af7be4 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 5 May 2024 21:16:52 +0200 Subject: [PATCH 5/7] format --- .../agenta_backend/services/llm_apps_service.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/agenta-backend/agenta_backend/services/llm_apps_service.py b/agenta-backend/agenta_backend/services/llm_apps_service.py index ef59ac2d1a..fd729297ed 100644 --- a/agenta-backend/agenta_backend/services/llm_apps_service.py +++ b/agenta-backend/agenta_backend/services/llm_apps_service.py @@ -8,6 +8,7 @@ from agenta_backend.models.db_models import InvokationResult, Result, Error from agenta_backend.utils import common + # Set logger logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -98,17 +99,16 @@ async def invoke_app( except aiohttp.ClientResponseError as e: error_message = f"HTTP error {e.status}: {e.message}" - logger.error( - f"HTTP error occurred during request: {error_message}") - common.capture_exception_in_sentry(e) - except aiohttp.ClientConnectionError as e: - error_message = f"Connection error: {str(e)}" - logger.error(error_message) + logger.error(f"HTTP error occurred during request: {error_message}") common.capture_exception_in_sentry(e) except aiohttp.ServerTimeoutError as e: error_message = "Request timed out" logger.error(error_message) common.capture_exception_in_sentry(e) + except aiohttp.ClientConnectionError as e: + error_message = f"Connection error: {str(e)}" + logger.error(error_message) + common.capture_exception_in_sentry(e) except json.JSONDecodeError as e: error_message = "Failed to decode JSON from response" logger.error(error_message) @@ -162,8 +162,7 @@ async def run_with_retry( return result except aiohttp.ClientError as e: last_exception = e - print( - f"Error in evaluation. Retrying in {retry_delay} seconds:", e) + print(f"Error in evaluation. Retrying in {retry_delay} seconds:", e) await asyncio.sleep(retry_delay) retries += 1 except Exception as e: From 4e13cbb70fe1ff01bf14c99e9422764ca689b61a Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 5 May 2024 21:30:54 +0200 Subject: [PATCH 6/7] improve error handling --- .../agenta_backend/services/llm_apps_service.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/agenta-backend/agenta_backend/services/llm_apps_service.py b/agenta-backend/agenta_backend/services/llm_apps_service.py index fd729297ed..88257f44c5 100644 --- a/agenta-backend/agenta_backend/services/llm_apps_service.py +++ b/agenta-backend/agenta_backend/services/llm_apps_service.py @@ -99,22 +99,28 @@ async def invoke_app( except aiohttp.ClientResponseError as e: error_message = f"HTTP error {e.status}: {e.message}" - logger.error(f"HTTP error occurred during request: {error_message}") + stacktrace = "".join(traceback.format_exception_only(type(e), e)) + logger.error( + f"HTTP error occurred during request: {error_message}") common.capture_exception_in_sentry(e) except aiohttp.ServerTimeoutError as e: error_message = "Request timed out" + stacktrace = "".join(traceback.format_exception_only(type(e), e)) logger.error(error_message) common.capture_exception_in_sentry(e) except aiohttp.ClientConnectionError as e: error_message = f"Connection error: {str(e)}" + stacktrace = "".join(traceback.format_exception_only(type(e), e)) logger.error(error_message) common.capture_exception_in_sentry(e) except json.JSONDecodeError as e: error_message = "Failed to decode JSON from response" + stacktrace = "".join(traceback.format_exception_only(type(e), e)) logger.error(error_message) common.capture_exception_in_sentry(e) except Exception as e: error_message = f"Unexpected error: {str(e)}" + stacktrace = "".join(traceback.format_exception_only(type(e), e)) logger.error(error_message) common.capture_exception_in_sentry(e) @@ -123,9 +129,7 @@ async def invoke_app( type="error", error=Error( message=error_message, - stacktrace="".join( - traceback.format_exception(None, e, e.__traceback__) - ), + stacktrace=stacktrace, ), ) ) @@ -162,13 +166,14 @@ async def run_with_retry( return result except aiohttp.ClientError as e: last_exception = e - print(f"Error in evaluation. Retrying in {retry_delay} seconds:", e) + print( + f"Error in evaluation. Retrying in {retry_delay} seconds:", e) await asyncio.sleep(retry_delay) retries += 1 except Exception as e: last_exception = e logger.info(f"Error processing datapoint: {input_data}. {str(e)}") - logger.info(traceback.format_exc()) + logger.info("".join(traceback.format_exception_only(type(e), e))) common.capture_exception_in_sentry(e) # If max retries is reached or an exception that isn't in the second block, From 9a5ced5756f2e31e0c08125970762549c0eda2da Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 5 May 2024 21:49:30 +0200 Subject: [PATCH 7/7] formaat --- agenta-backend/agenta_backend/services/llm_apps_service.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/agenta-backend/agenta_backend/services/llm_apps_service.py b/agenta-backend/agenta_backend/services/llm_apps_service.py index 88257f44c5..f9d7e1199e 100644 --- a/agenta-backend/agenta_backend/services/llm_apps_service.py +++ b/agenta-backend/agenta_backend/services/llm_apps_service.py @@ -100,8 +100,7 @@ async def invoke_app( except aiohttp.ClientResponseError as e: error_message = f"HTTP error {e.status}: {e.message}" stacktrace = "".join(traceback.format_exception_only(type(e), e)) - logger.error( - f"HTTP error occurred during request: {error_message}") + logger.error(f"HTTP error occurred during request: {error_message}") common.capture_exception_in_sentry(e) except aiohttp.ServerTimeoutError as e: error_message = "Request timed out" @@ -166,8 +165,7 @@ async def run_with_retry( return result except aiohttp.ClientError as e: last_exception = e - print( - f"Error in evaluation. Retrying in {retry_delay} seconds:", e) + print(f"Error in evaluation. Retrying in {retry_delay} seconds:", e) await asyncio.sleep(retry_delay) retries += 1 except Exception as e: