Merge branch 'main' into issue-1545/-rendering-issue-in-testset-view

.all-contributorsrc

@@ -399,6 +399,15 @@
       "contributions": [
         "doc"
       ]
+    },
+    {
+      "login": "vishalvanpariya",
+      "name": "Vishal Vanpariya",
+      "avatar_url": "https://avatars.githubusercontent.com/u/27823328?v=4",
+      "profile": "https://github.com/vishalvanpariya",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

README.md

@@ -159,7 +159,7 @@ Check out our [Contributing Guide](https://docs.agenta.ai/contributing/getting-s
 ## Contributors ✨
 
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-42-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-43-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
@@ -223,6 +223,9 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/Drewski2222"><img src="https://avatars.githubusercontent.com/u/39228951?v=4?s=100" width="100px;" alt="Drew Reisner"/><br /><sub><b>Drew Reisner</b></sub></a><br /><a href="https://github.com/Agenta-AI/agenta/commits?author=Drewski2222" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://speakerdeck.com/eltociear"><img src="https://avatars.githubusercontent.com/u/22633385?v=4?s=100" width="100px;" alt="Ikko Eltociear Ashimine"/><br /><sub><b>Ikko Eltociear Ashimine</b></sub></a><br /><a href="https://github.com/Agenta-AI/agenta/commits?author=eltociear" title="Documentation">📖</a></td>
     </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/vishalvanpariya"><img src="https://avatars.githubusercontent.com/u/27823328?v=4?s=100" width="100px;" alt="Vishal Vanpariya"/><br /><sub><b>Vishal Vanpariya</b></sub></a><br /><a href="https://github.com/Agenta-AI/agenta/commits?author=vishalvanpariya" title="Code">💻</a></td>
+    </tr>
   </tbody>
 </table>
 

agenta-backend/agenta_backend/routers/variants_router.py

@@ -298,17 +298,7 @@ async def start_variant(
     logger.debug("Starting variant %s", variant_id)
 
     # Inject env vars to docker container
-    if isCloudEE():
-        if not os.environ["OPENAI_API_KEY"]:
-            raise HTTPException(
-                status_code=400,
-                detail="Unable to start app container. Please file an issue by clicking on the button below.",
-            )
-        envvars = {
-            "OPENAI_API_KEY": os.environ["OPENAI_API_KEY"],
-        }
-    else:
-        envvars = {} if env_vars is None else env_vars.env_vars
+    envvars = {} if env_vars is None else env_vars.env_vars
 
     if action.action == VariantActionEnum.START:
         url: URI = await app_manager.start_variant(app_variant_db, envvars)

agenta-backend/agenta_backend/services/llm_apps_service.py

@@ -1,8 +1,7 @@
 import json
-import httpx
 import logging
 import asyncio
-import traceback
+import aiohttp
 from typing import Any, Dict, List
 
 
@@ -74,19 +73,17 @@ async def invoke_app(
         InvokationResult: The output of the app.
 
     Raises:
-        httpx.HTTPError: If the POST request fails.
+        aiohttp.ClientError: If the POST request fails.
     """
     url = f"{uri}/generate"
     payload = await make_payload(datapoint, parameters, openapi_parameters)
 
-    async with httpx.AsyncClient() as client:
+    async with aiohttp.ClientSession() as client:
         try:
             logger.debug(f"Invoking app {uri} with payload {payload}")
-            response = await client.post(
-                url, json=payload, timeout=httpx.Timeout(timeout=5, read=None, write=5)
-            )
+            response = await client.post(url, json=payload, timeout=5)
             response.raise_for_status()
-            app_response = response.json()
+            app_response = await response.json()
             return InvokationResult(
                 result=Result(
                     type="text",
@@ -97,17 +94,11 @@ async def invoke_app(
                 cost=app_response.get("cost"),
             )
 
-        except httpx.HTTPStatusError as e:
+        except aiohttp.ClientResponseError as e:
             # Parse error details from the API response
             error_message = "Error in invoking the LLM App:"
             try:
-                error_body = e.response.json()
-                if "message" in error_body:
-                    error_message = error_body["message"]
-                elif (
-                    "error" in error_body
-                ):  # Some APIs return error information under an 'error' key
-                    error_message = error_body["error"]
+                error_message = e.message
             except ValueError:
                 # Fallback if the error response is not JSON or doesn't have the expected structure
                 logger.error(f"Failed to parse error response: {e}")
@@ -117,20 +108,7 @@ async def invoke_app(
                 result=Result(
                     type="error",
                     error=Error(
-                        message=error_message,
-                        stacktrace=str(e),
-                    ),
-                )
-            )
-
-        except httpx.RequestError as e:
-            # Handle other request errors (e.g., network issues)
-            logger.error(f"Request error: {e}")
-            return InvokationResult(
-                result=Result(
-                    type="error",
-                    error=Error(
-                        message="Network error while invoking the LLM App",
+                        message=f"{e.code}: {error_message}",
                         stacktrace=str(e),
                     ),
                 )
@@ -179,19 +157,27 @@ async def run_with_retry(
         try:
             result = await invoke_app(uri, input_data, parameters, openapi_parameters)
             return result
-        except (httpx.TimeoutException, httpx.ConnectTimeout, httpx.ConnectError) as e:
+        except aiohttp.ClientError as e:
             last_exception = e
             print(f"Error in evaluation. Retrying in {retry_delay} seconds:", e)
             await asyncio.sleep(retry_delay)
             retries += 1
-
-    # If max retries reached, return the last exception
-    # return AppOutput(output=None, status=str(last_exception))
+        except Exception as e:
+            last_exception = e
+            logger.info(f"Error processing datapoint: {input_data}")
+
+    # If max retries is reached or an exception that isn't in the second block,
+    # update & return the last exception
+    exception_message = (
+        "Max retries reached"
+        if retries == max_retry_count
+        else f"Error processing {input_data} datapoint"
+    )
     return InvokationResult(
         result=Result(
             type="error",
             value=None,
-            error=Error(message="max retries reached", stacktrace=last_exception),
+            error=Error(message=exception_message, stacktrace=last_exception),
         )
     )
 
@@ -230,25 +216,27 @@ async def batch_invoke(
     openapi_parameters = await get_parameters_from_openapi(uri + "/openapi.json")
 
     async def run_batch(start_idx: int):
+        tasks = []
         print(f"Preparing {start_idx} batch...")
         end_idx = min(start_idx + batch_size, len(testset_data))
         for index in range(start_idx, end_idx):
-            try:
-                batch_output: InvokationResult = await run_with_retry(
+            task = asyncio.ensure_future(
+                run_with_retry(
                     uri,
                     testset_data[index],
                     parameters,
                     max_retries,
                     retry_delay,
                     openapi_parameters,
                 )
-                list_of_app_outputs.append(batch_output)
-                print(f"Adding outputs to batch {start_idx}")
-            except Exception as exc:
-                traceback.print_exc()
-                logger.info(
-                    f"Error processing batch[{start_idx}]:[{end_idx}] ==> {str(exc)}"
-                )
+            )
+            tasks.append(task)
+
+        # Gather results of all tasks
+        results = await asyncio.gather(*tasks)
+        for result in results:
+            list_of_app_outputs.append(result)
+            print(f"Adding outputs to batch {start_idx}")
 
         # Schedule the next batch with a delay
         next_batch_start_idx = end_idx
@@ -307,9 +295,8 @@ async def get_parameters_from_openapi(uri: str) -> List[Dict]:
 
 
 async def _get_openai_json_from_uri(uri):
-    async with httpx.AsyncClient() as client:
-        resp = await client.get(uri)
-        timeout = httpx.Timeout(timeout=5, read=None, write=5)
-        resp = await client.get(uri, timeout=timeout)
-        json_data = json.loads(resp.text)
+    async with aiohttp.ClientSession() as client:
+        resp = await client.get(uri, timeout=5)
+        resp_text = await resp.text()
+        json_data = json.loads(resp_text)
         return json_data

agenta-backend/agenta_backend/services/templates_manager.py

@@ -140,7 +140,7 @@ async def retrieve_templates_from_dockerhub(
     """
 
     async with httpx.AsyncClient() as client:
-        response = await client.get(f"{url}/{repo_name}/tags", timeout=10)
+        response = await client.get(f"{url}/{repo_name}/tags", timeout=90)
         if response.status_code == 200:
             response_data = response.json()
             return response_data
@@ -165,7 +165,7 @@ async def get_templates_info_from_s3(url: str) -> Dict[str, Dict[str, Any]]:
     """
 
     async with httpx.AsyncClient() as client:
-        response = await client.get(url, timeout=10)
+        response = await client.get(url, timeout=90)
         if response.status_code == 200:
             response_data = response.json()
             return response_data

agenta-backend/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "agenta_backend"
-version = "0.13.3"
+version = "0.13.5"
 description = ""
 authors = ["Mahmoud Mabrouk <[email protected]>"]
 readme = "README.md"

agenta-cli/agenta/__init__.py

@@ -1,3 +1,4 @@
+from .sdk.utils.preinit import PreInitObject
 from .sdk.agenta_decorator import app, entrypoint
 from .sdk.context import get_contexts, save_context
 from .sdk.types import (
@@ -14,7 +15,6 @@
     BinaryParam,
 )
 from .sdk.tracing.decorators import span
-from .sdk.utils.preinit import PreInitObject
 from .sdk.agenta_init import Config, init, llm_tracing
 from .sdk.utils.helper.openai_cost import calculate_token_usage
 

agenta-cli/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "agenta"
-version = "0.13.3"
+version = "0.13.5"
 description = "The SDK for agenta is an open-source LLMOps platform."
 readme = "README.md"
 authors = ["Mahmoud Mabrouk <[email protected]>"]

agenta-web/package.json

@@ -1,6 +1,6 @@
 {
     "name": "agenta",
-    "version": "0.13.3",
+    "version": "0.13.5",
     "private": true,
     "engines": {
         "node": ">=18"