Merge pull request #1479 from Agenta-AI/main

merge main into template url

agenta-backend/agenta_backend/models/api/evaluation_model.py

@@ -71,6 +71,8 @@ class Evaluation(BaseModel):
     testset_name: Optional[str]
     status: Result
     aggregated_results: List[AggregatedResult]
+    average_cost: Optional[Result]
+    average_latency: Optional[Result]
     created_at: datetime
     updated_at: datetime
 
@@ -100,6 +102,8 @@ class EvaluationScenarioInput(BaseModel):
 
 class EvaluationScenarioOutput(BaseModel):
     result: Result
+    cost: Optional[float]
+    latency: Optional[float]
 
 
 class HumanEvaluationScenarioInput(BaseModel):

agenta-backend/agenta_backend/models/converters.py

@@ -145,6 +145,8 @@ async def evaluation_db_to_pydantic(
         aggregated_results=aggregated_results,
         created_at=evaluation_db.created_at,
         updated_at=evaluation_db.updated_at,
+        average_cost=evaluation_db.average_cost,
+        average_latency=evaluation_db.average_latency,
     )
 
 

agenta-backend/agenta_backend/models/db_models.py

@@ -193,6 +193,8 @@ class Result(BaseModel):
 
 class InvokationResult(BaseModel):
     result: Result
+    cost: Optional[float] = None
+    latency: Optional[float] = None
 
 
 class EvaluationScenarioResult(BaseModel):
@@ -213,6 +215,8 @@ class EvaluationScenarioInputDB(BaseModel):
 
 class EvaluationScenarioOutputDB(BaseModel):
     result: Result
+    cost: Optional[float] = None
+    latency: Optional[float] = None
 
 
 class HumanEvaluationScenarioInput(BaseModel):
@@ -266,6 +270,8 @@ class EvaluationDB(Document):
     variant_revision: PydanticObjectId
     evaluators_configs: List[PydanticObjectId]
     aggregated_results: List[AggregatedResult]
+    average_cost: Optional[Result] = None
+    average_latency: Optional[Result] = None
     created_at: datetime = Field(default=datetime.now())
     updated_at: datetime = Field(default=datetime.now())
 
@@ -284,6 +290,8 @@ class EvaluationScenarioDB(Document):
     note: Optional[str]
     evaluators_configs: List[PydanticObjectId]
     results: List[EvaluationScenarioResult]
+    latency: Optional[int] = None
+    cost: Optional[int] = None
     created_at: datetime = Field(default=datetime.now())
     updated_at: datetime = Field(default=datetime.now())
 

agenta-backend/agenta_backend/services/aggregation_service.py

@@ -1,8 +1,8 @@
 import re
 import traceback
-from typing import List
+from typing import List, Optional
 
-from agenta_backend.models.db_models import Result, Error
+from agenta_backend.models.db_models import InvokationResult, Result, Error
 
 
 def aggregate_ai_critique(results: List[Result]) -> Result:
@@ -73,3 +73,29 @@ def aggregate_float(results: List[Result]) -> Result:
             value=None,
             error=Error(message=str(exc), stacktrace=str(traceback.format_exc())),
         )
+
+
+def aggregate_float_from_llm_app_response(
+    invocation_results: List[InvokationResult], key: Optional[str]
+) -> Result:
+    try:
+        if not key:
+            raise ValueError("Key is required to aggregate InvokationResult objects.")
+
+        values = [
+            getattr(inv_result, key)
+            for inv_result in invocation_results
+            if hasattr(inv_result, key) and getattr(inv_result, key) is not None
+        ]
+
+        if not values:
+            raise ValueError(f"No valid values found for {key} aggregation.")
+
+        average_value = sum(values) / len(values)
+        return Result(type=key, value=average_value)
+    except Exception as exc:
+        return Result(
+            type="error",
+            value=None,
+            error=Error(message=str(exc), stacktrace=str(traceback.format_exc())),
+        )

agenta-backend/agenta_backend/services/llm_apps_service.py

@@ -86,9 +86,15 @@ async def invoke_app(
                 url, json=payload, timeout=httpx.Timeout(timeout=5, read=None, write=5)
             )
             response.raise_for_status()
-            app_output = response.json()
+            app_response = response.json()
             return InvokationResult(
-                result=Result(type="text", value=app_output["message"], error=None)
+                result=Result(
+                    type="text",
+                    value=app_response["message"],
+                    error=None,
+                ),
+                latency=app_response.get("latency"),
+                cost=app_response.get("cost"),
             )
 
         except httpx.HTTPStatusError as e:

agenta-backend/agenta_backend/tasks/evaluations.py

@@ -267,7 +267,9 @@ def evaluate(
                     correct_answer=correct_answer,
                     outputs=[
                         EvaluationScenarioOutputDB(
-                            result=Result(type="text", value=app_output.result.value)
+                            result=Result(type="text", value=app_output.result.value),
+                            latency=app_output.latency,
+                            cost=app_output.cost,
                         )
                     ],
                     results=evaluators_results,
@@ -276,6 +278,20 @@ def evaluate(
                 )
             )
 
+        # Add average cost and latency
+        average_latency = aggregation_service.aggregate_float_from_llm_app_response(
+            app_outputs, "latency"
+        )
+        average_cost = aggregation_service.aggregate_float_from_llm_app_response(
+            app_outputs, "cost"
+        )
+        loop.run_until_complete(
+            update_evaluation(
+                evaluation_id,
+                {"average_latency": average_latency, "average_cost": average_cost},
+            )
+        )
+
     except Exception as e:
         logger.error(f"An error occurred during evaluation: {e}")
         traceback.print_exc()

agenta-backend/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "agenta_backend"
-version = "0.12.4"
+version = "0.12.6"
 description = ""
 authors = ["Mahmoud Mabrouk <[email protected]>"]
 readme = "README.md"

agenta-cli/agenta/cli/main.py

@@ -153,18 +153,21 @@ def init(app_name: str, backend_host: str):
             try:
                 key_prefix = api_key.split(".")[0]
                 client.validate_api_key(key_prefix=key_prefix)
-
-                # Make request to fetch user organizations after api key validation
+            except Exception as ex:
+                click.echo(
+                    click.style(
+                        f"Error: Unable to validate API key.\nError: {ex}", fg="red"
+                    )
+                )
+                sys.exit(1)
+            # Make request to fetch user organizations after api key validation
+            try:
                 organizations = client.list_organizations()
                 if len(organizations) >= 1:
                     user_organizations = organizations
             except Exception as ex:
-                if ex.status_code == 401:
-                    click.echo(click.style("Error: Invalid API key", fg="red"))
-                    sys.exit(1)
-                else:
-                    click.echo(click.style(f"Error: {ex}", fg="red"))
-                    sys.exit(1)
+                click.echo(click.style(f"Error: {ex}", fg="red"))
+                sys.exit(1)
 
         filtered_org = None
         if where_question == "On agenta cloud":

agenta-cli/agenta/client/backend/types/app_variant_output.py

@@ -18,7 +18,7 @@ class AppVariantOutput(pydantic.BaseModel):
     variant_name: str
     parameters: typing.Optional[typing.Dict[str, typing.Any]]
     previous_variant_name: typing.Optional[str]
-    organization_id: str
+    organization_id: typing.Optional[str]
     user_id: str
     base_name: str
     base_id: str

agenta-cli/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "agenta"
-version = "0.12.4"
+version = "0.12.6"
 description = "The SDK for agenta is an open-source LLMOps platform."
 readme = "README.md"
 authors = ["Mahmoud Mabrouk <[email protected]>"]

agenta-web/cypress/e2e/eval.comparison.cy.ts

@@ -32,8 +32,8 @@ describe("Evaluation Comparison Test", function () {
         })
 
         it("Should select 2 evaluations, click on the compare button, and successfully navigate to the comparison page", () => {
-            cy.get("#ag-33-input").check()
-            cy.get("#ag-39-input").check()
+            cy.get("div.ag-selection-checkbox input").eq(0).check()
+            cy.get("div.ag-selection-checkbox input").eq(1).check()
             cy.get('[data-cy="evaluation-results-compare-button"]').should("not.be.disabled")
             cy.get('[data-cy="evaluation-results-compare-button"]').click()
             cy.location("pathname").should("include", "/evaluations/compare")

agenta-web/cypress/e2e/eval.evaluations.cy.ts

@@ -33,7 +33,7 @@ describe("Evaluations CRUD Operations Test", function () {
 
         it("Should select evaluation and successfully delete it", () => {
             cy.get(".ag-root-wrapper").should("exist")
-            cy.get("#ag-33-input").check()
+            cy.get("div.ag-selection-checkbox input").eq(0).check()
             cy.get(":nth-child(1) > .ant-btn > .ant-btn-icon > .anticon > svg").click()
             cy.get(".ant-modal-confirm-btns > :nth-child(2) > span").click()
         })

agenta-web/package.json

@@ -1,6 +1,6 @@
 {
     "name": "agenta",
-    "version": "0.12.4",
+    "version": "0.12.6",
     "private": true,
     "engines": {
         "node": ">=18"
@@ -39,8 +39,8 @@
         "@types/react-highlight-words": "^0.16.4",
         "@types/react-syntax-highlighter": "^15.5.7",
         "@types/uuid": "^9.0.7",
-        "ag-grid-community": "^31.0.1",
-        "ag-grid-react": "^31.0.1",
+        "ag-grid-community": "^31.2.0",
+        "ag-grid-react": "^31.2.0",
         "antd": "^5.4.7",
         "autoprefixer": "10.4.14",
         "axios": "^1.4.0",

agenta-web/src/components/Playground/AddToTestSetDrawer/AddToTestSetDrawer.tsx

@@ -190,7 +190,7 @@ const AddToTestSetDrawer: React.FC<Props> = ({params, isChatVariant, ...props})
                     )
 
                     // if cols mismatch (playground cols not a superset of testset cols)
-                    if (missingColsTestset.length) {
+                    if (missingColsTestset.length && missingColsPlayground.length) {
                         AlertPopup({
                             type: "error",
                             title: "Columns mismatch",

agenta-web/src/components/Playground/Views/ParametersView.tsx

@@ -1,7 +1,7 @@
 import {Environment, IPromptRevisions, Parameter, Variant} from "@/lib/Types"
 import type {CollapseProps} from "antd"
 import {Button, Col, Collapse, Row, Space, Tooltip, message} from "antd"
-import React, {useContext, useEffect, useState} from "react"
+import React, {useEffect, useState} from "react"
 import {createUseStyles} from "react-jss"
 import {ModelParameters, ObjectParameters, StringParameters} from "./ParametersCards"
 import PublishVariantModal from "./PublishVariantModal"

agenta-web/src/components/Playground/Views/TestView.tsx

@@ -2,14 +2,7 @@ import React, {useContext, useEffect, useRef, useState} from "react"
 import {Button, Input, Card, Row, Col, Space, Form, Modal} from "antd"
 import {CaretRightOutlined, CloseCircleOutlined, PlusOutlined} from "@ant-design/icons"
 import {callVariant, promptRevision} from "@/lib/services/api"
-import {
-    ChatMessage,
-    ChatRole,
-    GenericObject,
-    IPromptVersioning,
-    Parameter,
-    Variant,
-} from "@/lib/Types"
+import {ChatMessage, ChatRole, GenericObject, Parameter, Variant} from "@/lib/Types"
 import {batchExecute, randString, removeKeys} from "@/lib/helpers/utils"
 import LoadTestsModal from "../LoadTestsModal"
 import AddToTestSetDrawer from "../AddToTestSetDrawer/AddToTestSetDrawer"
@@ -29,7 +22,6 @@ import dayjs from "dayjs"
 import relativeTime from "dayjs/plugin/relativeTime"
 import duration from "dayjs/plugin/duration"
 import {useQueryParam} from "@/hooks/useQuery"
-import {dynamicComponent} from "@/lib/helpers/dynamic"
 
 dayjs.extend(relativeTime)
 dayjs.extend(duration)