Latency cost in eval

agenta-backend/agenta_backend/models/api/evaluation_model.py

@@ -71,6 +71,8 @@ class Evaluation(BaseModel):
     testset_name: Optional[str]
     status: Result
     aggregated_results: List[AggregatedResult]
+    average_cost: Optional[Result]
+    average_latency: Optional[Result]
     created_at: datetime
     updated_at: datetime
 
@@ -100,6 +102,8 @@ class EvaluationScenarioInput(BaseModel):
 
 class EvaluationScenarioOutput(BaseModel):
     result: Result
+    cost: Optional[float]
+    latency: Optional[float]
 
 
 class HumanEvaluationScenarioInput(BaseModel):

agenta-backend/agenta_backend/models/converters.py

@@ -145,6 +145,8 @@ async def evaluation_db_to_pydantic(
         aggregated_results=aggregated_results,
         created_at=evaluation_db.created_at,
         updated_at=evaluation_db.updated_at,
+        average_cost=evaluation_db.average_cost,
+        average_latency=evaluation_db.average_latency,
     )
 
 

agenta-backend/agenta_backend/models/db_models.py

@@ -193,6 +193,8 @@ class Result(BaseModel):
 
 class InvokationResult(BaseModel):
     result: Result
+    cost: Optional[float] = None
+    latency: Optional[float] = None
 
 
 class EvaluationScenarioResult(BaseModel):
@@ -213,6 +215,8 @@ class EvaluationScenarioInputDB(BaseModel):
 
 class EvaluationScenarioOutputDB(BaseModel):
     result: Result
+    cost: Optional[float] = None
+    latency: Optional[float] = None
 
 
 class HumanEvaluationScenarioInput(BaseModel):
@@ -266,6 +270,8 @@ class EvaluationDB(Document):
     variant_revision: PydanticObjectId
     evaluators_configs: List[PydanticObjectId]
     aggregated_results: List[AggregatedResult]
+    average_cost: Optional[Result] = None
+    average_latency: Optional[Result] = None
     created_at: datetime = Field(default=datetime.now())
     updated_at: datetime = Field(default=datetime.now())
 
@@ -284,6 +290,8 @@ class EvaluationScenarioDB(Document):
     note: Optional[str]
     evaluators_configs: List[PydanticObjectId]
     results: List[EvaluationScenarioResult]
+    latency: Optional[int] = None
+    cost: Optional[int] = None
     created_at: datetime = Field(default=datetime.now())
     updated_at: datetime = Field(default=datetime.now())
 

agenta-backend/agenta_backend/services/aggregation_service.py

@@ -1,8 +1,8 @@
 import re
 import traceback
-from typing import List
+from typing import List, Optional
 
-from agenta_backend.models.db_models import Result, Error
+from agenta_backend.models.db_models import InvokationResult, Result, Error
 
 
 def aggregate_ai_critique(results: List[Result]) -> Result:
@@ -73,3 +73,29 @@ def aggregate_float(results: List[Result]) -> Result:
             value=None,
             error=Error(message=str(exc), stacktrace=str(traceback.format_exc())),
         )
+
+
+def aggregate_float_from_llm_app_response(
+    invocation_results: List[InvokationResult], key: Optional[str]
+) -> Result:
+    try:
+        if not key:
+            raise ValueError("Key is required to aggregate InvokationResult objects.")
+
+        values = [
+            getattr(inv_result, key)
+            for inv_result in invocation_results
+            if hasattr(inv_result, key) and getattr(inv_result, key) is not None
+        ]
+
+        if not values:
+            raise ValueError(f"No valid values found for {key} aggregation.")
+
+        average_value = sum(values) / len(values)
+        return Result(type=key, value=average_value)
+    except Exception as exc:
+        return Result(
+            type="error",
+            value=None,
+            error=Error(message=str(exc), stacktrace=str(traceback.format_exc())),
+        )

agenta-backend/agenta_backend/services/llm_apps_service.py

@@ -86,9 +86,15 @@ async def invoke_app(
                 url, json=payload, timeout=httpx.Timeout(timeout=5, read=None, write=5)
             )
             response.raise_for_status()
-            app_output = response.json()
+            app_response = response.json()
             return InvokationResult(
-                result=Result(type="text", value=app_output["message"], error=None)
+                result=Result(
+                    type="text",
+                    value=app_response["message"],
+                    error=None,
+                ),
+                latency=app_response.get("latency"),
+                cost=app_response.get("cost"),
             )
 
         except httpx.HTTPStatusError as e:

agenta-backend/agenta_backend/tasks/evaluations.py

@@ -267,7 +267,9 @@ def evaluate(
                     correct_answer=correct_answer,
                     outputs=[
                         EvaluationScenarioOutputDB(
-                            result=Result(type="text", value=app_output.result.value)
+                            result=Result(type="text", value=app_output.result.value),
+                            latency=app_output.latency,
+                            cost=app_output.cost,
                         )
                     ],
                     results=evaluators_results,
@@ -276,6 +278,20 @@ def evaluate(
                 )
             )
 
+        # Add average cost and latency
+        average_latency = aggregation_service.aggregate_float_from_llm_app_response(
+            app_outputs, "latency"
+        )
+        average_cost = aggregation_service.aggregate_float_from_llm_app_response(
+            app_outputs, "cost"
+        )
+        loop.run_until_complete(
+            update_evaluation(
+                evaluation_id,
+                {"average_latency": average_latency, "average_cost": average_cost},
+            )
+        )
+
     except Exception as e:
         logger.error(f"An error occurred during evaluation: {e}")
         traceback.print_exc()

agenta-web/cypress/e2e/eval.comparison.cy.ts

@@ -32,8 +32,8 @@ describe("Evaluation Comparison Test", function () {
         })
 
         it("Should select 2 evaluations, click on the compare button, and successfully navigate to the comparison page", () => {
-            cy.get("#ag-33-input").check()
-            cy.get("#ag-39-input").check()
+            cy.get("div.ag-selection-checkbox input").eq(0).check()
+            cy.get("div.ag-selection-checkbox input").eq(1).check()
             cy.get('[data-cy="evaluation-results-compare-button"]').should("not.be.disabled")
             cy.get('[data-cy="evaluation-results-compare-button"]').click()
             cy.location("pathname").should("include", "/evaluations/compare")

agenta-web/cypress/e2e/eval.evaluations.cy.ts

@@ -33,7 +33,7 @@ describe("Evaluations CRUD Operations Test", function () {
 
         it("Should select evaluation and successfully delete it", () => {
             cy.get(".ag-root-wrapper").should("exist")
-            cy.get("#ag-33-input").check()
+            cy.get("div.ag-selection-checkbox input").eq(0).check()
             cy.get(":nth-child(1) > .ant-btn > .ant-btn-icon > .anticon > svg").click()
             cy.get(".ant-modal-confirm-btns > :nth-child(2) > span").click()
         })

agenta-web/package.json

@@ -39,8 +39,8 @@
         "@types/react-highlight-words": "^0.16.4",
         "@types/react-syntax-highlighter": "^15.5.7",
         "@types/uuid": "^9.0.7",
-        "ag-grid-community": "^31.0.1",
-        "ag-grid-react": "^31.0.1",
+        "ag-grid-community": "^31.2.0",
+        "ag-grid-react": "^31.2.0",
         "antd": "^5.4.7",
         "autoprefixer": "10.4.14",
         "axios": "^1.4.0",

agenta-web/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx

@@ -25,6 +25,7 @@ import AgCustomHeader from "@/components/AgCustomHeader/AgCustomHeader"
 import {useAtom} from "jotai"
 import {evaluatorsAtom} from "@/lib/atoms/evaluation"
 import CompareOutputDiff from "@/components/CompareOutputDiff/CompareOutputDiff"
+import {formatCurrency, formatLatency} from "@/lib/helpers/formatters"
 
 const useStyles = createUseStyles((theme: JSSTheme) => ({
     table: {
@@ -218,6 +219,50 @@ const EvaluationCompareMode: React.FC<Props> = () => {
             })
         })
 
+        variants.forEach((variant, vi) => {
+            colDefs.push({
+                headerComponent: (props: any) => (
+                    <AgCustomHeader {...props}>
+                        <Space direction="vertical">
+                            <span>Latency</span>
+                            <Tag color={colors[vi]}>{variant.variantName}</Tag>
+                        </Space>
+                    </AgCustomHeader>
+                ),
+                minWidth: 120,
+                flex: 1,
+                valueGetter: (params) => {
+                    const latency = params.data?.variants.find(
+                        (item) => item.evaluationId === variant.evaluationId,
+                    )?.output?.latency
+                    return latency === undefined ? "-" : formatLatency(latency)
+                },
+                ...getFilterParams("text"),
+            })
+        })
+
+        variants.forEach((variant, vi) => {
+            colDefs.push({
+                headerComponent: (props: any) => (
+                    <AgCustomHeader {...props}>
+                        <Space direction="vertical">
+                            <span>Cost</span>
+                            <Tag color={colors[vi]}>{variant.variantName}</Tag>
+                        </Space>
+                    </AgCustomHeader>
+                ),
+                minWidth: 120,
+                flex: 1,
+                valueGetter: (params) => {
+                    const cost = params.data?.variants.find(
+                        (item) => item.evaluationId === variant.evaluationId,
+                    )?.output?.cost
+                    return cost === undefined ? "-" : formatCurrency(cost)
+                },
+                ...getFilterParams("text"),
+            })
+        })
+
         return colDefs
     }, [rows, showDiff, evalIds])
 
@@ -364,3 +409,6 @@ const EvaluationCompareMode: React.FC<Props> = () => {
 }
 
 export default EvaluationCompareMode
+function formatCost(cost: any) {
+    throw new Error("Function not implemented.")
+}

agenta-web/src/components/pages/evaluations/evaluationResults/EvaluationResults.tsx

@@ -246,6 +246,22 @@ const EvaluationResults: React.FC<Props> = () => {
                     statusMapper(token)[params.data?.status.value as EvaluationStatus].label,
                 cellRenderer: StatusRenderer,
             },
+            {
+                flex: 1,
+                field: "average_latency",
+                headerName: "Latency",
+                minWidth: 120,
+                ...getFilterParams("number"),
+                valueGetter: (params) => getTypedValue(params?.data?.average_latency),
+            },
+            {
+                flex: 1,
+                field: "average_cost",
+                headerName: "Cost",
+                minWidth: 120,
+                ...getFilterParams("number"),
+                valueGetter: (params) => getTypedValue(params?.data?.average_cost),
+            },
             {
                 flex: 1,
                 field: "created_at",

agenta-web/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx

@@ -23,6 +23,7 @@ import {useAtom} from "jotai"
 import {evaluatorsAtom} from "@/lib/atoms/evaluation"
 import CompareOutputDiff from "@/components/CompareOutputDiff/CompareOutputDiff"
 import {useQueryParam} from "@/hooks/useQuery"
+import {formatCurrency, formatLatency} from "@/lib/helpers/formatters"
 
 const useStyles = createUseStyles((theme: JSSTheme) => ({
     infoRow: {
@@ -139,6 +140,29 @@ const EvaluationScenarios: React.FC<Props> = () => {
                 },
             })
         })
+        colDefs.push({
+            flex: 1,
+            minWidth: 120,
+            headerName: "Cost",
+            ...getFilterParams("text"),
+            valueGetter: (params) => {
+                return params.data?.outputs[0].cost == undefined
+                    ? "-"
+                    : formatCurrency(params.data.outputs[0].cost)
+            },
+        })
+
+        colDefs.push({
+            flex: 1,
+            minWidth: 120,
+            headerName: "Latency",
+            ...getFilterParams("text"),
+            valueGetter: (params) => {
+                return params.data?.outputs[0].latency == undefined
+                    ? "-"
+                    : formatLatency(params.data.outputs[0].latency)
+            },
+        })
         return colDefs
     }, [evalaution, scenarios, showDiff])
 

agenta-web/src/lib/Types.ts

@@ -363,6 +363,8 @@ type ValueTypeOptions =
     | "regex"
     | "object"
     | "error"
+    | "cost"
+    | "latency"
 
 //evaluation revamp types
 export interface EvaluationSettingsTemplate {
@@ -442,6 +444,8 @@ export interface _Evaluation {
     updated_at?: string
     duration?: number
     revisions: string[]
+    average_latency?: TypedValue & {error: null | EvaluationError}
+    average_cost?: TypedValue & {error: null | EvaluationError}
     variant_revision_ids: string[]
 }
 
@@ -451,7 +455,7 @@ export interface _EvaluationScenario {
     evaluation: _Evaluation
     evaluators_configs: EvaluatorConfig[]
     inputs: (TypedValue & {name: string})[]
-    outputs: {result: TypedValue}[]
+    outputs: {result: TypedValue; cost?: number; latency?: number}[]
     correct_answer?: string
     is_pinned?: boolean
     note?: string
@@ -487,7 +491,7 @@ export type ComparisonResultRow = {
     variants: {
         variantId: string
         variantName: string
-        output: {result: TypedValue}
+        output: {result: TypedValue; cost?: number; latency?: number}
         evaluationId: string
         evaluatorConfigs: {
             evaluatorConfig: EvaluatorConfig