Latency cost in eval

agenta-backend/agenta_backend/models/api/evaluation_model.py

@@ -71,6 +71,8 @@ class Evaluation(BaseModel):
     testset_name: Optional[str]
     status: Result
     aggregated_results: List[AggregatedResult]
+    average_cost: Optional[Result]
+    average_latency: Optional[Result]
     created_at: datetime
     updated_at: datetime
 
@@ -100,6 +102,8 @@ class EvaluationScenarioInput(BaseModel):
 
 class EvaluationScenarioOutput(BaseModel):
     result: Result
+    cost: Optional[float]
+    latency: Optional[float]
 
 
 class HumanEvaluationScenarioInput(BaseModel):

agenta-backend/agenta_backend/models/converters.py

@@ -145,6 +145,8 @@ async def evaluation_db_to_pydantic(
         aggregated_results=aggregated_results,
         created_at=evaluation_db.created_at,
         updated_at=evaluation_db.updated_at,
+        average_cost=evaluation_db.average_cost,
+        average_latency=evaluation_db.average_latency,
     )
 
 

agenta-backend/agenta_backend/models/db_models.py

@@ -193,6 +193,8 @@ class Result(BaseModel):
 
 class InvokationResult(BaseModel):
     result: Result
+    cost: Optional[float] = None
+    latency: Optional[float] = None
 
 
 class EvaluationScenarioResult(BaseModel):
@@ -213,6 +215,8 @@ class EvaluationScenarioInputDB(BaseModel):
 
 class EvaluationScenarioOutputDB(BaseModel):
     result: Result
+    cost: Optional[float] = None
+    latency: Optional[float] = None
 
 
 class HumanEvaluationScenarioInput(BaseModel):
@@ -266,6 +270,8 @@ class EvaluationDB(Document):
     variant_revision: PydanticObjectId
     evaluators_configs: List[PydanticObjectId]
     aggregated_results: List[AggregatedResult]
+    average_cost: Optional[Result] = None
+    average_latency: Optional[Result] = None
     created_at: datetime = Field(default=datetime.now())
     updated_at: datetime = Field(default=datetime.now())
 
@@ -284,6 +290,8 @@ class EvaluationScenarioDB(Document):
     note: Optional[str]
     evaluators_configs: List[PydanticObjectId]
     results: List[EvaluationScenarioResult]
+    latency: Optional[int] = None
+    cost: Optional[int] = None
     created_at: datetime = Field(default=datetime.now())
     updated_at: datetime = Field(default=datetime.now())
 

agenta-backend/agenta_backend/services/aggregation_service.py

@@ -1,8 +1,8 @@
 import re
 import traceback
-from typing import List
+from typing import List, Optional
 
-from agenta_backend.models.db_models import Result, Error
+from agenta_backend.models.db_models import InvokationResult, Result, Error
 
 
 def aggregate_ai_critique(results: List[Result]) -> Result:
@@ -73,3 +73,29 @@ def aggregate_float(results: List[Result]) -> Result:
             value=None,
             error=Error(message=str(exc), stacktrace=str(traceback.format_exc())),
         )
+
+
+def aggregate_float_from_llm_app_response(
+    invocation_results: List[InvokationResult], key: Optional[str]
+) -> Result:
+    try:
+        if not key:
+            raise ValueError("Key is required to aggregate InvokationResult objects.")
+
+        values = [
+            getattr(inv_result, key)
+            for inv_result in invocation_results
+            if hasattr(inv_result, key) and getattr(inv_result, key) is not None
+        ]
+
+        if not values:
+            raise ValueError(f"No valid values found for {key} aggregation.")
+
+        average_value = sum(values) / len(values)
+        return Result(type=key, value=average_value)
+    except Exception as exc:
+        return Result(
+            type="error",
+            value=None,
+            error=Error(message=str(exc), stacktrace=str(traceback.format_exc())),
+        )

agenta-backend/agenta_backend/services/llm_apps_service.py

@@ -86,9 +86,15 @@ async def invoke_app(
                 url, json=payload, timeout=httpx.Timeout(timeout=5, read=None, write=5)
             )
             response.raise_for_status()
-            app_output = response.json()
+            app_response = response.json()
             return InvokationResult(
-                result=Result(type="text", value=app_output["message"], error=None)
+                result=Result(
+                    type="text",
+                    value=app_response["message"],
+                    error=None,
+                ),
+                latency=app_response["latency"],
+                cost=app_response["cost"],
             )
 
         except httpx.HTTPStatusError as e:

agenta-backend/agenta_backend/tasks/evaluations.py

@@ -218,9 +218,14 @@ def evaluate(
             for evaluator_config_db in evaluator_config_dbs:
                 logger.debug(f"Evaluating with evaluator: {evaluator_config_db}")
                 if correct_answer_column in data_point:
+                    output_value = (
+                        app_output.result.value["output"]
+                        if isinstance(app_output.result.value, dict)
+                        else app_output.result.value
+                    )
                     result = evaluators_service.evaluate(
                         evaluator_key=evaluator_config_db.evaluator_key,
-                        output=app_output.result.value,
+                        output=output_value,
                         correct_answer=data_point[correct_answer_column],
                         settings_values=evaluator_config_db.settings_values,
                         app_params=app_variant_parameters,
@@ -267,7 +272,9 @@ def evaluate(
                     correct_answer=correct_answer,
                     outputs=[
                         EvaluationScenarioOutputDB(
-                            result=Result(type="text", value=app_output.result.value)
+                            result=Result(type="text", value=app_output.result.value),
+                            latency=app_output.latency,
+                            cost=app_output.cost,
                         )
                     ],
                     results=evaluators_results,
@@ -276,6 +283,20 @@ def evaluate(
                 )
             )
 
+        # Add average cost and latency
+        average_latency = aggregation_service.aggregate_float_from_llm_app_response(
+            app_outputs, "latency"
+        )
+        average_cost = aggregation_service.aggregate_float_from_llm_app_response(
+            app_outputs, "cost"
+        )
+        loop.run_until_complete(
+            update_evaluation(
+                evaluation_id,
+                {"average_latency": average_latency, "average_cost": average_cost},
+            )
+        )
+
     except Exception as e:
         logger.error(f"An error occurred during evaluation: {e}")
         traceback.print_exc()

agenta-web/src/components/pages/evaluations/evaluationResults/EvaluationResults.tsx

@@ -246,6 +246,22 @@ const EvaluationResults: React.FC<Props> = () => {
                     statusMapper(token)[params.data?.status.value as EvaluationStatus].label,
                 cellRenderer: StatusRenderer,
             },
+            {
+                flex: 1,
+                field: "average_latency",
+                headerName: "Latency",
+                minWidth: 120,
+                ...getFilterParams("number"),
+                valueGetter: (params) => getTypedValue(params?.data?.average_latency),
+            },
+            {
+                flex: 1,
+                field: "average_cost",
+                headerName: "Cost",
+                minWidth: 120,
+                ...getFilterParams("number"),
+                valueGetter: (params) => getTypedValue(params?.data?.average_cost),
+            },
             {
                 flex: 1,
                 field: "created_at",

agenta-web/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx

@@ -23,6 +23,7 @@ import {useAtom} from "jotai"
 import {evaluatorsAtom} from "@/lib/atoms/evaluation"
 import CompareOutputDiff from "@/components/CompareOutputDiff/CompareOutputDiff"
 import {useQueryParam} from "@/hooks/useQuery"
+import {formatCurrency, formatLatency} from "@/lib/helpers/formatters"
 
 const useStyles = createUseStyles((theme: JSSTheme) => ({
     infoRow: {
@@ -139,6 +140,29 @@ const EvaluationScenarios: React.FC<Props> = () => {
                 },
             })
         })
+        colDefs.push({
+            flex: 1,
+            minWidth: 120,
+            headerName: "Cost",
+            ...getFilterParams("text"),
+            valueGetter: (params) => {
+                return params.data?.outputs[0].cost == undefined
+                    ? "-"
+                    : formatCurrency(params.data.outputs[0].cost)
+            },
+        })
+
+        colDefs.push({
+            flex: 1,
+            minWidth: 120,
+            headerName: "Latency",
+            ...getFilterParams("text"),
+            valueGetter: (params) => {
+                return params.data?.outputs[0].latency == undefined
+                    ? "-"
+                    : formatLatency(params.data.outputs[0].latency)
+            },
+        })
         return colDefs
     }, [evalaution, scenarios, showDiff])
 

agenta-web/src/lib/Types.ts

@@ -363,6 +363,8 @@ type ValueTypeOptions =
     | "regex"
     | "object"
     | "error"
+    | "cost"
+    | "latency"
 
 //evaluation revamp types
 export interface EvaluationSettingsTemplate {
@@ -442,6 +444,8 @@ export interface _Evaluation {
     updated_at?: string
     duration?: number
     revisions: string[]
+    average_latency?: TypedValue & {error: null | EvaluationError}
+    average_cost?: TypedValue & {error: null | EvaluationError}
     variant_revision_ids: string[]
 }
 
@@ -451,7 +455,7 @@ export interface _EvaluationScenario {
     evaluation: _Evaluation
     evaluators_configs: EvaluatorConfig[]
     inputs: (TypedValue & {name: string})[]
-    outputs: {result: TypedValue}[]
+    outputs: {result: TypedValue; cost?: number; latency?: number}[]
     correct_answer?: string
     is_pinned?: boolean
     note?: string

agenta-web/src/lib/helpers/dateTimeHelper.ts

@@ -3,3 +3,7 @@ import dayjs from "dayjs"
 export const formatDate = (date: dayjs.ConfigType): string => {
     return dayjs(date).format("DD MMM YYYY | h:m a")
 }
+
+export const formatDate24 = (date: dayjs.ConfigType, includeSeconds = false): string => {
+    return dayjs(date).format("DD MMM YY, HH:mm" + (includeSeconds ? ":ss" : ""))
+}

agenta-web/src/lib/helpers/evaluate.ts

@@ -14,6 +14,7 @@ import AlertPopup from "@/components/AlertPopup/AlertPopup"
 import {capitalize, round} from "lodash"
 import dayjs from "dayjs"
 import {runningStatuses} from "@/components/pages/evaluations/cellRenderers/cellRenderers"
+import {formatCurrency, formatLatency} from "./formatters"
 
 export const exportExactEvaluationData = (evaluation: Evaluation, rows: GenericObject[]) => {
     const exportRow = rows.map((data, ix) => {
@@ -269,11 +270,19 @@ export function getTypedValue(res?: TypedValue) {
 
     if (value === undefined) return "-"
 
-    return type === "number"
-        ? round(Number(value), 2)
-        : ["boolean", "bool"].includes(type as string)
-          ? capitalize(value?.toString())
-          : value?.toString()
+    switch (type) {
+        case "number":
+            return round(Number(value), 2)
+        case "boolean":
+        case "bool":
+            return capitalize(value?.toString())
+        case "cost":
+            return formatCurrency(Number(value))
+        case "latency":
+            return formatLatency(Number(value))
+        default:
+            return value?.toString()
+    }
 }
 
 type CellDataType = "number" | "text" | "date"

agenta-web/src/lib/helpers/formatters.ts

@@ -0,0 +1,21 @@
+const intlNumber = new Intl.NumberFormat("en-US", {
+    maximumFractionDigits: 2,
+})
+
+const intlCurrency = new Intl.NumberFormat("en-US", {
+    style: "currency",
+    currency: "USD",
+    maximumFractionDigits: 4,
+})
+
+export const formatNumber = (value = 0) => {
+    return intlNumber.format(value)
+}
+
+export const formatCurrency = (value = 0) => {
+    return intlCurrency.format(value)
+}
+
+export const formatLatency = (value = 0) => {
+    return `${intlNumber.format(value / 1000)}s`
+}

agenta-web/src/services/evaluations/index.ts

@@ -104,6 +104,8 @@ const evaluationTransformer = (item: any) => ({
     revisions: item.revisions,
     variant_revision_ids: item.variant_revision_ids,
     variant_ids: item.variant_ids,
+    average_cost: item.average_cost,
+    average_latency: item.average_latency,
 })
 export const fetchAllEvaluations = async (appId: string) => {
     const response = await axios.get(`/api/evaluations/`, {params: {app_id: appId}})