Merge pull request #1155 from Agenta-AI/bugfix/evaluation_new_sdk

Make new SDK work with evaluation
Agenta-AI · Jan 5, 2024 · e9b5fad · e9b5fad
2 parents a294424 + e8caf7e
commit e9b5fad
Show file tree

Hide file tree

Showing 10 changed files with 48 additions and 11 deletions.
diff --git a/agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx
@@ -21,6 +21,7 @@ import {testsetRowToChatMessages} from "@/lib/helpers/testset"
 import EvaluationVotePanel from "../Evaluations/EvaluationCardView/EvaluationVotePanel"
 import VariantAlphabet from "../Evaluations/EvaluationCardView/VariantAlphabet"
 import {ParamsFormWithRun} from "./SingleModelEvaluationTable"
+import {PassThrough} from "stream"
 
 const {Title} = Typography
 
@@ -238,6 +239,9 @@ const ABTestingEvaluationTable: React.FC<EvaluationTableProps> = ({
                             ? testsetRowToChatMessages(evaluation.testset.csvdata[rowIndex], false)
                             : [],
                     )
+                    if (typeof result !== "string") {
+                        result = result.message
+                    }
 
                     setRowValue(rowIndex, variant.variantId, result)
                     ;(outputs as KeyValuePair)[variant.variantId] = result

diff --git a/agenta-web/src/components/EvaluationTable/AICritiqueEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/AICritiqueEvaluationTable.tsx
@@ -271,7 +271,9 @@ Answer ONLY with one of the given grading or evaluation options.
                         ? testsetRowToChatMessages(evaluation.testset.csvdata[rowIndex], false)
                         : [],
                 )
-
+                if (typeof result !== "string") {
+                    result = result.message
+                }
                 if (variantData[idx].isChatVariant) {
                     result = contentToChatMessageString(result)
                 }

diff --git a/agenta-web/src/components/EvaluationTable/CustomCodeRunEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/CustomCodeRunEvaluationTable.tsx
@@ -249,6 +249,10 @@ const CustomCodeRunEvaluationTable: React.FC<CustomCodeEvaluationTableProps> = (
                     ? testsetRowToChatMessages(evaluation.testset.csvdata[rowIndex], false)
                     : [],
             )
+            if (typeof result !== "string") {
+                result = result.message
+            }
+
             if (variantData[idx].isChatVariant) result = contentToChatMessageString(result)
 
             setRowValue(rowIndex, columnName as any, result)

diff --git a/agenta-web/src/components/EvaluationTable/ExactMatchEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/ExactMatchEvaluationTable.tsx
@@ -192,6 +192,10 @@ const ExactMatchEvaluationTable: React.FC<ExactMatchEvaluationTableProps> = ({
                         ? testsetRowToChatMessages(evaluation.testset.csvdata[rowIndex], false)
                         : [],
                 )
+                if (typeof result !== "string") {
+                    result = result.message
+                }
+
                 if (variantData[idx].isChatVariant) result = contentToChatMessageString(result)
 
                 setRowValue(rowIndex, columnName, result)

diff --git a/agenta-web/src/components/EvaluationTable/RegexEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/RegexEvaluationTable.tsx
@@ -220,6 +220,10 @@ const RegexEvaluationTable: React.FC<RegexEvaluationTableProps> = ({
                         ? testsetRowToChatMessages(evaluation.testset.csvdata[rowIndex], false)
                         : [],
                 )
+                if (typeof result !== "string") {
+                    result = result.message
+                }
+
                 if (variantData[idx].isChatVariant) result = contentToChatMessageString(result)
 
                 const {regexPattern, regexShouldMatch} = form.getFieldsValue()

diff --git a/agenta-web/src/components/EvaluationTable/SimilarityMatchEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/SimilarityMatchEvaluationTable.tsx
@@ -215,6 +215,10 @@ const SimilarityMatchEvaluationTable: React.FC<SimilarityMatchEvaluationTablePro
                         ? testsetRowToChatMessages(evaluation.testset.csvdata[rowIndex], false)
                         : [],
                 )
+                if (typeof result !== "string") {
+                    result = result.message
+                }
+
                 if (variantData[idx].isChatVariant) result = contentToChatMessageString(result)
 
                 const {similarityThreshold} = form.getFieldsValue()

diff --git a/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx
@@ -298,6 +298,9 @@ const SingleModelEvaluationTable: React.FC<EvaluationTableProps> = ({
                             ? testsetRowToChatMessages(evaluation.testset.csvdata[rowIndex], false)
                             : [],
                     )
+                    if (typeof result !== "string") {
+                        result = result.message
+                    }
 
                     setRowValue(rowIndex, variant.variantId, result)
                     ;(outputs as KeyValuePair)[variant.variantId] = result

diff --git a/agenta-web/src/components/EvaluationTable/WebhookEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/WebhookEvaluationTable.tsx
@@ -199,6 +199,10 @@ const WebhookEvaluationTable: React.FC<WebhookEvaluationTableProps> = ({
                         ? testsetRowToChatMessages(evaluation.testset.csvdata[rowIndex], false)
                         : [],
                 )
+                if (typeof result !== "string") {
+                    result = result.message
+                }
+
                 if (variantData[idx].isChatVariant) result = contentToChatMessageString(result)
 
                 const {webhookUrl} = form.getFieldsValue()

diff --git a/agenta-web/src/components/Playground/Views/TestView.tsx b/agenta-web/src/components/Playground/Views/TestView.tsx
@@ -166,7 +166,7 @@ const BoxComponent: React.FC<BoxComponentProps> = ({
                     imageSize="large"
                 />
             </Row>
-            {additionalData && (
+            {additionalData?.cost || additionalData?.latency ? (
                 <Space>
                     <p>
                         Tokens:{" "}
@@ -187,6 +187,8 @@ const BoxComponent: React.FC<BoxComponentProps> = ({
                             : "0ms"}
                     </p>
                 </Space>
+            ) : (
+                ""
             )}
             <Row className={classes.row2} style={{marginBottom: isChatVariant ? 12 : 0}}>
                 <Col span={24} className={classes.row2Col} id={variant.variantId}>
@@ -340,13 +342,17 @@ const App: React.FC<TestViewProps> = ({
                 variant.baseId || "",
                 isChatVariant ? testItem.chat : [],
             )
-
-            setResultForIndex(res.message, index)
-            setAdditionalDataList((prev) => {
-                const newDataList = [...prev]
-                newDataList[index] = {cost: res.cost, latency: res.latency, usage: res.usage}
-                return newDataList
-            })
+            // check if res is an object or string
+            if (typeof res === "string") {
+                setResultForIndex(res, index)
+            } else {
+                setResultForIndex(res.message, index)
+                setAdditionalDataList((prev) => {
+                    const newDataList = [...prev]
+                    newDataList[index] = {cost: res.cost, latency: res.latency, usage: res.usage}
+                    return newDataList
+                })
+            }
         } catch (e) {
             setResultForIndex(
                 "The code has resulted in the following error: \n\n --------------------- \n" +

diff --git a/agenta-web/src/lib/services/api.ts b/agenta-web/src/lib/services/api.ts
@@ -72,8 +72,10 @@ export function restartAppVariantContainer(variantId: string) {
  * @param inputParametersDict A dictionary of the input parameters to be passed to the variant endpoint
  * @param inputParamDefinition A list of the parameters that are defined in the openapi.json (these are only part of the input params, the rest is defined by the user in the optparms)
  * @param optionalParameters The optional parameters (prompt, models, AND DICTINPUTS WHICH ARE TO BE USED TO ADD INPUTS )
- * @param URIPath
- * @returns
+ * @param appId - The ID of the app.
+ * @param baseId - The base ID.
+ * @param chatMessages - An optional array of chat messages.
+ * @returns A Promise that resolves with the response data from the POST request.
  */
 export async function callVariant(
     inputParametersDict: KeyValuePair,