Merge branch 'gh/vision-gpt-llm-app-temp' of https://github.com/Agent…

…a-AI/agenta into feat/file-input-support-playground
Agenta-AI · Dec 1, 2023 · 9caa502 · 9caa502
2 parents cc39449 + 521d2e8
commit 9caa502
Show file tree

Hide file tree

Showing 14 changed files with 128 additions and 27 deletions.
diff --git a/.all-contributorsrc b/.all-contributorsrc
@@ -345,6 +345,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "brockWith",
+      "name": "brockWith",
+      "avatar_url": "https://avatars.githubusercontent.com/u/105627491?v=4",
+      "profile": "https://github.com/brockWith",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

diff --git a/README.md b/README.md
@@ -231,7 +231,7 @@ Check out our [Contributing Guide](https://docs.agenta.ai/contributing/getting-s
 ## Contributors ✨
 
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-36-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-37-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
@@ -288,6 +288,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     </tr>
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://diegolikescode.me/"><img src="https://avatars.githubusercontent.com/u/57499868?v=4?s=100" width="100px;" alt="diego"/><br /><sub><b>diego</b></sub></a><br /><a href="https://github.com/Agenta-AI/agenta/commits?author=diegolikescode" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/brockWith"><img src="https://avatars.githubusercontent.com/u/105627491?v=4?s=100" width="100px;" alt="brockWith"/><br /><sub><b>brockWith</b></sub></a><br /><a href="https://github.com/Agenta-AI/agenta/commits?author=brockWith" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

diff --git a/agenta-web/prod.Dockerfile b/agenta-web/prod.Dockerfile
@@ -12,8 +12,8 @@ COPY public ./public
 COPY next.config.js .
 COPY tsconfig.json .
 COPY postcss.config.js .
-COPY .env .
-# used in cloud 
+COPY .env.production .
+# used in cloud
 COPY sentry.* .
 # Build the Next.js app for production
 RUN npm run build

diff --git a/agenta-web/src/components/EvaluationTable/AICritiqueEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/AICritiqueEvaluationTable.tsx
@@ -31,7 +31,7 @@ import {createUseStyles} from "react-jss"
 import {exportAICritiqueEvaluationData} from "@/lib/helpers/evaluate"
 import SecondaryButton from "../SecondaryButton/SecondaryButton"
 import {useAppTheme} from "../Layout/ThemeContextProvider"
-import {testsetRowToChatMessages} from "@/lib/helpers/testset"
+import {contentToChatMessageString, testsetRowToChatMessages} from "@/lib/helpers/testset"
 
 const {Title} = Typography
 
@@ -257,6 +257,7 @@ Answer ONLY with one of the given grading or evaluation options.
                     ? testsetRowToChatMessages(evaluation.testset.csvdata[rowIndex], false)
                     : [],
             )
+            if (variantData[idx].isChatVariant) result = contentToChatMessageString(result)
 
             setRowValue(rowIndex, columnName as any, result)
             await evaluate(rowIndex)

diff --git a/agenta-web/src/components/EvaluationTable/CustomCodeRunEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/CustomCodeRunEvaluationTable.tsx
@@ -35,7 +35,7 @@ import {createUseStyles} from "react-jss"
 import SecondaryButton from "../SecondaryButton/SecondaryButton"
 import {exportCustomCodeEvaluationData} from "@/lib/helpers/evaluate"
 import CodeBlock from "../DynamicCodeBlock/CodeBlock"
-import {testsetRowToChatMessages} from "@/lib/helpers/testset"
+import {contentToChatMessageString, testsetRowToChatMessages} from "@/lib/helpers/testset"
 
 const {Title} = Typography
 
@@ -251,6 +251,7 @@ const CustomCodeRunEvaluationTable: React.FC<CustomCodeEvaluationTableProps> = (
                     ? testsetRowToChatMessages(evaluation.testset.csvdata[rowIndex], false)
                     : [],
             )
+            if (variantData[idx].isChatVariant) result = contentToChatMessageString(result)
 
             setRowValue(rowIndex, columnName as any, result)
             await evaluate(rowIndex)

diff --git a/agenta-web/src/components/EvaluationTable/ExactMatchEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/ExactMatchEvaluationTable.tsx
@@ -28,7 +28,7 @@ import {evaluateWithExactMatch} from "@/lib/services/evaluations"
 import {createUseStyles} from "react-jss"
 import {exportExactEvaluationData} from "@/lib/helpers/evaluate"
 import SecondaryButton from "../SecondaryButton/SecondaryButton"
-import {testsetRowToChatMessages} from "@/lib/helpers/testset"
+import {contentToChatMessageString, testsetRowToChatMessages} from "@/lib/helpers/testset"
 import {Evaluation} from "@/lib/Types"
 
 const {Title} = Typography
@@ -198,6 +198,7 @@ const ExactMatchEvaluationTable: React.FC<ExactMatchEvaluationTableProps> = ({
                         ? testsetRowToChatMessages(evaluation.testset.csvdata[rowIndex], false)
                         : [],
                 )
+                if (variantData[idx].isChatVariant) result = contentToChatMessageString(result)
 
                 setRowValue(rowIndex, columnName, result)
                 setRowValue(rowIndex, "evaluationFlow", EvaluationFlow.COMPARISON_RUN_STARTED)

diff --git a/agenta-web/src/components/EvaluationTable/RegexEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/RegexEvaluationTable.tsx
@@ -29,7 +29,7 @@ import {globalErrorHandler} from "@/lib/helpers/errorHandler"
 import SecondaryButton from "../SecondaryButton/SecondaryButton"
 import {exportRegexEvaluationData} from "@/lib/helpers/evaluate"
 import {isValidRegex} from "@/lib/helpers/validators"
-import {testsetRowToChatMessages} from "@/lib/helpers/testset"
+import {contentToChatMessageString, testsetRowToChatMessages} from "@/lib/helpers/testset"
 
 const {Title} = Typography
 
@@ -227,6 +227,7 @@ const RegexEvaluationTable: React.FC<RegexEvaluationTableProps> = ({
                         ? testsetRowToChatMessages(evaluation.testset.csvdata[rowIndex], false)
                         : [],
                 )
+                if (variantData[idx].isChatVariant) result = contentToChatMessageString(result)
 
                 const {regexPattern, regexShouldMatch} = form.getFieldsValue()
                 const isCorrect = evaluateWithRegex(result, regexPattern, regexShouldMatch)

diff --git a/agenta-web/src/components/EvaluationTable/SimilarityMatchEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/SimilarityMatchEvaluationTable.tsx
@@ -25,7 +25,7 @@ import {Typography} from "antd"
 import {createUseStyles} from "react-jss"
 import {exportSimilarityEvaluationData} from "@/lib/helpers/evaluate"
 import SecondaryButton from "../SecondaryButton/SecondaryButton"
-import {testsetRowToChatMessages} from "@/lib/helpers/testset"
+import {contentToChatMessageString, testsetRowToChatMessages} from "@/lib/helpers/testset"
 
 const {Title} = Typography
 
@@ -223,6 +223,7 @@ const SimilarityMatchEvaluationTable: React.FC<SimilarityMatchEvaluationTablePro
                         ? testsetRowToChatMessages(evaluation.testset.csvdata[rowIndex], false)
                         : [],
                 )
+                if (variantData[idx].isChatVariant) result = contentToChatMessageString(result)
 
                 const {similarityThreshold} = form.getFieldsValue()
                 const similarity = evaluateWithSimilarityMatch(result, rows[rowIndex].correctAnswer)

diff --git a/agenta-web/src/components/EvaluationTable/WebhookEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/WebhookEvaluationTable.tsx
@@ -28,7 +28,7 @@ import {globalErrorHandler} from "@/lib/helpers/errorHandler"
 import {isValidUrl} from "@/lib/helpers/validators"
 import SecondaryButton from "../SecondaryButton/SecondaryButton"
 import {exportWebhookEvaluationData} from "@/lib/helpers/evaluate"
-import {testsetRowToChatMessages} from "@/lib/helpers/testset"
+import {contentToChatMessageString, testsetRowToChatMessages} from "@/lib/helpers/testset"
 
 const {Title} = Typography
 
@@ -206,6 +206,7 @@ const WebhookEvaluationTable: React.FC<WebhookEvaluationTableProps> = ({
                         ? testsetRowToChatMessages(evaluation.testset.csvdata[rowIndex], false)
                         : [],
                 )
+                if (variantData[idx].isChatVariant) result = contentToChatMessageString(result)
 
                 const {webhookUrl} = form.getFieldsValue()
                 const score = await evaluateWithWebhook(webhookUrl, {

diff --git a/agenta-web/src/lib/helpers/axiosConfig.ts b/agenta-web/src/lib/helpers/axiosConfig.ts
@@ -1,5 +1,7 @@
 import axiosApi from "axios"
 import {getErrorMessage, globalErrorHandler} from "./errorHandler"
+import {signOut} from "supertokens-auth-react/recipe/thirdpartypasswordless"
+import router from "next/router"
 
 const axios = axiosApi.create({
     headers: {
@@ -20,6 +22,14 @@ axios.interceptors.response.use(
             }`
         error.message = msg
 
+        if (error.response?.status === 401) {
+            signOut()
+                .then(() => {
+                    router.push("/auth")
+                })
+                .catch(console.error)
+        }
+
         globalErrorHandler(error)
 
         throw error

diff --git a/agenta-web/src/lib/helpers/evaluate.ts b/agenta-web/src/lib/helpers/evaluate.ts
@@ -2,10 +2,12 @@ import {HumanEvaluationListTableDataType} from "@/components/Evaluations/HumanEv
 import {Evaluation, GenericObject, Variant} from "../Types"
 import {convertToCsv, downloadCsv} from "./utils"
 
-export const exportExactEvaluationData = (evaluation: any, rows: any[]) => {
-    const exportRow = rows.map((data) => {
+export const exportExactEvaluationData = (evaluation: Evaluation, rows: GenericObject[]) => {
+    const exportRow = rows.map((data, ix) => {
         return {
-            ["Inputs"]: data.inputs[0].input_value,
+            ["Inputs"]:
+                evaluation.testset.csvdata[ix]?.[evaluation.testset.testsetChatColumn] ||
+                data.inputs[0].input_value,
             [`App Variant ${evaluation.variants[0].variantName} Output`]: data?.columnData0
                 ? data?.columnData0
                 : data.outputs[0]?.variant_output,
@@ -20,10 +22,12 @@ export const exportExactEvaluationData = (evaluation: any, rows: any[]) => {
     downloadCsv(csvData, filename)
 }
 
-export const exportSimilarityEvaluationData = (evaluation: any, rows: any[]) => {
-    const exportRow = rows.map((data) => {
+export const exportSimilarityEvaluationData = (evaluation: Evaluation, rows: GenericObject[]) => {
+    const exportRow = rows.map((data, ix) => {
         return {
-            ["Inputs"]: data.inputs[0].input_value,
+            ["Inputs"]:
+                evaluation.testset.csvdata[ix]?.[evaluation.testset.testsetChatColumn] ||
+                data.inputs[0].input_value,
             [`App Variant ${evaluation.variants[0].variantName} Output`]: data?.columnData0
                 ? data?.columnData0
                 : data.outputs[0]?.variant_output,
@@ -39,10 +43,12 @@ export const exportSimilarityEvaluationData = (evaluation: any, rows: any[]) =>
     downloadCsv(csvData, filename)
 }
 
-export const exportAICritiqueEvaluationData = (evaluation: any, rows: any[]) => {
-    const exportRow = rows.map((data) => {
+export const exportAICritiqueEvaluationData = (evaluation: Evaluation, rows: GenericObject[]) => {
+    const exportRow = rows.map((data, ix) => {
         return {
-            ["Inputs"]: data.inputs[0].input_value,
+            ["Inputs"]:
+                evaluation.testset.csvdata[ix]?.[evaluation.testset.testsetChatColumn] ||
+                data.inputs[0].input_value,
             [`App Variant ${evaluation.variants[0].variantName} Output`]: data?.columnData0
                 ? data?.columnData0
                 : data.outputs[0]?.variant_output,
@@ -101,13 +107,19 @@ export const exportSingleModelEvaluationData = (evaluation: Evaluation, rows: Ge
     downloadCsv(csvData, filename)
 }
 
-export const exportRegexEvaluationData = (evaluation: any, rows: any[], settings: any) => {
-    const exportRow = rows.map((data) => {
+export const exportRegexEvaluationData = (
+    evaluation: Evaluation,
+    rows: GenericObject[],
+    settings: GenericObject,
+) => {
+    const exportRow = rows.map((data, ix) => {
         const isCorrect = data.score === "correct"
         const isMatch = settings.regexShouldMatch ? isCorrect : !isCorrect
 
         return {
-            ["Inputs"]: data.inputs[0].input_value,
+            ["Inputs"]:
+                evaluation.testset.csvdata[ix]?.[evaluation.testset.testsetChatColumn] ||
+                data.inputs[0].input_value,
             [`App Variant ${evaluation.variants[0].variantName} Output`]: data?.columnData0
                 ? data?.columnData0
                 : data.outputs[0]?.variant_output,
@@ -122,10 +134,12 @@ export const exportRegexEvaluationData = (evaluation: any, rows: any[], settings
     downloadCsv(csvData, filename)
 }
 
-export const exportWebhookEvaluationData = (evaluation: any, rows: any[]) => {
-    const exportRow = rows.map((data) => {
+export const exportWebhookEvaluationData = (evaluation: Evaluation, rows: GenericObject[]) => {
+    const exportRow = rows.map((data, ix) => {
         return {
-            ["Inputs"]: data.inputs[0].input_value,
+            ["Inputs"]:
+                evaluation.testset.csvdata[ix]?.[evaluation.testset.testsetChatColumn] ||
+                data.inputs[0].input_value,
             [`App Variant ${evaluation.variants[0].variantName} Output`]: data?.columnData0
                 ? data?.columnData0
                 : data.outputs[0]?.variant_output,
@@ -140,10 +154,12 @@ export const exportWebhookEvaluationData = (evaluation: any, rows: any[]) => {
     downloadCsv(csvData, filename)
 }
 
-export const exportCustomCodeEvaluationData = (evaluation: any, rows: any[]) => {
-    const exportRow = rows.map((data) => {
+export const exportCustomCodeEvaluationData = (evaluation: Evaluation, rows: GenericObject[]) => {
+    const exportRow = rows.map((data, ix) => {
         return {
-            ["Inputs"]: data.inputs[0].input_value,
+            ["Inputs"]:
+                evaluation.testset.csvdata[ix]?.[evaluation.testset.testsetChatColumn] ||
+                data.inputs[0].input_value,
             [`App Variant ${evaluation.variants[0].variantName} Output`]: data?.columnData0
                 ? data?.columnData0
                 : data.outputs[0]?.variant_output,

diff --git a/agenta-web/src/lib/helpers/testset.ts b/agenta-web/src/lib/helpers/testset.ts
@@ -52,3 +52,10 @@ export function testsetRowToChatMessages(rowData: KeyValuePair, includeCorrectAn
 
     return chat
 }
+
+export function contentToChatMessageString(content: string, role: ChatRole = ChatRole.Assistant) {
+    return JSON.stringify({
+        content,
+        role,
+    })
+}
diff --git a/examples/vision_gpt_explain_image/app.py b/examples/vision_gpt_explain_image/app.py
@@ -0,0 +1,50 @@
+import agenta as ag
+from openai import OpenAI
+
+
+client = OpenAI()
+
+
+SYSTEM_PROMPT = "You are an expert in reading images you look into details, you answer in accurate language."
+HUMAN_PROMPT = "Please compare two images"
+
+ag.init(app_name="explain_image", base_name="app")
+ag.config.default(
+    temperature=ag.FloatParam(0.5, 0, 1),
+    max_tokens=ag.IntParam(300, 1, 4000),
+    prompt_system=ag.TextParam(SYSTEM_PROMPT),
+    prompt_human=ag.TextParam(HUMAN_PROMPT),
+)
+
+
+@ag.entrypoint
+def explain(
+    image_one: ag.FileInputURL,
+    image_two: ag.FileInputURL,
+) -> str:
+    messages = [{"role": "system", "content": ag.config.prompt_system}] + [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": ag.config.prompt_human},
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": image_one,
+                    },
+                },
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": image_two,
+                    },
+                },
+            ],
+        }
+    ]
+    chat_completion = client.chat.completions.create(
+        model="gpt-4-vision-preview",
+        messages=messages,
+        max_tokens=max_tokens,
+    )
+    return chat_completion.choices[0].message.content
diff --git a/examples/vision_gpt_explain_image/requirements.txt b/examples/vision_gpt_explain_image/requirements.txt
@@ -0,0 +1,2 @@
+agenta
+openai