feat: multiple LLMs keys implemented

.gitignore

@@ -54,4 +54,5 @@ docker-compose.cloud.prod.yml
 agenta-web/src/pages/auth/[[...path]].tsx
 agenta-web/cypress/screenshots/
 agenta-web/cypress/videos/
-.nextjs_cache/
+.nextjs_cache/
+

agenta-web/src/components/AppSelector/AppSelector.tsx

@@ -8,7 +8,7 @@ import {useAppTheme} from "../Layout/ThemeContextProvider"
 import {CloseCircleFilled} from "@ant-design/icons"
 import TipsAndFeatures from "./TipsAndFeatures"
 import Welcome from "./Welcome"
-import {getOpenAIKey, isAppNameInputValid, isDemo} from "@/lib/helpers/utils"
+import {getLlmProviderKey, isAppNameInputValid, isDemo} from "@/lib/helpers/utils"
 import {
     createAndStartTemplate,
     getTemplates,
@@ -178,7 +178,8 @@ const AppSelector: React.FC = () => {
         handleCreateAppModalCancel()
 
         // warn the user and redirect if openAI key is not present
-        const openAIKey = getOpenAIKey()
+        // TODO: must be changed for multiples LLM keys
+        const openAIKey = getLlmProviderKey("OpenAI")
         if (!openAIKey && !isDemo()) {
             notification.error({
                 message: "OpenAI API Key Missing",

agenta-web/src/components/EvaluationTable/AICritiqueEvaluationTable.tsx

@@ -26,7 +26,7 @@ import {
 import {useVariants} from "@/lib/hooks/useVariant"
 import {useRouter} from "next/router"
 import {EvaluationFlow, EvaluationType} from "@/lib/enums"
-import {getOpenAIKey} from "@/lib/helpers/utils"
+import {getLlmProviderKey} from "@/lib/helpers/utils"
 import {createUseStyles} from "react-jss"
 import {exportAICritiqueEvaluationData} from "@/lib/helpers/evaluate"
 import SecondaryButton from "../SecondaryButton/SecondaryButton"
@@ -272,7 +272,7 @@ Answer ONLY with one of the given grading or evaluation options.
                 inputs: rows[rowNumber].inputs,
                 outputs: data.outputs,
                 evaluation_prompt_template: evaluationPromptTemplate,
-                open_ai_key: getOpenAIKey(),
+                open_ai_key: getLlmProviderKey("OpenAI"),
             })
 
             try {

agenta-web/src/components/EvaluationTable/CustomCodeRunEvaluationTable.tsx

@@ -30,7 +30,7 @@ import {
 import {useVariants} from "@/lib/hooks/useVariant"
 import {useRouter} from "next/router"
 import {EvaluationFlow, EvaluationType} from "@/lib/enums"
-import {getOpenAIKey} from "@/lib/helpers/utils"
+import {getLlmProviderKey} from "@/lib/helpers/utils"
 import {createUseStyles} from "react-jss"
 import SecondaryButton from "../SecondaryButton/SecondaryButton"
 import {exportCustomCodeEvaluationData} from "@/lib/helpers/evaluate"
@@ -292,7 +292,7 @@ const CustomCodeRunEvaluationTable: React.FC<CustomCodeEvaluationTableProps> = (
                 outputs: [{variant_id: variants[0].variantId, variant_output: outputVariantX}],
                 inputs: rows[rowNumber].inputs,
                 correct_answer: correctAnswer(rows[rowNumber].inputs),
-                open_ai_key: getOpenAIKey(),
+                open_ai_key: getLlmProviderKey("OpenAI"),
             }
 
             try {

agenta-web/src/components/Evaluations/Evaluations.tsx

@@ -20,7 +20,7 @@ import {
     useLoadTestsetsList,
     fetchCustomEvaluations,
 } from "@/lib/services/api"
-import {dynamicComponent, getOpenAIKey, isDemo} from "@/lib/helpers/utils"
+import {dynamicComponent, getLlmProviderKey, isDemo} from "@/lib/helpers/utils"
 import {useRouter} from "next/router"
 import {Variant, Parameter, GenericObject, SingleCustomEvaluation} from "@/lib/Types"
 import {EvaluationType} from "@/lib/enums"
@@ -337,7 +337,10 @@ export default function Evaluations() {
         } else if (selectedTestset?.name === "Select a Test set") {
             message.error("Please select a testset")
             return
-        } else if (!getOpenAIKey() && selectedEvaluationType === EvaluationType.auto_ai_critique) {
+        } else if (
+            !getLlmProviderKey("OpenAI") &&
+            selectedEvaluationType === EvaluationType.auto_ai_critique
+        ) {
             setError({
                 message:
                     "In order to run an AI Critique evaluation, please set your OpenAI API key in the API Keys page.",

agenta-web/src/components/pages/settings/Secrets/Secrets.tsx

@@ -1,4 +1,10 @@
-import {getOpenAIKey, removeOpenAIKey, saveOpenAIKey} from "@/lib/helpers/utils"
+import {
+    getLlmProviderKey,
+    saveLlmProviderKey,
+    removeSingleLlmProviderKey,
+    getAllProviderLlmKeys,
+    LlmProvider,
+} from "@/lib/helpers/utils"
 import {Button, Input, Space, Typography, message} from "antd"
 import {useState} from "react"
 import {createUseStyles} from "react-jss"
@@ -16,18 +22,19 @@ const useStyles = createUseStyles({
         margin: "0px 0",
     },
     input: {
-        minWidth: 400,
+        display: "flex",
+        alignItems: "center",
+        width: 420,
+        marginBottom: 8,
+        marginLeft: 8,
     },
 })
 
 export default function Secrets() {
     const classes = useStyles()
-    const savedOpenAiKey = getOpenAIKey()
-    const [openAiKey, setOpenAiKey] = useState(savedOpenAiKey)
+    const [llmProviderKeys, setLlmProviderKeys] = useState(getAllProviderLlmKeys())
     const [messageAPI, contextHolder] = message.useMessage()
 
-    const saveDisabled = openAiKey === savedOpenAiKey
-
     return (
         <div data-cy="secrets">
             {contextHolder}
@@ -41,38 +48,52 @@ export default function Secrets() {
             </Text>
 
             <div className={classes.container}>
-                <Title level={5}>Providers</Title>
+                <Title level={5}>Available Providers</Title>
 
                 <div className={classes.apiContainer}>
-                    <Space direction="horizontal">
-                        <Input.Password
-                            data-cy="openai-api-input"
-                            value={openAiKey}
-                            onChange={(e) => setOpenAiKey(e.target.value)}
-                            addonBefore="OpenAI API Key"
-                            visibilityToggle={false}
-                            className={classes.input}
-                        />
-                        <Button
-                            data-cy="openai-api-save"
-                            disabled={saveDisabled}
-                            onClick={() => {
-                                saveOpenAIKey(openAiKey)
-                                messageAPI.success("The secret is saved")
-                            }}
-                        >
-                            Save
-                        </Button>
-                        <Button
-                            onClick={() => {
-                                removeOpenAIKey()
-                                setOpenAiKey("")
-                                messageAPI.warning("The secret is deleted")
-                            }}
-                        >
-                            Delete
-                        </Button>
-                    </Space>
+                    {llmProviderKeys.map(({title, key}: LlmProvider, i: number) => (
+                        <Space direction="horizontal" key={i}>
+                            <Input.Password
+                                data-cy="openai-api-input"
+                                value={key}
+                                onChange={(e) => {
+                                    const newLlmProviderKeys = [...llmProviderKeys]
+                                    newLlmProviderKeys[i].key = e.target.value
+                                    setLlmProviderKeys(newLlmProviderKeys)
+                                }}
+                                addonBefore={`${title}`}
+                                visibilityToggle={false}
+                                className={classes.input}
+                            />
+                            <Button
+                                data-cy="openai-api-save"
+                                disabled={
+                                    llmProviderKeys[i].key ===
+                                        getLlmProviderKey(llmProviderKeys[i].key) ||
+                                    !llmProviderKeys[i].key
+                                }
+                                onClick={() => {
+                                    saveLlmProviderKey(i, key)
+                                    messageAPI.success("The secret is saved")
+                                }}
+                            >
+                                Save
+                            </Button>
+                            <Button
+                                onClick={() => {
+                                    removeSingleLlmProviderKey(i)
+
+                                    const newLlmProviderKeys = [...llmProviderKeys]
+                                    newLlmProviderKeys[i].key = ""
+                                    setLlmProviderKeys(newLlmProviderKeys)
+
+                                    messageAPI.warning("The secret is deleted")
+                                }}
+                            >
+                                Delete
+                            </Button>
+                        </Space>
+                    ))}
                 </div>
             </div>
         </div>

agenta-web/src/lib/helpers/utils.ts

@@ -1,7 +1,25 @@
 import dynamic from "next/dynamic"
 import {EvaluationType} from "../enums"
 
-const openAItoken = "openAiToken"
+const llmAvailableProvidersToken = "llmAvailableProvidersToken"
+
+export type LlmProvider = {
+    title: string
+    key: string
+}
+
+export const llmAvailableProviders: LlmProvider[] = [
+    "OpenAI",
+    "Replicate",
+    "Hugging Face",
+    "Cohere",
+    "Anthropic",
+    "Azure",
+    "TogetherAI",
+].map((provider) => ({
+    title: provider,
+    key: "",
+}))
 
 export const renameVariables = (name: string) => {
     return name.charAt(0).toUpperCase() + name.slice(1).replace(/_/g, " ")
@@ -26,25 +44,45 @@ export const EvaluationTypeLabels: Record<EvaluationType, string> = {
     [EvaluationType.auto_webhook_test]: "Webhook Test",
 }
 
-export const saveOpenAIKey = (key: string) => {
+export const saveLlmProviderKey = (providerIdx: number, keyValue: string) => {
     if (typeof window !== "undefined") {
         // TODO: add encryption here
-        localStorage.setItem(openAItoken, key)
+        const keys = JSON.parse(localStorage.getItem(llmAvailableProvidersToken) ?? "[{}]")
+        keys[providerIdx].key = keyValue
+        localStorage.setItem(llmAvailableProvidersToken, JSON.stringify(keys))
     }
 }
 
-export const getOpenAIKey = (): string => {
+export const getLlmProviderKey = (providerName: string) => {
     // precedence order: local storage, env variable, empty string
-    let key
     if (typeof window !== "undefined") {
-        key = localStorage.getItem(openAItoken)
+        const inStorage = localStorage.getItem(llmAvailableProvidersToken)
+        if (inStorage) {
+            return JSON.parse(inStorage).find((prov: LlmProvider) => prov.title == providerName)
+        }
+    }
+    // the "NEXT_PUBLIC_OPENIA_API_KEY logic here must be rethought
+    // return key ?? process.env.NEXT_PUBLIC_OPENAI_API_KEY ?? ""
+    return ""
+}
+
+export const getAllProviderLlmKeys = () => {
+    if (typeof window !== "undefined") {
+        const inStorage = localStorage.getItem(llmAvailableProvidersToken)
+        if (inStorage) {
+            return JSON.parse(inStorage)
+        }
+        // if doesn't have the localStorage variable
+        localStorage.setItem(llmAvailableProvidersToken, JSON.stringify(llmAvailableProviders))
     }
-    return key ?? process.env.NEXT_PUBLIC_OPENAI_API_KEY ?? ""
+    return llmAvailableProviders
 }
 
-export const removeOpenAIKey = () => {
+export const removeSingleLlmProviderKey = (providerIdx: number) => {
     if (typeof window !== "undefined") {
-        localStorage.removeItem(openAItoken)
+        const keys = JSON.parse(localStorage.getItem(llmAvailableProvidersToken) ?? "[{}]")
+        keys[providerIdx].key = ""
+        localStorage.setItem(llmAvailableProvidersToken, JSON.stringify(keys))
     }
 }