feat: multiple LLMs keys implemented

.gitignore

@@ -54,4 +54,6 @@ docker-compose.cloud.prod.yml
 agenta-web/src/pages/auth/[[...path]].tsx
 agenta-web/cypress/screenshots/
 agenta-web/cypress/videos/
-.nextjs_cache/
+.nextjs_cache/
+node_modules
+

agenta-web/src/components/AppSelector/AppSelector.tsx

@@ -8,7 +8,12 @@ import {useAppTheme} from "../Layout/ThemeContextProvider"
 import {CloseCircleFilled} from "@ant-design/icons"
 import TipsAndFeatures from "./TipsAndFeatures"
 import Welcome from "./Welcome"
-import {getOpenAIKey, isAppNameInputValid, isDemo} from "@/lib/helpers/utils"
+import {
+    getSingleLlmProviderKey,
+    isAppNameInputValid,
+    isDemo,
+    llmAvailableProviders,
+} from "@/lib/helpers/utils"
 import {
     createAndStartTemplate,
     getTemplates,
@@ -178,7 +183,8 @@ const AppSelector: React.FC = () => {
         handleCreateAppModalCancel()
 
         // warn the user and redirect if openAI key is not present
-        const openAIKey = getOpenAIKey()
+        // TODO: must be changed for multiples LLM keys
+        const openAIKey = getSingleLlmProviderKey("OpenAI")
         if (!openAIKey && !isDemo()) {
             notification.error({
                 message: "OpenAI API Key Missing",

agenta-web/src/components/EvaluationTable/AICritiqueEvaluationTable.tsx

@@ -26,7 +26,7 @@ import {
 import {useVariants} from "@/lib/hooks/useVariant"
 import {useRouter} from "next/router"
 import {EvaluationFlow, EvaluationType} from "@/lib/enums"
-import {getOpenAIKey} from "@/lib/helpers/utils"
+import {getSingleLlmProviderKey} from "@/lib/helpers/utils"
 import {createUseStyles} from "react-jss"
 import {exportAICritiqueEvaluationData} from "@/lib/helpers/evaluate"
 import SecondaryButton from "../SecondaryButton/SecondaryButton"
@@ -272,7 +272,7 @@ Answer ONLY with one of the given grading or evaluation options.
                 inputs: rows[rowNumber].inputs,
                 outputs: data.outputs,
                 evaluation_prompt_template: evaluationPromptTemplate,
-                open_ai_key: getOpenAIKey(),
+                open_ai_key: getSingleLlmProviderKey("OpenAI"),
             })
 
             try {

agenta-web/src/components/EvaluationTable/CustomCodeRunEvaluationTable.tsx

@@ -30,7 +30,7 @@ import {
 import {useVariants} from "@/lib/hooks/useVariant"
 import {useRouter} from "next/router"
 import {EvaluationFlow, EvaluationType} from "@/lib/enums"
-import {getOpenAIKey} from "@/lib/helpers/utils"
+import {getSingleLlmProviderKey} from "@/lib/helpers/utils"
 import {createUseStyles} from "react-jss"
 import SecondaryButton from "../SecondaryButton/SecondaryButton"
 import {exportCustomCodeEvaluationData} from "@/lib/helpers/evaluate"
@@ -292,7 +292,7 @@ const CustomCodeRunEvaluationTable: React.FC<CustomCodeEvaluationTableProps> = (
                 outputs: [{variant_id: variants[0].variantId, variant_output: outputVariantX}],
                 inputs: rows[rowNumber].inputs,
                 correct_answer: correctAnswer(rows[rowNumber].inputs),
-                open_ai_key: getOpenAIKey(),
+                open_ai_key: getSingleLlmProviderKey("OpenAI"),
             }
 
             try {

agenta-web/src/components/Evaluations/Evaluations.tsx

@@ -20,7 +20,7 @@ import {
     useLoadTestsetsList,
     fetchCustomEvaluations,
 } from "@/lib/services/api"
-import {dynamicComponent, getOpenAIKey, isDemo} from "@/lib/helpers/utils"
+import {dynamicComponent, getSingleLlmProviderKey, isDemo} from "@/lib/helpers/utils"
 import {useRouter} from "next/router"
 import {Variant, Parameter, GenericObject, SingleCustomEvaluation} from "@/lib/Types"
 import {EvaluationType} from "@/lib/enums"
@@ -337,7 +337,10 @@ export default function Evaluations() {
         } else if (selectedTestset?.name === "Select a Test set") {
             message.error("Please select a testset")
             return
-        } else if (!getOpenAIKey() && selectedEvaluationType === EvaluationType.auto_ai_critique) {
+        } else if (
+            !getSingleLlmProviderKey("OpenAI") &&
+            selectedEvaluationType === EvaluationType.auto_ai_critique
+        ) {
             setError({
                 message:
                     "In order to run an AI Critique evaluation, please set your OpenAI API key in the API Keys page.",

agenta-web/src/components/pages/settings/Secrets/Secrets.tsx

@@ -1,4 +1,10 @@
-import {getOpenAIKey, removeOpenAIKey, saveOpenAIKey} from "@/lib/helpers/utils"
+import {
+    getLlmProviderKeys,
+    saveLlmProviderKey,
+    llmAvailableProviders,
+    removeSingleLlmProviderKey,
+    getSingleLlmProviderKey,
+} from "@/lib/helpers/utils"
 import {Button, Input, Space, Typography, message} from "antd"
 import {useState} from "react"
 import {createUseStyles} from "react-jss"
@@ -16,18 +22,21 @@ const useStyles = createUseStyles({
         margin: "0px 0",
     },
     input: {
-        minWidth: 400,
+        display: "flex",
+        alignItems: "center",
+        width: 420,
+        marginBottom: 8,
+        marginLeft: 8,
     },
 })
 
 export default function Secrets() {
     const classes = useStyles()
-    const savedOpenAiKey = getOpenAIKey()
-    const [openAiKey, setOpenAiKey] = useState(savedOpenAiKey)
+    const [llmProviderKeys, setLlmProviderKeys] = useState<{[key: string]: string}>(
+        getLlmProviderKeys(),
+    )
     const [messageAPI, contextHolder] = message.useMessage()
 
-    const saveDisabled = openAiKey === savedOpenAiKey
-
     return (
         <div data-cy="secrets">
             {contextHolder}
@@ -41,38 +50,52 @@ export default function Secrets() {
             </Text>
 
             <div className={classes.container}>
-                <Title level={5}>Providers</Title>
+                <Title level={5}>Providers API Key</Title>
 
                 <div className={classes.apiContainer}>
-                    <Space direction="horizontal">
-                        <Input.Password
-                            data-cy="openai-api-input"
-                            value={openAiKey}
-                            onChange={(e) => setOpenAiKey(e.target.value)}
-                            addonBefore="OpenAI API Key"
-                            visibilityToggle={false}
-                            className={classes.input}
-                        />
-                        <Button
-                            data-cy="openai-api-save"
-                            disabled={saveDisabled}
-                            onClick={() => {
-                                saveOpenAIKey(openAiKey)
-                                messageAPI.success("The secret is saved")
-                            }}
-                        >
-                            Save
-                        </Button>
-                        <Button
-                            onClick={() => {
-                                removeOpenAIKey()
-                                setOpenAiKey("")
-                                messageAPI.warning("The secret is deleted")
-                            }}
-                        >
-                            Delete
-                        </Button>
-                    </Space>
+                    {llmAvailableProviders.map((provider, i) => (
+                        <Space direction="horizontal" key={`space-${i}`}>
+                            <Input.Password
+                                data-cy="openai-api-input"
+                                value={llmProviderKeys[provider]}
+                                onChange={(e) => {
+                                    const newLlmProviderKeys = {...llmProviderKeys}
+                                    newLlmProviderKeys[provider] = e.target.value
+                                    setLlmProviderKeys(newLlmProviderKeys)
+                                }}
+                                addonBefore={`${provider}`}
+                                visibilityToggle={false}
+                                className={classes.input}
+                            />
+                            <Button
+                                data-cy="openai-api-save"
+                                disabled={
+                                    llmProviderKeys[provider] ===
+                                        getSingleLlmProviderKey(provider) ||
+                                    !llmProviderKeys[provider]
+                                }
+                                onClick={() => {
+                                    saveLlmProviderKey(provider, llmProviderKeys[provider])
+                                    messageAPI.success("The secret is saved")
+                                }}
+                            >
+                                Save
+                            </Button>
+                            <Button
+                                onClick={() => {
+                                    removeSingleLlmProviderKey(provider)
+
+                                    const newLlmProviderKeys = {...llmProviderKeys}
+                                    newLlmProviderKeys[provider] = ""
+                                    setLlmProviderKeys(newLlmProviderKeys)
+
+                                    messageAPI.warning("The secret is deleted")
+                                }}
+                            >
+                                Delete
+                            </Button>
+                        </Space>
+                    ))}
                 </div>
             </div>
         </div>

agenta-web/src/lib/helpers/utils.ts

@@ -1,7 +1,16 @@
 import dynamic from "next/dynamic"
 import {EvaluationType} from "../enums"
 
-const openAItoken = "openAiToken"
+const llmAvailableProvidersToken = "llmAvailableProvidersToken"
+export const llmAvailableProviders = [
+    "OpenAI",
+    "Replicate",
+    "Hugging Face",
+    "Cohere",
+    "Anthropic",
+    "Azure",
+    "TogetherAI",
+]
 
 export const renameVariables = (name: string) => {
     return name.charAt(0).toUpperCase() + name.slice(1).replace(/_/g, " ")
@@ -26,25 +35,44 @@ export const EvaluationTypeLabels: Record<EvaluationType, string> = {
     [EvaluationType.auto_webhook_test]: "Webhook Test",
 }
 
-export const saveOpenAIKey = (key: string) => {
+export const saveLlmProviderKey = (providerName: string, keyValue: string) => {
     if (typeof window !== "undefined") {
         // TODO: add encryption here
-        localStorage.setItem(openAItoken, key)
+        const keys = JSON.parse(localStorage.getItem(llmAvailableProvidersToken) ?? "{}")
+        keys[providerName] = keyValue
+        localStorage.setItem(llmAvailableProvidersToken, JSON.stringify(keys))
     }
 }
 
-export const getOpenAIKey = (): string => {
+export const getLlmProviderKeys = () => {
     // precedence order: local storage, env variable, empty string
-    let key
+    let keys: {[key: string]: string} = {}
+    for (const prov of llmAvailableProviders) {
+        keys[prov] = ""
+    }
+
+    if (typeof window !== "undefined") {
+        keys = JSON.parse(localStorage.getItem(llmAvailableProvidersToken) ?? "{}")
+    }
+    // the "NEXT_PUBLIC_OPENIA_API_KEY logic here must be rethought
+    // return key ?? process.env.NEXT_PUBLIC_OPENAI_API_KEY ?? ""
+    return keys ?? {}
+}
+
+export const getSingleLlmProviderKey = (providerName: string): string => {
+    let key = ""
     if (typeof window !== "undefined") {
-        key = localStorage.getItem(openAItoken)
+        const keys = JSON.parse(localStorage.getItem(llmAvailableProvidersToken) ?? "{}")
+        key = keys[providerName] ?? ""
     }
-    return key ?? process.env.NEXT_PUBLIC_OPENAI_API_KEY ?? ""
+    return key
 }
 
-export const removeOpenAIKey = () => {
+export const removeSingleLlmProviderKey = (providerName: string) => {
     if (typeof window !== "undefined") {
-        localStorage.removeItem(openAItoken)
+        const keys = JSON.parse(localStorage.getItem(llmAvailableProvidersToken) ?? "{}")
+        keys[providerName] = ""
+        localStorage.setItem(llmAvailableProvidersToken, JSON.stringify(keys))
     }
 }