Merge branch 'agenta/feature/multiple-llm-api-keys' into pr/790

agenta-web/cypress/e2e/ai-critic-evaluation.cy.ts

@@ -14,7 +14,7 @@ describe("AI Critics Evaluation workflow", () => {
     context("When you select evaluation without an API key", () => {
         beforeEach(() => {
             cy.visit(`/apps/${app_id}/evaluations`)
-            cy.clearLocalStorage("openAiToken")
+            cy.clearLocalStorage("llmAvailableProvidersToken")
 
             cy.get('[data-cy="evaluation-error-modal"]').should("not.exist")
             cy.get('[data-cy="ai-critic-button"]').click()
@@ -81,7 +81,7 @@ describe("AI Critics Evaluation workflow", () => {
         })
 
         it("Should execute evaluation workflow with error", () => {
-            cy.clearLocalStorage("openAiToken")
+            cy.clearLocalStorage("llmAvailableProvidersToken")
             cy.wait(1000)
             cy.clickLinkAndWait('[data-cy="ai-critic-run-evaluation"]')
 

agenta-web/cypress/support/commands/evaluations.ts

@@ -1,4 +1,4 @@
-import {randString, removeOpenAIKey} from "../../../src/lib/helpers/utils"
+import {randString, removeLlmProviderKey} from "../../../src/lib/helpers/utils"
 
 let app_id
 
@@ -51,7 +51,7 @@ Cypress.Commands.add("createVariant", () => {
         cy.wrap(app_id).as("app_id")
     })
     cy.contains(/modify parameters/i)
-    cy.removeOpenAiKey()
+    cy.removeLlmProviderKey()
 })
 
 Cypress.Commands.add("createVariantsAndTestsets", () => {
@@ -98,15 +98,15 @@ Cypress.Commands.add("cleanupVariantAndTestset", () => {
         },
     })
 
-    cy.removeOpenAiKey()
+    cy.removeLlmProviderKey()
 })
 
 Cypress.Commands.add("addingOpenaiKey", () => {
     cy.visit("/settings")
-    cy.get('[data-cy="openai-api-input"]').type(apiKey)
-    cy.get('[data-cy="openai-api-save"]').click()
+    cy.get('[data-cy="openai-api-input"]').eq(0).type(apiKey)
+    cy.get('[data-cy="openai-api-save"]').eq(0).click()
 })
 
-Cypress.Commands.add("removeOpenAiKey", () => {
-    removeOpenAIKey()
+Cypress.Commands.add("removeLlmProviderKey", () => {
+    removeLlmProviderKey()
 })

agenta-web/cypress/support/commands/index.ts

@@ -11,7 +11,7 @@ declare global {
             cleanupVariantAndTestset(): Chainable<void>
             createVariant(): Chainable<void>
             saveOpenAiKey(): Chainable<void>
-            removeOpenAiKey(): Chainable<void>
+            removeLlmProviderKey(): Chainable<void>
             addingOpenaiKey(): Chainable<void>
         }
     }

agenta-web/src/components/AppSelector/AppSelector.tsx

@@ -4,12 +4,12 @@ import {usePostHog} from "posthog-js/react"
 import {PlusOutlined} from "@ant-design/icons"
 import {Input, Modal, ConfigProvider, theme, Spin, Card, Button, notification, Divider} from "antd"
 import AppCard from "./AppCard"
-import {Template, GenericObject, LlmProvidersKeys} from "@/lib/Types"
+import {Template, GenericObject} from "@/lib/Types"
 import {useAppTheme} from "../Layout/ThemeContextProvider"
 import {CloseCircleFilled} from "@ant-design/icons"
 import TipsAndFeatures from "./TipsAndFeatures"
 import Welcome from "./Welcome"
-import {getAllLlmProviderKeysAsEnvVariable, isAppNameInputValid, isDemo} from "@/lib/helpers/utils"
+import {getApikeys, isAppNameInputValid, isDemo} from "@/lib/helpers/utils"
 import {
     createAndStartTemplate,
     getProfile,
@@ -183,7 +183,7 @@ const AppSelector: React.FC = () => {
 
         // warn the user and redirect if openAI key is not present
         // TODO: must be changed for multiples LLM keys
-        const providerKeys = getAllLlmProviderKeysAsEnvVariable()
+        const providerKeys = getApikeys()
         if (!providerKeys && !isDemo()) {
             notification.error({
                 message: "OpenAI API Key Missing",
@@ -202,9 +202,7 @@ const AppSelector: React.FC = () => {
             appName: newApp,
             templateId: template_id,
             orgId: selectedOrg?.id!,
-            env_vars: isDemo()
-                ? getAllLlmProviderKeysAsEnvVariable()
-                : (providerKeys as LlmProvidersKeys),
+            providerKey: isDemo() ? "" : (providerKeys as string),
             timeout,
             onStatusChange: (status, details, appId) => {
                 setStatusData((prev) => ({status, details, appId: appId || prev.appId}))

agenta-web/src/components/EvaluationTable/AICritiqueEvaluationTable.tsx

@@ -26,7 +26,7 @@ import {
 import {useVariants} from "@/lib/hooks/useVariant"
 import {useRouter} from "next/router"
 import {EvaluationFlow, EvaluationType} from "@/lib/enums"
-import {getLlmProviderKey} from "@/lib/helpers/utils"
+import {getApikeys} from "@/lib/helpers/utils"
 import {createUseStyles} from "react-jss"
 import {exportAICritiqueEvaluationData} from "@/lib/helpers/evaluate"
 import SecondaryButton from "../SecondaryButton/SecondaryButton"
@@ -279,7 +279,7 @@ Answer ONLY with one of the given grading or evaluation options.
                 inputs: rows[rowNumber].inputs,
                 outputs: data.outputs,
                 evaluation_prompt_template: evaluationPromptTemplate,
-                open_ai_key: getLlmProviderKey("OpenAI"),
+                open_ai_key: getApikeys(),
             })
 
             try {

agenta-web/src/components/EvaluationTable/CustomCodeRunEvaluationTable.tsx

@@ -30,7 +30,7 @@ import {
 import {useVariants} from "@/lib/hooks/useVariant"
 import {useRouter} from "next/router"
 import {EvaluationFlow, EvaluationType} from "@/lib/enums"
-import {getLlmProviderKey} from "@/lib/helpers/utils"
+import {getApikeys} from "@/lib/helpers/utils"
 import {createUseStyles} from "react-jss"
 import SecondaryButton from "../SecondaryButton/SecondaryButton"
 import {exportCustomCodeEvaluationData} from "@/lib/helpers/evaluate"
@@ -292,7 +292,7 @@ const CustomCodeRunEvaluationTable: React.FC<CustomCodeEvaluationTableProps> = (
                 outputs: [{variant_id: variants[0].variantId, variant_output: outputVariantX}],
                 inputs: rows[rowNumber].inputs,
                 correct_answer: correctAnswer(rows[rowNumber].inputs),
-                open_ai_key: getLlmProviderKey("OpenAI"),
+                open_ai_key: getApikeys(),
             }
 
             try {

agenta-web/src/components/Evaluations/Evaluations.tsx

@@ -20,7 +20,7 @@ import {
     useLoadTestsetsList,
     fetchCustomEvaluations,
 } from "@/lib/services/api"
-import {dynamicComponent, getLlmProviderKey, isDemo} from "@/lib/helpers/utils"
+import {dynamicComponent, getApikeys, isDemo} from "@/lib/helpers/utils"
 import {useRouter} from "next/router"
 import {Variant, Parameter, GenericObject, SingleCustomEvaluation} from "@/lib/Types"
 import {EvaluationType} from "@/lib/enums"
@@ -337,10 +337,7 @@ export default function Evaluations() {
         } else if (selectedTestset?.name === "Select a Test set") {
             message.error("Please select a testset")
             return
-        } else if (
-            !getLlmProviderKey("OpenAI") &&
-            selectedEvaluationType === EvaluationType.auto_ai_critique
-        ) {
+        } else if (!getApikeys() && selectedEvaluationType === EvaluationType.auto_ai_critique) {
             setError({
                 message:
                     "In order to run an AI Critique evaluation, please set your OpenAI API key in the API Keys page.",

agenta-web/src/lib/Types.ts

@@ -229,7 +229,9 @@ export interface LlmProvidersKeys {
 export interface AppTemplate {
     app_name: string
     template_id: string
-    env_vars?: LlmProvidersKeys
+    env_vars?: {
+        OPENAI_API_KEY: string | null
+    }
     organization_id?: string
 }
 

agenta-web/src/lib/helpers/utils.ts

@@ -53,6 +53,28 @@ export const EvaluationTypeLabels: Record<EvaluationType, string> = {
     [EvaluationType.auto_webhook_test]: "Webhook Test",
 }
 
+export const getApikeys = () => {
+    if (typeof window !== "undefined") {
+        const llmAvailableProvidersTokenString = localStorage.getItem(llmAvailableProvidersToken)
+
+        if (llmAvailableProvidersTokenString !== null) {
+            const llmAvailableProvidersTokenArray = JSON.parse(llmAvailableProvidersTokenString)
+
+            if (
+                Array.isArray(llmAvailableProvidersTokenArray) &&
+                llmAvailableProvidersTokenArray.length > 0
+            ) {
+                for (let i = 0; i < llmAvailableProvidersTokenArray.length; i++) {
+                    if (llmAvailableProvidersTokenArray[i].key !== "") {
+                        return llmAvailableProvidersTokenArray[i].key
+                    }
+                }
+            }
+        }
+        return ""
+    }
+}
+
 export const saveLlmProviderKey = (providerIdx: number, keyValue: string) => {
     if (typeof window !== "undefined") {
         // TODO: add encryption here
@@ -86,6 +108,12 @@ export const removeSingleLlmProviderKey = (providerIdx: number) => {
     }
 }
 
+export const removeLlmProviderKey = () => {
+    if (typeof window !== "undefined") {
+        localStorage.removeItem(llmAvailableProvidersToken)
+    }
+}
+
 export const capitalize = (s: string) => {
     if (typeof s !== "string") return ""
     return s

agenta-web/src/lib/services/api.ts

@@ -8,13 +8,11 @@ import {
     Evaluation,
     AppTemplate,
     GenericObject,
-    TemplateImage,
     Environment,
     CreateCustomEvaluation,
     ExecuteCustomEvalCode,
     ListAppsItem,
     AICritiqueCreate,
-    LlmProvidersKeys,
 } from "@/lib/Types"
 import {
     fromEvaluationResponseToEvaluation,
@@ -614,16 +612,14 @@ export const waitForAppToStart = async ({
 
 export const createAndStartTemplate = async ({
     appName,
-    env_vars,
+    providerKey,
     templateId,
     orgId,
     timeout,
     onStatusChange,
 }: {
     appName: string
-    env_vars: {
-        [key in keyof LlmProvidersKeys]: {title: string; key: string}
-    }
+    providerKey: string
     templateId: string
     orgId: string
     timeout?: number
@@ -642,13 +638,7 @@ export const createAndStartTemplate = async ({
                     app_name: appName,
                     template_id: templateId,
                     env_vars: {
-                        OPENAI_API_KEY: env_vars.OPENAI_API_KEY.key,
-                        COHERE_API_KEY: env_vars.COHERE_API_KEY.key,
-                        ANTHROPIC_API_KEY: env_vars.ANTHROPIC_API_KEY.key,
-                        AZURE_API_KEY: env_vars.AZURE_API_KEY.key,
-                        REPLICATE_API_KEY: env_vars.REPLICATE_API_KEY.key,
-                        TOGETHERAI_API_KEY: env_vars.TOGETHERAI_API_KEY.key,
-                        HUGGING_FACE_API_KEY: env_vars.HUGGING_FACE_API_KEY.key,
+                        OPENAI_API_KEY: providerKey,
                     },
                     organization_id: orgId,
                 },