diff --git a/agenta-backend/agenta_backend/routers/evaluation_router.py b/agenta-backend/agenta_backend/routers/evaluation_router.py index 96ced112cc..edd223ba2b 100644 --- a/agenta-backend/agenta_backend/routers/evaluation_router.py +++ b/agenta-backend/agenta_backend/routers/evaluation_router.py @@ -174,6 +174,7 @@ async def create_evaluation_scenario( "/{evaluation_id}/evaluation_scenario/{evaluation_scenario_id}/{evaluation_type}/" ) async def update_evaluation_scenario_router( + evaluation_id: str, evaluation_scenario_id: str, evaluation_type: EvaluationType, evaluation_scenario: EvaluationScenarioUpdate, diff --git a/agenta-web/cypress.config.ts b/agenta-web/cypress.config.ts index 264a79bf05..6824bd5527 100644 --- a/agenta-web/cypress.config.ts +++ b/agenta-web/cypress.config.ts @@ -11,7 +11,6 @@ export default defineConfig({ baseUrl: "http://localhost", defaultCommandTimeout: 30000, requestTimeout: 10000, - specPattern: ["*/e2e/smoke-tests.cy.ts", "*/e2e/app-navigation.cy.ts"], }, env: { baseApiURL: "http://localhost/api", diff --git a/agenta-web/cypress/e2e/ab-testing-evaluation.cy.ts b/agenta-web/cypress/e2e/ab-testing-evaluation.cy.ts index a54245d469..2e58da24ab 100644 --- a/agenta-web/cypress/e2e/ab-testing-evaluation.cy.ts +++ b/agenta-web/cypress/e2e/ab-testing-evaluation.cy.ts @@ -69,15 +69,16 @@ describe("A/B Testing Evaluation workflow", () => { cy.clickLinkAndWait('[data-cy="start-new-evaluation-button"]') cy.url().should("include", "/human_a_b_testing") + cy.get('[data-cy="evalInstructionsShown-ok-btn"]').click() + + cy.get('[data-cy="abTesting-app-variant-vote-button"]').should("not.exist") + cy.get('[data-cy="abTesting-both-bad-vote-button"]').should("not.exist") cy.wait(1000) cy.get('[data-cy="abTesting-run-all-button"]').click() - cy.get('[data-cy^="abTesting-app-variant-1-vote-button"]').should("not.be.disabled") - cy.get('[data-cy^="abTesting-app-variant-2-vote-button"]').should("not.be.disabled") - cy.get('[data-cy^="abTesting-both-bad-vote-button"]').should("not.be.disabled") - cy.get('[data-cy="abTesting-app-variant-1-vote-button-0"]').click() - cy.get('[data-cy="abTesting-app-variant-2-vote-button-1"]').click() - cy.get('[data-cy="abTesting-both-bad-vote-button-2"]').click() + cy.get('[data-cy="abTesting-app-variant-vote-button"]').eq(0).click() + cy.get('[data-cy="abTesting-app-variant-vote-button"]').eq(1).click() + cy.get('[data-cy="abTesting-both-bad-vote-button"]').click() }) }) diff --git a/agenta-web/cypress/e2e/playground.cy.ts b/agenta-web/cypress/e2e/playground.cy.ts index da8a2aad3a..a486e47257 100644 --- a/agenta-web/cypress/e2e/playground.cy.ts +++ b/agenta-web/cypress/e2e/playground.cy.ts @@ -4,7 +4,7 @@ describe("Playground Prompt Test", function () { }) it("Should test prompt functionality in the Playground", () => { - cy.get('[data-cy="testview-input-parameters-0"]').type("Germany") + cy.get('[data-cy^="testview-input-parameters"]').eq(0).type("Germany") cy.get('[data-cy="testview-input-parameters-run-button"]').click() cy.get('[data-cy="testview-input-parameters-result"]').should("contain.text", "Loading...") cy.get('[data-cy="testview-input-parameters-result"]').should( diff --git a/agenta-web/cypress/e2e/testset.cy.ts b/agenta-web/cypress/e2e/testset.cy.ts index 14f7e10aa5..839138ff92 100644 --- a/agenta-web/cypress/e2e/testset.cy.ts +++ b/agenta-web/cypress/e2e/testset.cy.ts @@ -35,24 +35,12 @@ describe("Testsets crud and UI functionality", () => { cy.get('[data-cy="testset-name-input"]').type(testsetName) cy.get(".ag-row").should("have.length", 3) countries.forEach((country, index) => { - cy.get(".ag-row") - .eq(index) - .within(() => { - cy.get("div.ag-cell") - .eq(1) - .within(() => { - cy.get("span").eq(0).dblclick() - cy.get(".ag-input-field-input").type(country.country) - }) - cy.get("div.ag-cell") - .eq(2) - .within(() => { - cy.get("span").eq(0).dblclick() - cy.get(".ag-input-field-input").type( - `The capital of ${country.country} is ${country.capital}.`, - ) - }) - }) + cy.get(`.ag-center-cols-container .ag-row[row-index="${index}"]`).within(() => { + cy.get(".ag-cell").eq(1).type(country.country) + cy.get(".ag-cell") + .eq(2) + .type(`The capital of ${country.country} is ${country.capital}.`) + }) }) cy.intercept("/api/testsets/*").as("saveTestsetRequest") cy.get('[data-cy="testset-save-button"]').click() diff --git a/agenta-web/cypress/support/commands/evaluations.ts b/agenta-web/cypress/support/commands/evaluations.ts index 8959fdf7d4..71c47a5e99 100644 --- a/agenta-web/cypress/support/commands/evaluations.ts +++ b/agenta-web/cypress/support/commands/evaluations.ts @@ -33,7 +33,7 @@ Cypress.Commands.add("createVariant", () => { }) }) - cy.get('[data-cy="create-app-button"]').first().click() + cy.get('[data-cy^="create-app-button"]').eq(1).click() const appName = randString(5) cy.get('[data-cy="enter-app-name-modal"]') @@ -66,24 +66,12 @@ Cypress.Commands.add("createVariantsAndTestsets", () => { cy.get(".ag-row").should("have.length", 3) countries.forEach((country, index) => { - cy.get(".ag-row") - .eq(index) - .within(() => { - cy.get("div.ag-cell") - .eq(1) - .within(() => { - cy.get("span").eq(0).dblclick() - cy.get(".ag-input-field-input").type(country.country) - }) - cy.get("div.ag-cell") - .eq(2) - .within(() => { - cy.get("span").eq(0).dblclick() - cy.get(".ag-input-field-input").type( - `The capital of ${country.country} is ${country.capital}.`, - ) - }) - }) + cy.get(`.ag-center-cols-container .ag-row[row-index="${index}"]`).within(() => { + cy.get(".ag-cell").eq(1).type(country.country) + cy.get(".ag-cell") + .eq(2) + .type(`The capital of ${country.country} is ${country.capital}.`) + }) }) cy.get('[data-cy="testset-save-button"]').click() diff --git a/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx index 2280d13824..77bc9ebb4c 100644 --- a/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx +++ b/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx @@ -443,7 +443,7 @@ const SingleModelEvaluationTable: React.FC = ({ type="primary" onClick={runAllEvaluations} size="large" - data-cy="abTesting-run-all-button" + data-cy="single-model-run-all-button" > Run All diff --git a/agenta-web/src/components/Evaluations/EvaluationCardView/EvaluationVotePanel.tsx b/agenta-web/src/components/Evaluations/EvaluationCardView/EvaluationVotePanel.tsx index a8ae9830d4..06875a67f1 100644 --- a/agenta-web/src/components/Evaluations/EvaluationCardView/EvaluationVotePanel.tsx +++ b/agenta-web/src/components/Evaluations/EvaluationCardView/EvaluationVotePanel.tsx @@ -92,6 +92,7 @@ const ComparisonVote: React.FC = ({variants, onChange, valu onClick={getOnClick(variant.variantId)} type={value === variant.variantId ? "primary" : undefined} danger + data-cy="abTesting-app-variant-vote-button" > {String.fromCharCode(65 + ix)}: {variant.variantName} @@ -106,6 +107,7 @@ const ComparisonVote: React.FC = ({variants, onChange, valu type={value === badId ? "primary" : undefined} key={badId} onClick={getOnClick(badId)} + data-cy="abTesting-both-bad-vote-button" > Both are bad diff --git a/agenta-web/src/components/Evaluations/EvaluationCardView/index.tsx b/agenta-web/src/components/Evaluations/EvaluationCardView/index.tsx index acbb06228a..7da02bd60e 100644 --- a/agenta-web/src/components/Evaluations/EvaluationCardView/index.tsx +++ b/agenta-web/src/components/Evaluations/EvaluationCardView/index.tsx @@ -221,7 +221,7 @@ const EvaluationCardView: React.FC = ({ ), - okText: "Ok", + okText: Ok, cancelText: null, width: 500, onCancel: () => (opened.current = false), diff --git a/docs/cli/init.mdx b/docs/cli/init.mdx new file mode 100644 index 0000000000..8cd93f7e6c --- /dev/null +++ b/docs/cli/init.mdx @@ -0,0 +1,53 @@ +--- +title: "agenta init" +description: "Initialize a new project" +--- + + +```bash +agenta init +``` + +# Description + + +The init command initializes a new agenta project. Users can define the application's name and select the agenta platform host (either local or cloud). + +This command creates a blank project in the backend and generates a config.toml file for configuration settings. + +This command creates a `project.toml` file containing all the information about your project + +# Usage + +When running `agenta init` you will prompted the following questions: + +### Project name +```bash +? Please enter the app name +``` + +Enter the app/project name + +### Agenta host +```bash +? Where are you running agenta? (Use arrow keys) + » On agenta cloud + On my local machine + On a remote machine +``` + +Here you can select where the agenta platform is hosted. +- Select cloud, in case you are using agenta cloud ([https://cloud.agenta.ai](https://cloud.agenta.ai)). In that case you will prompted to enter your API key. You can get these from the [agenta cloud configuration](https://cloud.agenta.ai/settings?tab=apiKeys). +- Select local machine if you are running the OSS version of agenta on your local machine ([http://localhost](http://localhost)) +- Select remote machine, in case you are running the OSS version of agenta on a remote machine. In that case you will prompted to enter the remote machine URL. + + In case you are running agenta enterprise, please refer to the enterprise documentation on how to set up your agenta project + +### Start from scratch or from a template +```bash +? How do you want to initialize your app? (Use arrow keys) + » Blank App + Start from template +``` + +Chose blank app to start from scratch (write your own code) or start from template to use one of the agenta templates. diff --git a/docs/cli/install.mdx b/docs/cli/install.mdx new file mode 100644 index 0000000000..5611bb8911 --- /dev/null +++ b/docs/cli/install.mdx @@ -0,0 +1,24 @@ +--- +title: "Install" +description: "Agenta's CLI enables developers to easily create, evaluate, and deploy LLM apps from code. Install it here" +--- + +# Installation + +The agenta CLI can be easily installed through pip: + +```bash +pip install -U agenta +``` + +# Quick usage guide + + + + Get an overview of the main commands and capabilities of agenta CLI + + + Jump into a tutorial deploying an LLM app from code using agenta CLI + + + \ No newline at end of file diff --git a/docs/cli/quick-usage.mdx b/docs/cli/quick-usage.mdx new file mode 100644 index 0000000000..0c48f77afa --- /dev/null +++ b/docs/cli/quick-usage.mdx @@ -0,0 +1,46 @@ +--- +title: "Quick usage" +description: "Deploy and test LLM apps using agenta CLI" +--- + + +The Agenta CLI is a tool used to manage LLM app variants used in Agenta. It allows you to create new apps and deploy variants to the Agenta platform. + +## Create an application / project + +To use the agenta CLI, first create a new folder for each project / application. + +```bash +mkdir my_app +``` + +Next, initialize the project in agenta by running + +```bash +agenta init +``` + +agenta init creates an empty project in the agenta platform (or populates it on one of the templates). + +## Write the code [optional] +Depending whether you initialized the project with a template or not, you may need to write the code for your variant. +The code for your new variant in a .py file. The file should contain a function marked with the `@ag.entrypoint` decorator. + +Here is an example +```python + +import agenta as ag + +ag.config.register_default(prompt="Translate {sentence} to {language}) + +@ag.entrypoint +def translate(sentence:str, language:str): + ### add here openai call logic +``` +## Serve the application + +```bash +agenta variant serve myapp.py +``` + +This command deploys a new variant to the Agenta platform. It processes the code in the specified folder, with `myapp.py` as the entrypoint. This command builds a Docker image and deploys a container based on it. As a result, the variant becomes accessible in the web UI, allowing for prediction generation and API calls. The variant is named as `myapp.default` in the UI. diff --git a/docs/cli/variant_list.mdx b/docs/cli/variant_list.mdx new file mode 100644 index 0000000000..22d3b9d122 --- /dev/null +++ b/docs/cli/variant_list.mdx @@ -0,0 +1,14 @@ +--- +title: "agenta variant list" +description: "List the variants for an application" +--- + +```bash +agenta variant list +``` + +# Description +The `list` command displays all the variants of your app that are currently available in the backend. + + The `variant list` command can only be run in a directory where the `config.toml` generated by `agenta init` is present. + diff --git a/docs/cli/variant_remove.mdx b/docs/cli/variant_remove.mdx new file mode 100644 index 0000000000..2cf1ca84d7 --- /dev/null +++ b/docs/cli/variant_remove.mdx @@ -0,0 +1,15 @@ +--- +title: "agenta variant remove" +description: "Removes a variant from an app/project" +--- + +```bash +agenta variant remove +``` + +# Description + +`variant remove` removes a variant from an app/project. It is called without any argument. The list of variants in the app/project is displayed and the user is prompted to select one of them to remove. + + + The `variant remove` command can only be run in a directory where the `config.toml` generated by `agenta init` is present. diff --git a/docs/cli/variant_serve.mdx b/docs/cli/variant_serve.mdx new file mode 100644 index 0000000000..40fb445a64 --- /dev/null +++ b/docs/cli/variant_serve.mdx @@ -0,0 +1,31 @@ +--- +title: "agenta variant serve" +description: "Serve an application to the agenta platform" +--- + +```bash +agenta variant serve app_name.py +``` + +# Description +The `serve` command deploys the code of an app to the agenta platform. The command packages the code in the .py file along with any additional files in the same directory and deploy them on the agenta platform. Once deployed, the variant becomes accessible the web UI. It can ehtn be further deployed to a staging or production environment as an API endpoint. + +In the agenta UI, the deployed variant is initially named `app_name.default` where `app_name` is the name of the python file deployed and `default` the default configuration. Creating a new configuration `someconfig` (either from the UI or from CLI), will result in the creation of corresponding new variant, named `app_name.someconfig`. + + + The code in `app_name.py` needs to include an entrypoint function. This function should be marked with the `@agenta.entrypoint` decorator. + +Below is a brief example of a valid `app_name.py` file: + +```python + +import agenta as ag + +ag.config.register_default(prompt="Translate {sentence} to {language}) + +@ag.entrypoint +def translate(sentence:str, language:str): + ### add here openai call logic +``` + + The `variant serve` command can only be run in a directory where the `config.toml` generated by `agenta init` is present. diff --git a/docs/contributing/file-issue.mdx b/docs/contributing/file-issue.mdx index 7bee0631db..9cfece8b0a 100644 --- a/docs/contributing/file-issue.mdx +++ b/docs/contributing/file-issue.mdx @@ -2,8 +2,6 @@ title: How to File an Issue --- -# How to File an Issue - Filing an issue is an important step in helping us improve our project. Please follow these steps and provide the requested information when reporting an issue. ## Reporting a Bug diff --git a/docs/howto/use-a-custom-llm.mdx b/docs/howto/use-a-custom-llm.mdx new file mode 100644 index 0000000000..668177e95c --- /dev/null +++ b/docs/howto/use-a-custom-llm.mdx @@ -0,0 +1,42 @@ +--- +title: How to use a Custom LLM in agenta +description: 'Learn how to write an LLM application that uses a custom LLM' +--- + +Using a custom LLM in Agenta is straightforward. The process involves writing the code for a custom application using the SDK, which then calls the LLM. + +Below is the structure of a custom application that calls a [vllm hosted model on an API server](https://docs.vllm.ai/en/latest/getting_started/quickstart.html#api-server): + +```python + +import agenta as ag +import requests + +default_prompt = "Please write a joke about {subject}" + +url = "https:///generate" +ag.config.default(prompt=default_prompt, + temperature=0.8) + +@ag.entrypoint +def generate(subject:str)->str: + prompt = config.prompt.format(subject=subject) + data = { + "prompt": prompt, + "temperature": config.temperature + } + response = requests.post(url, data=json.dumps(data)) + return response.json() +``` + +The above code is a simple LLM app that generates jokes about a given subject, using a vLLM hosted model. It is structured as follows: + +`ag.config.default` sets the default values for the configuration of the LLM application. In this example, the default prompt is "Please write a joke about {subject}", and the temperature is set at 0.8. + +The `@ag.entrypoint` decorator marks the function that will be called. The function `generate` accepts a subject as input and returns a joke as output. It calls the vLLM hosted model using the requests library. + +To call any other LLM, you need to set up the configuration for the LLM (prompt, temperature, etc.) and then call the LLM in the main function. + +After writing the code, it can be deployed using the CLI, as described in the [command line reference](/cli/quick-usage). This can be done by running `agenta init` followed by `agenta variant serve app.py` in the code folder. + + Note that if the LLM is hosted on your local machine and not accessible from outside, you will need to [self-host agenta locally](self-host/host-locally) to be able to call the LLM from the LLm app. \ No newline at end of file diff --git a/docs/learn/llm_app_architectures.mdx b/docs/learn/llm_app_architectures.mdx index f67b42ca76..4ceb3adf92 100644 --- a/docs/learn/llm_app_architectures.mdx +++ b/docs/learn/llm_app_architectures.mdx @@ -1,6 +1,6 @@ --- title: 'LLM App Architectures' -description: 'The different types of LLM applications.' +description: 'The different types of LLM applications (that can be used in agenta).' --- @@ -20,10 +20,8 @@ In agenta you can [create such LLM apps from the UI](/quickstart/getting-started The chain of prompt architecture as its name suggest is based on calling an LLM and then injecting the output into a second call as shown in the figure. -
- - -
+ + ## The Retrieval Augment Generation Architecture diff --git a/docs/learn/the_llmops_workflow.mdx b/docs/learn/the_llmops_workflow.mdx index 405a340289..397d7976b9 100644 --- a/docs/learn/the_llmops_workflow.mdx +++ b/docs/learn/the_llmops_workflow.mdx @@ -6,7 +6,9 @@ description: 'How the best teams build robust LLM applications.' ## The Difference between AI Applications and Traditional Software -Building AI applications powered with Large Language Models is very different than building traditional software. Traditional software is deterministic, running the command in python `1+1` will always return 2. However if we were to create a prompt asking about the result of `1+1` the result could be anything from `2`, `the answer is two`, `two`, or `I am an Large Language Model and I cannot answer mathematical question`. +Building AI applications powered with Large Language Models is very different than building traditional software. + +Traditional software is deterministic, running the command in python `1+1` will always return 2. However if we were to create a prompt asking about the result of `1+1` the result could be anything from `2`, `the answer is two`, `two`, or `I am an Large Language Model and I cannot answer mathematical question`. The main issue is that at the moment of designining the software, we have no idea how the LLM would respond to the question. The only way to know is to test the prompt. diff --git a/docs/mint.json b/docs/mint.json index e7db2c00f2..daa7bd2305 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -8,9 +8,9 @@ }, "favicon": "/logo/favicon.png", "colors": { - "primary": "#32bf40", - "light": "#00FF19", - "dark": "#00FF19" + "primary": "#34d399", + "light": "#34d399", + "dark": "#34d399" }, "modeToggle": { "default": "dark", @@ -43,7 +43,7 @@ "url": "https://github.com/agenta-ai/agenta" }, { - "name": "Book a Free Consulation", + "name": "Book a Demo", "icon": "phone", "url": "https://cal.com/mahmoud-mabrouk-ogzgey/demo" } @@ -73,26 +73,66 @@ { "group": "How-to Guides", "pages": [ + "howto/use-a-custom-llm", "howto/creating-multiple-app-variants", "howto/how-to-debug" ] }, { - "group": "Conceptual Guides", + "group": "Learn", "pages": [ + "learn/the_llmops_workflow", "conceptual/evaluating_llm_apps", "conceptual/concepts", "conceptual/architecture" ] }, + { + "group": "Python SDK", + "pages": [ + "sdk/quick_start", + { + "group": "Core Functions", + "pages": [ + "sdk/init", + "sdk/config_object", + "sdk/config_default", + "sdk/config_pull", + "sdk/config_push", + "sdk/config_datatypes" + ] + } + ] + }, + { + "group": "Command Line", + "pages": [ + "cli/install", + "cli/quick-usage", + { + "group": "Core commands", + "pages": [ + "cli/init", + "cli/variant_serve", + "cli/variant_list", + "cli/variant_remove" + ] + } + ] + }, { "group": "Self-host agenta", "pages": [ "self-host/host-locally", - "self-host/host-remotely", - "self-host/host-on-aws", - "self-host/host-on-gcp", - "self-host/host-on-kubernetes" + { + "group": "Deploy Remotely", + "pages": [ + "self-host/host-remotely", + "self-host/host-on-aws", + "self-host/host-on-gcp", + "self-host/host-on-kubernetes" + ] + } ] }, { @@ -106,8 +146,6 @@ { "group": "Reference", "pages": [ - "reference/agenta-cli", - "reference/sdk", { "group": "Backend API", "pages": [ diff --git a/docs/quickstart/getting-started-code.mdx b/docs/quickstart/getting-started-code.mdx index 39efbab01c..bcea8e32be 100644 --- a/docs/quickstart/getting-started-code.mdx +++ b/docs/quickstart/getting-started-code.mdx @@ -15,7 +15,7 @@ Prefer video tutorial? Watch our 4-minute video [here](https://youtu.be/nggaRwDZ This guide outlines building a basic LLM app with **langchain** and **agenta**. By the end, you'll have a functional LLM app and knowledge of how to use the agenta CLI. -To learn more about creating an LLM app from scratch, please visit our [advanced tutorial](/tutorials/your-first-llm-app). +To learn more about creating an LLM app from scratch, please visit our [advanced tutorial](/tutorials/first-app-with-langchain). ## Step 0: Installation diff --git a/docs/quickstart/getting-started-ui.mdx b/docs/quickstart/getting-started-ui.mdx index 8f942d32f1..569199b676 100644 --- a/docs/quickstart/getting-started-ui.mdx +++ b/docs/quickstart/getting-started-ui.mdx @@ -15,17 +15,20 @@ Welcome! This beginner-friendly guide will walk you through building a simple LL ## Prerequisites -Make sure you have installed the Agenta web platform or are using the cloud version. If you haven't done this yet, follow our [installation guide](/installation). +This guide assumes you are using [agenta cloud](https://cloud.agenta.ai). If you would like to self-host the agenta platform, please refer to [the self-hosting guide](/self-host/host-locally) for deployment instructions. + + ## Step 0: Add your OpenAI API keys -1. Access the Agenta web platform. -2. Select API keys from the left menu. +1. Access the agenta in your webbrowser. +2. Select "Settings" from the left sidebar. +3. Select "LLM Keys" from the top menu. 3. Add your OpenAI API key. Your OpenAI API keys are saved locally on your browser and are sent to the Agenta server only when you create a new application (as an environment variable in the Docker container). - + ## Step 1: Create a New LLM App @@ -47,7 +50,9 @@ The provided application template already includes a prompt that states the capi ## Step 3: Deploy the application -The application was deployed as an API the moment you created it. You can find the API endpoint in the "Endpoints" menu. Copy and paste the code from the "Endpoints" menu to use it in your software. +To deploy the application as an API. You need to click on the button "Publish" in the playground and select an environemnt. + +You can now find the API endpoint in the "Endpoints" menu. Copy and paste the code from the "Endpoints" menu to use it in your software. diff --git a/docs/quickstart/introduction.mdx b/docs/quickstart/introduction.mdx index 68a3a6bb8b..fa401976fe 100644 --- a/docs/quickstart/introduction.mdx +++ b/docs/quickstart/introduction.mdx @@ -19,38 +19,11 @@ With agenta, you can: Agenta focuses on increasing the speed of the development cycle of LLM applications by increasing the speed of iteration. Agenta integrates with all frameworks and model providers in the ecosystem, such as [Langchain](https://langchain.com), [LlamaIndex](https://www.llamaindex.ai/), [OpenAI](https://openai.com), [Cohere](https://cohere.ai), [Mistral](https://mistral.ai/), [Huggingface](https://huggingface.co/), and self-hosted open source LLMs such as the one served using [vLLM](https://github.com/vllm-project/vllm) -## Overview of agenta - - - - -## Why use agenta? - -- If you need to **collaborate with domain experts** and want their feedback on your LLM apps, as well as their help experimenting with prompts and parameters without having to modify your code. -- If you want the flexibility of **using code for writing LLM app**, without being restricted by libraries, models, or frameworks. -- If you need to **save, version, and compare** different variants of your LLM apps **on your own data**. -- If you need a systematic way to **programmatically evaluate your LLM apps**. -- If you **care about your data privacy** and do not want to be proxied through third-party services. - -## Features - -- **Parameter Playground:** Define your app's parameters within your code and experiment with them through a user-friendly web platform. -- **Test Sets:** Build test sets using the UI, by uploading CSVs, or by connecting to your own data via our API. -- **Evaluation:** Evaluate your app on your test sets using different strategies (e.g., exact match, AI Critic, human evaluation, etc.). -- **Deployment:** Deploy your app as an API in just one click. -- **Collaboration:** Share your app with collaborators and receive feedback on it. ## Getting Started - - Install Agenta on your local machine to get started. - - - Dive in to explore the platform. - - ## Getting Help @@ -82,6 +47,10 @@ Agenta integrates with all frameworks and model providers in the ecosystem, such If you have questions or need support, here's how you can reach us. We'd also ❤️ your support. + + Book a call with a founder for one-on-one guidance on using agenta. + + Use the #support channel on Slack to ask questions and get assistance with Agenta. - - Follow us on Twitter to get the latest updates and news. - - - Book a call with a founder for one-on-one guidance on building your first LLM app. - - - Find us on Github at agenta-ai/agenta - diff --git a/docs/reference/agenta-cli.mdx b/docs/reference/agenta-cli.mdx deleted file mode 100644 index 0e40658a96..0000000000 --- a/docs/reference/agenta-cli.mdx +++ /dev/null @@ -1,44 +0,0 @@ ---- -title: "Command Line Interface" -description: "Overview of the Agenta CLI" ---- - - -The Agenta CLI is a tool used to manage LLM app variants used in Agenta. It allows you to create new apps and deploy variants to the Agenta platform. - -## Commands - -### 1. `agenta init` - -```bash -agenta init -``` - -The init command initializes a new Agenta app. Users can define the application's name and select the Agenta platform host (either local or cloud). - -This command creates a blank application in the backend and generates a config.toml file for configuration settings. - -### 2. `agenta variant serve` - -```bash -agenta variant serve variant_name.py -``` - -The `serve` command deploys a new variant to the Agenta platform. It processes the code in the specified folder, with `variant_name.py` as the entrypoint. This command builds a Docker image and deploys a container based on it. As a result, the variant becomes accessible in the web UI, allowing for prediction generation and API calls. The variant is named as `variant_name.default` in the UI. - -### 3. `agenta variant list` - -```bash -agenta variant list -``` - -The `list` command displays all the variants of your app that are currently available in the backend. - -### 4. `agenta variant remove` - -```bash -agenta variant remove -``` - -The `remove` command allows you to delete an existing variant from the platform. - diff --git a/docs/reference/sdk.mdx b/docs/reference/sdk.mdx deleted file mode 100644 index ece2aa835a..0000000000 --- a/docs/reference/sdk.mdx +++ /dev/null @@ -1,136 +0,0 @@ ---- -title: "Python SDK" -description: "You can use our SDK to build your own applications" ---- - -The agenta SDK allows you to experiment with AI applications with LLMs (and in the future any Foundational Model) in a simple way. - -The most commenly used functions are: - -- agenta.init - initialize your variant -- agenta.config.default - set the default configuration - - -## `agenta.init()` - -Initialize your variant. -You need to call this function once at the entry point of your code. - -## `agenta.config` - -`config` is an object that contains the parameters for your variant and methods that enable you to pull and push these parameters to the backend. Use the config to save the usual parameters for an LLM variant, such as the prompt, temperature, LLM model, chunk size (for RAG), etc. Everything that you plan to experiment with in the playground should be saved in the config. - -The parameters in the configuration are accessible in your code at any module importing agenta in this way: - -```python -agenta.config.parameter_name -``` - -The only requirement is that `agenta.init()` is called before in the entry point module. - -## `agenta.config.default(**kwargs)` - -Set the default configuration for your variant. For instance if you set - -```python -agenta.config.default(prompt = ag.TextParam("Hello world")) -``` - -This will set the default value of the prompt to "Hello World". This means that the default configuration (the configuration in the variant variant_name.default) will have the value "Hello World". - -agenta.config.default should be used to set the parameters that you are planning to test and iterate on in the playground. For instance if your application has two different prompts in addition to temperature, you might want to use the following: - -```python -agenta.config.default(prompt1 = ag.TextParam("my prompt1"), - prompt2 = ag.TextParam("my prompt2"), - temperature = ag.FloatParam(0.5), - temperature2 = ag.FloatParam(0.5)) -``` - -Using this configuration will result in: - -1. A playground with the two prompts and temperature sliders -2. An API endpoint /generate wich takes the parameters prompt1, prompt2 and temperature as input - -You can later use the last parameters from the configuration (from the backend) the following way: - -```python -agenta.config.prompt1 -``` - -## `agenta.config.pull(config_name: str = "default", environment_name: str = None)` - -This function pulls the configuration from the backend and save them in the config object. The configuration contains all the parameters for the variant. - -The configuration can be pulled by name. For instance -```python -agenta.config.pull("my_config") -``` - -This would pull the parameters from the backend for the configuration `my_config` (that is for `variant_name.py`, the configuration in the playground for the variant `variant_name.my_config`) - -The configuration can also be pulled by the environment name. For instance, you can always use the parameters from the 'production' environment by using: - -```python -agenta.config.pull(environment_name = "production") -``` - -After pulling the configuration, the parameters are accessible in the following way: - -```python -agenta.config.parameter_name -``` - -## `agenta.config.push(self, config_name: str, overwrite=True, **kwargs)`` - -This function pushes the configuration to the backend. The configuration contains all the parameters for the variant. - -The configuration is pushed to config_name for the code base being run. Basically if you are running `variant_name.py`, the configuration will be pushed to `variant_name.config_name`. - -If overwrite is set to True, the configuration will overwrite the existing configuration. If overwrite is set to False, the configuration will be pushed only if the configuration does not exist yet. - -## Data types for parameters - -There are multiple data types that can be used for the parameters in the configuration. Each data type translates into an element in the playground. - -### TextParam - -This display a text area widget in the playground. - -```python -agenta.config.default(prompt_system = ag.TextParam("You are an expert in geography."), - prompt_user = ag.TextParam("What is the capital of {country}?")) -``` - -### IntParam - -This displays a slider widget in the playground with a step 1 (integers). - -```python -agenta.config.default(intval1 = ag.IntParam(default=1, minval=0, maxval=2), - intval2 = ag.IntParam(1) -``` - -### FloatParam - -This displays a slider widget in the playground with a step 0.1 (float). - - -```python -agenta.config.default(temperature = ag.IntParam(default=0.5, minval=0, maxval=2), - temperature2 = ag.IntParam(0.5) -``` - - -## Data types for inputs - -Inputs in contrast to parameters are given as argument to the function decorated with `@agenta.entrypoint`. They are not part of the configuration but instead are the input in the call to the LLM app. - -```python -def my_entrypoint(name:str): -``` - -Since the agenta sdk uses fastapi under the hood, the data types for the inputs are the same as the ones used in fastapi. In addition, there are some special data types. - -### DictInput -This is data type is used to create variants that have a dynamic number of inputs. When used \ No newline at end of file diff --git a/docs/sdk/config_datatypes.mdx b/docs/sdk/config_datatypes.mdx new file mode 100644 index 0000000000..3f0de2e078 --- /dev/null +++ b/docs/sdk/config_datatypes.mdx @@ -0,0 +1,48 @@ +--- +title: "Config Data Types" +description: "Data types allow you to configure the playground" +--- + +## Data types for parameters + +There are multiple data types that can be used for the parameters in the configuration. Each data type translates into an element in the playground. + +### TextParam + +This display a text area widget in the playground. + +```python +agenta.config.default(prompt_system = ag.TextParam("You are an expert in geography."), + prompt_user = ag.TextParam("What is the capital of {country}?")) +``` + +### IntParam + +This displays a slider widget in the playground with a step 1 (integers). + +```python +agenta.config.default(intval1 = ag.IntParam(default=1, minval=0, maxval=2), + intval2 = ag.IntParam(1) +``` + +### FloatParam + +This displays a slider widget in the playground with a step 0.1 (float). + + +```python +agenta.config.default(temperature = ag.IntParam(default=0.5, minval=0, maxval=2), + temperature2 = ag.IntParam(0.5) +``` + + +## Data types for inputs + +Inputs in contrast to parameters are given as argument to the function decorated with `@agenta.entrypoint`. They are not part of the configuration but instead are the input in the call to the LLM app. + +```python +def my_entrypoint(name:str): +``` + +Since the agenta sdk uses fastapi under the hood, the data types for the inputs are the same as the ones used in fastapi. In addition, there are some special data types. + diff --git a/docs/sdk/config_default.mdx b/docs/sdk/config_default.mdx new file mode 100644 index 0000000000..fdfea3b474 --- /dev/null +++ b/docs/sdk/config_default.mdx @@ -0,0 +1,34 @@ +--- +title: "config.default()" +description: "Register the default configuration for your application" +--- + +`agenta.config.default(**kwargs)` + +Set the default configuration for your variant. For instance if you set + +```python +agenta.config.default(prompt = ag.TextParam("Hello world")) +``` + +This will set the default value of the prompt to "Hello World". This means that the default configuration (the configuration in the variant variant_name.default) will have the value "Hello World". + +agenta.config.default should be used to set the parameters that you are planning to test and iterate on in the playground. For instance if your application has two different prompts in addition to temperature, you might want to use the following: + +```python +agenta.config.default(prompt1 = ag.TextParam("my prompt1"), + prompt2 = ag.TextParam("my prompt2"), + temperature = ag.FloatParam(0.5), + temperature2 = ag.FloatParam(0.5)) +``` + +Using this configuration will result in: + +1. A playground with the two prompts and temperature sliders +2. An API endpoint /generate wich takes the parameters prompt1, prompt2 and temperature as input + +You can later use the last parameters from the configuration (from the backend) the following way: + +```python +agenta.config.prompt1 +``` diff --git a/docs/sdk/config_object.mdx b/docs/sdk/config_object.mdx new file mode 100644 index 0000000000..892e8fb7c8 --- /dev/null +++ b/docs/sdk/config_object.mdx @@ -0,0 +1,16 @@ +--- +title: "The Config Object" +description: "Use the Config Object to Access the Configuration Parameters of your Variant" +--- + +## `agenta.config` + +`config` is an object that contains the parameters for your variant and methods that enable you to pull and push these parameters to the backend. Use the config to save the usual parameters for an LLM variant, such as the prompt, temperature, LLM model, chunk size (for RAG), etc. Everything that you plan to experiment with in the playground should be saved in the config. + +The parameters in the configuration are accessible in your code at any module importing agenta in this way: + +```python +agenta.config.parameter_name +``` + +The only requirement is that `agenta.init()` is called before in the entry point module. diff --git a/docs/sdk/config_pull.mdx b/docs/sdk/config_pull.mdx new file mode 100644 index 0000000000..1c5c3b04ae --- /dev/null +++ b/docs/sdk/config_pull.mdx @@ -0,0 +1,27 @@ +--- +title: "config.pull()" +description: "Pull a configuration from the backend" +--- + +`agenta.config.pull(config_name: str = "default", environment_name: str = None)` + +This function pulls the configuration from the backend and save them in the config object. The configuration contains all the parameters for the variant. + +The configuration can be pulled by name. For instance +```python +agenta.config.pull("my_config") +``` + +This would pull the parameters from the backend for the configuration `my_config` (that is for `variant_name.py`, the configuration in the playground for the variant `variant_name.my_config`) + +The configuration can also be pulled by the environment name. For instance, you can always use the parameters from the 'production' environment by using: + +```python +agenta.config.pull(environment_name = "production") +``` + +After pulling the configuration, the parameters are accessible in the following way: + +```python +agenta.config.parameter_name +``` diff --git a/docs/sdk/config_push.mdx b/docs/sdk/config_push.mdx new file mode 100644 index 0000000000..4f98f72a28 --- /dev/null +++ b/docs/sdk/config_push.mdx @@ -0,0 +1,13 @@ +--- +title: "config.push()" +description: "Pushes a configuration to the backend" +--- + +`agenta.config.push(self, config_name: str, overwrite=True, **kwargs)`` + +This function pushes the configuration to the backend. The configuration contains all the parameters for the variant. + +The configuration is pushed to config_name for the code base being run. Basically if you are running `variant_name.py`, the configuration will be pushed to `variant_name.config_name`. + +If overwrite is set to True, the configuration will overwrite the existing configuration. If overwrite is set to False, the configuration will be pushed only if the configuration does not exist yet. + diff --git a/docs/sdk/init.mdx b/docs/sdk/init.mdx new file mode 100644 index 0000000000..962ed7cee5 --- /dev/null +++ b/docs/sdk/init.mdx @@ -0,0 +1,10 @@ +--- +title: "init()" +description: "Call at the start of your code to initialize your variant" +--- + + +`agenta.init()` + +Initialize your variant. +You need to call this function once at the entry point of your code. diff --git a/docs/sdk/quick_start.mdx b/docs/sdk/quick_start.mdx new file mode 100644 index 0000000000..3907013df6 --- /dev/null +++ b/docs/sdk/quick_start.mdx @@ -0,0 +1,14 @@ +--- +title: "Python SDK" +description: "You can use our SDK to build your own applications" +--- + +The agenta SDK allows you to experiment with AI applications with LLMs (and in the future any Foundational Model) in a simple way. + +The most commenly used functions are: + +- `agenta.init` - initialize your variant +- `agenta.config.default` - set the default configuration + + + diff --git a/docs/self-host/host-locally.mdx b/docs/self-host/host-locally.mdx index f055df2675..97465ef470 100644 --- a/docs/self-host/host-locally.mdx +++ b/docs/self-host/host-locally.mdx @@ -9,11 +9,12 @@ You'll need Docker installed on your machine. If you don't have it yet, download Before you start, make sure that the port 80 is not being used by other applications. To free up resources, you can run docker system prune. -## Starting Agenta in Production Mode +## Starting agenta in Production Mode ```bash mkdir agenta && cd agenta -curl -L https://raw.githubusercontent.com/Agenta-AI/agenta/main/docker-compose.gh.yml -o docker-compose.yml && docker compose -f docker-compose.yml up -d +curl -L https://raw.githubusercontent.com/agenta-ai/agenta/main/docker-compose.gh.yml -o docker-compose.gh.yml +docker compose -f docker-compose.gh.yml up -d ```` @@ -37,6 +38,14 @@ docker compose -f "docker-compose.yml" up -d --build Open your browser and go to [http://localhost](http://localhost). If you see the Agenta web interface, you're good to go. + + The most likely reason for failure is a port conflict. To check whether this is the case please check whether the containers for agenta are all running (agenta-web, agenta-backend, mongodb and redis) and check the logs for the containers agenta-web and agenta-backend. + + If that is not the problem, please [file an issue in github](https://github.com/Agenta-AI/agenta/issues/new/choose) or reach out on [#support on Slack](https://join.slack.com/t/agenta-hq/shared_invite/zt-1zsafop5i-Y7~ZySbhRZvKVPV5DO_7IA). We are very active there and **should answer within minutes** (European time). + + You can also check the most common problems in the [troubleshooting section](/howto/how-to-debug) of the documentation. + + ## What's next? You're all set to start using Agenta! diff --git a/docs/self-host/host-on-aws.mdx b/docs/self-host/host-on-aws.mdx index 181d1bf0b9..b6e5508d1e 100644 --- a/docs/self-host/host-on-aws.mdx +++ b/docs/self-host/host-on-aws.mdx @@ -1,7 +1,9 @@ --- title: Deploy on AWS EC2 -description: 'Installing Agenta on AWS' +description: 'Installing Agenta on AWS using terraform' + --- + This guide is for provisioning and deploying agenta on AWS EC2 **using Terraform**. If you prefer to provision the instance manually, you can use the [guide for deploying on a remote machine](/self-host/host-remotely/). ## AWS We provide a Terraform module to deploy Agenta on AWS. You can find it [here](https://github.com/Agenta-AI/agenta/tree/main/self-host/aws). diff --git a/docs/self-host/host-on-gcp.mdx b/docs/self-host/host-on-gcp.mdx index 27ed39cba9..b6a083ea18 100644 --- a/docs/self-host/host-on-gcp.mdx +++ b/docs/self-host/host-on-gcp.mdx @@ -1,8 +1,9 @@ --- title: Deploy on Google Cloud Engine -description: 'Installing Agenta on GCP' +description: 'Installing Agenta on GCP Compute Engine using terraform' --- + This guide is for provisioning and deploying agenta on GCP Compute Engine **using Terraform**. If you prefer to provision the instance manually, you can use the [guide for deploying on a remote machine](/self-host/host-remotely/). ## GCP We provide a Terraform module to deploy Agenta on GCP. You can find it [here](https://github.com/Agenta-AI/agenta/tree/main/self-host/gcp). The terraform module will in the in us-central1 region create; @@ -15,6 +16,9 @@ You can customize the module according to your needs. ### Prerequisites 1. **Install Terraform**: First, install [Terraform](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli). + + Step 2 until 6 can be skipped if you already have a project with billing enabled and a service account with the compute admin role. + 2. **Setup Project**; In the [Google cloud Console](https://console.cloud.google.com), on the project selector page, select or [create a Google Cloud project](https://cloud.google.com/resource-manager/docs/creating-managing-projects), then copy and save the Project ID for later. 3. **Confirm Billing**; [Make sure that billing is enabled for your Google Cloud project.](https://cloud.google.com/billing/docs/how-to/verify-billing-enabled#console) 4. **Create a Service account**; Click [here to Create a Service account](https://cloud.google.com/iam/docs/service-accounts-create) and specify the compute admin role. @@ -22,8 +26,9 @@ You can customize the module according to your needs. 5. **Get Service account key**; Click [here to obtain the keys for the service account](https://console.cloud.google.com/iam-admin/serviceaccounts?walkthrough_id=iam--create-service-account-keys&start_index=1#step_index=1) you created. -6. Copy the content of the downloaded key file and replace the content of [credentials.json](https://github.com/Agenta-AI/agenta/blob/main/self-host/gcp/credentials.json) with it. -7. Enable Google Compute Engine API: Navigate to [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) and make sure to select the correct project at the top, then enable Compute Engine API. +6. Enable Google Compute Engine API: Navigate to [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) and make sure to select the correct project at the top, then enable Compute Engine API. + +7. Copy the content of the downloaded key file and replace the content of [credentials.json](https://github.com/Agenta-AI/agenta/blob/main/self-host/gcp/credentials.json) with it. 8. **Initialize and Apply Terraform**: Navigate to the correct directory and run: ```bash cd self-host/gcp diff --git a/docs/self-host/host-on-kubernetes.mdx b/docs/self-host/host-on-kubernetes.mdx index 6b309cf968..687b137ce8 100644 --- a/docs/self-host/host-on-kubernetes.mdx +++ b/docs/self-host/host-on-kubernetes.mdx @@ -2,6 +2,6 @@ title: 'Deploy on Kubernetes' --- -Kubernetes deployment are available part of our Enterprise Edition. Agenta Enterprise is the best way to run Agenta on premise. It is highly scalable and the data never leaves your environment. It provides the tools to manage multiple users and teams all in one place. +Kubernetes deployment are available part of our Enterprise Edition. Agenta Enterprise is the best way to self-host Agenta. It is highly scalable and the data never leaves your environment. It provides the tools to manage multiple users and teams all in one place. -Agenta Enterprise is an early access stage for select partners. [Reach out](https://cal.com/mahmoud-mabrouk-ogzgey/demo) to inquire for more details. \ No newline at end of file +Agenta Enterprise is an early access stage for select partners. [Reach out](https://cal.com/mahmoud-mabrouk-ogzgey/demo) to inquire for more details.