Merge pull request #1140 from Agenta-AI/sync-main-with-evaluations

Sync evaluation with main

.all-contributorsrc

@@ -363,6 +363,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "romainrbr",
+      "name": "Romain Brucker",
+      "avatar_url": "https://avatars.githubusercontent.com/u/10381609?v=4",
+      "profile": "https://github.com/romainrbr",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

README.md

@@ -20,12 +20,24 @@
 </br>
 <p align="center">
   <img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="MIT license." />
+  <a href="https://docs.agenta.ai/">
+    <img src="https://img.shields.io/badge/Doc-online-green" alt="Doc">
+  </a>
+
   <a href="https://github.com/Agenta-AI/agenta/blob/main/CONTRIBUTING.md">
     <img src="https://img.shields.io/badge/PRs-Welcome-brightgreen" alt="PRs welcome" />
   </a>
   <img src="https://img.shields.io/github/contributors/Agenta-AI/agenta" alt="Contributors">
   <img src="https://img.shields.io/github/last-commit/Agenta-AI/agenta" alt="Last Commit">
-  <img src="https://img.shields.io/github/commit-activity/m/agenta-ai/agenta" alt="Commits per month">
+    <img src="https://img.shields.io/github/commit-activity/m/agenta-ai/agenta" alt="Commits per month">
+
+  <a href="https://pypi.org/project/agenta/">
+    <img src="https://img.shields.io/pypi/dm/agenta" alt="PyPI - Downloads">
+  </a>
+
+  <a href="https://repo-rater.eddiehub.io/rate?owner=Agenta-AI&name=agenta">
+    <img src="https://repo-rater.eddiehub.io/api/badge?owner=Agenta-AI&name=agenta" alt="RepoRater">
+  </a>
 </br>
 </p>
 
@@ -71,12 +83,11 @@
 
 <h3 align="center">
   <a href="#ℹ️-about"><b>About</b></a> &bull;
-  <a href="#demo"><b>Demo</b></a> &bull;
   <a href="#quick-start"><b>Quick Start</b></a> &bull;
   <a href="https://docs.agenta.ai/installation"><b>Installation</b></a> &bull;
   <a href="#features"><b>Features</b></a> &bull;
   <a href="https://docs.agenta.ai"><b>Documentation</b></a> &bull;
-  <a href="#support"><b>Support</b></a> &bull;
+  <a href="#enterprise-support"><b>Enterprise</b></a> &bull;
   <a href="https://join.slack.com/t/agenta-hq/shared_invite/zt-1zsafop5i-Y7~ZySbhRZvKVPV5DO_7IA"><b>Community</b></a> &bull;
   <a href="#contributing"><b>Contributing</b></a>
 </h3>
@@ -85,54 +96,26 @@
 
 # ℹ️ About
 
-Building production-ready LLM-powered applications is currently very difficult. It involves countless iterations of prompt engineering, parameter tuning, and architectures.
-
-Agenta provides you with the tools to quickly do prompt engineering and 🧪 **experiment**, ⚖️ **evaluate**, and :rocket: **deploy** your LLM apps. All without imposing any restrictions on your choice of framework, library, or model. 
-<br /><br />
-<div  align="center" >
-<picture >
-  <source media="(prefers-color-scheme: dark)" srcset="https://github.com/Agenta-AI/agenta/assets/4510758/01edf2af-2cef-44d7-bdec-7b033cae39cf"  >
-  <source media="(prefers-color-scheme: light)" srcset="https://github.com/Agenta-AI/agenta/assets/4510758/1d6c9627-8b7d-4357-b79f-aa9fced63772"  >
-  <img alt="Overview agenta" src="https://github.com/Agenta-AI/agenta/assets/4510758/1d6c9627-8b7d-4357-b79f-aa9fced63772" >
-</picture>
-</div>
+Agenta is an end-to-end LLMOps platform. It provides the tools for **prompt engineering and management**,  ⚖️ **evaluation**, and :rocket: **deployment**. All without imposing any restrictions on your choice of framework, library, or model. 
 
+Agenta allows developers and product teams to collaborate and build robust AI applications in less time. 
 
-# Demo
-https://github.com/Agenta-AI/agenta/assets/57623556/99733147-2b78-4b95-852f-67475e4ce9ed
+## 🔨 How does it work?
 
-# Quick Start
+| Using an LLM App Template (For Non-Technical Users)  | Starting from Code |
+| ------------- | ------------- |
+|1. [Create an application using a pre-built template from our UI](https://cloud.agenta.ai?utm_source=github&utm_medium=readme&utm_campaign=github)<br />2. Access a playground where you can test and compare different prompts and configurations side-by-side.<br /> 3. Systematically evaluate your application using pre-built or custom evaluators.<br /> 4. Deploy the application to production with one click. |1. [Add a few lines to any LLM application code to automatically create a playground for it](https://docs.agenta.ai/tutorials/first-app-with-langchain) <br />2. Experiment with prompts and configurations, and compare them side-by-side in the playground. <br />3. Systematically evaluate your application using pre-built or custom evaluators. <br />4. Deploy the application to production with one click. |
 
+<br /><br />
 
-<div  align="center" >
-<a href="https://cloud.agenta.ai">
-  <picture>
-  <img src="https://github.com/Agenta-AI/agenta/assets/4510758/a3024fac-2945-4208-ae12-4cc51ecfc970" />
-  </picture>
-</a>
-</br>
-  <a href="https://docs.agenta.ai/installation">
-    <picture>
-      <img src="https://github.com/Agenta-AI/agenta/assets/4510758/29f3fc9a-8a7c-4b39-81a8-776a472a2baf" />
-    </picture>
-  </a>
-  <a href="https://docs.agenta.ai/getting-started">
-    <picture>
-      <img src="https://github.com/Agenta-AI/agenta/assets/4510758/18decb9c-eba5-4d2f-9735-048a1e9d9386" />
-    </picture>
-  </a>
-  <a href="https://docs.agenta.ai/tutorials">
-    <picture>
-      <img src="https://github.com/Agenta-AI/agenta/assets/4510758/e665d66d-d0fe-4b0a-ae7b-5e08350e5e6c" />
-    </picture>
-  </a>
-<a href="https://docs.agenta.ai/installation/self-hosting/self-hosted-installation-aws">
-  <picture>
-    <img src="https://github.com/Agenta-AI/agenta/assets/4510758/65c1487e-5233-4509-8e8d-d967235ccfd7"  />
-  </picture>
-</a>
+# Quick Start
 
-</div>
+### [Try the cloud version](https://cloud.agenta.ai?utm_source=github&utm_medium=readme&utm_campaign=github)
+### [Create your first application in one-minute](https://docs.agenta.ai/quickstart/getting-started-ui)
+### [Create an application using Langchain](https://docs.agenta.ai/tutorials/first-app-with-langchain)
+### [Self-host agenta](https://docs.agenta.ai/self-host/host-locally)
+### [Read the Documentation](https://docs.agenta.ai)
+### [Check the Cookbook](https://docs.agenta.ai/cookbook)
 
 # Features
 
@@ -207,8 +190,8 @@ Now your team can 🔄 iterate, 🧪 experiment, and ⚖️ evaluate different v
 <img width="900" alt="Screenshot 2023-06-25 at 21 08 53" src="https://github.com/Agenta-AI/agenta/assets/57623556/7e07a988-a36a-4fb5-99dd-9cc13a678434">
 
 
-# Support
-Talk with the founders for any commercial inquiries. <br/><br/>
+# Enterprise Support
+Contact us here for enterprise support and early access to agenta self-managed enterprise with Kubernetes support. <br/><br/>
 <a href="https://cal.com/mahmoud-mabrouk-ogzgey/demo"><img src="https://cal.com/book-with-cal-dark.svg" alt="Book us"></a>
 
 # Disabling Anonymized Tracking
@@ -231,7 +214,7 @@ Check out our [Contributing Guide](https://docs.agenta.ai/contributing/getting-s
 ## Contributors ✨
 
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-38-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-39-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
@@ -290,6 +273,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://diegolikescode.me/"><img src="https://avatars.githubusercontent.com/u/57499868?v=4?s=100" width="100px;" alt="diego"/><br /><sub><b>diego</b></sub></a><br /><a href="https://github.com/Agenta-AI/agenta/commits?author=diegolikescode" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/brockWith"><img src="https://avatars.githubusercontent.com/u/105627491?v=4?s=100" width="100px;" alt="brockWith"/><br /><sub><b>brockWith</b></sub></a><br /><a href="https://github.com/Agenta-AI/agenta/commits?author=brockWith" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="http://denniszelada.wordpress.com/"><img src="https://avatars.githubusercontent.com/u/219311?v=4?s=100" width="100px;" alt="Dennis Zelada"/><br /><sub><b>Dennis Zelada</b></sub></a><br /><a href="https://github.com/Agenta-AI/agenta/commits?author=denniszelada" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/romainrbr"><img src="https://avatars.githubusercontent.com/u/10381609?v=4?s=100" width="100px;" alt="Romain Brucker"/><br /><sub><b>Romain Brucker</b></sub></a><br /><a href="https://github.com/Agenta-AI/agenta/commits?author=romainrbr" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

agenta-backend/agenta_backend/main.py

@@ -88,3 +88,8 @@ async def lifespan(application: FastAPI, cache=True):
 app.include_router(organization_router.router, prefix="/organizations")
 app.include_router(bases_router.router, prefix="/bases")
 app.include_router(configs_router.router, prefix="/configs")
+
+if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]:
+    import agenta_backend.cloud.main as cloud
+
+    app = cloud.extend_app_schema(app)

agenta-backend/agenta_backend/routers/app_router.py

@@ -4,7 +4,8 @@
 from fastapi.responses import JSONResponse
 from agenta_backend.config import settings
 from typing import List, Optional
-from fastapi import APIRouter, HTTPException, Request
+from fastapi import HTTPException, Request
+from agenta_backend.utils.common import APIRouter
 from agenta_backend.services.selectors import get_user_own_org
 from agenta_backend.services import (
     app_manager,
@@ -49,7 +50,11 @@
 logger.setLevel(logging.DEBUG)
 
 
-@router.get("/{app_id}/variants/", response_model=List[AppVariantOutput])
+@router.get(
+    "/{app_id}/variants/",
+    response_model=List[AppVariantOutput],
+    operation_id="list_app_variants",
+)
 async def list_app_variants(
     app_id: str,
     request: Request,
@@ -90,7 +95,11 @@ async def list_app_variants(
         raise HTTPException(status_code=500, detail=str(e))
 
 
-@router.get("/get_variant_by_env/", response_model=AppVariantOutput)
+@router.get(
+    "/get_variant_by_env/",
+    response_model=AppVariantOutput,
+    operation_id="get_variant_by_env",
+)
 async def get_variant_by_env(
     app_id: str,
     environment: str,
@@ -134,7 +143,7 @@ async def get_variant_by_env(
         raise HTTPException(status_code=500, detail=str(e))
 
 
-@router.post("/", response_model=CreateAppOutput)
+@router.post("/", response_model=CreateAppOutput, operation_id="create_app")
 async def create_app(
     payload: CreateApp,
     request: Request,
@@ -180,7 +189,7 @@ async def create_app(
         raise HTTPException(status_code=500, detail=str(e))
 
 
-@router.get("/", response_model=List[App])
+@router.get("/", response_model=List[App], operation_id="list_apps")
 async def list_apps(
     request: Request,
     app_name: Optional[str] = None,
@@ -209,7 +218,7 @@ async def list_apps(
         raise HTTPException(status_code=500, detail=str(e))
 
 
-@router.post("/{app_id}/variant/from-image/")
+@router.post("/{app_id}/variant/from-image/", operation_id="add_variant_from_image")
 async def add_variant_from_image(
     app_id: str,
     payload: AddVariantFromImagePayload,
@@ -271,7 +280,7 @@ async def add_variant_from_image(
         raise HTTPException(status_code=500, detail=str(e))
 
 
-@router.delete("/{app_id}/")
+@router.delete("/{app_id}/", operation_id="remove_app")
 async def remove_app(app_id: str, request: Request):
     """Remove app, all its variant, containers and images
 
@@ -301,7 +310,10 @@ async def remove_app(app_id: str, request: Request):
         raise HTTPException(status_code=500, detail=detail)
 
 
-@router.post("/app_and_variant_from_template/")
+@router.post(
+    "/app_and_variant_from_template/",
+    operation_id="create_app_and_variant_from_template",
+)
 async def create_app_and_variant_from_template(
     payload: CreateAppVariant,
     request: Request,
@@ -406,7 +418,11 @@ async def create_app_and_variant_from_template(
         raise HTTPException(status_code=500, detail=str(e))
 
 
-@router.get("/{app_id}/environments/", response_model=List[EnvironmentOutput])
+@router.get(
+    "/{app_id}/environments/",
+    response_model=List[EnvironmentOutput],
+    operation_id="list_environments",
+)
 async def list_environments(
     app_id: str,
     request: Request,

agenta-backend/agenta_backend/routers/bases_router.py

@@ -1,6 +1,7 @@
 import os
 from typing import List, Optional
-from fastapi import APIRouter, Request, HTTPException
+from fastapi import Request, HTTPException
+from agenta_backend.utils.common import APIRouter
 from agenta_backend.models.api.api_models import BaseOutput
 from fastapi.responses import JSONResponse
 from agenta_backend.services import db_manager
@@ -22,7 +23,7 @@
 router = APIRouter()
 
 
-@router.get("/", response_model=List[BaseOutput])
+@router.get("/", response_model=List[BaseOutput], operation_id="list_bases")
 async def list_bases(
     request: Request,
     app_id: Optional[str] = None,

agenta-backend/agenta_backend/routers/configs_router.py

@@ -1,6 +1,7 @@
 import os
 from typing import Optional
-from fastapi import APIRouter, Request, HTTPException
+from fastapi import Request, HTTPException
+from agenta_backend.utils.common import APIRouter
 import logging
 
 from agenta_backend.models.api.api_models import (
@@ -26,7 +27,7 @@
 router = APIRouter()
 
 
-@router.post("/")
+@router.post("/", operation_id="save_config")
 async def save_config(
     payload: SaveConfigPayload,
     request: Request,
@@ -73,7 +74,7 @@ async def save_config(
         raise HTTPException(status_code=500, detail=str(e)) from e
 
 
-@router.get("/", response_model=GetConfigReponse)
+@router.get("/", response_model=GetConfigReponse, operation_id="get_config")
 async def get_config(
     request: Request,
     base_id: str,

agenta-backend/agenta_backend/routers/container_router.py

@@ -7,7 +7,8 @@
     Template,
 )
 from agenta_backend.services import db_manager
-from fastapi import APIRouter, Request, UploadFile, HTTPException
+from fastapi import Request, UploadFile, HTTPException
+from agenta_backend.utils.common import APIRouter
 from fastapi.responses import JSONResponse
 
 if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]:
@@ -35,7 +36,7 @@
 
 
 # TODO: We need to improve this to use the introduced abstraction to also use start and stop service
-@router.post("/build_image/")
+@router.post("/build_image/", operation_id="build_image")
 async def build_image(
     app_id: str,
     base_name: str,
@@ -71,7 +72,7 @@ async def build_image(
     return image_result
 
 
-@router.post("/restart_container/")
+@router.post("/restart_container/", operation_id="restart_container")
 async def restart_docker_container(
     payload: RestartAppContainer,
     request: Request,
@@ -100,7 +101,7 @@ async def restart_docker_container(
         return JSONResponse({"message": str(ex)}, status_code=500)
 
 
-@router.get("/templates/")
+@router.get("/templates/", operation_id="container_templates")
 async def container_templates(
     request: Request,
 ) -> Union[List[Template], str]:
@@ -121,7 +122,7 @@ async def container_templates(
     return templates
 
 
-@router.get("/container_url/")
+@router.get("/container_url/", operation_id="construct_app_container_url")
 async def construct_app_container_url(
     request: Request,
     base_id: Optional[str] = None,

agenta-backend/agenta_backend/routers/environment_router.py

@@ -3,7 +3,8 @@
 
 from fastapi.responses import JSONResponse
 from agenta_backend.services import db_manager
-from fastapi import APIRouter, Request, HTTPException
+from fastapi import Request, HTTPException
+from agenta_backend.utils.common import APIRouter
 from agenta_backend.utils.common import check_access_to_app, check_access_to_variant
 from agenta_backend.models.api.api_models import (
     EnvironmentOutput,
@@ -24,7 +25,7 @@
 router = APIRouter()
 
 
-@router.post("/deploy/")
+@router.post("/deploy/", operation_id="deploy_to_environment")
 async def deploy_to_environment(
     payload: DeployToEnvironmentPayload,
     request: Request,