Evaluations in backend

.github/workflows/run-backend-tests.yml

@@ -8,23 +8,33 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Start Docker Compose
-        run: docker-compose -f "docker-compose.test.yml" up -d --build
+      - name: Set Environment Variables
+        run: |
+          echo "OPENAI_API_KEY=${{ secrets.NEXT_PUBLIC_OPENAI_API_KEY }}" >> $GITHUB_ENV
 
       - name: Install Curl
         run: sudo apt install curl -y
 
+      - name: Start Docker Compose
+        run: OPENAI_API_KEY=${{ secrets.NEXT_PUBLIC_OPENAI_API_KEY }} ENVIRONMENT=github docker-compose -f "docker-compose.test.yml" up -d --build
+
+      - name: Restart Backend Service To Fetch Template Images
+        run: docker container restart agenta-backend-test
+
+      - name: Check Templates Exists
+        run: |
+          sleep 10 && curl -i http://localhost/api/containers/templates/
+
       - name: Wait for Backend Service
         run: |
-          while true; do
-            if curl -s http://localhost:8000/openapi.json; then
-              break
-            fi
-            sleep 5
-          done
+          sleep 10 && curl -i http://localhost/api/openapi.json
 
       - name: Run tests
-        run: docker exec agenta-backend-test pytest
+        run: sleep 10 && docker exec agenta-backend-test pytest
+
+      - name: Docker logs
+        if: always() #
+        run: docker ps -q | xargs -I {} docker logs {}
 
       - name: Stop Docker Compose
         run: docker-compose down

.gitignore

@@ -56,3 +56,4 @@ agenta-web/cypress/screenshots/
 agenta-web/cypress/videos/
 .nextjs_cache/
 
+rabbitmq_data

agenta-backend/agenta_backend/celery_config.py

@@ -0,0 +1,17 @@
+import os
+from kombu import Exchange, Queue
+
+BROKER_URL = os.getenv("CELERY_BROKER_URL")
+CELERY_RESULT_BACKEND = os.getenv("CELERY_RESULT_BACKEND")
+CELERY_TASK_SERIALIZER = "json"
+CELERY_ACCEPT_CONTENT = ["json"]
+CELERY_RESULT_SERIALIZER = "json"
+CELERY_TIMEZONE = "UTC"
+
+CELERY_QUEUES = (
+    Queue(
+        "agenta_backend.tasks.evaluations.evaluate",
+        Exchange("agenta_backend.tasks.evaluations.evaluate"),
+        routing_key="agenta_backend.tasks.evaluations.evaluate",
+    ),
+)

agenta-backend/agenta_backend/main.py

@@ -1,12 +1,16 @@
 import os
+import asyncio
 from contextlib import asynccontextmanager
 
 from agenta_backend.config import settings
+from agenta_backend import celery_config
 from agenta_backend.routers import (
     app_router,
     container_router,
     environment_router,
     evaluation_router,
+    human_evaluation_router,
+    evaluators_router,
     observability_router,
     organization_router,
     testset_router,
@@ -16,6 +20,7 @@
     configs_router,
     health_router,
 )
+from agenta_backend.models.db_engine import DBEngine
 
 if os.environ["FEATURE_FLAG"] in ["cloud", "ee"]:
     from agenta_backend.commons.services import templates_manager
@@ -25,13 +30,19 @@
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
+from celery import Celery
+
+
 origins = [
     "http://localhost:3000",
     "http://localhost:3001",
     "http://0.0.0.0:3000",
     "http://0.0.0.0:3001",
 ]
 
+celery_app = Celery("agenta_app")
+celery_app.config_from_object(celery_config)
+
 
 @asynccontextmanager
 async def lifespan(application: FastAPI, cache=True):
@@ -41,6 +52,8 @@ async def lifespan(application: FastAPI, cache=True):
         application: FastAPI application.
         cache: A boolean value that indicates whether to use the cached data or not.
     """
+    # initialize the database
+    await DBEngine().init_db()
     await templates_manager.update_and_sync_templates(cache=cache)
     yield
 
@@ -72,6 +85,8 @@ async def lifespan(application: FastAPI, cache=True):
 app.include_router(app_router.router, prefix="/apps")
 app.include_router(variants_router.router, prefix="/variants")
 app.include_router(evaluation_router.router, prefix="/evaluations")
+app.include_router(human_evaluation_router.router, prefix="/human-evaluations")
+app.include_router(evaluators_router.router, prefix="/evaluators")
 app.include_router(testset_router.router, prefix="/testsets")
 app.include_router(container_router.router, prefix="/containers")
 app.include_router(environment_router.router, prefix="/environments")