integrated versioning into evaluation

Agenta-AI · Jan 23, 2024 · b9d261c · b9d261c
1 parent e4d6ccf
commit b9d261c
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 2 deletions.
diff --git a/agenta-backend/agenta_backend/models/api/evaluation_model.py b/agenta-backend/agenta_backend/models/api/evaluation_model.py
@@ -63,6 +63,8 @@ class Evaluation(BaseModel):
     user_username: str
     variant_ids: List[str]
     variant_names: List[str]
+    variant_revision_ids: List[str]
+    revisions: List[str]
     testset_id: str
     testset_name: str
     status: str

diff --git a/agenta-backend/agenta_backend/models/converters.py b/agenta-backend/agenta_backend/models/converters.py
@@ -79,14 +79,19 @@ async def evaluation_db_to_pydantic(
         str(evaluation_db.variant)
     )
     variant_name = variant.variant_name if variant else str(evaluation_db.variant)
-
+    variant_revision = await db_manager.get_app_variant_revision_by_id(
+        str(evaluation_db.variant_revision)
+    )
+    revision = str(variant_revision.revision)
     return Evaluation(
         id=str(evaluation_db.id),
         app_id=str(evaluation_db.app.id),
         user_id=str(evaluation_db.user.id),
         user_username=evaluation_db.user.username or "",
         status=evaluation_db.status,
         variant_ids=[str(evaluation_db.variant)],
+        variant_revision_ids=[str(evaluation_db.variant_revision)],
+        revisions=[revision],
         variant_names=[variant_name],
         testset_id=str(evaluation_db.testset.id),
         testset_name=evaluation_db.testset.name,

diff --git a/agenta-backend/agenta_backend/models/db_models.py b/agenta-backend/agenta_backend/models/db_models.py
@@ -287,6 +287,7 @@ class EvaluationDB(Document):
     status: str = Field(default="EVALUATION_INITIALIZED")
     testset: Link[TestSetDB]
     variant: PydanticObjectId
+    variant_revision: PydanticObjectId
     evaluators_configs: List[PydanticObjectId]
     aggregated_results: List[AggregatedResult]
     created_at: datetime = Field(default=datetime.utcnow())

diff --git a/agenta-backend/agenta_backend/services/db_manager.py b/agenta-backend/agenta_backend/services/db_manager.py
@@ -1653,7 +1653,8 @@ async def create_new_evaluation(
     user: UserDB,
     testset: TestSetDB,
     status: str,
-    variant: AppVariantDB,
+    variant: str,
+    variant_revision: str,
     evaluators_configs: List[str],
 ) -> EvaluationDB:
     """Create a new evaluation scenario.
@@ -1667,6 +1668,7 @@ async def create_new_evaluation(
         testset=testset,
         status=status,
         variant=variant,
+        variant_revision=variant_revision,
         evaluators_configs=evaluators_configs,
         aggregated_results=[],
         created_at=datetime.now().isoformat(),

diff --git a/agenta-backend/agenta_backend/services/evaluation_service.py b/agenta-backend/agenta_backend/services/evaluation_service.py
@@ -688,6 +688,10 @@ async def create_new_evaluation(
     app = await db_manager.fetch_app_by_id(app_id=app_id)
 
     testset = await db_manager.fetch_testset_by_id(testset_id)
+    variant_db = await db_manager.get_app_variant_instance_by_id(variant_id)
+    variant_revision = await db_manager.fetch_app_variant_revision_by_variant(
+        variant_id, variant_db.revision
+    )
 
     evaluation_db = await db_manager.create_new_evaluation(
         app=app,
@@ -696,6 +700,7 @@ async def create_new_evaluation(
         testset=testset,
         status=EvaluationStatusEnum.EVALUATION_STARTED,
         variant=variant_id,
+        variant_revision=str(variant_revision.id),
         evaluators_configs=evaluator_config_ids,
     )
     return await converters.evaluation_db_to_pydantic(evaluation_db)