create direct_use evaluators with default correct answers

Agenta-AI · May 19, 2024 · 0aa62e1 · 0aa62e1
1 parent f610819
commit 0aa62e1
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/agenta-backend/agenta_backend/resources/evaluators/evaluators.py b/agenta-backend/agenta_backend/resources/evaluators/evaluators.py
@@ -2,7 +2,7 @@
     {
         "name": "Exact Match",
         "key": "auto_exact_match",
-        "direct_use": False,
+        "direct_use": True,
         "settings_template": {
             "label": "Exact Match Settings",
             "description": "Settings for the Exact Match evaluator",

diff --git a/agenta-backend/agenta_backend/services/evaluator_manager.py b/agenta-backend/agenta_backend/services/evaluator_manager.py
@@ -148,14 +148,20 @@ async def create_ready_to_use_evaluators(app: AppDB):
     ]
 
     for evaluator in direct_use_evaluators:
+        settings_values = {}
+        settings_template = evaluator.get("settings_template", {})
+        if "correct_answer_keys" in settings_template:
+            settings_values["correct_answer_keys"] = settings_template[
+                "correct_answer_keys"
+            ].get("default", [])
         await db_manager.create_evaluator_config(
             app=app,
             organization=app.organization if isCloudEE() else None,  # noqa,
             workspace=app.workspace if isCloudEE() else None,  # noqa,
             user=app.user,
             name=evaluator["name"],
             evaluator_key=evaluator["key"],
-            settings_values={},
+            settings_values=settings_values,
         )