add a direct-use attribute for evaluators

Agenta-AI · Jan 2, 2024 · 1c6ced4 · 1c6ced4
1 parent 3ff30b2
commit 1c6ced4
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 0 deletions.
diff --git a/agenta-backend/agenta_backend/models/api/evaluation_model.py b/agenta-backend/agenta_backend/models/api/evaluation_model.py
@@ -8,6 +8,7 @@
 class Evaluator(BaseModel):
     name: str
     key: str
+    direct_use: bool
     settings_template: dict
 
 

diff --git a/agenta-backend/agenta_backend/resources/evaluators/evaluators.json b/agenta-backend/agenta_backend/resources/evaluators/evaluators.json
@@ -2,6 +2,7 @@
     {
         "name": "Exact Match",
         "key": "auto_exact_match",
+        "direct_use": true,
         "settings_template": {
             "label": "Exact Match Settings",
             "description": "Settings for the Exact Match evaluator"
@@ -10,6 +11,7 @@
     {
         "name": "Similarity Match",
         "key": "auto_similarity_match",
+        "direct_use": false,
         "settings_template": {
             "similarity_threshold": {
                 "label": "Similarity Threshold",
@@ -22,6 +24,7 @@
     {
         "name": "Regex Test",
         "key": "auto_regex_test",
+        "direct_use": false,
         "settings_template": {
             "regex_pattern": {
                 "label": "Regex Pattern",
@@ -40,6 +43,7 @@
     {
         "name": "AI Critique",
         "key": "auto_ai_critique",
+        "direct_use": false,
         "settings_template": {
             "prompt_template": {
                 "label": "Prompt Template",
@@ -52,6 +56,7 @@
     {
         "name": "Code Evaluation",
         "key": "auto_custom_code_run",
+        "direct_use": false,
         "settings_template": {
             "code": {
                 "label": "Evaluation Code",
@@ -64,6 +69,7 @@
     {
         "name": "Webhook test",
         "key": "auto_webhook_test",
+        "direct_use": false,
         "settings_template": {
             "webhook_url": {
                 "label": "Webhook URL",
@@ -76,6 +82,7 @@
     {
         "name": "A/B Test",
         "key": "human_a_b_testing",
+        "direct_use": false,
         "settings_template": {
             "label": "A/B Testing Settings",
             "description": "Settings for A/B testing configurations"
@@ -84,6 +91,7 @@
     {
         "name": "Single Model Test",
         "key": "human_single_model_test",
+        "direct_use": false,
         "settings_template": {
             "label": "Single Model Testing Settings",
             "description": "Settings for single model testing configurations"