Merge pull request #1373 from Agenta-AI/main

Merge main into add-revisions-endpoints
Agenta-AI · Feb 15, 2024 · df7b4f9 · df7b4f9
2 parents 9eaf666 + 23fbf31
commit df7b4f9
Show file tree

Hide file tree

Showing 34 changed files with 111 additions and 243 deletions.
diff --git a/agenta-backend/agenta_backend/migrations/README.md b/agenta-backend/agenta_backend/migrations/README.md
diff --git a/...tions/20240110001454_initial_migration.py → ...0_8_0/20240110001454_initial_migration.py b/...tions/20240110001454_initial_migration.py → ...0_8_0/20240110001454_initial_migration.py
diff --git a/...47_create_exact_match_evaluator_config.py → ...47_create_exact_match_evaluator_config.py b/...47_create_exact_match_evaluator_config.py → ...47_create_exact_match_evaluator_config.py
diff --git a/...ions/20240110165900_evaluations_revamp.py → ..._8_0/20240110165900_evaluations_revamp.py b/...ions/20240110165900_evaluations_revamp.py → ..._8_0/20240110165900_evaluations_revamp.py
diff --git a/...0112120721_evaluation_scenarios_revamp.py → ...0112120721_evaluation_scenarios_revamp.py b/...0112120721_evaluation_scenarios_revamp.py → ...0112120721_evaluation_scenarios_revamp.py
diff --git a/...2120740_human_a_b_evaluation_scenarios.py → ...2120740_human_a_b_evaluation_scenarios.py b/...2120740_human_a_b_evaluation_scenarios.py → ...2120740_human_a_b_evaluation_scenarios.py
diff --git a/...uman_single_model_evaluation_scenarios.py → ...uman_single_model_evaluation_scenarios.py b/...uman_single_model_evaluation_scenarios.py → ...uman_single_model_evaluation_scenarios.py
diff --git a/...802_new_evaluation_results_aggregation.py → ...802_new_evaluation_results_aggregation.py b/...802_new_evaluation_results_aggregation.py → ...802_new_evaluation_results_aggregation.py
diff --git a/...4909_change_odmantic_reference_to_link.py → ...4909_change_odmantic_reference_to_link.py b/...4909_change_odmantic_reference_to_link.py → ...4909_change_odmantic_reference_to_link.py
diff --git a/...ng/20240130133518_updating_app_variant.py → ..._0/20240130133518_updating_app_variant.py b/...ng/20240130133518_updating_app_variant.py → ..._0/20240130133518_updating_app_variant.py
diff --git a/...0130133544_create_app_variant_revision.py → ...0130133544_create_app_variant_revision.py b/...0130133544_create_app_variant_revision.py → ...0130133544_create_app_variant_revision.py
diff --git a/...0240130133603_updating_app_environment.py → ...0240130133603_updating_app_environment.py b/...0240130133603_updating_app_environment.py → ...0240130133603_updating_app_environment.py
diff --git a/...g/20240130140202_connection_evaluation.py → ...0/20240130140202_connection_evaluation.py b/...g/20240130140202_connection_evaluation.py → ...0/20240130140202_connection_evaluation.py
diff --git a/...240124174954_evaluation_error_handling.py → ...240124174954_evaluation_error_handling.py b/...240124174954_evaluation_error_handling.py → ...240124174954_evaluation_error_handling.py
diff --git a/...808_evaluation_scenario_error_handling.py → ...808_evaluation_scenario_error_handling.py b/...808_evaluation_scenario_error_handling.py → ...808_evaluation_scenario_error_handling.py
diff --git a/agenta-backend/agenta_backend/models/api/organization_models.py b/agenta-backend/agenta_backend/models/api/organization_models.py
diff --git a/agenta-backend/agenta_backend/routers/organization_router.py b/agenta-backend/agenta_backend/routers/organization_router.py
diff --git a/agenta-backend/agenta_backend/tasks/evaluations.py b/agenta-backend/agenta_backend/tasks/evaluations.py
@@ -96,6 +96,9 @@ def evaluate(
         loop.run_until_complete(DBEngine().init_db())
         app = loop.run_until_complete(fetch_app_by_id(app_id))
         app_variant_db = loop.run_until_complete(fetch_app_variant_by_id(variant_id))
+        assert (
+            app_variant_db is not None
+        ), f"App variant with id {variant_id} not found!"
         app_variant_parameters = app_variant_db.config.parameters
         testset_db = loop.run_until_complete(fetch_testset_by_id(testset_id))
         new_evaluation_db = loop.run_until_complete(
@@ -180,10 +183,12 @@ def evaluate(
                     if correct_answer_column in data_point
                     else ""
                 )
+
                 loop.run_until_complete(
                     create_new_evaluation_scenario(
                         user=app.user,
-                        organization=app.organization,
+                        organization=app.organization if isCloudEE() else None,
+                        workspace=app.workspace if isCloudEE() else None,
                         evaluation=new_evaluation_db,
                         variant_id=variant_id,
                         evaluators_configs=new_evaluation_db.evaluators_configs,

diff --git a/agenta-backend/pyproject.toml b/agenta-backend/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "agenta_backend"
-version = "0.2.2"
+version = "0.12.0"
 description = ""
 authors = ["Mahmoud Mabrouk <[email protected]>"]
 readme = "README.md"

diff --git a/agenta-cli/pyproject.toml b/agenta-cli/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "agenta"
-version = "0.10.3"
+version = "0.12.0"
 description = "The SDK for agenta is an open-source LLMOps platform."
 readme = "README.md"
 authors = ["Mahmoud Mabrouk <[email protected]>"]

diff --git a/agenta-web/cypress/e2e/eval.comparison.cy.ts b/agenta-web/cypress/e2e/eval.comparison.cy.ts
@@ -12,35 +12,26 @@ describe("Evaluation Comparison Test", function () {
         beforeEach(() => {
             cy.visit(`/apps/${app_id}/evaluations`)
             cy.location("pathname").should("include", "/evaluations")
+            cy.get('[data-cy="evaluation-results-compare-button"]').should("exist")
+            cy.get('[data-cy="evaluation-results-delete-button"]').should("exist")
+            cy.get('[data-cy="new-evaluation-button"]').should("exist")
         })
 
         it("Should create 2 new Evaluations", () => {
             Array.from({length: 2}).map((_) => {
-                cy.get('[data-cy="new-evaluation-button"]').click()
-                cy.get(".ant-modal-content").should("exist")
-
-                cy.get('[data-cy="select-testset-group"]').click()
-                cy.get('[data-cy="select-testset-option"]').click()
-
-                cy.get('[data-cy="select-variant-group"]').click()
-                cy.get('[data-cy="select-variant-option"]').eq(0).click()
-                cy.get('[data-cy="select-variant-group"]').click()
-
-                cy.get('[data-cy="select-evaluators-group"]').click()
-                cy.get('[data-cy="select-evaluators-option"]').eq(0).click()
-                cy.get('[data-cy="select-evaluators-group"]').click()
-
-                cy.get(
-                    ".ant-modal-footer > .ant-btn-primary > .ant-btn-icon > .anticon > svg",
-                ).click()
-                cy.wait(1000)
+                cy.createNewEvaluation()
             })
         })
 
         it("Should verify that there are completed evaluations in the list", () => {
             cy.get('.ag-row[row-index="0"]').should("exist")
             cy.get('.ag-row[row-index="1"]').should("exist")
-            cy.get('.ag-cell[col-id="status"]').should("contain.text", "Completed")
+            cy.get('.ag-cell[col-id="status"]', {timeout: 60000})
+                .eq(0)
+                .should("contain.text", "Completed")
+            cy.get('.ag-cell[col-id="status"]', {timeout: 60000})
+                .eq(1)
+                .should("contain.text", "Completed")
         })
 
         it("Should select 2 evaluations, click on the compare button, and successfully navigate to the comparison page", () => {

diff --git a/agenta-web/cypress/e2e/eval.evaluations.cy.ts b/agenta-web/cypress/e2e/eval.evaluations.cy.ts
@@ -12,24 +12,13 @@ describe("Evaluations CRUD Operations Test", function () {
         beforeEach(() => {
             cy.visit(`/apps/${app_id}/evaluations`)
             cy.location("pathname").should("include", "/evaluations")
+            cy.get('[data-cy="evaluation-results-compare-button"]').should("exist")
+            cy.get('[data-cy="evaluation-results-delete-button"]').should("exist")
+            cy.get('[data-cy="new-evaluation-button"]').should("exist")
         })
 
         it("Should successfully create an Evaluation", () => {
-            cy.get('[data-cy="new-evaluation-button"]').click()
-            cy.get(".ant-modal-content").should("exist")
-
-            cy.get('[data-cy="select-testset-group"]').click()
-            cy.get('[data-cy="select-testset-option"]').click()
-
-            cy.get('[data-cy="select-variant-group"]').click()
-            cy.get('[data-cy="select-variant-option"]').eq(0).click()
-            cy.get('[data-cy="select-variant-group"]').click()
-
-            cy.get('[data-cy="select-evaluators-group"]').click()
-            cy.get('[data-cy="select-evaluators-option"]').eq(0).click()
-            cy.get('[data-cy="select-evaluators-group"]').click()
-
-            cy.get(".ant-modal-footer > .ant-btn-primary > .ant-btn-icon > .anticon > svg").click()
+            cy.createNewEvaluation()
         })
 
         it("Should throw warning when creating an evaluation without selecting testset, variants, and evaluators", () => {

diff --git a/agenta-web/cypress/e2e/eval.evaluators.cy.ts b/agenta-web/cypress/e2e/eval.evaluators.cy.ts
@@ -14,6 +14,9 @@ describe("Evaluators CRUD Operations Test", function () {
         beforeEach(() => {
             cy.visit(`/apps/${app_id}/evaluations`)
             cy.location("pathname").should("include", "/evaluations")
+            cy.get('[data-cy="evaluation-results-compare-button"]').should("exist")
+            cy.get('[data-cy="evaluation-results-delete-button"]').should("exist")
+            cy.get('[data-cy="new-evaluation-button"]').should("exist")
             cy.get("#rc-tabs-1-tab-evaluators > :nth-child(2)").click()
         })
 

diff --git a/agenta-web/cypress/e2e/eval.scenarios.cy.ts b/agenta-web/cypress/e2e/eval.scenarios.cy.ts
@@ -12,24 +12,13 @@ describe("Evaluation Scenarios Test", function () {
         beforeEach(() => {
             cy.visit(`/apps/${app_id}/evaluations`)
             cy.location("pathname").should("include", "/evaluations")
+            cy.get('[data-cy="evaluation-results-compare-button"]').should("exist")
+            cy.get('[data-cy="evaluation-results-delete-button"]').should("exist")
+            cy.get('[data-cy="new-evaluation-button"]').should("exist")
         })
 
         it("Should successfully create an Evaluation", () => {
-            cy.get('[data-cy="new-evaluation-button"]').click()
-            cy.get(".ant-modal-content").should("exist")
-
-            cy.get('[data-cy="select-testset-group"]').click()
-            cy.get('[data-cy="select-testset-option"]').click()
-
-            cy.get('[data-cy="select-variant-group"]').click()
-            cy.get('[data-cy="select-variant-option"]').eq(0).click()
-            cy.get('[data-cy="select-variant-group"]').click()
-
-            cy.get('[data-cy="select-evaluators-group"]').click()
-            cy.get('[data-cy="select-evaluators-option"]').eq(0).click()
-            cy.get('[data-cy="select-evaluators-group"]').click()
-
-            cy.get(".ant-modal-footer > .ant-btn-primary > .ant-btn-icon > .anticon > svg").click()
+            cy.createNewEvaluation()
         })
 
         it("Should verify that evalaution was created and completed successfully", () => {

diff --git a/agenta-web/cypress/support/commands/evaluations.ts b/agenta-web/cypress/support/commands/evaluations.ts
@@ -104,3 +104,22 @@ Cypress.Commands.add("addingOpenaiKey", () => {
 Cypress.Commands.add("removeLlmProviderKey", () => {
     removeLlmProviderKey()
 })
+
+Cypress.Commands.add("createNewEvaluation", () => {
+    cy.get('[data-cy="new-evaluation-button"]').click()
+    cy.get(".ant-modal-content").should("exist")
+
+    cy.get('[data-cy="select-testset-group"]').click()
+    cy.get('[data-cy="select-testset-option"]').eq(0).click()
+
+    cy.get('[data-cy="select-variant-group"]').click()
+    cy.get('[data-cy="select-variant-option"]').eq(0).click()
+    cy.get('[data-cy="select-variant-group"]').click()
+
+    cy.get('[data-cy="select-evaluators-group"]').click()
+    cy.get('[data-cy="select-evaluators-option"]').eq(0).click()
+    cy.get('[data-cy="select-evaluators-group"]').click()
+
+    cy.get(".ant-modal-footer > .ant-btn-primary > .ant-btn-icon > .anticon > svg").click()
+    cy.wait(1000)
+})
diff --git a/agenta-web/cypress/support/commands/index.ts b/agenta-web/cypress/support/commands/index.ts
@@ -13,6 +13,7 @@ declare global {
             saveOpenAiKey(): Chainable<void>
             removeLlmProviderKey(): Chainable<void>
             addingOpenaiKey(): Chainable<void>
+            createNewEvaluation(): Chainable<void>
         }
     }
 }
diff --git a/agenta-web/package.json b/agenta-web/package.json
@@ -1,6 +1,6 @@
 {
-    "name": "dashboard",
-    "version": "0.1.0",
+    "name": "agenta",
+    "version": "0.12.0",
     "private": true,
     "engines": {
         "node": ">=18"