Merge branch 'main' into saving-variant-only-saves-last-unsaved-variant

.all-contributorsrc

@@ -354,6 +354,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "denniszelada",
+      "name": "Dennis Zelada",
+      "avatar_url": "https://avatars.githubusercontent.com/u/219311?v=4",
+      "profile": "http://denniszelada.wordpress.com/",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.github/workflows/run-frontend-tests.yml

@@ -40,18 +40,16 @@ jobs:
           node-version: 18
 
       - name: Install Frontend Depedencies
-        env:
-          NEXT_PUBLIC_OPENAI_API_KEY: ${{ secrets.NEXT_PUBLIC_OPENAI_API_KEY }}
         run: |
           cd agenta-web/ && npm install
 
       - name: Run Cypress
         env:
           NEXT_PUBLIC_OPENAI_API_KEY: ${{ secrets.NEXT_PUBLIC_OPENAI_API_KEY }}
-        run: cd agenta-web/ && npm run cypress:run
+        run: cd agenta-web/ && npm run test
 
       - name: Docker logs
-        if: always()  #
+        if: always() #
         run: docker ps -q | xargs -I {} docker logs {}
 
       - name: Stop Docker Compose

README.md

@@ -231,7 +231,7 @@ Check out our [Contributing Guide](https://docs.agenta.ai/contributing/getting-s
 ## Contributors ✨
 
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-37-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-38-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
@@ -289,6 +289,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://diegolikescode.me/"><img src="https://avatars.githubusercontent.com/u/57499868?v=4?s=100" width="100px;" alt="diego"/><br /><sub><b>diego</b></sub></a><br /><a href="https://github.com/Agenta-AI/agenta/commits?author=diegolikescode" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/brockWith"><img src="https://avatars.githubusercontent.com/u/105627491?v=4?s=100" width="100px;" alt="brockWith"/><br /><sub><b>brockWith</b></sub></a><br /><a href="https://github.com/Agenta-AI/agenta/commits?author=brockWith" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://denniszelada.wordpress.com/"><img src="https://avatars.githubusercontent.com/u/219311?v=4?s=100" width="100px;" alt="Dennis Zelada"/><br /><sub><b>Dennis Zelada</b></sub></a><br /><a href="https://github.com/Agenta-AI/agenta/commits?author=denniszelada" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

agenta-backend/agenta_backend/routers/evaluation_router.py

@@ -1,5 +1,5 @@
 import os
-import random
+import secrets
 from typing import List, Dict
 
 from fastapi.responses import JSONResponse
@@ -174,6 +174,7 @@ async def create_evaluation_scenario(
     "/{evaluation_id}/evaluation_scenario/{evaluation_scenario_id}/{evaluation_type}/"
 )
 async def update_evaluation_scenario_router(
+    evaluation_id: str,
     evaluation_scenario_id: str,
     evaluation_type: EvaluationType,
     evaluation_scenario: EvaluationScenarioUpdate,
@@ -577,4 +578,4 @@ async def webhook_example_fake():
     """
 
     # return a random score b/w 0 and 1
-    return {"score": random.random()}
+    return {"score": secrets.SystemRandom.random()}

agenta-backend/agenta_backend/services/app_manager.py

@@ -392,7 +392,7 @@ async def add_variant_based_on_image(
     variants = await db_manager.list_app_variants_for_app_id(
         app_id=str(app.id), **user_org_data
     )
-    already_exists = any([av for av in variants if av.variant_name == variant_name])
+    already_exists = any(av for av in variants if av.variant_name == variant_name)
     if already_exists:
         logger.error("App variant with the same name already exists")
         raise ValueError("App variant with the same name already exists")

agenta-backend/agenta_backend/services/db_manager.py

@@ -753,7 +753,7 @@ async def add_variant_from_base_and_config(
     app_variant_for_base = await list_variants_for_base(base_db)
 
     already_exists = any(
-        [av for av in app_variant_for_base if av.config_name == new_config_name]
+        av for av in app_variant_for_base if av.config_name == new_config_name
     )
     if already_exists:
         raise ValueError("App variant with the same name already exists")

agenta-backend/agenta_backend/services/docker_utils.py

@@ -105,13 +105,15 @@ def start_container(
             labels.update(development_labels)
 
         env_vars = {} if env_vars is None else env_vars
+        extra_hosts = {"host.docker.internal": "host-gateway"}
         container = client.containers.run(
             image,
             detach=True,
             labels=labels,
             network="agenta-network",
             name=container_name,
             environment=env_vars,
+            extra_hosts=extra_hosts,
         )
         # Check the container's status
         sleep(0.5)

agenta-web/cypress.config.ts

@@ -8,10 +8,17 @@ export default defineConfig({
     video: false,
     screenshotOnRunFailure: false,
     e2e: {
-        baseUrl: "http://localhost",
+        baseUrl: "http://localhost:3000",
         defaultCommandTimeout: 30000,
         requestTimeout: 10000,
-        specPattern: ["*/e2e/smoke-tests.cy.ts", "*/e2e/app-navigation.cy.ts"],
+        setupNodeEvents(on) {
+            on("task", {
+                log(message) {
+                    console.log(message)
+                    return null
+                },
+            })
+        },
     },
     env: {
         baseApiURL: "http://localhost/api",

agenta-web/cypress/e2e/ab-testing-evaluation.cy.ts

@@ -69,15 +69,16 @@ describe("A/B Testing Evaluation workflow", () => {
 
             cy.clickLinkAndWait('[data-cy="start-new-evaluation-button"]')
             cy.url().should("include", "/human_a_b_testing")
+            cy.get('[data-cy="evalInstructionsShown-ok-btn"]').click()
+
+            cy.get('[data-cy="abTesting-app-variant-vote-button"]').should("not.exist")
+            cy.get('[data-cy="abTesting-both-bad-vote-button"]').should("not.exist")
             cy.wait(1000)
             cy.get('[data-cy="abTesting-run-all-button"]').click()
-            cy.get('[data-cy^="abTesting-app-variant-1-vote-button"]').should("not.be.disabled")
-            cy.get('[data-cy^="abTesting-app-variant-2-vote-button"]').should("not.be.disabled")
-            cy.get('[data-cy^="abTesting-both-bad-vote-button"]').should("not.be.disabled")
 
-            cy.get('[data-cy="abTesting-app-variant-1-vote-button-0"]').click()
-            cy.get('[data-cy="abTesting-app-variant-2-vote-button-1"]').click()
-            cy.get('[data-cy="abTesting-both-bad-vote-button-2"]').click()
+            cy.get('[data-cy="abTesting-app-variant-vote-button"]').eq(0).click()
+            cy.get('[data-cy="abTesting-app-variant-vote-button"]').eq(1).click()
+            cy.get('[data-cy="abTesting-both-bad-vote-button"]').click()
         })
     })
 

agenta-web/cypress/e2e/playground.cy.ts

@@ -4,7 +4,7 @@ describe("Playground Prompt Test", function () {
     })
 
     it("Should test prompt functionality in the Playground", () => {
-        cy.get('[data-cy="testview-input-parameters-0"]').type("Germany")
+        cy.get('[data-cy^="testview-input-parameters"]').eq(0).type("Germany")
         cy.get('[data-cy="testview-input-parameters-run-button"]').click()
         cy.get('[data-cy="testview-input-parameters-result"]').should("contain.text", "Loading...")
         cy.get('[data-cy="testview-input-parameters-result"]').should(

agenta-web/cypress/e2e/testset.cy.ts

@@ -35,24 +35,12 @@ describe("Testsets crud and UI functionality", () => {
             cy.get('[data-cy="testset-name-input"]').type(testsetName)
             cy.get(".ag-row").should("have.length", 3)
             countries.forEach((country, index) => {
-                cy.get(".ag-row")
-                    .eq(index)
-                    .within(() => {
-                        cy.get("div.ag-cell")
-                            .eq(1)
-                            .within(() => {
-                                cy.get("span").eq(0).dblclick()
-                                cy.get(".ag-input-field-input").type(country.country)
-                            })
-                        cy.get("div.ag-cell")
-                            .eq(2)
-                            .within(() => {
-                                cy.get("span").eq(0).dblclick()
-                                cy.get(".ag-input-field-input").type(
-                                    `The capital of ${country.country} is ${country.capital}.`,
-                                )
-                            })
-                    })
+                cy.get(`.ag-center-cols-container .ag-row[row-index="${index}"]`).within(() => {
+                    cy.get(".ag-cell").eq(1).type(country.country)
+                    cy.get(".ag-cell")
+                        .eq(2)
+                        .type(`The capital of ${country.country} is ${country.capital}.`)
+                })
             })
             cy.intercept("/api/testsets/*").as("saveTestsetRequest")
             cy.get('[data-cy="testset-save-button"]').click()

agenta-web/cypress/support/commands/evaluations.ts

@@ -33,8 +33,14 @@ Cypress.Commands.add("createVariant", () => {
         })
     })
 
-    cy.get('[data-cy="create-app-button"]').first().click()
+    cy.contains("Single Prompt")
+        .parentsUntil('[data-cy^="app-template-card"]')
+        .last()
+        .contains("create app", {matchCase: false})
+        .click()
+
     const appName = randString(5)
+    cy.task("log", `App name: ${appName}`)
 
     cy.get('[data-cy="enter-app-name-modal"]')
         .should("exist")
@@ -66,24 +72,12 @@ Cypress.Commands.add("createVariantsAndTestsets", () => {
 
     cy.get(".ag-row").should("have.length", 3)
     countries.forEach((country, index) => {
-        cy.get(".ag-row")
-            .eq(index)
-            .within(() => {
-                cy.get("div.ag-cell")
-                    .eq(1)
-                    .within(() => {
-                        cy.get("span").eq(0).dblclick()
-                        cy.get(".ag-input-field-input").type(country.country)
-                    })
-                cy.get("div.ag-cell")
-                    .eq(2)
-                    .within(() => {
-                        cy.get("span").eq(0).dblclick()
-                        cy.get(".ag-input-field-input").type(
-                            `The capital of ${country.country} is ${country.capital}.`,
-                        )
-                    })
-            })
+        cy.get(`.ag-center-cols-container .ag-row[row-index="${index}"]`).within(() => {
+            cy.get(".ag-cell").eq(1).type(country.country)
+            cy.get(".ag-cell")
+                .eq(2)
+                .type(`The capital of ${country.country} is ${country.capital}.`)
+        })
     })
 
     cy.get('[data-cy="testset-save-button"]').click()