Feature: Batching and Retrying for bulk operations

.github/workflows/run-frontend-tests.yml

@@ -40,18 +40,16 @@ jobs:
           node-version: 18
 
       - name: Install Frontend Depedencies
-        env:
-          NEXT_PUBLIC_OPENAI_API_KEY: ${{ secrets.NEXT_PUBLIC_OPENAI_API_KEY }}
         run: |
           cd agenta-web/ && npm install
 
       - name: Run Cypress
         env:
           NEXT_PUBLIC_OPENAI_API_KEY: ${{ secrets.NEXT_PUBLIC_OPENAI_API_KEY }}
-        run: cd agenta-web/ && npm run cypress:run
+        run: cd agenta-web/ && npm run test
 
       - name: Docker logs
-        if: always()  #
+        if: always() #
         run: docker ps -q | xargs -I {} docker logs {}
 
       - name: Stop Docker Compose

agenta-web/cypress.config.ts

@@ -8,9 +8,17 @@ export default defineConfig({
     video: false,
     screenshotOnRunFailure: false,
     e2e: {
-        baseUrl: "http://localhost",
+        baseUrl: "http://localhost:3000",
         defaultCommandTimeout: 30000,
         requestTimeout: 10000,
+        setupNodeEvents(on) {
+            on("task", {
+                log(message) {
+                    console.log(message)
+                    return null
+                },
+            })
+        },
     },
     env: {
         baseApiURL: "http://localhost/api",

agenta-web/cypress/support/commands/evaluations.ts

@@ -33,8 +33,14 @@ Cypress.Commands.add("createVariant", () => {
         })
     })
 
-    cy.get('[data-cy^="create-app-button"]').eq(1).click()
+    cy.contains("Single Prompt")
+        .parentsUntil('[data-cy^="app-template-card"]')
+        .last()
+        .contains("create app", {matchCase: false})
+        .click()
+
     const appName = randString(5)
+    cy.task("log", `App name: ${appName}`)
 
     cy.get('[data-cy="enter-app-name-modal"]')
         .should("exist")

agenta-web/package.json

@@ -2,6 +2,9 @@
     "name": "dashboard",
     "version": "0.1.0",
     "private": true,
+    "engines": {
+        "node": ">=18"
+    },
     "scripts": {
         "dev": "next dev",
         "build": "next build",
@@ -30,6 +33,7 @@
         "@types/lodash": "^4.14.201",
         "@types/mdx": "^2.0.6",
         "@types/papaparse": "^5.3.14",
+        "@types/promise-retry": "^1.1.6",
         "@types/react-dom": "18.2.1",
         "@types/react-highlight-words": "^0.16.4",
         "@types/react-syntax-highlighter": "^15.5.7",
@@ -51,6 +55,7 @@
         "papaparse": "^5.4.1",
         "postcss": "8.4.23",
         "posthog-js": "^1.87.2",
+        "promise-retry": "^2.0.1",
         "react": "18.2.0",
         "react-dom": "18.2.0",
         "react-error-boundary": "^4.0.11",

agenta-web/src/components/AppSelector/AppTemplateCard.tsx

@@ -47,7 +47,7 @@ interface Props {
 const AppTemplateCard: React.FC<Props> = ({title, tag, onClick, body, noTemplate}) => {
     const classes = useStyles({tag} as StylesProp)
     return (
-        <Card className={classes.card}>
+        <Card className={classes.card} data-cy="app-template-card">
             {tag && (
                 <Tag color="blue" className={classes.tag}>
                     {tag}
@@ -68,12 +68,7 @@ const AppTemplateCard: React.FC<Props> = ({title, tag, onClick, body, noTemplate
                     <Text type="secondary">
                         <p>{body}</p>
                     </Text>
-                    <Button
-                        shape="round"
-                        onClick={onClick}
-                        className={classes.createBtn}
-                        data-cy="create-app-button"
-                    >
+                    <Button shape="round" onClick={onClick} className={classes.createBtn}>
                         Create App
                     </Button>
                 </div>

agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx

@@ -16,7 +16,7 @@ import SecondaryButton from "../SecondaryButton/SecondaryButton"
 import {useQueryParam} from "@/hooks/useQuery"
 import EvaluationCardView, {VARIANT_COLORS} from "../Evaluations/EvaluationCardView"
 import {Evaluation, EvaluationScenario, KeyValuePair, Variant} from "@/lib/Types"
-import {EvaluationTypeLabels, camelToSnake} from "@/lib/helpers/utils"
+import {EvaluationTypeLabels, batchExecute, camelToSnake} from "@/lib/helpers/utils"
 import {testsetRowToChatMessages} from "@/lib/helpers/testset"
 import EvaluationVotePanel from "../Evaluations/EvaluationCardView/EvaluationVotePanel"
 import VariantAlphabet from "../Evaluations/EvaluationCardView/VariantAlphabet"
@@ -200,7 +200,10 @@ const ABTestingEvaluationTable: React.FC<EvaluationTableProps> = ({
 
     const runAllEvaluations = async () => {
         setEvaluationStatus(EvaluationFlow.EVALUATION_STARTED)
-        Promise.all(rows.map((row) => runEvaluation(row.id!, rows.length - 1, false)))
+        batchExecute(
+            rows.map((row) => () => runEvaluation(row.id!, rows.length - 1, false)),
+            {allowRetry: true, batchSize: 10},
+        )
             .then(() => {
                 setEvaluationStatus(EvaluationFlow.EVALUATION_FINISHED)
                 message.success("Evaluations Updated!")

agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx

@@ -24,7 +24,7 @@ import SecondaryButton from "../SecondaryButton/SecondaryButton"
 import {useQueryParam} from "@/hooks/useQuery"
 import EvaluationCardView from "../Evaluations/EvaluationCardView"
 import {Evaluation, EvaluationScenario, KeyValuePair, Variant} from "@/lib/Types"
-import {EvaluationTypeLabels, camelToSnake} from "@/lib/helpers/utils"
+import {EvaluationTypeLabels, batchExecute, camelToSnake} from "@/lib/helpers/utils"
 import {testsetRowToChatMessages} from "@/lib/helpers/testset"
 import {debounce} from "lodash"
 import EvaluationVotePanel from "../Evaluations/EvaluationCardView/EvaluationVotePanel"
@@ -260,7 +260,10 @@ const SingleModelEvaluationTable: React.FC<EvaluationTableProps> = ({
 
     const runAllEvaluations = async () => {
         setEvaluationStatus(EvaluationFlow.EVALUATION_STARTED)
-        Promise.all(rows.map((row) => runEvaluation(row.id!, rows.length - 1, false)))
+        batchExecute(
+            rows.map((row) => () => runEvaluation(row.id!, rows.length - 1, false)),
+            {allowRetry: true, batchSize: 10},
+        )
             .then(() => {
                 setEvaluationStatus(EvaluationFlow.EVALUATION_FINISHED)
                 message.success("Evaluations Updated!")

agenta-web/src/components/Playground/Views/TestView.tsx

@@ -3,7 +3,7 @@ import {Button, Input, Card, Row, Col, Space, Form} from "antd"
 import {CaretRightOutlined, PlusOutlined} from "@ant-design/icons"
 import {callVariant} from "@/lib/services/api"
 import {ChatMessage, ChatRole, GenericObject, Parameter, Variant} from "@/lib/Types"
-import {randString, removeKeys} from "@/lib/helpers/utils"
+import {batchExecute, randString, removeKeys} from "@/lib/helpers/utils"
 import LoadTestsModal from "../LoadTestsModal"
 import AddToTestSetDrawer from "../AddToTestSetDrawer/AddToTestSetDrawer"
 import {DeleteOutlined} from "@ant-design/icons"
@@ -278,9 +278,12 @@ const App: React.FC<TestViewProps> = ({inputParams, optParams, variant, isChatVa
     }
 
     const handleRunAll = () => {
+        const funcs: Function[] = []
         rootRef.current
             ?.querySelectorAll("[data-cy=testview-input-parameters-run-button]")
-            .forEach((btn) => (btn as HTMLButtonElement).click())
+            .forEach((btn) => funcs.push(() => (btn as HTMLButtonElement).click()))
+
+        batchExecute(funcs, {allowRetry: true, batchSize: 10})
     }
 
     const handleAddRow = () => {

agenta-web/src/lib/helpers/utils.ts

@@ -1,6 +1,8 @@
 import dynamic from "next/dynamic"
 import {EvaluationType} from "../enums"
 import {GenericObject} from "../Types"
+import promiseRetry from "promise-retry"
+import {getErrorMessage} from "./errorHandler"
 
 const llmAvailableProvidersToken = "llmAvailableProvidersToken"
 
@@ -219,3 +221,81 @@ export const getAgentaApiUrl = () => {
 
     return apiUrl
 }
+
+export function promisifyFunction(fn: Function, ...args: any[]) {
+    return async () => {
+        return fn(...args)
+    }
+}
+
+export const withRetry = (
+    fn: Function,
+    options?: Parameters<typeof promiseRetry>[0] & {logErrors?: boolean},
+) => {
+    const {logErrors = true, ...config} = options || {}
+    const func = promisifyFunction(fn)
+
+    return promiseRetry(
+        (retry, attempt) =>
+            func().catch((e) => {
+                if (logErrors) {
+                    console.log("Error: ", getErrorMessage(e))
+                    console.log("Retry attempt: ", attempt)
+                }
+                retry(e)
+            }),
+        {
+            retries: 3,
+            ...config,
+        },
+    )
+}
+
+export async function batchExecute(
+    functions: Function[],
+    options?: {
+        batchSize?: number
+        supressErrors?: boolean
+        batchDelayMs?: number
+        logErrors?: boolean
+        allowRetry?: boolean
+        retryConfig?: Parameters<typeof promiseRetry>[0]
+    },
+) {
+    const {
+        batchSize = 20,
+        supressErrors = false,
+        batchDelayMs = 2000,
+        logErrors = true,
+        allowRetry = false,
+        retryConfig,
+    } = options || {}
+
+    functions = functions.map((f) => async () => {
+        try {
+            return await (allowRetry ? withRetry(f, {logErrors, ...(retryConfig || {})}) : f())
+        } catch (e) {
+            if (supressErrors) {
+                if (logErrors) console.log("Ignored error:", getErrorMessage(e))
+                return {__error: e}
+            }
+            throw e
+        }
+    })
+
+    if (!batchSize || !Number.isInteger(batchSize) || batchSize <= 0)
+        return Promise.all(functions.map((f) => f()))
+
+    let position = 0
+    let results: any[] = []
+
+    while (position < functions.length) {
+        const batch = functions.slice(position, position + batchSize)
+        results = [...results, ...(await Promise.all(batch.map((f) => f())))]
+        position += batchSize
+        if (batchDelayMs) {
+            await delay(batchDelayMs)
+        }
+    }
+    return results
+}