From 167691578765617f98c72ad06614d575db2ca3f6 Mon Sep 17 00:00:00 2001 From: Abram Date: Wed, 6 Dec 2023 12:50:06 +0100 Subject: [PATCH 1/4] Update - implemented batch-run-with-retry functionality --- .../components/Playground/Views/TestView.tsx | 46 ++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/agenta-web/src/components/Playground/Views/TestView.tsx b/agenta-web/src/components/Playground/Views/TestView.tsx index ba34c49c78..250e5d1ab6 100644 --- a/agenta-web/src/components/Playground/Views/TestView.tsx +++ b/agenta-web/src/components/Playground/Views/TestView.tsx @@ -271,10 +271,52 @@ const App: React.FC = ({inputParams, optParams, variant, isChatVa } } - const handleRunAll = () => { - testList.forEach((_, index) => handleRun(index)) + const handleRunAll = async () => { + const batchSize = 10 // Number of requests to make in each batch + const maxRetryCount = 3 // Maximum number of times to retry a failed request + const retryDelay = 3000 // Delay before retrying a failed request (in milliseconds) + + for (let startIdx = 0; startIdx < testList.length; startIdx += batchSize) { + const endIdx = Math.min(startIdx + batchSize, testList.length) + const batchPromises = [] + + for (let index = startIdx; index < endIdx; index++) { + batchPromises.push(runWithRetry(index, maxRetryCount, retryDelay)) + } + + await Promise.all(batchPromises) + } } + const runWithRetry = async (index: number, maxRetryCount: number, retryDelay: number) => { + let retryCount = 0 + + while (retryCount <= maxRetryCount) { + try { + await handleRun(index) + // If the request is successful, break out of the retry loop + break + } catch (error) { + console.error(`Error in request ${index + 1}, retrying...`) + retryCount++ + if (retryCount <= maxRetryCount) { + // Add a delay before retrying + await delay(retryDelay) + } else { + console.error(`Max retry count reached for request ${index + 1}.`) + setResultForIndex( + `Error in request ${index + 1} after ${maxRetryCount} retries: ${getErrorMessage(error)}`, + index + ) + } + } + } + } + + const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) + + + const handleAddRow = () => { setTestList([...testList, {_id: randString(6)}]) setResultsList([...resultsList, ""]) From 9d9c981b4b4cb5611db16aca517c265e6acbfab7 Mon Sep 17 00:00:00 2001 From: Abram Date: Wed, 6 Dec 2023 16:07:12 +0100 Subject: [PATCH 2/4] Update - introduce sequential requests handling in ab testing and single model evaluation --- .../ABTestingEvaluationTable.tsx | 66 ++++++++++++++++-- .../SingleModelEvaluationTable.tsx | 68 +++++++++++++++++-- 2 files changed, 126 insertions(+), 8 deletions(-) diff --git a/agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx index adc4e7ad37..fd357b6485 100644 --- a/agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx +++ b/agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx @@ -211,14 +211,72 @@ const ABTestingEvaluationTable: React.FC = ({ .catch(console.error) } + const runEvaluationWithRetry = async ( + id: string, + count: number = 1, + showNotification: boolean = true, + maxRetryCount: number, + retryDelay: number + ) => { + let retryCount = 0 + while (retryCount <= maxRetryCount) { + try { + await runEvaluation(id, count, showNotification) + // If the evaluation is successful, break out of the retry loop + break; + } catch (error) { + console.error(`Error in evaluation for ID ${id}, retrying...`) + retryCount++ + if (retryCount <= maxRetryCount) { + // Add a delay before retrying + await delay(retryDelay) + } else { + console.error(`Max retry count reached for evaluation ID ${id}.`) + // Handle the failure after max retries (if needed) + } + } + } + } + + const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms)) + const runAllEvaluations = async () => { + const batchSize = 10 // Number of evaluations to make in each batch + const maxRetryCount = 3 // Maximum number of times to retry a failed evaluation + const retryDelay = 3000 // Delay before retrying a failed evaluation (in milliseconds) + const delayBetweenBatches = 5000 // Delay between batches (in milliseconds) + setEvaluationStatus(EvaluationFlow.EVALUATION_STARTED) - Promise.all(rows.map((row) => runEvaluation(row.id!, rows.length - 1, false))) - .then(() => { + const runBatch = async (startIdx) => { + const endIdx = Math.min(startIdx + batchSize, rows.length) + const batchPromises = [] + + for (let index = startIdx; index < endIdx; index++) { + batchPromises.push( + runEvaluationWithRetry(rows[index].id!, rows.length - 1, false, maxRetryCount, retryDelay) + ) + } + + try { + await Promise.all(batchPromises) + } catch (err) { + console.error("Error in batch:", err) + // Handle the failure of the entire batch (if needed) + } + + // Schedule the next batch with a delay + const nextBatchStartIdx = endIdx + if (nextBatchStartIdx < rows.length) { + setTimeout(() => runBatch(nextBatchStartIdx), delayBetweenBatches) + } else { + // All batches completed setEvaluationStatus(EvaluationFlow.EVALUATION_FINISHED) message.success("Evaluations Updated!") - }) - .catch((err) => console.error("An error occurred:", err)) + } + } + + // Start the first batch + runBatch(0) } const runEvaluation = async ( diff --git a/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx index 5a1fdaad36..1ab2a1fda7 100644 --- a/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx +++ b/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx @@ -213,16 +213,76 @@ const SingleModelEvaluationTable: React.FC = ({ .catch(console.error) } + const runEvaluationWithRetry = async ( + id: string, + count: number = 1, + showNotification: boolean = true, + maxRetryCount: number, + retryDelay: number + ) => { + let retryCount = 0 + while (retryCount <= maxRetryCount) { + try { + await runEvaluation(id, count, showNotification) + // If the evaluation is successful, break out of the retry loop + break; + } catch (error) { + console.error(`Error in evaluation for ID ${id}, retrying...`) + retryCount++ + if (retryCount <= maxRetryCount) { + // Add a delay before retrying + await delay(retryDelay) + } else { + console.error(`Max retry count reached for evaluation ID ${id}.`) + // Handle the failure after max retries (if needed) + } + } + } + } + + const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms)) + const runAllEvaluations = async () => { + const batchSize = 10 // Number of evaluations to make in each batch + const maxRetryCount = 3 // Maximum number of times to retry a failed evaluation + const retryDelay = 3000 // Delay before retrying a failed evaluation (in milliseconds) + const delayBetweenBatches = 5000 // Delay between batches (in milliseconds) + setEvaluationStatus(EvaluationFlow.EVALUATION_STARTED) - Promise.all(rows.map((row) => runEvaluation(row.id!, rows.length - 1, false))) - .then(() => { + const runBatch = async (startIdx) => { + const endIdx = Math.min(startIdx + batchSize, rows.length) + const batchPromises = [] + + for (let index = startIdx; index < endIdx; index++) { + batchPromises.push( + runEvaluationWithRetry(rows[index].id!, rows.length - 1, false, maxRetryCount, retryDelay) + ) + } + + try { + await Promise.all(batchPromises) + } catch (err) { + console.error("Error in batch:", err) + // Handle the failure of the entire batch (if needed) + } + + // Schedule the next batch with a delay + const nextBatchStartIdx = endIdx + if (nextBatchStartIdx < rows.length) { + setTimeout(() => runBatch(nextBatchStartIdx), delayBetweenBatches) + } else { + // All batches completed setEvaluationStatus(EvaluationFlow.EVALUATION_FINISHED) message.success("Evaluations Updated!") - }) - .catch((err) => console.error("An error occurred:", err)) + } + } + + // Start the first batch + runBatch(0) } + + const runEvaluation = async ( id: string, count: number = 1, From b678f8ff3e9564e3e279a13fcb37366d3faba8b8 Mon Sep 17 00:00:00 2001 From: Abram Date: Wed, 6 Dec 2023 16:10:18 +0100 Subject: [PATCH 3/4] art: Format - ran format-fix --- .../ABTestingEvaluationTable.tsx | 20 +++++++++++------ .../SingleModelEvaluationTable.tsx | 22 +++++++++++-------- .../components/Playground/Views/TestView.tsx | 10 ++++----- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx index fd357b6485..a47fcdb0c1 100644 --- a/agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx +++ b/agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx @@ -216,14 +216,14 @@ const ABTestingEvaluationTable: React.FC = ({ count: number = 1, showNotification: boolean = true, maxRetryCount: number, - retryDelay: number + retryDelay: number, ) => { let retryCount = 0 while (retryCount <= maxRetryCount) { try { await runEvaluation(id, count, showNotification) // If the evaluation is successful, break out of the retry loop - break; + break } catch (error) { console.error(`Error in evaluation for ID ${id}, retrying...`) retryCount++ @@ -238,22 +238,28 @@ const ABTestingEvaluationTable: React.FC = ({ } } - const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms)) + const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) const runAllEvaluations = async () => { const batchSize = 10 // Number of evaluations to make in each batch const maxRetryCount = 3 // Maximum number of times to retry a failed evaluation const retryDelay = 3000 // Delay before retrying a failed evaluation (in milliseconds) const delayBetweenBatches = 5000 // Delay between batches (in milliseconds) - + setEvaluationStatus(EvaluationFlow.EVALUATION_STARTED) const runBatch = async (startIdx) => { const endIdx = Math.min(startIdx + batchSize, rows.length) const batchPromises = [] - + for (let index = startIdx; index < endIdx; index++) { batchPromises.push( - runEvaluationWithRetry(rows[index].id!, rows.length - 1, false, maxRetryCount, retryDelay) + runEvaluationWithRetry( + rows[index].id!, + rows.length - 1, + false, + maxRetryCount, + retryDelay, + ), ) } @@ -263,7 +269,7 @@ const ABTestingEvaluationTable: React.FC = ({ console.error("Error in batch:", err) // Handle the failure of the entire batch (if needed) } - + // Schedule the next batch with a delay const nextBatchStartIdx = endIdx if (nextBatchStartIdx < rows.length) { diff --git a/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx index 1ab2a1fda7..a27026b068 100644 --- a/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx +++ b/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx @@ -218,14 +218,14 @@ const SingleModelEvaluationTable: React.FC = ({ count: number = 1, showNotification: boolean = true, maxRetryCount: number, - retryDelay: number + retryDelay: number, ) => { let retryCount = 0 while (retryCount <= maxRetryCount) { try { await runEvaluation(id, count, showNotification) // If the evaluation is successful, break out of the retry loop - break; + break } catch (error) { console.error(`Error in evaluation for ID ${id}, retrying...`) retryCount++ @@ -240,22 +240,28 @@ const SingleModelEvaluationTable: React.FC = ({ } } - const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms)) + const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) const runAllEvaluations = async () => { const batchSize = 10 // Number of evaluations to make in each batch const maxRetryCount = 3 // Maximum number of times to retry a failed evaluation const retryDelay = 3000 // Delay before retrying a failed evaluation (in milliseconds) const delayBetweenBatches = 5000 // Delay between batches (in milliseconds) - + setEvaluationStatus(EvaluationFlow.EVALUATION_STARTED) const runBatch = async (startIdx) => { const endIdx = Math.min(startIdx + batchSize, rows.length) const batchPromises = [] - + for (let index = startIdx; index < endIdx; index++) { batchPromises.push( - runEvaluationWithRetry(rows[index].id!, rows.length - 1, false, maxRetryCount, retryDelay) + runEvaluationWithRetry( + rows[index].id!, + rows.length - 1, + false, + maxRetryCount, + retryDelay, + ), ) } @@ -265,7 +271,7 @@ const SingleModelEvaluationTable: React.FC = ({ console.error("Error in batch:", err) // Handle the failure of the entire batch (if needed) } - + // Schedule the next batch with a delay const nextBatchStartIdx = endIdx if (nextBatchStartIdx < rows.length) { @@ -281,8 +287,6 @@ const SingleModelEvaluationTable: React.FC = ({ runBatch(0) } - - const runEvaluation = async ( id: string, count: number = 1, diff --git a/agenta-web/src/components/Playground/Views/TestView.tsx b/agenta-web/src/components/Playground/Views/TestView.tsx index 250e5d1ab6..7b7e785f7c 100644 --- a/agenta-web/src/components/Playground/Views/TestView.tsx +++ b/agenta-web/src/components/Playground/Views/TestView.tsx @@ -305,17 +305,17 @@ const App: React.FC = ({inputParams, optParams, variant, isChatVa } else { console.error(`Max retry count reached for request ${index + 1}.`) setResultForIndex( - `Error in request ${index + 1} after ${maxRetryCount} retries: ${getErrorMessage(error)}`, - index + `Error in request ${ + index + 1 + } after ${maxRetryCount} retries: ${getErrorMessage(error)}`, + index, ) } } } } - const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) - - + const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) const handleAddRow = () => { setTestList([...testList, {_id: randString(6)}]) From 8565ddac9a7603ff09222d281bdd441faa0d6fa1 Mon Sep 17 00:00:00 2001 From: Abram Date: Wed, 6 Dec 2023 16:23:10 +0100 Subject: [PATCH 4/4] Update - resolve lint and build error --- .../components/EvaluationTable/ABTestingEvaluationTable.tsx | 4 ++-- .../components/EvaluationTable/SingleModelEvaluationTable.tsx | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx index a47fcdb0c1..ed06dca072 100644 --- a/agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx +++ b/agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx @@ -238,7 +238,7 @@ const ABTestingEvaluationTable: React.FC = ({ } } - const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) const runAllEvaluations = async () => { const batchSize = 10 // Number of evaluations to make in each batch @@ -247,7 +247,7 @@ const ABTestingEvaluationTable: React.FC = ({ const delayBetweenBatches = 5000 // Delay between batches (in milliseconds) setEvaluationStatus(EvaluationFlow.EVALUATION_STARTED) - const runBatch = async (startIdx) => { + const runBatch = async (startIdx: number) => { const endIdx = Math.min(startIdx + batchSize, rows.length) const batchPromises = [] diff --git a/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx b/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx index a27026b068..eff6acec8c 100644 --- a/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx +++ b/agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx @@ -240,7 +240,7 @@ const SingleModelEvaluationTable: React.FC = ({ } } - const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) const runAllEvaluations = async () => { const batchSize = 10 // Number of evaluations to make in each batch @@ -249,7 +249,7 @@ const SingleModelEvaluationTable: React.FC = ({ const delayBetweenBatches = 5000 // Delay between batches (in milliseconds) setEvaluationStatus(EvaluationFlow.EVALUATION_STARTED) - const runBatch = async (startIdx) => { + const runBatch = async (startIdx: number) => { const endIdx = Math.min(startIdx + batchSize, rows.length) const batchPromises = []