Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improvement: Resolves OpenAI rate-limiting issue when running large test sets in the playground/evaluation #1007

Closed
wants to merge 4 commits into from
Closed

Improvement: Resolves OpenAI rate-limiting issue when running large test sets in the playground/evaluation #1007

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1676915
Update - implemented batch-run-with-retry functionality
aybruhm Dec 6, 2023
9d9c981
Update - introduce sequential requests handling in ab testing and sin…
aybruhm Dec 6, 2023
b678f8f
art: Format - ran format-fix
aybruhm Dec 6, 2023
8565dda
Update - resolve lint and build error
aybruhm Dec 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 68 additions & 4 deletions agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,14 +211,78 @@ const ABTestingEvaluationTable: React.FC<EvaluationTableProps> = ({
.catch(console.error)
}

const runEvaluationWithRetry = async (
id: string,
count: number = 1,
showNotification: boolean = true,
maxRetryCount: number,
retryDelay: number,
) => {
let retryCount = 0
while (retryCount <= maxRetryCount) {
try {
await runEvaluation(id, count, showNotification)
// If the evaluation is successful, break out of the retry loop
break
} catch (error) {
console.error(`Error in evaluation for ID ${id}, retrying...`)
retryCount++
if (retryCount <= maxRetryCount) {
// Add a delay before retrying
await delay(retryDelay)
} else {
console.error(`Max retry count reached for evaluation ID ${id}.`)
// Handle the failure after max retries (if needed)
}
}
}
}

const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))

const runAllEvaluations = async () => {
const batchSize = 10 // Number of evaluations to make in each batch
const maxRetryCount = 3 // Maximum number of times to retry a failed evaluation
const retryDelay = 3000 // Delay before retrying a failed evaluation (in milliseconds)
const delayBetweenBatches = 5000 // Delay between batches (in milliseconds)

setEvaluationStatus(EvaluationFlow.EVALUATION_STARTED)
Promise.all(rows.map((row) => runEvaluation(row.id!, rows.length - 1, false)))
.then(() => {
const runBatch = async (startIdx: number) => {
const endIdx = Math.min(startIdx + batchSize, rows.length)
const batchPromises = []

for (let index = startIdx; index < endIdx; index++) {
batchPromises.push(
runEvaluationWithRetry(
rows[index].id!,
rows.length - 1,
false,
maxRetryCount,
retryDelay,
),
)
}

try {
await Promise.all(batchPromises)
} catch (err) {
console.error("Error in batch:", err)
// Handle the failure of the entire batch (if needed)
}

// Schedule the next batch with a delay
const nextBatchStartIdx = endIdx
if (nextBatchStartIdx < rows.length) {
setTimeout(() => runBatch(nextBatchStartIdx), delayBetweenBatches)
} else {
// All batches completed
setEvaluationStatus(EvaluationFlow.EVALUATION_FINISHED)
message.success("Evaluations Updated!")
})
.catch((err) => console.error("An error occurred:", err))
}
}

// Start the first batch
runBatch(0)
}

const runEvaluation = async (
Expand Down
72 changes: 68 additions & 4 deletions agenta-web/src/components/EvaluationTable/SingleModelEvaluationTable.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,14 +213,78 @@ const SingleModelEvaluationTable: React.FC<EvaluationTableProps> = ({
.catch(console.error)
}

const runEvaluationWithRetry = async (
id: string,
count: number = 1,
showNotification: boolean = true,
maxRetryCount: number,
retryDelay: number,
) => {
let retryCount = 0
while (retryCount <= maxRetryCount) {
try {
await runEvaluation(id, count, showNotification)
// If the evaluation is successful, break out of the retry loop
break
} catch (error) {
console.error(`Error in evaluation for ID ${id}, retrying...`)
retryCount++
if (retryCount <= maxRetryCount) {
// Add a delay before retrying
await delay(retryDelay)
} else {
console.error(`Max retry count reached for evaluation ID ${id}.`)
// Handle the failure after max retries (if needed)
}
}
}
}

const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))

const runAllEvaluations = async () => {
const batchSize = 10 // Number of evaluations to make in each batch
const maxRetryCount = 3 // Maximum number of times to retry a failed evaluation
const retryDelay = 3000 // Delay before retrying a failed evaluation (in milliseconds)
const delayBetweenBatches = 5000 // Delay between batches (in milliseconds)

setEvaluationStatus(EvaluationFlow.EVALUATION_STARTED)
Promise.all(rows.map((row) => runEvaluation(row.id!, rows.length - 1, false)))
.then(() => {
const runBatch = async (startIdx: number) => {
const endIdx = Math.min(startIdx + batchSize, rows.length)
const batchPromises = []

for (let index = startIdx; index < endIdx; index++) {
batchPromises.push(
runEvaluationWithRetry(
rows[index].id!,
rows.length - 1,
false,
maxRetryCount,
retryDelay,
),
)
}

try {
await Promise.all(batchPromises)
} catch (err) {
console.error("Error in batch:", err)
// Handle the failure of the entire batch (if needed)
}

// Schedule the next batch with a delay
const nextBatchStartIdx = endIdx
if (nextBatchStartIdx < rows.length) {
setTimeout(() => runBatch(nextBatchStartIdx), delayBetweenBatches)
} else {
// All batches completed
setEvaluationStatus(EvaluationFlow.EVALUATION_FINISHED)
message.success("Evaluations Updated!")
})
.catch((err) => console.error("An error occurred:", err))
}
}

// Start the first batch
runBatch(0)
}

const runEvaluation = async (
Expand Down
46 changes: 44 additions & 2 deletions agenta-web/src/components/Playground/Views/TestView.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,10 +271,52 @@ const App: React.FC<TestViewProps> = ({inputParams, optParams, variant, isChatVa
}
}

const handleRunAll = () => {
testList.forEach((_, index) => handleRun(index))
const handleRunAll = async () => {
const batchSize = 10 // Number of requests to make in each batch
const maxRetryCount = 3 // Maximum number of times to retry a failed request
const retryDelay = 3000 // Delay before retrying a failed request (in milliseconds)

for (let startIdx = 0; startIdx < testList.length; startIdx += batchSize) {
const endIdx = Math.min(startIdx + batchSize, testList.length)
const batchPromises = []

for (let index = startIdx; index < endIdx; index++) {
batchPromises.push(runWithRetry(index, maxRetryCount, retryDelay))
}

await Promise.all(batchPromises)
}
}

const runWithRetry = async (index: number, maxRetryCount: number, retryDelay: number) => {
let retryCount = 0

while (retryCount <= maxRetryCount) {
try {
await handleRun(index)
// If the request is successful, break out of the retry loop
break
} catch (error) {
console.error(`Error in request ${index + 1}, retrying...`)
retryCount++
if (retryCount <= maxRetryCount) {
// Add a delay before retrying
await delay(retryDelay)
} else {
console.error(`Max retry count reached for request ${index + 1}.`)
setResultForIndex(
`Error in request ${
index + 1
} after ${maxRetryCount} retries: ${getErrorMessage(error)}`,
index,
)
}
}
}
}

const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))

const handleAddRow = () => {
setTestList([...testList, {_id: randString(6)}])
setResultsList([...resultsList, ""])
Expand Down
Loading