Add the option to mark both as correct in A\B testing human evaluation

agenta-backend/agenta_backend/services/results_service.py

@@ -47,10 +47,23 @@ async def _compute_stats_for_human_a_b_testing_evaluation(evaluation_scenarios:
     results = {}
     results["variants_votes_data"] = {}
     results["flag_votes"] = {}
+    results["positive_votes"] = {}
 
     flag_votes_nb = [
         scenario for scenario in evaluation_scenarios if scenario.vote == "0"
     ]
+
+    positive_votes_nb = [
+        scenario for scenario in evaluation_scenarios if scenario.vote == "1"
+    ]
+
+    results["positive_votes"]["number_of_votes"] = len(positive_votes_nb)
+    results["positive_votes"]["percentage"] = (
+        round(len(positive_votes_nb) / len(evaluation_scenarios) * 100, 2)
+        if len(evaluation_scenarios)
+        else 0
+    )
+
     results["flag_votes"]["number_of_votes"] = len(flag_votes_nb)
     results["flag_votes"]["percentage"] = (
         round(len(flag_votes_nb) / len(evaluation_scenarios) * 100, 2)

agenta-web/cypress/tsconfig.json

@@ -2,7 +2,7 @@
     "compilerOptions": {
         "target": "es5",
         "lib": ["es5", "dom"],
-        "types": ["cypress", "node"]
+        "types": ["cypress", "node"],
     },
-    "include": ["**/*.ts"]
+    "include": ["**/*.ts"],
 }

agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx

@@ -15,7 +15,7 @@ import {exportABTestingEvaluationData} from "@/lib/helpers/evaluate"
 import SecondaryButton from "../SecondaryButton/SecondaryButton"
 import {useQueryParam} from "@/hooks/useQuery"
 import EvaluationCardView, {VARIANT_COLORS} from "../Evaluations/EvaluationCardView"
-import {Evaluation, EvaluationScenario, KeyValuePair, Variant} from "@/lib/Types"
+import {Evaluation, EvaluationResult, EvaluationScenario, KeyValuePair, Variant} from "@/lib/Types"
 import {EvaluationTypeLabels, batchExecute, camelToSnake} from "@/lib/helpers/utils"
 import {testsetRowToChatMessages} from "@/lib/helpers/testset"
 import EvaluationVotePanel from "../Evaluations/EvaluationCardView/EvaluationVotePanel"
@@ -105,11 +105,12 @@ const ABTestingEvaluationTable: React.FC<EvaluationTableProps> = ({
 
     const [rows, setRows] = useState<ABTestingEvaluationTableRow[]>([])
     const [evaluationStatus, setEvaluationStatus] = useState<EvaluationFlow>(evaluation.status)
-    const [evaluationResults, setEvaluationResults] = useState<any>(null)
+    const [evaluationResults, setEvaluationResults] = useState<EvaluationResult | null>(null)
     const [viewMode, setViewMode] = useQueryParam("viewMode", "card")
 
     let num_of_rows = evaluationResults?.votes_data.nb_of_rows || 0
     let flag_votes = evaluationResults?.votes_data.flag_votes?.number_of_votes || 0
+    let positive_votes = evaluationResults?.votes_data.positive_votes.number_of_votes || 0
     let appVariant1 =
         evaluationResults?.votes_data?.variants_votes_data?.[evaluation.variants[0]?.variantId]
             ?.number_of_votes || 0
@@ -423,6 +424,13 @@ const ABTestingEvaluationTable: React.FC<EvaluationTableProps> = ({
                                         className={classes.statWrong}
                                     />
                                 </Col>
+                                <Col span={4}>
+                                    <Statistic
+                                        title="Both are good:"
+                                        value={`${positive_votes} out of ${num_of_rows}`}
+                                        className={classes.statCorrect}
+                                    />
+                                </Col>
                             </Row>
                         </Card>
                     </Col>

agenta-web/src/components/Evaluations/EvaluationCardView/EvaluationVotePanel.tsx

@@ -3,6 +3,7 @@ import {Button, ConfigProvider, InputNumber, Spin, Typography, theme} from "antd
 import React from "react"
 import {createUseStyles} from "react-jss"
 import {VARIANT_COLORS} from "."
+import {v4 as uuidv4} from "uuid"
 
 const useStyles = createUseStyles({
     root: {
@@ -76,6 +77,7 @@ const ComparisonVote: React.FC<ComparisonVoteProps> = ({variants, onChange, valu
     const classes = useStyles()
     const {token} = theme.useToken()
     const badId = "0"
+    const goodId = "1"
 
     const getOnClick = (variantId: string) => () => {
         onChange(variantId)
@@ -111,6 +113,18 @@ const ComparisonVote: React.FC<ComparisonVoteProps> = ({variants, onChange, valu
             >
                 Both are bad
             </Button>
+
+            <ConfigProvider theme={{token: {colorError: VARIANT_COLORS[2]}}}>
+                <Button
+                    danger
+                    type={value === goodId ? "primary" : undefined}
+                    key={goodId}
+                    onClick={getOnClick(goodId)}
+                    data-cy="evaluation-vote-panel-comparison-both-good-vote-button-button"
+                >
+                    Both are good
+                </Button>
+            </ConfigProvider>
         </div>
     )
 }

agenta-web/src/components/Evaluations/EvaluationCardView/index.tsx

@@ -26,6 +26,7 @@ import {useVariants} from "@/lib/hooks/useVariant"
 export const VARIANT_COLORS = [
     "#297F87", // "#722ed1",
     "#F6D167", //"#13c2c2",
+    "#4caf50",
 ]
 
 const useStyles = createUseStyles({

agenta-web/src/lib/Types.ts

@@ -106,6 +106,10 @@ export interface EvaluationResult {
             number_of_votes: number
             percentage: number
         }
+        positive_votes: {
+            number_of_votes: number
+            percentage: number
+        }
         variants: string[]
         variant_names: string[]
         variants_votes_data: {

agenta-web/tsconfig.json

@@ -15,9 +15,9 @@
         "jsx": "preserve",
         "incremental": true,
         "paths": {
-            "@/*": ["./src/*"]
-        }
+            "@/*": ["./src/*"],
+        },
     },
     "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
-    "exclude": ["node_modules", "cypress.config.ts", "cypress/**/*"]
+    "exclude": ["node_modules", "cypress.config.ts", "cypress/**/*"],
 }