Add the option to mark both as correct in A\B testing human evaluation

agenta-backend/agenta_backend/services/results_service.py

@@ -47,10 +47,23 @@ async def _compute_stats_for_human_a_b_testing_evaluation(evaluation_scenarios:
     results = {}
     results["variants_votes_data"] = {}
     results["flag_votes"] = {}
+    results["positive_votes"] = {}
 
     flag_votes_nb = [
         scenario for scenario in evaluation_scenarios if scenario.vote == "0"
     ]
+
+    positive_votes_nb = [
+        scenario for scenario in evaluation_scenarios if scenario.vote == "1"
+    ]
+
+    results["positive_votes"]["number_of_votes"] = len(positive_votes_nb)
+    results["positive_votes"]["percentage"] = (
+        round(len(positive_votes_nb) / len(evaluation_scenarios) * 100, 2)
+        if len(evaluation_scenarios)
+        else 0
+    )
+
     results["flag_votes"]["number_of_votes"] = len(flag_votes_nb)
     results["flag_votes"]["percentage"] = (
         round(len(flag_votes_nb) / len(evaluation_scenarios) * 100, 2)

agenta-web/cypress/tsconfig.json

@@ -2,7 +2,7 @@
     "compilerOptions": {
         "target": "es5",
         "lib": ["es5", "dom"],
-        "types": ["cypress", "node"]
+        "types": ["cypress", "node"],
     },
-    "include": ["**/*.ts"]
+    "include": ["**/*.ts"],
 }

agenta-web/src/components/EvaluationTable/ABTestingEvaluationTable.tsx

@@ -15,7 +15,7 @@ import {exportABTestingEvaluationData} from "@/lib/helpers/evaluate"
 import SecondaryButton from "../SecondaryButton/SecondaryButton"
 import {useQueryParam} from "@/hooks/useQuery"
 import EvaluationCardView, {VARIANT_COLORS} from "../Evaluations/EvaluationCardView"
-import {Evaluation, EvaluationScenario, KeyValuePair, Variant} from "@/lib/Types"
+import {Evaluation, EvaluationResult, EvaluationScenario, KeyValuePair, Variant} from "@/lib/Types"
 import {EvaluationTypeLabels, batchExecute, camelToSnake} from "@/lib/helpers/utils"
 import {testsetRowToChatMessages} from "@/lib/helpers/testset"
 import EvaluationVotePanel from "../Evaluations/EvaluationCardView/EvaluationVotePanel"
@@ -77,6 +77,11 @@ const useStyles = createUseStyles({
             color: "#3f8600",
         },
     },
+    stat: {
+        "& .ant-statistic-content-value": {
+            color: "#1677ff",
+        },
+    },
     statWrong: {
         "& .ant-statistic-content-value": {
             color: "#cf1322",
@@ -105,11 +110,12 @@ const ABTestingEvaluationTable: React.FC<EvaluationTableProps> = ({
 
     const [rows, setRows] = useState<ABTestingEvaluationTableRow[]>([])
     const [evaluationStatus, setEvaluationStatus] = useState<EvaluationFlow>(evaluation.status)
-    const [evaluationResults, setEvaluationResults] = useState<any>(null)
+    const [evaluationResults, setEvaluationResults] = useState<EvaluationResult | null>(null)
     const [viewMode, setViewMode] = useQueryParam("viewMode", "card")
 
     let num_of_rows = evaluationResults?.votes_data.nb_of_rows || 0
     let flag_votes = evaluationResults?.votes_data.flag_votes?.number_of_votes || 0
+    let positive_votes = evaluationResults?.votes_data.positive_votes.number_of_votes || 0
     let appVariant1 =
         evaluationResults?.votes_data?.variants_votes_data?.[evaluation.variants[0]?.variantId]
             ?.number_of_votes || 0
@@ -410,7 +416,7 @@ const ABTestingEvaluationTable: React.FC<EvaluationTableProps> = ({
                                             evaluation.variants[0]?.variantName || ""
                                         } is better:`}
                                         value={`${appVariant1} out of ${num_of_rows}`}
-                                        className={classes.statCorrect}
+                                        className={classes.stat}
                                     />
                                 </Col>
                                 <Col span={10}>
@@ -419,6 +425,13 @@ const ABTestingEvaluationTable: React.FC<EvaluationTableProps> = ({
                                             evaluation.variants[1]?.variantName || ""
                                         } is better:`}
                                         value={`${appVariant2} out of ${num_of_rows}`}
+                                        className={classes.stat}
+                                    />
+                                </Col>
+                                <Col span={4}>
+                                    <Statistic
+                                        title="Both are good:"
+                                        value={`${positive_votes} out of ${num_of_rows}`}
                                         className={classes.statCorrect}
                                     />
                                 </Col>

agenta-web/src/components/Evaluations/EvaluationCardView/EvaluationVotePanel.tsx

@@ -3,6 +3,7 @@ import {Button, ConfigProvider, InputNumber, Spin, Typography, theme} from "antd
 import React from "react"
 import {createUseStyles} from "react-jss"
 import {VARIANT_COLORS} from "."
+import {v4 as uuidv4} from "uuid"
 
 const useStyles = createUseStyles({
     root: {
@@ -76,6 +77,7 @@ const ComparisonVote: React.FC<ComparisonVoteProps> = ({variants, onChange, valu
     const classes = useStyles()
     const {token} = theme.useToken()
     const badId = "0"
+    const goodId = "1"
 
     const getOnClick = (variantId: string) => () => {
         onChange(variantId)
@@ -102,6 +104,17 @@ const ComparisonVote: React.FC<ComparisonVoteProps> = ({variants, onChange, valu
                 className={vertical ? classes.btnsDividerVertical : classes.btnsDividerHorizontal}
                 style={{borderColor: token.colorBorder}}
             />
+            <ConfigProvider theme={{token: {colorError: VARIANT_COLORS[2]}}}>
+                <Button
+                    danger
+                    type={value === goodId ? "primary" : undefined}
+                    key={goodId}
+                    onClick={getOnClick(goodId)}
+                    data-cy="evaluation-vote-panel-comparison-both-good-vote-button-button"
+                >
+                    Both are good
+                </Button>
+            </ConfigProvider>
             <Button
                 danger
                 type={value === badId ? "primary" : undefined}

agenta-web/src/components/Evaluations/EvaluationCardView/index.tsx

@@ -26,6 +26,7 @@ import {useVariants} from "@/lib/hooks/useVariant"
 export const VARIANT_COLORS = [
     "#297F87", // "#722ed1",
     "#F6D167", //"#13c2c2",
+    "#4caf50",
 ]
 
 const useStyles = createUseStyles({

agenta-web/src/components/Evaluations/HumanEvaluationResult.tsx

@@ -38,6 +38,10 @@ export interface HumanEvaluationListTableDataType {
             number_of_votes: number
             percentage: number
         }
+        positive_votes: {
+            number_of_votes: number
+            percentage: number
+        }
         variants_votes_data: Record<string, VariantVotesData>
     }
     createdAt: string
@@ -84,6 +88,16 @@ const useStyles = createUseStyles({
             color: "#1677ff",
         },
     },
+    statGood: {
+        "& .ant-statistic-content-value": {
+            fontSize: 20,
+            color: "#3f8600",
+        },
+        "& .ant-statistic-content-suffix": {
+            fontSize: 20,
+            color: "#3f8600",
+        },
+    },
 })
 
 const {Title} = Typography
@@ -209,11 +223,29 @@ export default function HumanEvaluationResult() {
                 )
             },
         },
+        {
+            title: "Both are good",
+            dataIndex: "positive",
+            key: "positive",
+            render: (value: any, record: HumanEvaluationListTableDataType) => {
+                let percentage = record.votesData.positive_votes.percentage
+                return (
+                    <span>
+                        <Statistic
+                            className={classes.statGood}
+                            value={percentage}
+                            precision={percentage <= 99 ? 2 : 1}
+                            suffix="%"
+                        />
+                    </span>
+                )
+            },
+        },
         {
             title: "Flag",
             dataIndex: "flag",
             key: "flag",
-            render: (value: any, record: HumanEvaluationListTableDataType, index: number) => {
+            render: (value: any, record: HumanEvaluationListTableDataType) => {
                 let percentage = record.votesData.flag_votes.percentage
                 return (
                     <span>

agenta-web/src/lib/Types.ts

@@ -106,6 +106,10 @@ export interface EvaluationResult {
             number_of_votes: number
             percentage: number
         }
+        positive_votes: {
+            number_of_votes: number
+            percentage: number
+        }
         variants: string[]
         variant_names: string[]
         variants_votes_data: {

agenta-web/tsconfig.json

@@ -15,9 +15,9 @@
         "jsx": "preserve",
         "incremental": true,
         "paths": {
-            "@/*": ["./src/*"]
-        }
+            "@/*": ["./src/*"],
+        },
     },
     "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
-    "exclude": ["node_modules", "cypress.config.ts", "cypress/**/*"]
+    "exclude": ["node_modules", "cypress.config.ts", "cypress/**/*"],
 }