Merge pull request #1361 from Agenta-AI/issue-1309/-show-diff-of-outp…

…uts-in-the-evaluation-view Highlight output diff in the evaluation comparison view
Agenta-AI · Feb 16, 2024 · 7d7e324 · 7d7e324
2 parents 3928d75 + 4e24624
commit 7d7e324
Show file tree

Hide file tree

Showing 5 changed files with 128 additions and 15 deletions.
diff --git a/agenta-web/package-lock.json b/agenta-web/package-lock.json
diff --git a/agenta-web/package.json b/agenta-web/package.json
@@ -29,6 +29,7 @@
         "@monaco-editor/react": "^4.5.2",
         "@next/mdx": "^13.4.13",
         "@sentry/nextjs": "^7.77.0",
+        "@types/diff": "^5.0.9",
         "@types/js-beautify": "^1.14.0",
         "@types/lodash": "^4.14.201",
         "@types/mdx": "^2.0.6",
@@ -45,6 +46,7 @@
         "axios": "^1.4.0",
         "classnames": "^2.3.2",
         "dayjs": "^1.11.10",
+        "diff": "^5.2.0",
         "dotenv": "^16.3.1",
         "eslint": "8.39.0",
         "eslint-config-next": "13.3.4",

diff --git a/agenta-web/src/components/CompareOutputDiff/CompareOutputDiff.tsx b/agenta-web/src/components/CompareOutputDiff/CompareOutputDiff.tsx
@@ -0,0 +1,52 @@
+import React from "react"
+import {diffWords} from "diff"
+import {useAppTheme} from "../Layout/ThemeContextProvider"
+
+interface CompareOutputDiffProps {
+    variantOutput: any
+    expectedOutput: any
+}
+
+const CompareOutputDiff = ({variantOutput, expectedOutput}: CompareOutputDiffProps) => {
+    const {appTheme} = useAppTheme()
+    const results = diffWords(variantOutput, expectedOutput)
+
+    const display = results.map((part, index) => {
+        if (part.removed) {
+            return (
+                <span
+                    key={index}
+                    style={{
+                        backgroundColor: "#ccffd8",
+                        color: "#000",
+                    }}
+                >
+                    {part.value}
+                </span>
+            )
+        } else if (!part.added) {
+            return <span key={index}>{part.value}</span>
+        } else if (part.added) {
+            return (
+                <>
+                    {" "}
+                    <span
+                        key={index}
+                        style={{
+                            backgroundColor: "#ff818266",
+                            textDecoration: "line-through",
+                            color: appTheme === "dark" ? "#f1f5f8" : "#000",
+                        }}
+                    >
+                        {part.value}
+                    </span>
+                </>
+            )
+        }
+        return null
+    })
+
+    return <span>{display}</span>
+}
+
+export default CompareOutputDiff
diff --git a/agenta-web/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx b/agenta-web/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx
@@ -11,7 +11,7 @@ import {
 import {fetchAllComparisonResults} from "@/services/evaluations"
 import {ColDef} from "ag-grid-community"
 import {AgGridReact} from "ag-grid-react"
-import {Space, Spin, Tag, Tooltip, Typography} from "antd"
+import {Button, Space, Spin, Switch, Tag, Tooltip, Typography} from "antd"
 import React, {useEffect, useMemo, useRef, useState} from "react"
 import {createUseStyles} from "react-jss"
 import {getFilterParams, getTypedValue} from "@/lib/helpers/evaluate"
@@ -24,6 +24,7 @@ import Link from "next/link"
 import AgCustomHeader from "@/components/AgCustomHeader/AgCustomHeader"
 import {useAtom} from "jotai"
 import {evaluatorsAtom} from "@/lib/atoms/evaluation"
+import CompareOutputDiff from "@/components/CompareOutputDiff/CompareOutputDiff"
 
 const useStyles = createUseStyles((theme: JSSTheme) => ({
     table: {
@@ -55,6 +56,7 @@ const EvaluationCompareMode: React.FC<Props> = () => {
     const classes = useStyles()
     const {appTheme} = useAppTheme()
     const [evaluationIdsStr = "", setEvaluationIdsStr] = useQueryParam("evaluations")
+    const [showDiff, setShowDiff] = useQueryParam("showDiff", "show")
     const [fetching, setFetching] = useState(false)
     const [rows, setRows] = useState<ComparisonResultRow[]>([])
     const [testset, setTestset] = useState<TestSet>()
@@ -124,10 +126,28 @@ const EvaluationCompareMode: React.FC<Props> = () => {
                 field: `variants.${vi}.output` as any,
                 ...getFilterParams("text"),
                 valueGetter: (params) => {
-                    return getTypedValue(
-                        params.data?.variants.find(
-                            (item) => item.evaluationId === variant.evaluationId,
-                        )?.output?.result,
+                    return (
+                        <>
+                            {showDiff === "show" ? (
+                                <span>
+                                    <CompareOutputDiff
+                                        variantOutput={getTypedValue(
+                                            params.data?.variants.find(
+                                                (item) =>
+                                                    item.evaluationId === variant.evaluationId,
+                                            )?.output?.result,
+                                        )}
+                                        expectedOutput={params.data?.correctAnswer}
+                                    />
+                                </span>
+                            ) : (
+                                getTypedValue(
+                                    params.data?.variants.find(
+                                        (item) => item.evaluationId === variant.evaluationId,
+                                    )?.output?.result,
+                                )
+                            )}
+                        </>
                     )
                 },
                 cellRenderer: LongTextCellRenderer,
@@ -182,7 +202,7 @@ const EvaluationCompareMode: React.FC<Props> = () => {
         })
 
         return colDefs
-    }, [rows])
+    }, [rows, showDiff])
 
     const fetcher = () => {
         setFetching(true)
@@ -257,9 +277,20 @@ const EvaluationCompareMode: React.FC<Props> = () => {
                         </Space>
                     </Spin>
                 </Space>
-                <Tooltip title="Export as CSV">
-                    <DownloadOutlined onClick={onExport} style={{fontSize: 16}} />
-                </Tooltip>
+                <Space size={10}>
+                    <Space>
+                        <Typography.Text>Show Difference: </Typography.Text>
+                        <Switch
+                            value={showDiff === "show"}
+                            onClick={() => setShowDiff(showDiff === "show" ? "hide" : "show")}
+                        />
+                    </Space>
+                    <Tooltip title="Export as CSV">
+                        <Button icon={<DownloadOutlined />} onClick={onExport}>
+                            Export
+                        </Button>
+                    </Tooltip>
+                </Space>
             </div>
 
             <Spin spinning={fetching}>

diff --git a/agenta-web/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx b/agenta-web/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx
@@ -9,7 +9,7 @@ import {
 import {DeleteOutlined, DownloadOutlined} from "@ant-design/icons"
 import {ColDef} from "ag-grid-community"
 import {AgGridReact} from "ag-grid-react"
-import {Space, Spin, Tag, Tooltip, Typography} from "antd"
+import {Space, Spin, Switch, Tag, Tooltip, Typography} from "antd"
 import {useRouter} from "next/router"
 import React, {useEffect, useMemo, useRef, useState} from "react"
 import {createUseStyles} from "react-jss"
@@ -21,6 +21,8 @@ import {LongTextCellRenderer, ResultRenderer} from "../cellRenderers/cellRendere
 import AgCustomHeader from "@/components/AgCustomHeader/AgCustomHeader"
 import {useAtom} from "jotai"
 import {evaluatorsAtom} from "@/lib/atoms/evaluation"
+import CompareOutputDiff from "@/components/CompareOutputDiff/CompareOutputDiff"
+import {useQueryParam} from "@/hooks/useQuery"
 
 const useStyles = createUseStyles((theme: JSSTheme) => ({
     infoRow: {
@@ -53,6 +55,7 @@ const EvaluationScenarios: React.FC<Props> = () => {
     const [evaluators, setEvaluators] = useAtom(evaluatorsAtom)
     const gridRef = useRef<AgGridReact<_EvaluationScenario>>()
     const evalaution = scenarios[0]?.evaluation
+    const [showDiff, setShowDiff] = useQueryParam("showDiff", "show")
 
     const colDefs = useMemo(() => {
         const colDefs: ColDef<_EvaluationScenario>[] = []
@@ -94,7 +97,18 @@ const EvaluationScenarios: React.FC<Props> = () => {
                     if (result && result.type == "error") {
                         return `${result?.error?.message}\n${result?.error?.stacktrace}`
                     }
-                    return result?.value
+                    return (
+                        <>
+                            {showDiff === "show" ? (
+                                <CompareOutputDiff
+                                    variantOutput={result?.value}
+                                    expectedOutput={params.data?.correct_answer}
+                                />
+                            ) : (
+                                result?.value
+                            )}
+                        </>
+                    )
                 },
                 cellRenderer: LongTextCellRenderer,
             })
@@ -126,7 +140,7 @@ const EvaluationScenarios: React.FC<Props> = () => {
             })
         })
         return colDefs
-    }, [evalaution, scenarios])
+    }, [evalaution, scenarios, showDiff])
 
     const fetcher = () => {
         setFetching(true)
@@ -200,6 +214,13 @@ const EvaluationScenarios: React.FC<Props> = () => {
                     </Space>
                 </Space>
                 <Space size="middle" align="center">
+                    <Space>
+                        <Typography.Text>Show Difference: </Typography.Text>
+                        <Switch
+                            value={showDiff === "show"}
+                            onClick={() => setShowDiff(showDiff === "show" ? "hide" : "show")}
+                        />
+                    </Space>
                     <Tooltip title="Export as CSV">
                         <DownloadOutlined onClick={onExport} style={{fontSize: 16}} />
                     </Tooltip>