improved error handling in evaluation views

Agenta-AI · May 26, 2024 · cc4fe9c · cc4fe9c
1 parent 96f26cc
commit cc4fe9c
Show file tree

Hide file tree

Showing 4 changed files with 95 additions and 11 deletions.
diff --git a/agenta-web/src/components/pages/evaluations/cellRenderers/cellRenderers.tsx b/agenta-web/src/components/pages/evaluations/cellRenderers/cellRenderers.tsx
@@ -13,7 +13,7 @@ import {
     InfoCircleOutlined,
 } from "@ant-design/icons"
 import {ICellRendererParams} from "ag-grid-community"
-import {GlobalToken, Space, Tooltip, Typography, message, theme} from "antd"
+import {Button, GlobalToken, Space, Tooltip, Typography, message, theme} from "antd"
 import dayjs from "dayjs"
 import relativeTime from "dayjs/plugin/relativeTime"
 import duration from "dayjs/plugin/duration"
@@ -137,19 +137,40 @@ export function LongTextCellRenderer(params: ICellRendererParams, output?: any)
 }
 
 export const ResultRenderer = React.memo(
-    (params: ICellRendererParams<_EvaluationScenario> & {config: EvaluatorConfig}) => {
+    (
+        params: ICellRendererParams<_EvaluationScenario> & {
+            config: EvaluatorConfig
+            setIsErrorModalOpen: React.Dispatch<React.SetStateAction<boolean>>
+            setModalErrorMsg: React.Dispatch<
+                React.SetStateAction<{
+                    message: string
+                    stackTrace: string
+                }>
+            >
+        },
+    ) => {
+        const {setIsErrorModalOpen, setModalErrorMsg} = params
         const result = params.data?.results.find(
             (item) => item.evaluator_config === params.config.id,
         )?.result
-        let errorMsg = ""
-        if (result?.type === "error") {
-            errorMsg = `${result?.error?.message}\n${result?.error?.stacktrace}`
+        if (result?.type === "error" && result.error) {
+            setModalErrorMsg({message: result.error.message, stackTrace: result.error.stacktrace})
         }
 
-        return (
-            <Typography.Text type={errorMsg ? "danger" : undefined}>
-                {errorMsg || getTypedValue(result)}
+        return result?.type === "error" && result.error ? (
+            <Typography.Text type={"danger"} strong>
+                Failed to invoke LLM app{" "}
+                <Button
+                    size="small"
+                    className="text-xs"
+                    type="text"
+                    onClick={() => setIsErrorModalOpen(true)}
+                >
+                    (more details)
+                </Button>
             </Typography.Text>
+        ) : (
+            <Typography.Text>{getTypedValue(result)}</Typography.Text>
         )
     },
     (prev, next) => prev.value === next.value,

diff --git a/agenta-web/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx b/agenta-web/src/components/pages/evaluations/evaluationCompare/EvaluationCompare.tsx
@@ -9,7 +9,7 @@ import {
     _EvaluationScenario,
 } from "@/lib/Types"
 import {fetchAllComparisonResults} from "@/services/evaluations"
-import {ColDef} from "ag-grid-community"
+import {ColDef, ValueGetterParams} from "ag-grid-community"
 import {AgGridReact} from "ag-grid-react"
 import {Button, Dropdown, DropdownProps, Space, Spin, Switch, Tag, Tooltip, Typography} from "antd"
 import React, {useEffect, useMemo, useRef, useState} from "react"
@@ -33,6 +33,7 @@ import {evaluatorsAtom} from "@/lib/atoms/evaluation"
 import CompareOutputDiff from "@/components/CompareOutputDiff/CompareOutputDiff"
 import {formatCurrency, formatLatency} from "@/lib/helpers/formatters"
 import {useLocalStorage} from "usehooks-ts"
+import EvaluationErrorModal from "../EvaluationErrorModal/EvaluationErrorModal"
 
 const useStyles = createUseStyles((theme: JSSTheme) => ({
     table: {
@@ -89,6 +90,8 @@ const EvaluationCompareMode: React.FC<Props> = () => {
     const [evaluators] = useAtom(evaluatorsAtom)
     const gridRef = useRef<AgGridReact<_EvaluationScenario>>()
     const [filterColsDropdown, setFilterColsDropdown] = useState(false)
+    const [modalErrorMsg, setModalErrorMsg] = useState({message: "", stackTrace: ""})
+    const [isErrorModalOpen, setIsErrorModalOpen] = useState(false)
 
     const handleOpenChange: DropdownProps["onOpenChange"] = (nextOpen, info) => {
         if (info.source === "trigger" || nextOpen) {
@@ -254,6 +257,36 @@ const EvaluationCompareMode: React.FC<Props> = () => {
                     hide:
                         !evalIds.includes(variant.evaluationId) ||
                         hiddenVariants.includes(config.name),
+                    cellRenderer: (params: ValueGetterParams<ComparisonResultRow, any>) => {
+                        const result = params.data?.variants
+                            .find((item) => item.evaluationId === variant.evaluationId)
+                            ?.evaluatorConfigs.find(
+                                (item) => item.evaluatorConfig.id === config.id,
+                            )?.result
+
+                        if (result?.error && result.type === "error") {
+                            setModalErrorMsg({
+                                message: result.error.message,
+                                stackTrace: result.error.stacktrace,
+                            })
+                        }
+
+                        return result?.type === "error" && result.error ? (
+                            <Typography.Text type={"danger"} strong>
+                                Failed to invoke LLM app{" "}
+                                <Button
+                                    size="small"
+                                    className="text-xs"
+                                    type="text"
+                                    onClick={() => setIsErrorModalOpen(true)}
+                                >
+                                    (more details)
+                                </Button>
+                            </Typography.Text>
+                        ) : (
+                            <Typography.Text>{getTypedValue(result)}</Typography.Text>
+                        )
+                    },
                     valueGetter: (params) => {
                         return getTypedValue(
                             params.data?.variants
@@ -504,6 +537,12 @@ const EvaluationCompareMode: React.FC<Props> = () => {
                     />
                 </div>
             </Spin>
+
+            <EvaluationErrorModal
+                isErrorModalOpen={isErrorModalOpen}
+                setIsErrorModalOpen={setIsErrorModalOpen}
+                modalErrorMsg={modalErrorMsg}
+            />
         </div>
     )
 }

diff --git a/agenta-web/src/components/pages/evaluations/evaluationResults/EvaluationResults.tsx b/agenta-web/src/components/pages/evaluations/evaluationResults/EvaluationResults.tsx
@@ -1,9 +1,9 @@
 import React, {useEffect, useMemo, useRef, useState} from "react"
 import {AgGridReact} from "ag-grid-react"
 import {useAppTheme} from "@/components/Layout/ThemeContextProvider"
-import {ColDef} from "ag-grid-community"
+import {ColDef, ValueGetterParams} from "ag-grid-community"
 import {createUseStyles} from "react-jss"
-import {Button, Dropdown, DropdownProps, Space, Spin, Tag, Tooltip, theme} from "antd"
+import {Button, Dropdown, DropdownProps, Space, Spin, Tag, Tooltip, Typography, theme} from "antd"
 import {
     CheckOutlined,
     DeleteOutlined,
@@ -266,6 +266,19 @@ const EvaluationResults: React.FC<Props> = () => {
                         ),
                         autoHeaderHeight: true,
                         ...getFilterParams("number"),
+                        cellRenderer: (params: ValueGetterParams<_Evaluation, any>) => {
+                            const result = params.data?.aggregated_results.find(
+                                (item) => item.evaluator_config.id === config.id,
+                            )?.result
+
+                            return result?.error ? (
+                                <Typography.Text type="danger" strong>
+                                    Error
+                                </Typography.Text>
+                            ) : (
+                                <Typography.Text>{getTypedValue(result)}</Typography.Text>
+                            )
+                        },
                         valueGetter: (params) =>
                             getTypedValue(
                                 params.data?.aggregated_results.find(

diff --git a/agenta-web/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx b/agenta-web/src/components/pages/evaluations/evaluationScenarios/EvaluationScenarios.tsx
@@ -24,6 +24,7 @@ import {evaluatorsAtom} from "@/lib/atoms/evaluation"
 import CompareOutputDiff from "@/components/CompareOutputDiff/CompareOutputDiff"
 import {formatCurrency, formatLatency} from "@/lib/helpers/formatters"
 import {useLocalStorage} from "usehooks-ts"
+import EvaluationErrorModal from "../EvaluationErrorModal/EvaluationErrorModal"
 
 const useStyles = createUseStyles((theme: JSSTheme) => ({
     infoRow: {
@@ -57,6 +58,8 @@ const EvaluationScenarios: React.FC<Props> = () => {
     const gridRef = useRef<AgGridReact<_EvaluationScenario>>()
     const evalaution = scenarios[0]?.evaluation
     const [showDiff, setShowDiff] = useLocalStorage("showDiff", "show")
+    const [modalErrorMsg, setModalErrorMsg] = useState({message: "", stackTrace: ""})
+    const [isErrorModalOpen, setIsErrorModalOpen] = useState(false)
 
     const colDefs = useMemo(() => {
         const colDefs: ColDef<_EvaluationScenario>[] = []
@@ -137,6 +140,8 @@ const EvaluationScenarios: React.FC<Props> = () => {
                 cellRenderer: ResultRenderer,
                 cellRendererParams: {
                     config,
+                    setIsErrorModalOpen,
+                    setModalErrorMsg,
                 },
                 valueGetter: (params) => {
                     return params.data?.results[index].result.value
@@ -272,6 +277,12 @@ const EvaluationScenarios: React.FC<Props> = () => {
                     />
                 </div>
             </Spin>
+
+            <EvaluationErrorModal
+                isErrorModalOpen={isErrorModalOpen}
+                setIsErrorModalOpen={setIsErrorModalOpen}
+                modalErrorMsg={modalErrorMsg}
+            />
         </div>
     )
 }