diff --git a/agenta-web/src/components/pages/evaluations/NewEvaluation/NewEvaluationModal.tsx b/agenta-web/src/components/pages/evaluations/NewEvaluation/NewEvaluationModal.tsx
index d48c78399e..c0eb9a86ac 100644
--- a/agenta-web/src/components/pages/evaluations/NewEvaluation/NewEvaluationModal.tsx
+++ b/agenta-web/src/components/pages/evaluations/NewEvaluation/NewEvaluationModal.tsx
@@ -43,12 +43,16 @@ const useStyles = createUseStyles((theme: JSSTheme) => ({
             alignItems: "center !important",
         },
         "& .ant-collapse-content": {
-            height: 500,
+            maxHeight: 400,
+            height: "100%",
             overflowY: "auto",
             "& .ant-collapse-content-box": {
                 padding: 0,
             },
         },
+        "& .ant-input-group-addon button": {
+            height: 30,
+        },
     },
 }))
 
@@ -70,6 +74,11 @@ const NewEvaluationModal: React.FC<Props> = ({onSuccess, ...props}) => {
     const [selectedVariantIds, setSelectedVariantIds] = useState<string[]>([])
     const [selectedEvalConfigs, setSelectedEvalConfigs] = useState<string[]>([])
 
+    const [activePanel, setActivePanel] = useState<string | null>("testsetPanel")
+    const handlePanelChange = (key: string | string[]) => {
+        setActivePanel((prevKey) => (prevKey === key ? null : (key as string)))
+    }
+
     useEffect(() => {
         const fetchData = async () => {
             setFetching(true)
@@ -195,12 +204,16 @@ const NewEvaluationModal: React.FC<Props> = ({onSuccess, ...props}) => {
             <Spin spinning={fetching} className="w-full">
                 <Space direction="vertical" size={16} className="w-full">
                     <SelectTestsetSection
+                        activePanel={activePanel}
+                        handlePanelChange={handlePanelChange}
                         testSets={testSets}
                         selectedTestsetId={selectedTestsetId}
                         setSelectedTestsetId={setSelectedTestsetId}
                         className={classes.collapseContainer}
                     />
                     <SelectVariantSection
+                        activePanel={activePanel}
+                        handlePanelChange={handlePanelChange}
                         variants={variants}
                         usernames={usernames}
                         selectedVariantIds={selectedVariantIds}
@@ -208,6 +221,8 @@ const NewEvaluationModal: React.FC<Props> = ({onSuccess, ...props}) => {
                         className={classes.collapseContainer}
                     />
                     <SelectEvaluatorSection
+                        activePanel={activePanel}
+                        handlePanelChange={handlePanelChange}
                         evaluators={evaluators}
                         evaluatorConfigs={evaluatorConfigs}
                         selectedEvalConfigs={selectedEvalConfigs}
diff --git a/agenta-web/src/components/pages/evaluations/NewEvaluation/SelectEvaluatorSection.tsx b/agenta-web/src/components/pages/evaluations/NewEvaluation/SelectEvaluatorSection.tsx
index 70bbe77f5f..13204ee7e7 100644
--- a/agenta-web/src/components/pages/evaluations/NewEvaluation/SelectEvaluatorSection.tsx
+++ b/agenta-web/src/components/pages/evaluations/NewEvaluation/SelectEvaluatorSection.tsx
@@ -10,6 +10,8 @@ type SelectEvaluatorSectionProps = {
     evaluators: Evaluator[]
     selectedEvalConfigs: string[]
     setSelectedEvalConfigs: React.Dispatch<React.SetStateAction<string[]>>
+    handlePanelChange: (key: string | string[]) => void
+    activePanel: string | null
 } & React.ComponentProps<typeof Collapse>
 
 const SelectEvaluatorSection = ({
@@ -17,6 +19,8 @@ const SelectEvaluatorSection = ({
     evaluators,
     selectedEvalConfigs,
     setSelectedEvalConfigs,
+    activePanel,
+    handlePanelChange,
     ...props
 }: SelectEvaluatorSectionProps) => {
     const [searchTerm, setSearchTerm] = useState("")
@@ -89,11 +93,12 @@ const SelectEvaluatorSection = ({
     return (
         <>
             <Collapse
-                defaultActiveKey={["1"]}
+                activeKey={activePanel === "evaluatorPanel" ? "evaluatorPanel" : undefined}
+                onChange={() => handlePanelChange("evaluatorPanel")}
                 {...props}
                 items={[
                     {
-                        key: "1",
+                        key: "evaluatorPanel",
                         label: (
                             <div className="flex items-center gap-2">
                                 <div>Select Evaluator</div>
diff --git a/agenta-web/src/components/pages/evaluations/NewEvaluation/SelectTestsetSection.tsx b/agenta-web/src/components/pages/evaluations/NewEvaluation/SelectTestsetSection.tsx
index 84093eeed5..81460bc2a0 100644
--- a/agenta-web/src/components/pages/evaluations/NewEvaluation/SelectTestsetSection.tsx
+++ b/agenta-web/src/components/pages/evaluations/NewEvaluation/SelectTestsetSection.tsx
@@ -10,12 +10,16 @@ type SelectTestsetSectionProps = {
     testSets: testset[]
     selectedTestsetId: string
     setSelectedTestsetId: React.Dispatch<React.SetStateAction<string>>
+    handlePanelChange: (key: string | string[]) => void
+    activePanel: string | null
 } & React.ComponentProps<typeof Collapse>
 
 const SelectTestsetSection = ({
     testSets,
     selectedTestsetId,
     setSelectedTestsetId,
+    activePanel,
+    handlePanelChange,
     ...props
 }: SelectTestsetSectionProps) => {
     const [searchTerm, setSearchTerm] = useState("")
@@ -77,11 +81,12 @@ const SelectTestsetSection = ({
 
     return (
         <Collapse
-            defaultActiveKey={["1"]}
+            activeKey={activePanel === "testsetPanel" ? "testsetPanel" : undefined}
+            onChange={() => handlePanelChange("testsetPanel")}
             {...props}
             items={[
                 {
-                    key: "1",
+                    key: "testsetPanel",
                     label: (
                         <Space>
                             <div>Select Testset</div>
@@ -115,6 +120,7 @@ const SelectTestsetSection = ({
                                 selectedRowKeys: [selectedTestset?._id as React.Key],
                                 onChange: (selectedRowKeys) => {
                                     setSelectedTestsetId(selectedRowKeys[0] as string)
+                                    handlePanelChange("variantPanel")
                                 },
                             }}
                             data-cy="evaluation-testset-table"
diff --git a/agenta-web/src/components/pages/evaluations/NewEvaluation/SelectVariantSection.tsx b/agenta-web/src/components/pages/evaluations/NewEvaluation/SelectVariantSection.tsx
index 3d53b47526..6cb7695f3a 100644
--- a/agenta-web/src/components/pages/evaluations/NewEvaluation/SelectVariantSection.tsx
+++ b/agenta-web/src/components/pages/evaluations/NewEvaluation/SelectVariantSection.tsx
@@ -10,6 +10,8 @@ type SelectVariantSectionProps = {
     usernames: Record<string, string>
     selectedVariantIds: string[]
     setSelectedVariantIds: React.Dispatch<React.SetStateAction<string[]>>
+    handlePanelChange: (key: string | string[]) => void
+    activePanel: string | null
 } & React.ComponentProps<typeof Collapse>
 
 const SelectVariantSection = ({
@@ -17,6 +19,8 @@ const SelectVariantSection = ({
     usernames,
     selectedVariantIds,
     setSelectedVariantIds,
+    activePanel,
+    handlePanelChange,
     ...props
 }: SelectVariantSectionProps) => {
     const [searchTerm, setSearchTerm] = useState("")
@@ -115,11 +119,12 @@ const SelectVariantSection = ({
 
     return (
         <Collapse
-            defaultActiveKey={["1"]}
+            activeKey={activePanel === "variantPanel" ? "variantPanel" : undefined}
+            onChange={() => handlePanelChange("variantPanel")}
             {...props}
             items={[
                 {
-                    key: "1",
+                    key: "variantPanel",
                     label: (
                         <Space>
                             <div>Select Variant</div>