diff --git a/agenta-backend/agenta_backend/services/evaluator_manager.py b/agenta-backend/agenta_backend/services/evaluator_manager.py
index de262987cc..d5b22279c7 100644
--- a/agenta-backend/agenta_backend/services/evaluator_manager.py
+++ b/agenta-backend/agenta_backend/services/evaluator_manager.py
@@ -80,15 +80,6 @@ async def create_evaluator_config(
         EvaluatorConfigDB: The newly created evaluator configuration object.
     """
     app = await db_manager.fetch_app_by_id(app_id)
-    evaluator_config = evaluators.get_evaluator_by_key(evaluator_key)
-
-    if evaluator_config is not None:
-        if "correct_answer_keys" in evaluator_config.get("settings_template", {}):
-            if settings_values is None:
-                settings_values = {}
-            settings_values["correct_answer_keys"] = evaluator_config[
-                "settings_template"
-            ]["correct_answer_keys"]["default"]
 
     evaluator_config = await db_manager.create_evaluator_config(
         app=app,
diff --git a/agenta-web/src/components/pages/evaluations/evaluators/AdvancedSettings.tsx b/agenta-web/src/components/pages/evaluations/evaluators/AdvancedSettings.tsx
new file mode 100644
index 0000000000..b86c54d8b7
--- /dev/null
+++ b/agenta-web/src/components/pages/evaluations/evaluators/AdvancedSettings.tsx
@@ -0,0 +1,59 @@
+import React from "react"
+import {Form, Input, Collapse} from "antd"
+import {CaretRightOutlined} from "@ant-design/icons"
+import {createUseStyles} from "react-jss"
+
+type AdvancedSettings = {
+    correct_answer_keys: string[]
+}
+
+const useStyles = createUseStyles((theme: any) => ({
+    label: {
+        display: "flex",
+        alignItems: "center",
+        gap: "0.5rem",
+    },
+}))
+
+type AdvancedSettingsProps = {
+    settings: Record<string, any>[]
+}
+
+const AdvancedSettings: React.FC<AdvancedSettingsProps> = ({settings}) => {
+    const classes = useStyles()
+
+    const initialValues = settings.reduce((acc, field) => {
+        acc[field.key] = field.default
+        return acc
+    }, {})
+
+    return (
+        <Collapse
+            bordered={false}
+            expandIcon={({isActive}) => <CaretRightOutlined rotate={isActive ? 90 : 0} />}
+            className={"my-[10px]"}
+        >
+            <Collapse.Panel key="1" header="Advanced Settings">
+                {settings.map((field) => (
+                    <Form.Item
+                        key={field.key}
+                        initialValue={initialValues.correct_answer_keys[0]}
+                        name={["settings_values", "correct_answer_keys"]}
+                        label={
+                            <div className={classes.label}>
+                                <span>{field.label}</span>
+                            </div>
+                        }
+                        rules={[
+                            {required: field.required ?? true, message: "This field is required"},
+                        ]}
+                    >
+                        {field.key === "correct_answer_keys" && <Input />}
+                    </Form.Item>
+                ))}
+            </Collapse.Panel>
+        </Collapse>
+    )
+}
+
+export default AdvancedSettings
diff --git a/agenta-web/src/components/pages/evaluations/evaluators/NewEvaluatorModal.tsx b/agenta-web/src/components/pages/evaluations/evaluators/NewEvaluatorModal.tsx
index 1cf22cd2d7..8114845751 100644
--- a/agenta-web/src/components/pages/evaluations/evaluators/NewEvaluatorModal.tsx
+++ b/agenta-web/src/components/pages/evaluations/evaluators/NewEvaluatorModal.tsx
@@ -35,6 +35,7 @@ import Link from "next/link"
 import React, {useEffect, useMemo, useState} from "react"
 import {createUseStyles} from "react-jss"
 import {ColumnsType} from "antd/es/table"
+import AdvancedSettings from "./AdvancedSettings"
 
 const useStyles = createUseStyles((theme: JSSTheme) => ({
     label: {
@@ -170,7 +171,7 @@ const DynamicFormField: React.FC<DynamicFormFieldProps> = ({
 
     return (
         <>
-            {label !== "Correct Answer" ? (
+            {label !== "Correct Answer" && (
                 <Form.Item
                     name={name}
                     label={
@@ -213,36 +214,6 @@ const DynamicFormField: React.FC<DynamicFormFieldProps> = ({
                         />
                     ) : null}
                 </Form.Item>
-            ) : (
-                <>
-                    <Collapse
-                        bordered={false}
-                        expandIcon={({isActive}) => (
-                            <CaretRightOutlined rotate={isActive ? 90 : 0} />
-                        )}
-                        className={"my-[10px]"}
-                        items={[
-                            {
-                                key: "1",
-                                label: "Advanced Settings",
-                                children: (
-                                    <Form.Item
-                                        name={name}
-                                        label={
-                                            <div className={classes.label}>
-                                                <span>{label}</span>
-                                            </div>
-                                        }
-                                        initialValue={defaultVal}
-                                        rules={rules}
-                                    >
-                                        <Input className="w-[50%]" />
-                                    </Form.Item>
-                                ),
-                            },
-                        ]}
-                    />
-                </>
             )}
 
             {ExternalHelpInfo}
@@ -309,13 +280,30 @@ const NewEvaluatorModal: React.FC<Props> = ({
         }
     }, [newEvalModalConfigOpen])
 
+    const advancedSettingsFields = evalFields.filter((field) => field.key === "correct_answer_keys")
+
     const onSubmit = (values: CreateEvaluationConfigData) => {
         setSubmitLoading(true)
         if (!selectedEval?.key) throw new Error("No selected key")
+        const settingsValues = values.settings_values || {}
+
+        if (settingsValues.correct_answer_keys) {
+            settingsValues.correct_answer_keys = Array.isArray(settingsValues.correct_answer_keys)
+                ? settingsValues.correct_answer_keys
+                : [settingsValues.correct_answer_keys]
+        }
+
+        if (
+            !settingsValues.correct_answer_keys &&
+            selectedEval.settings_template.correct_answer_keys
+        ) {
+            settingsValues["correct_answer_keys"] = ["correct_answer"]
+        }
+
         const data = {
             ...values,
             evaluator_key: selectedEval.key,
-            settings_values: values.settings_values || {},
+            settings_values: settingsValues,
         }
         ;(editMode
             ? updateEvaluatorConfig(initialValues?.id!, data)
@@ -453,6 +441,10 @@ const NewEvaluatorModal: React.FC<Props> = ({
                         />
                     ))}
 
+                    {advancedSettingsFields.length > 0 && (
+                        <AdvancedSettings settings={advancedSettingsFields} />
+                    )}
+
                     <Form.Item style={{marginBottom: 0}}>
                         <div className={classes.evalBtnContainer}>
                             {!editMode && (