diff --git a/agenta-backend/agenta_backend/services/evaluator_manager.py b/agenta-backend/agenta_backend/services/evaluator_manager.py
index de262987cc..d5b22279c7 100644
--- a/agenta-backend/agenta_backend/services/evaluator_manager.py
+++ b/agenta-backend/agenta_backend/services/evaluator_manager.py
@@ -80,15 +80,6 @@ async def create_evaluator_config(
         EvaluatorConfigDB: The newly created evaluator configuration object.
     """
     app = await db_manager.fetch_app_by_id(app_id)
-    evaluator_config = evaluators.get_evaluator_by_key(evaluator_key)
-
-    if evaluator_config is not None:
-        if "correct_answer_keys" in evaluator_config.get("settings_template", {}):
-            if settings_values is None:
-                settings_values = {}
-            settings_values["correct_answer_keys"] = evaluator_config[
-                "settings_template"
-            ]["correct_answer_keys"]["default"]
 
     evaluator_config = await db_manager.create_evaluator_config(
         app=app,
diff --git a/agenta-web/src/components/pages/evaluations/evaluators/NewEvaluatorModal.tsx b/agenta-web/src/components/pages/evaluations/evaluators/NewEvaluatorModal.tsx
index ba7da1bfb9..bac6654836 100644
--- a/agenta-web/src/components/pages/evaluations/evaluators/NewEvaluatorModal.tsx
+++ b/agenta-web/src/components/pages/evaluations/evaluators/NewEvaluatorModal.tsx
@@ -219,7 +219,6 @@ const DynamicFormField: React.FC<DynamicFormFieldProps> = ({
                         expandIcon={({isActive}) => (
                             <CaretRightOutlined rotate={isActive ? 90 : 0} />
                         )}
-                        defaultActiveKey={["1"]}
                         className={"my-[10px]"}
                         items={[
                             {
@@ -321,6 +320,13 @@ const NewEvaluatorModal: React.FC<Props> = ({
                 : [settingsValues.correct_answer_keys]
         }
 
+        if (
+            !settingsValues.correct_answer_keys &&
+            selectedEval.settings_template.correct_answer_keys
+        ) {
+            settingsValues["correct_answer_keys"] = ["correct_answer"]
+        }
+
         const data = {
             ...values,
             evaluator_key: selectedEval.key,