tune hyperparams

CALIPSO-project · Sep 20, 2024 · 095303b · 095303b
1 parent e00b304
commit 095303b
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 20 deletions.
diff --git a/DEF_Trunk/config.py b/DEF_Trunk/config.py
@@ -3,8 +3,8 @@
 logfile = "log.MLacc_Trunk"
 tasks = [
     # 2,
-    # 4,
-    5
+    4,
+    5,
 ]  # 1=test clustering, 2=clustering, 3=compress forcing, 4=ML, 5=evaluation
 results_dir = "./EXE_DIR/"
 reference_dir = "/home/surface10/mrasolon/files_for_zenodo/reference/EXE_DIR/"
@@ -15,12 +15,12 @@
 max_kmeans_clusters = 9
 random_seed = 1000
 algorithms = [
+    "gbm",
     "nn",
     "bt",
     "rf",
-    "gbm",
-    "lasso",
-    "stack",
+    # "lasso",
+    # "stack",
 ]  # bt: BaggingTrees, rf: RandomForest, nn: MLPRegressor, gbm: XGBRegressor, lasso: Lasso, stack: StackingRegressor
 leave_one_out_cv = False
 repro_test_task_1 = False

diff --git a/Tools/__init__.py b/Tools/__init__.py
@@ -44,7 +44,7 @@
 from sklearn.ensemble import BaggingRegressor, RandomForestRegressor, StackingRegressor
 from sklearn.linear_model import Lasso
 from sklearn.metrics import mean_squared_error, r2_score
-from sklearn.model_selection import LeaveOneOut
+from sklearn.model_selection import LeaveOneOut, GridSearchCV
 from sklearn.pipeline import Pipeline
 from sklearn.preprocessing import StandardScaler
 from sklearn.tree import DecisionTreeRegressor

diff --git a/Tools/train.py b/Tools/train.py
@@ -127,8 +127,12 @@ def training_BAT(XY_train, logfile, config, alg="gbm"):
     if alg == "nn":
         model = MLPRegressor(
             hidden_layer_sizes=(32, 32),
-            max_iter=1000,
-            learning_rate_init=0.05,
+            activation="tanh",
+            solver="lbfgs" if len(Ytrain) < 800 else "adam",
+            learning_rate="invscaling",
+            learning_rate_init=0.6,
+            power_t=0.99,
+            max_iter=int(1e6 / len(Ytrain)),
             random_state=1000,
         )
     elif alg == "bt":
@@ -140,18 +144,21 @@ def training_BAT(XY_train, logfile, config, alg="gbm"):
         )
     elif alg == "rf":
         model = RandomForestRegressor(
+            n_estimators=500,
             max_samples=0.8,
-            n_estimators=300,
+            max_depth=30,
             random_state=1000,
         )
     elif alg == "gbm":
         model = XGBRegressor(
-            n_estimators=300,
+            n_estimators=500,
+            # max_depth=30,
+            # learning_rate=0.001,
             random_state=1000,
         )
     elif alg == "lasso":
         model = Lasso(
-            alpha=0.1,
+            alpha=0.3,
         )
     elif alg == "stack":
         model = StackingRegressor(
@@ -166,27 +173,41 @@ def training_BAT(XY_train, logfile, config, alg="gbm"):
                 #     ),
                 # ),
                 (
+                    "nn",
+                    MLPRegressor(
+                        hidden_layer_sizes=(32, 32),
+                        activation="tanh",
+                        solver="lbfgs" if len(Ytrain) < 800 else "adam",
+                        learning_rate="invscaling",
+                        learning_rate_init=0.6,
+                        power_t=0.99,
+                        max_iter=int(1e6 / len(Ytrain)),
+                        random_state=1000,
+                    ),
+                )(
                     "rf",
                     RandomForestRegressor(
+                        n_estimators=500,
                         max_samples=0.8,
-                        n_estimators=300,
+                        max_depth=30,
                         random_state=1000,
                     ),
                 ),
                 (
                     "xgb",
                     XGBRegressor(
-                        n_estimators=300,
-                        max_depth=8,
+                        n_estimators=500,
+                        # max_depth=30,
+                        # learning_rate=0.001,
                         random_state=1000,
                     ),
                 ),
-                (
-                    "lasso",
-                    Lasso(
-                        alpha=0.1,
-                    ),
-                ),
+                # (
+                #     "lasso",
+                #     Lasso(
+                #         alpha=0.1,
+                #     ),
+                # ),
             ]
         )
     else: