removed parallel memory reduction

supsi-dacd-isaac · Oct 18, 2023 · ebc7df4 · ebc7df4
1 parent 0c4e848
commit ebc7df4
Showing 1 changed file with 8 additions and 5 deletions.
diff --git a/pyforecaster/formatter.py b/pyforecaster/formatter.py
@@ -93,7 +93,8 @@ def add_target_transform(self, names, functions=None, agg_freq=None, lags=None,
         self.target_transformers.append(transformer)
         return self
 
-    def transform(self, x, time_features=True, holidays=False, return_target=True, global_form=False, parallel=True, **holidays_kwargs):
+    def transform(self, x, time_features=True, holidays=False, return_target=True, global_form=False, parallel=False,
+                  reduce_memory=True, **holidays_kwargs):
         """
         Takes the DataFrame x and applies the specified transformations stored in the transformers in order to obtain
         the pre-fold-transformed dataset: this dataset has the correct final dimensions, but fold-specific
@@ -151,16 +152,18 @@ def transform(self, x, time_features=True, holidays=False, return_target=True, g
                 for i in tqdm(range(n_folds)):
                     x, y = fdf_parallel(f=partial(self._transform, time_features=time_features, holidays=holidays,
                                         return_target=return_target, **holidays_kwargs), df=dfs[n_cpu * i:n_cpu * (i + 1)])
-                    x = reduce_mem_usage(x, use_ray=True)
-                    y = reduce_mem_usage(y, use_ray=True)
+                    if reduce_memory:
+                        x = reduce_mem_usage(x, use_ray=True)
+                        y = reduce_mem_usage(y, use_ray=True)
                     xs.append(x)
                     ys.append(y)
             else:
                 for df_i in dfs:
                     x, y = self._transform(df_i,time_features=time_features, holidays=holidays,
                                         return_target=return_target, **holidays_kwargs)
-                    x = reduce_mem_usage(x, use_ray=True, parallel=False)
-                    y = reduce_mem_usage(y, use_ray=True, parallel=False)
+                    if reduce_memory:
+                        x = reduce_mem_usage(x, use_ray=False, parallel=False)
+                        y = reduce_mem_usage(y, use_ray=False, parallel=False)
                     xs.append(x)
                     ys.append(y)
             x = pd.concat(xs)