From 30e8a773052fbfe4953996830cf2bb91586b4b6f Mon Sep 17 00:00:00 2001 From: Chess_champion Date: Thu, 28 Sep 2023 13:11:50 -0700 Subject: [PATCH 01/11] Update base_model.py NhiTS covariate fix --- pytorch_forecasting/models/base_model.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pytorch_forecasting/models/base_model.py b/pytorch_forecasting/models/base_model.py index 7c858255..0c3832eb 100644 --- a/pytorch_forecasting/models/base_model.py +++ b/pytorch_forecasting/models/base_model.py @@ -393,6 +393,7 @@ def forward(self, x): def __init__( self, + dataset_parameters: Dict[str, Any] = None, log_interval: Union[int, float] = -1, log_val_interval: Union[int, float] = None, learning_rate: Union[float, List[float]] = 1e-3, @@ -467,7 +468,8 @@ def __init__( self.output_transformer = output_transformer if not hasattr(self, "optimizer"): # callables are removed from hyperparameters, so better to save them self.optimizer = self.hparams.optimizer - + if not hasattr(self, "dataset_parameters"): + self.dataset_parameters = dataset_parameters # delete everything from hparams that cannot be serialized with yaml.dump # which is particularly important for tensorboard logging hparams_to_delete = [] @@ -1235,8 +1237,10 @@ def from_dataset(cls, dataset: TimeSeriesDataSet, **kwargs) -> LightningModule: """ if "output_transformer" not in kwargs: kwargs["output_transformer"] = dataset.target_normalizer + if "dataset_parameters" not in kwargs: + kwargs["dataset_parameters"] = dataset.get_parameters() net = cls(**kwargs) - net.dataset_parameters = dataset.get_parameters() + # net.dataset_parameters = dataset.get_parameters() if dataset.multi_target: assert isinstance( net.loss, MultiLoss From d395aca96f7cacbd2c8dc03efefce52c7854538d Mon Sep 17 00:00:00 2001 From: Chess_champion Date: Thu, 28 Sep 2023 13:35:46 -0700 Subject: [PATCH 02/11] Update __init__.py NHiTS covariate fix --- pytorch_forecasting/models/nhits/__init__.py | 32 ++++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/pytorch_forecasting/models/nhits/__init__.py b/pytorch_forecasting/models/nhits/__init__.py index 914a5f11..c2d77e5e 100644 --- a/pytorch_forecasting/models/nhits/__init__.py +++ b/pytorch_forecasting/models/nhits/__init__.py @@ -179,7 +179,9 @@ def __init__( prediction_length=self.hparams.prediction_length, output_size=to_list(output_size), static_size=self.static_size, - covariate_size=self.covariate_size, + # covariate_size=self.covariate_size, + encoder_covariate_size=self.encoder_covariate_size, + decoder_covariate_size=self.decoder_covariate_size, static_hidden_size=self.hparams.static_hidden_size, n_blocks=self.hparams.n_blocks, n_layers=self.hparams.n_layers, @@ -197,13 +199,26 @@ def __init__( ) @property - def covariate_size(self) -> int: - """Covariate size. + # def covariate_size(self) -> int: + # """Covariate size. + @property + def decoder_covariate_size(self) -> int: + """Decoder covariates size. Returns: - int: size of time-dependent covariates + int: size of time-dependent covariates used by the decoder. """ return len(set(self.hparams.time_varying_reals_decoder) - set(self.target_names)) + sum( + self.embeddings.output_size[name] for name in self.hparams.time_varying_categoricals_decoder + ) + + @property + def encoder_covariate_size(self) -> int: + """Encoder covariate size. + Returns: + int: size of time-dependent covariates used by the encoder + """ + return len(set(self.hparams.time_varying_reals_encoder) - set(self.target_names)) + sum( self.embeddings.output_size[name] for name in self.hparams.time_varying_categoricals_encoder ) @@ -239,16 +254,21 @@ def forward(self, x: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: Dict[str, torch.Tensor]: output of model """ # covariates - if self.covariate_size > 0: + # if self.covariate_size > 0: + if self.encoder_covariate_size > 0: encoder_features = self.extract_features(x, self.embeddings, period="encoder") encoder_x_t = torch.concat( [encoder_features[name] for name in self.encoder_variables if name not in self.target_names], dim=2, ) + else: + encoder_x_t = None + + if self.decoder_covariate_size > 0: decoder_features = self.extract_features(x, self.embeddings, period="decoder") decoder_x_t = torch.concat([decoder_features[name] for name in self.decoder_variables], dim=2) else: - encoder_x_t = None + # encoder_x_t = None decoder_x_t = None # statics From 7e8a1dd8a0c56f9a1e78b6c0874f4b67e9d18811 Mon Sep 17 00:00:00 2001 From: Chess_champion Date: Thu, 28 Sep 2023 13:52:08 -0700 Subject: [PATCH 03/11] Update sub_modules.py NhiTS covariate Change --- .../models/nhits/sub_modules.py | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/pytorch_forecasting/models/nhits/sub_modules.py b/pytorch_forecasting/models/nhits/sub_modules.py index 66bd4845..417c9546 100644 --- a/pytorch_forecasting/models/nhits/sub_modules.py +++ b/pytorch_forecasting/models/nhits/sub_modules.py @@ -91,7 +91,9 @@ def __init__( context_length: int, prediction_length: int, output_size: int, - covariate_size: int, + # covariate_size: int, + encoder_covariate_size: int, + decoder_covariate_size: int, static_size: int, static_hidden_size: int, n_theta: int, @@ -119,14 +121,18 @@ def __init__( self.prediction_length = prediction_length self.static_size = static_size self.static_hidden_size = static_hidden_size - self.covariate_size = covariate_size + # self.covariate_size = covariate_size + self.encoder_covariate_size = encoder_covariate_size + self.decoder_covariate_size = decoder_covariate_size self.pooling_sizes = pooling_sizes self.batch_normalization = batch_normalization self.dropout = dropout self.hidden_size = [ self.context_length_pooled * len(self.output_size) - + (self.context_length + self.prediction_length) * self.covariate_size + # + (self.context_length + self.prediction_length) * self.covariate_size + + self.context_length * self.encoder_covariate_size + + self.prediction_length * self.decoder_covariate_size + self.static_hidden_size ] + hidden_size @@ -173,11 +179,19 @@ def forward( encoder_y = self.pooling_layer(encoder_y) encoder_y = encoder_y.transpose(1, 2).reshape(batch_size, -1) - if self.covariate_size > 0: + if self.encoder_covariate_size > 0: encoder_y = torch.cat( ( encoder_y, encoder_x_t.reshape(batch_size, -1), + # decoder_x_t.reshape(batch_size, -1), + ), + 1, + ) + if self.decoder_covariate_size > 0: + encoder_y = torch.cat( + ( + encoder_y, decoder_x_t.reshape(batch_size, -1), ), 1, @@ -209,7 +223,9 @@ def __init__( context_length, prediction_length, output_size: int, - static_size, + # static_size, + encoder_covariate_size, + decoder_covariate_size, covariate_size, static_hidden_size, n_blocks: list, @@ -238,7 +254,9 @@ def __init__( context_length=context_length, prediction_length=prediction_length, output_size=output_size, - covariate_size=covariate_size, + # covariate_size=covariate_size, + encoder_covariate_size=encoder_covariate_size, + decoder_covariate_size=decoder_covariate_size, static_size=static_size, static_hidden_size=static_hidden_size, n_layers=n_layers, @@ -261,7 +279,9 @@ def create_stack( context_length, prediction_length, output_size, - covariate_size, + # covariate_size, + encoder_covariate_size, + decoder_covariate_size, static_size, static_hidden_size, n_layers, @@ -300,7 +320,9 @@ def create_stack( context_length=context_length, prediction_length=prediction_length, output_size=output_size, - covariate_size=covariate_size, + # covariate_size=covariate_size, + encoder_covariate_size=encoder_covariate_size, + decoder_covariate_size=decoder_covariate_size, static_size=static_size, static_hidden_size=static_hidden_size, n_theta=n_theta, From 30812ede112969e30f1b92b48b1f492dffa96361 Mon Sep 17 00:00:00 2001 From: Chess_champion Date: Thu, 28 Sep 2023 13:55:12 -0700 Subject: [PATCH 04/11] Update conftest.py NHitS Covariate fix unit test --- tests/test_models/conftest.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_models/conftest.py b/tests/test_models/conftest.py index 40614282..edc225d2 100644 --- a/tests/test_models/conftest.py +++ b/tests/test_models/conftest.py @@ -130,6 +130,36 @@ def make_dataloaders(data_with_covariates, **kwargs): def multiple_dataloaders_with_covariates(data_with_covariates, request): return make_dataloaders(data_with_covariates, **request.param) +@pytest.fixture(scope="session") +def dataloaders_with_different_encoder_decoder_length(data_with_covariates): + return make_dataloaders( + data_with_covariates.copy(), + target="target", + time_varying_known_categoricals=["special_days", "month"], + variable_groups=dict( + special_days=[ + "easter_day", + "good_friday", + "new_year", + "christmas", + "labor_day", + "independence_day", + "revolution_day_memorial", + "regional_games", + "fifa_u_17_world_cup", + "football_gold_cup", + "beer_capital", + "music_fest", + ] + ), + time_varying_known_reals=["time_idx", "price_regular", "price_actual", "discount", "discount_in_percent"], + time_varying_unknown_categoricals=[], + time_varying_unknown_reals=["target", "volume", "log_volume", "industry_volume", "soda_volume", "avg_max_temp"], + static_categoricals=["agency"], + add_relative_time_idx=False, + target_normalizer=GroupNormalizer(groups=["agency", "sku"], center=False), + ) + @pytest.fixture(scope="session") def dataloaders_with_covariates(data_with_covariates): From bec668de6b4eba0ef34397dbf103a110500c9c10 Mon Sep 17 00:00:00 2001 From: Chess_champion Date: Thu, 28 Sep 2023 13:59:36 -0700 Subject: [PATCH 05/11] Update test_nhits.py NHiTS covariate change test --- tests/test_models/test_nhits.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_models/test_nhits.py b/tests/test_models/test_nhits.py index 92425a65..8698a345 100644 --- a/tests/test_models/test_nhits.py +++ b/tests/test_models/test_nhits.py @@ -77,6 +77,7 @@ def _integration(dataloader, tmp_path, trainer_kwargs=None, **kwargs): "dataloader", [ "with_covariates", + "different_encoder_decoder_size", "fixed_window_without_covariates", "multi_target", "quantiles", @@ -86,6 +87,7 @@ def _integration(dataloader, tmp_path, trainer_kwargs=None, **kwargs): ) def test_integration( dataloaders_with_covariates, + dataloaders_with_different_encoder_decoder_length, dataloaders_fixed_window_without_covariates, dataloaders_multi_target, tmp_path, @@ -95,6 +97,8 @@ def test_integration( if dataloader == "with_covariates": dataloader = dataloaders_with_covariates kwargs["backcast_loss_ratio"] = 0.5 + elif dataloader == "different_encoder_decoder_size": + dataloader = dataloaders_with_different_encoder_decoder_length elif dataloader == "fixed_window_without_covariates": dataloader = dataloaders_fixed_window_without_covariates elif dataloader == "multi_target": From 064c4e0005d52e26b313a9f6d340bb81faeaefb0 Mon Sep 17 00:00:00 2001 From: Chess_champion Date: Thu, 28 Sep 2023 15:31:45 -0700 Subject: [PATCH 06/11] Update __init__.py Remove Additional property line --- pytorch_forecasting/models/nhits/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytorch_forecasting/models/nhits/__init__.py b/pytorch_forecasting/models/nhits/__init__.py index c2d77e5e..b7790e92 100644 --- a/pytorch_forecasting/models/nhits/__init__.py +++ b/pytorch_forecasting/models/nhits/__init__.py @@ -198,7 +198,6 @@ def __init__( naive_level=self.hparams.naive_level, ) - @property # def covariate_size(self) -> int: # """Covariate size. @property From 2101e06e35d99ac75bcd64bb278fad13c2c2db08 Mon Sep 17 00:00:00 2001 From: Chess_champion Date: Thu, 28 Sep 2023 16:39:10 -0700 Subject: [PATCH 07/11] Update __init__.py NhiTS Fix --- pytorch_forecasting/models/nhits/__init__.py | 31 ++++---------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/pytorch_forecasting/models/nhits/__init__.py b/pytorch_forecasting/models/nhits/__init__.py index b7790e92..914a5f11 100644 --- a/pytorch_forecasting/models/nhits/__init__.py +++ b/pytorch_forecasting/models/nhits/__init__.py @@ -179,9 +179,7 @@ def __init__( prediction_length=self.hparams.prediction_length, output_size=to_list(output_size), static_size=self.static_size, - # covariate_size=self.covariate_size, - encoder_covariate_size=self.encoder_covariate_size, - decoder_covariate_size=self.decoder_covariate_size, + covariate_size=self.covariate_size, static_hidden_size=self.hparams.static_hidden_size, n_blocks=self.hparams.n_blocks, n_layers=self.hparams.n_layers, @@ -198,26 +196,14 @@ def __init__( naive_level=self.hparams.naive_level, ) - # def covariate_size(self) -> int: - # """Covariate size. @property - def decoder_covariate_size(self) -> int: - """Decoder covariates size. + def covariate_size(self) -> int: + """Covariate size. Returns: - int: size of time-dependent covariates used by the decoder. + int: size of time-dependent covariates """ return len(set(self.hparams.time_varying_reals_decoder) - set(self.target_names)) + sum( - self.embeddings.output_size[name] for name in self.hparams.time_varying_categoricals_decoder - ) - - @property - def encoder_covariate_size(self) -> int: - """Encoder covariate size. - Returns: - int: size of time-dependent covariates used by the encoder - """ - return len(set(self.hparams.time_varying_reals_encoder) - set(self.target_names)) + sum( self.embeddings.output_size[name] for name in self.hparams.time_varying_categoricals_encoder ) @@ -253,21 +239,16 @@ def forward(self, x: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: Dict[str, torch.Tensor]: output of model """ # covariates - # if self.covariate_size > 0: - if self.encoder_covariate_size > 0: + if self.covariate_size > 0: encoder_features = self.extract_features(x, self.embeddings, period="encoder") encoder_x_t = torch.concat( [encoder_features[name] for name in self.encoder_variables if name not in self.target_names], dim=2, ) - else: - encoder_x_t = None - - if self.decoder_covariate_size > 0: decoder_features = self.extract_features(x, self.embeddings, period="decoder") decoder_x_t = torch.concat([decoder_features[name] for name in self.decoder_variables], dim=2) else: - # encoder_x_t = None + encoder_x_t = None decoder_x_t = None # statics From 6e9ea6cf155ab83cf3100a6c161b83014c3e5320 Mon Sep 17 00:00:00 2001 From: Chess_champion Date: Thu, 28 Sep 2023 16:39:59 -0700 Subject: [PATCH 08/11] Update __init__.py NhiTS Fix --- pytorch_forecasting/models/nhits/__init__.py | 27 +++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/pytorch_forecasting/models/nhits/__init__.py b/pytorch_forecasting/models/nhits/__init__.py index 914a5f11..59c3c454 100644 --- a/pytorch_forecasting/models/nhits/__init__.py +++ b/pytorch_forecasting/models/nhits/__init__.py @@ -179,7 +179,8 @@ def __init__( prediction_length=self.hparams.prediction_length, output_size=to_list(output_size), static_size=self.static_size, - covariate_size=self.covariate_size, + encoder_covariate_size=self.encoder_covariate_size, + decoder_covariate_size=self.decoder_covariate_size, static_hidden_size=self.hparams.static_hidden_size, n_blocks=self.hparams.n_blocks, n_layers=self.hparams.n_layers, @@ -197,13 +198,24 @@ def __init__( ) @property - def covariate_size(self) -> int: - """Covariate size. + def decoder_covariate_size(self) -> int: + """Decoder covariates size. Returns: - int: size of time-dependent covariates + int: size of time-dependent covariates used by the decoder """ return len(set(self.hparams.time_varying_reals_decoder) - set(self.target_names)) + sum( + self.embeddings.output_size[name] for name in self.hparams.time_varying_categoricals_decoder + ) + + @property + def encoder_covariate_size(self) -> int: + """Encoder covariate size. + + Returns: + int: size of time-dependent covariates used by the encoder + """ + return len(set(self.hparams.time_varying_reals_encoder) - set(self.target_names)) + sum( self.embeddings.output_size[name] for name in self.hparams.time_varying_categoricals_encoder ) @@ -239,16 +251,19 @@ def forward(self, x: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: Dict[str, torch.Tensor]: output of model """ # covariates - if self.covariate_size > 0: + if self.encoder_covariate_size > 0: encoder_features = self.extract_features(x, self.embeddings, period="encoder") encoder_x_t = torch.concat( [encoder_features[name] for name in self.encoder_variables if name not in self.target_names], dim=2, ) + else: + encoder_x_t = None + + if self.decoder_covariate_size > 0: decoder_features = self.extract_features(x, self.embeddings, period="decoder") decoder_x_t = torch.concat([decoder_features[name] for name in self.decoder_variables], dim=2) else: - encoder_x_t = None decoder_x_t = None # statics From 42c97e679b07f954100a32b07fb8485c9aa4fce3 Mon Sep 17 00:00:00 2001 From: Chess_champion Date: Thu, 28 Sep 2023 17:29:08 -0700 Subject: [PATCH 09/11] Update base_model.py --- pytorch_forecasting/models/base_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_forecasting/models/base_model.py b/pytorch_forecasting/models/base_model.py index 0c3832eb..50a4ad77 100644 --- a/pytorch_forecasting/models/base_model.py +++ b/pytorch_forecasting/models/base_model.py @@ -470,6 +470,7 @@ def __init__( self.optimizer = self.hparams.optimizer if not hasattr(self, "dataset_parameters"): self.dataset_parameters = dataset_parameters + # delete everything from hparams that cannot be serialized with yaml.dump # which is particularly important for tensorboard logging hparams_to_delete = [] @@ -1240,7 +1241,6 @@ def from_dataset(cls, dataset: TimeSeriesDataSet, **kwargs) -> LightningModule: if "dataset_parameters" not in kwargs: kwargs["dataset_parameters"] = dataset.get_parameters() net = cls(**kwargs) - # net.dataset_parameters = dataset.get_parameters() if dataset.multi_target: assert isinstance( net.loss, MultiLoss From e5a48f2eff3b3399694e3dfa690c5a3c6e3eb2aa Mon Sep 17 00:00:00 2001 From: Chess_champion Date: Thu, 28 Sep 2023 18:47:12 -0700 Subject: [PATCH 10/11] Update sub_modules.py --- pytorch_forecasting/models/nhits/sub_modules.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/pytorch_forecasting/models/nhits/sub_modules.py b/pytorch_forecasting/models/nhits/sub_modules.py index 417c9546..a9ec4338 100644 --- a/pytorch_forecasting/models/nhits/sub_modules.py +++ b/pytorch_forecasting/models/nhits/sub_modules.py @@ -91,7 +91,6 @@ def __init__( context_length: int, prediction_length: int, output_size: int, - # covariate_size: int, encoder_covariate_size: int, decoder_covariate_size: int, static_size: int, @@ -121,7 +120,6 @@ def __init__( self.prediction_length = prediction_length self.static_size = static_size self.static_hidden_size = static_hidden_size - # self.covariate_size = covariate_size self.encoder_covariate_size = encoder_covariate_size self.decoder_covariate_size = decoder_covariate_size self.pooling_sizes = pooling_sizes @@ -130,7 +128,6 @@ def __init__( self.hidden_size = [ self.context_length_pooled * len(self.output_size) - # + (self.context_length + self.prediction_length) * self.covariate_size + self.context_length * self.encoder_covariate_size + self.prediction_length * self.decoder_covariate_size + self.static_hidden_size @@ -184,10 +181,10 @@ def forward( ( encoder_y, encoder_x_t.reshape(batch_size, -1), - # decoder_x_t.reshape(batch_size, -1), ), 1, ) + if self.decoder_covariate_size > 0: encoder_y = torch.cat( ( @@ -223,10 +220,9 @@ def __init__( context_length, prediction_length, output_size: int, - # static_size, + static_size, encoder_covariate_size, decoder_covariate_size, - covariate_size, static_hidden_size, n_blocks: list, n_layers: list, @@ -254,7 +250,6 @@ def __init__( context_length=context_length, prediction_length=prediction_length, output_size=output_size, - # covariate_size=covariate_size, encoder_covariate_size=encoder_covariate_size, decoder_covariate_size=decoder_covariate_size, static_size=static_size, @@ -279,7 +274,6 @@ def create_stack( context_length, prediction_length, output_size, - # covariate_size, encoder_covariate_size, decoder_covariate_size, static_size, @@ -320,7 +314,6 @@ def create_stack( context_length=context_length, prediction_length=prediction_length, output_size=output_size, - # covariate_size=covariate_size, encoder_covariate_size=encoder_covariate_size, decoder_covariate_size=decoder_covariate_size, static_size=static_size, From f972a8c607dccbb39ab558fa8c1d0d376ca03cba Mon Sep 17 00:00:00 2001 From: Chess_champion Date: Sun, 5 Nov 2023 19:26:52 -0800 Subject: [PATCH 11/11] Remove Unsqueeze operation to solve the mismatch operation --- pytorch_forecasting/metrics/point.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_forecasting/metrics/point.py b/pytorch_forecasting/metrics/point.py index d0f812c5..cc666af4 100644 --- a/pytorch_forecasting/metrics/point.py +++ b/pytorch_forecasting/metrics/point.py @@ -203,7 +203,7 @@ def update( # weight samples if weight is not None: - losses = losses * weight.unsqueeze(-1) + losses = losses * weight self._update_losses_and_lengths(losses, lengths)