Change val data distance. Add scripts to new finetune.

cli/conf/new_eval/default.yaml

@@ -1,6 +1,6 @@
 hydra:
   run:
-    dir: outputs/eval/${hydra:runtime.choices.model}/${exp_name}/${data.dataset_name}/${data.mode}/cl${model.context_length}_pl${data.prediction_length}
+    dir: outputs/new_eval/${hydra:runtime.choices.model}/${exp_name}/${data.dataset_name}/${data.mode}/cl${model.context_length}_pl${data.prediction_length}
 defaults:
   - model: ???
   - data: ???

cli/conf/new_finetune/data/electricity.yaml

@@ -1,2 +1,6 @@
-_target_: uni2ts.data.builder.simple.SimpleDatasetBuilder
+_target_: uni2ts.data.builder.simple.generate_finetune_builder
 dataset: electricity
+train_length: 18412
+prediction_length: ???
+context_length: ???
+patch_size: ???

cli/conf/new_finetune/data/etth1.yaml

@@ -1,10 +1,6 @@
-#_target_: uni2ts.data.builder.simple.SimpleDatasetBuilder
-#dataset: ETTh1
-#weight: 1000
-
 _target_: uni2ts.data.builder.simple.generate_finetune_builder
 dataset: ETTh1
 train_length: 8640
-prediction_length: 96
-context_length: 3000
-patch_size: 64
+prediction_length: ???
+context_length: ???
+patch_size: ???

cli/conf/new_finetune/data/etth2.yaml

@@ -1,2 +1,6 @@
-_target_: uni2ts.data.builder.simple.SimpleDatasetBuilder
+_target_: uni2ts.data.builder.simple.generate_finetune_builder
 dataset: ETTh2
+train_length: 8640
+prediction_length: ???
+context_length: ???
+patch_size: ???

cli/conf/new_finetune/data/ettm1.yaml

@@ -1,2 +1,6 @@
-_target_: uni2ts.data.builder.simple.SimpleDatasetBuilder
+_target_: uni2ts.data.builder.simple.generate_finetune_builder
 dataset: ETTm1
+train_length: 34560
+prediction_length: ???
+context_length: ???
+patch_size: ???

cli/conf/new_finetune/data/ettm2.yaml

@@ -1,2 +1,6 @@
-_target_: uni2ts.data.builder.simple.SimpleDatasetBuilder
+_target_: uni2ts.data.builder.simple.generate_finetune_builder
 dataset: ETTm2
+train_length: 34560
+prediction_length: ???
+context_length: ???
+patch_size: ???

cli/conf/new_finetune/data/weather.yaml

@@ -1,2 +1,6 @@
-_target_: uni2ts.data.builder.simple.SimpleDatasetBuilder
+_target_: uni2ts.data.builder.simple.generate_finetune_builder
 dataset: weather
+train_length: 36887
+prediction_length: ???
+context_length: ???
+patch_size: ???

cli/conf/new_finetune/default.yaml

@@ -36,7 +36,7 @@ trainer:
     - _target_: uni2ts.callbacks.earlystop.WarmupEarlyStopping  # lightning.pytorch.callbacks.EarlyStopping
       monitor: val/PackedNLLLoss
       min_delta: 0.0
-      patience: 30
+      patience: 3  # Set to a small value as now each epoch has many batches.
       mode: min
       strict: false
       verbose: true
@@ -48,9 +48,9 @@ trainer:
   gradient_clip_algorithm: norm
 train_dataloader:
   _target_: uni2ts.data.loader.DataLoader
-  batch_size: 64
+  batch_size: 512 # Can use a large batch size after disabling sequence packing.
   batch_size_factor: 2.0
-  cycle: false # true
+  cycle: false  # Set it as false to loop over all batches per epoch
   num_batches_per_epoch: null
   shuffle: true
   num_workers: 11

cli/conf/new_finetune/model/moirai_1.1_R_small.yaml

@@ -29,11 +29,11 @@ val_metric:
   - _target_: uni2ts.loss.packed.PackedMSELoss
   - _target_: uni2ts.loss.packed.PackedNRMSELoss
     normalize: absolute_target_squared
-lr: 1e-4
+lr: 5e-7  #  On ETT dataset, using 1e-6/5e-7 converge within 1-2 epochs. 1e-7 converge in tens of epochs
 weight_decay: 1e-1
 beta1: 0.9
 beta2: 0.98
-num_training_steps: null  # ${mul:${trainer.max_epochs},${train_dataloader.num_batches_per_epoch}}
+num_training_steps: null
 num_warmup_steps: 0
 patch_size: null
 context_length: null

cli/conf/new_finetune/val_data/electricity.yaml

@@ -1,13 +1,7 @@
-_target_: uni2ts.data.builder.ConcatDatasetBuilder
-_args_:
-  _target_: uni2ts.data.builder.simple.generate_eval_builders
-  dataset: electricity_eval
-  offset: 18412 # Same as _lsf_dataset.py
-  eval_length: 2630 # Same as _lsf_dataset.py, test_length=5260
-  prediction_lengths: ???
-  context_lengths: ???
-  patch_sizes: ???
-
-#  prediction_lengths: [96, 192, 336, 720]
-#  context_lengths: [3000]
-#  patch_sizes: [32, 64]  # freq='h'
+_target_: uni2ts.data.builder.simple.generate_eval_builder
+dataset: electricity_eval
+offset: 18412 # Same as _lsf_dataset.py
+eval_length: 2630 # Same as _lsf_dataset.py, test_length=5260
+prediction_length: ???
+context_length: ???
+patch_size: ???

cli/conf/new_finetune/val_data/etth1.yaml

@@ -1,9 +1,7 @@
-_target_: uni2ts.data.builder.ConcatDatasetBuilder
-_args_:
-  _target_: uni2ts.data.builder.simple.generate_eval_builders
-  dataset: ETTh1_eval
-  offset: 8640
-  eval_length: 2880
-  prediction_lengths: [96, 192, 336, 720]
-  context_lengths: [1000, 2000, 3000, 4000, 5000]
-  patch_sizes: [32, 64]
+_target_: uni2ts.data.builder.simple.generate_eval_builder
+dataset: ETTh1_eval
+offset: 8640
+eval_length: 2880
+prediction_length: ???
+context_length: ???
+patch_size: ???

cli/conf/new_finetune/val_data/etth2.yaml

@@ -1,13 +1,7 @@
-_target_: uni2ts.data.builder.ConcatDatasetBuilder
-_args_:
-  _target_: uni2ts.data.builder.simple.generate_eval_builders
-  dataset: ETTh2_eval
-  offset: 8640  # Same as _lsf_dataset.py
-  eval_length: 2880  # Same as _lsf_dataset.py
-  prediction_lengths: ???
-  context_lengths: ???
-  patch_sizes: ???
-
-#  prediction_lengths: [ 96, 192, 336, 720 ]
-#  context_lengths: [ 3000 ]
-#  patch_sizes: [ 32, 64 ]
+_target_: uni2ts.data.builder.simple.generate_eval_builder
+dataset: ETTh2_eval
+offset: 8640  # Same as _lsf_dataset.py
+eval_length: 2880  # Same as _lsf_dataset.py
+prediction_length: ???
+context_length: ???
+patch_size: ???

cli/conf/new_finetune/val_data/ettm1.yaml

@@ -1,14 +1,7 @@
-_target_: uni2ts.data.builder.ConcatDatasetBuilder
-_args_:
-  _target_: uni2ts.data.builder.simple.generate_eval_builders
-  dataset: ETTm1_eval
-  offset: 34560  # Same as _lsf_dataset.py
-  eval_length: 11520  # Same as _lsf_dataset.py
-  prediction_lengths: ???
-  context_lengths: ???
-  patch_sizes: ???
-
-
-#  prediction_lengths: [96, 192, 336, 720]
-#  context_lengths: [ 3000 ]
-#  patch_sizes: [ 32, 64, 128 ]  # freq="15T"
+_target_: uni2ts.data.builder.simple.generate_eval_builder
+dataset: ETTm1_eval
+offset: 34560  # Same as _lsf_dataset.py
+eval_length: 11520  # Same as _lsf_dataset.py
+prediction_length: ???
+context_length: ???
+patch_size: ???

cli/conf/new_finetune/val_data/ettm2.yaml

@@ -1,14 +1,7 @@
-_target_: uni2ts.data.builder.ConcatDatasetBuilder
-_args_:
-  _target_: uni2ts.data.builder.simple.generate_eval_builders
-  dataset: ETTm2_eval
-  offset: 34560  # Same as _lsf_dataset.py
-  eval_length: 11520  # Same as _lsf_dataset.py
-  prediction_lengths: ???
-  context_lengths: ???
-  patch_sizes: ???
-
-
-#  prediction_lengths: [96, 192, 336, 720]
-#  context_lengths: [3000]
-#  patch_sizes: [32, 64, 128]  # "freq=15T"
+_target_: uni2ts.data.builder.simple.generate_eval_builder
+dataset: ETTm2_eval
+offset: 34560  # Same as _lsf_dataset.py
+eval_length: 11520  # Same as _lsf_dataset.py
+prediction_length: ???
+context_length: ???
+patch_size: ???

cli/conf/new_finetune/val_data/weather.yaml

@@ -1,9 +1,7 @@
-_target_: uni2ts.data.builder.ConcatDatasetBuilder
-_args_:
-  _target_: uni2ts.data.builder.simple.generate_eval_builders
-  dataset: weather_eval
-  offset: 36887 # Same as _lsf_dataset.py
-  eval_length: 5269 # Same as _lsf_dataset.py; test_length=10539
-  prediction_lengths: ???
-  context_lengths: ???
-  patch_sizes: ???
+_target_: uni2ts.data.builder.simple.generate_eval_builder
+dataset: weather_eval
+offset: 36887 # Same as _lsf_dataset.py
+eval_length: 5269 # Same as _lsf_dataset.py; test_length=10539
+prediction_length: ???
+context_length: ???
+patch_size: ???

cli/train.py

@@ -128,7 +128,7 @@ def main(cfg: DictConfig):
 
     model: L.LightningModule = instantiate(cfg.model, _convert_="all")
 
-    if 'collate_fn' not in cfg.train_dataloader:
+    if "collate_fn" not in cfg.train_dataloader:
         model.seq_fields = model.seq_fields + ("sample_id",)
 
     if cfg.compile:
@@ -151,8 +151,26 @@ def main(cfg: DictConfig):
     )
     L.seed_everything(cfg.seed + trainer.logger.version, workers=True)
 
-    print("Number of windows in train: ", train_dataset.dataset_weight * train_dataset.num_ts)
+    print(
+        "Number of windows in train: ",
+        train_dataset.dataset_weight * train_dataset.num_ts,
+    )
+    print("Batch size for train: ", cfg.train_dataloader.batch_size)
+    print(
+        "Number of batches in a epoch: ",
+        train_dataset.dataset_weight
+        * train_dataset.num_ts
+        // cfg.train_dataloader.batch_size,
+    )
 
+    print("Number of windows in val: ", val_dataset.dataset_weight * val_dataset.num_ts)
+    print("Batch size for val: ", cfg.val_dataloader.batch_size)
+    print(
+        "Number of batches in a epoch: ",
+        val_dataset.dataset_weight
+        * val_dataset.num_ts
+        // cfg.val_dataloader.batch_size,
+    )
 
     # Validate before training, check the performance of original pretrained model.
     trainer.validate(model, datamodule=DataModule(cfg, train_dataset, val_dataset))

project/multi_scale_finetune/eval/small/run_multi.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
-bash project/multi_scale_fintune/eval/small/etth1.sh
-bash project/multi_scale_fintune/eval/small/etth2.sh
-bash project/multi_scale_fintune/eval/small/ettm1.sh
-bash project/multi_scale_fintune/eval/small/ettm2.sh
-bash project/multi_scale_fintune/eval/small/weather.sh
+bash project/multi_scale_finetune/eval/small/etth1.sh
+bash project/multi_scale_finetune/eval/small/etth2.sh
+bash project/multi_scale_finetune/eval/small/ettm1.sh
+bash project/multi_scale_finetune/eval/small/ettm2.sh
+bash project/multi_scale_finetune/eval/small/weather.sh
 

project/new_finetune/eval/small/electricity.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+export HYDRA_FULL_ERROR=1
+export CUDA_VISIBLE_DEVICES=0
+
+mode=S
+cp=conf/new_eval
+exp_name=lsf_finetune
+cl=3000
+model=moirai_lightning_ckpt
+
+
+cpp1=''
+cpp2=''
+cpp3=''
+cpp4=''
+index=1
+for pl in 96 192 336 720; do
+  case $index in
+    1) cpp=$cpp1 ;;
+    2) cpp=$cpp2 ;;
+    3) cpp=$cpp3 ;;
+    4) cpp=$cpp4 ;;
+  esac
+
+  pretrained_model=$(echo $cpp | cut -d'/' -f4)
+  ft_pattern=$(echo $cpp | cut -d'/' -f6)
+
+  python -m cli.eval \
+    -cp $cp \
+    exp_name=$exp_name/$pretrained_model/$ft_pattern  \
+    model=$model \
+    model.patch_size=64 \
+    model.context_length=$cl \
+    model.checkpoint_path=$cpp \
+    model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \
+    data=lsf_test \
+    data.dataset_name=electricity \
+    data.mode=$mode \
+    data.prediction_length=$pl
+
+  index=$((index+1))
+done
+

project/new_finetune/eval/small/etth1.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+export HYDRA_FULL_ERROR=1
+export CUDA_VISIBLE_DEVICES=0
+
+mode=S
+cp=conf/new_eval
+exp_name=lsf_finetune
+cl=3000
+model=moirai_lightning_ckpt
+
+cpp1='./outputs/multi_scale_finetune/moirai_1.1_R_small/dev/full/etth1/cl3000_pl96/checkpoints/epoch_3-step_40.ckpt'
+cpp2='./outputs/multi_scale_finetune/moirai_1.1_R_small/lsf/full/etth1/cl3000_pl192/checkpoints/epoch_1-step_20.ckpt'
+cpp3='./outputs/multi_scale_finetune/moirai_1.1_R_small/lsf/full/etth1/cl3000_pl336/checkpoints/epoch_1-step_20.ckpt'
+cpp4='./outputs/multi_scale_finetune/moirai_1.1_R_small/lsf/full/etth1/cl3000_pl720/checkpoints/epoch_1-step_20.ckpt'
+
+index=1
+for pl in 96 192 336 720; do
+  case $index in
+    1) cpp=$cpp1 ;;
+    2) cpp=$cpp2 ;;
+    3) cpp=$cpp3 ;;
+    4) cpp=$cpp4 ;;
+  esac
+
+  pretrained_model=$(echo $cpp | cut -d'/' -f4)
+  ft_pattern=$(echo $cpp | cut -d'/' -f6)
+
+  python -m cli.eval \
+    -cp $cp \
+    exp_name=$exp_name/$pretrained_model/$ft_pattern  \
+    model=$model \
+    model.patch_size=64 \
+    model.context_length=$cl \
+    model.checkpoint_path=$cpp \
+    model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \
+    data=lsf_test \
+    data.dataset_name=ETTh1 \
+    data.mode=$mode \
+    data.prediction_length=$pl
+
+  index=$((index+1))
+done