From d7a09007363177daca76ee8da0da9f84c02b4a15 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Thu, 11 Nov 2021 12:17:52 +0200 Subject: [PATCH 01/30] better experiment cleanup + remove initial printing of cfg --- experiment/run.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/experiment/run.py b/experiment/run.py index 760ac412..51dc0e41 100644 --- a/experiment/run.py +++ b/experiment/run.py @@ -273,7 +273,7 @@ def hydra_create_framework(framework_cfg: DisentConfigurable.cfg, datamodule, cf # ========================================================================= # -# ACTIONS # +# ACTIONS # # ========================================================================= # @@ -301,9 +301,6 @@ def action_train(cfg: DictConfig): time_string = datetime.today().strftime('%Y-%m-%d--%H-%M-%S') log.info(f'Starting run at time: {time_string}') - # print initial config - log.info(f'Initial Config For Action: {cfg.action}\n\nCONFIG:{make_box_str(OmegaConf.to_yaml(cfg), char_v=":", char_h=".")}') - # -~-~-~-~-~-~-~-~-~-~-~-~- # # cleanup from old runs: @@ -390,6 +387,16 @@ def action_train(cfg: DictConfig): # initialising the training process we cannot capture it! trainer.fit(framework, datamodule=datamodule) + # -~-~-~-~-~-~-~-~-~-~-~-~- # + + # cleanup this run + try: + wandb.finish() + except: + pass + + # -~-~-~-~-~-~-~-~-~-~-~-~- # + # available actions ACTIONS = { From ef73b7565f5ba42753f754171f5dbde4e8b9d0c8 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Thu, 11 Nov 2021 13:59:30 +0200 Subject: [PATCH 02/30] fixes + extra info --- disent/frameworks/vae/_unsupervised__vae.py | 4 ++++ .../util/lightning/callbacks/_callbacks_vae.py | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/disent/frameworks/vae/_unsupervised__vae.py b/disent/frameworks/vae/_unsupervised__vae.py index 22a32d47..366bd4e9 100644 --- a/disent/frameworks/vae/_unsupervised__vae.py +++ b/disent/frameworks/vae/_unsupervised__vae.py @@ -158,6 +158,10 @@ def do_training_step(self, batch, batch_idx): 'recon_loss': recon_loss, 'reg_loss': reg_loss, 'aug_loss': aug_loss, + # ratios + 'ratio_reg': (reg_loss / loss) if (loss != 0) else 0, + 'ratio_rec': (recon_loss / loss) if (loss != 0) else 0, + 'ratio_aug': (aug_loss / loss) if (loss != 0) else 0, } # --------------------------------------------------------------------- # diff --git a/disent/util/lightning/callbacks/_callbacks_vae.py b/disent/util/lightning/callbacks/_callbacks_vae.py index f033a1c6..1ba8c6c9 100644 --- a/disent/util/lightning/callbacks/_callbacks_vae.py +++ b/disent/util/lightning/callbacks/_callbacks_vae.py @@ -488,6 +488,19 @@ def do_step(self, trainer: pl.Trainer, pl_module: pl.LightningModule): plt.show() +def _normalized_numeric_metrics(items: dict): + results = {} + for k, v in items.items(): + if isinstance(v, (float, int)): + results[k] = v + else: + try: + results[k] = float(v) + except: + log.warning(f'SKIPPED: metric with key: {repr(k)}, result has invalid type: {type(v)} with value: {repr(v)}') + return results + + class VaeMetricLoggingCallback(BaseCallbackPeriodic): def __init__( @@ -521,10 +534,11 @@ def _compute_metrics_and_log(self, trainer: pl.Trainer, pl_module: pl.LightningM scores = metric(dataset, lambda x: vae.encode(x.to(vae.device))) metric_results = ' '.join(f'{k}{c.GRY}={c.lMGT}{v:.3f}{c.RST}' for k, v in scores.items()) log.info(f'| {metric.__name__:<{pad}} - time{c.GRY}={c.lYLW}{timer.pretty:<9}{c.RST} - {metric_results}') + # log to trainer prefix = 'final_metric' if is_final else 'epoch_metric' prefixed_scores = {f'{prefix}/{k}': v for k, v in scores.items()} - log_metrics(trainer.logger, prefixed_scores) + log_metrics(trainer.logger, _normalized_numeric_metrics(prefixed_scores)) # log summary for WANDB # this is kinda hacky... the above should work for parallel coordinate plots From e4564b17a9102febb79acdb16ec6efdac052b5e7 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Fri, 12 Nov 2021 01:11:07 +0200 Subject: [PATCH 03/30] fix distance callback to use reconstruction loss --- .../lightning/callbacks/_callbacks_vae.py | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/disent/util/lightning/callbacks/_callbacks_vae.py b/disent/util/lightning/callbacks/_callbacks_vae.py index 1ba8c6c9..28d5c269 100644 --- a/disent/util/lightning/callbacks/_callbacks_vae.py +++ b/disent/util/lightning/callbacks/_callbacks_vae.py @@ -127,20 +127,20 @@ def _to_dmat( @torch.no_grad() -def _get_dists_ae(ae: Ae, recon_loss: ReconLossHandler, x_a: torch.Tensor, x_b: torch.Tensor): +def _get_dists_ae(ae: Ae, x_a: torch.Tensor, x_b: torch.Tensor): # feed forware z_a, z_b = ae.encode(x_a), ae.encode(x_b) r_a, r_b = ae.decode(z_a), ae.decode(z_b) # distances return [ - recon_loss.compute_pairwise_loss(x_a, x_b), + ae.recon_handler.compute_pairwise_loss(x_a, x_b), torch.norm(z_a - z_b, p=1, dim=-1), # l1 dist - recon_loss.compute_pairwise_loss(r_a, r_b), + ae.recon_handler.compute_pairwise_loss(r_a, r_b), ] @torch.no_grad() -def _get_dists_vae(vae: Vae, recon_loss: ReconLossHandler, x_a: torch.Tensor, x_b: torch.Tensor): +def _get_dists_vae(vae: Vae, x_a: torch.Tensor, x_b: torch.Tensor): from torch.distributions import kl_divergence # feed forward (z_post_a, z_prior_a), (z_post_b, z_prior_b) = vae.encode_dists(x_a), vae.encode_dists(x_b) @@ -150,19 +150,19 @@ def _get_dists_vae(vae: Vae, recon_loss: ReconLossHandler, x_a: torch.Tensor, x_ kl_ab = 0.5 * kl_divergence(z_post_a, z_post_b) + 0.5 * kl_divergence(z_post_b, z_post_a) # distances return [ - recon_loss.compute_pairwise_loss(x_a, x_b), + vae.recon_handler.compute_pairwise_loss(x_a, x_b), torch.norm(z_a - z_b, p=1, dim=-1), # l1 dist - recon_loss._pairwise_reduce(kl_ab), - recon_loss.compute_pairwise_loss(r_a, r_b), + vae.recon_handler._pairwise_reduce(kl_ab), + vae.recon_handler.compute_pairwise_loss(r_a, r_b), ] -def _get_dists_fn(model, recon_loss: ReconLossHandler) -> Tuple[Optional[Tuple[str, ...]], Optional[Callable[[object, object], Sequence[Sequence[float]]]]]: +def _get_dists_fn(model: Ae) -> Tuple[Optional[Tuple[str, ...]], Optional[Callable[[object, object], Sequence[Sequence[float]]]]]: # get aggregate function if isinstance(model, Vae): - dists_names, dists_fn = _VAE_DIST_NAMES, wrapped_partial(_get_dists_vae, model, recon_loss) + dists_names, dists_fn = _VAE_DIST_NAMES, wrapped_partial(_get_dists_vae, model) elif isinstance(model, Ae): - dists_names, dists_fn = _AE_DIST_NAMES, wrapped_partial(_get_dists_ae, model, recon_loss) + dists_names, dists_fn = _AE_DIST_NAMES, wrapped_partial(_get_dists_ae, model) else: dists_names, dists_fn = None, None return dists_names, dists_fn @@ -303,7 +303,6 @@ def __init__( assert traversal_repeats > 0 self._traversal_repeats = traversal_repeats self._seed = seed - self._recon_loss = make_reconstruction_loss('mse', 'mean') self._plt_block_size = plt_block_size self._plt_show = plt_show self._log_wandb = log_wandb @@ -321,7 +320,7 @@ def do_step(self, trainer: pl.Trainer, pl_module: pl.LightningModule): log.warning(f'cannot run {self.__class__.__name__} over non-ground-truth data, skipping!') return # get aggregate function - dists_names, dists_fn = _get_dists_fn(vae, self._recon_loss) + dists_names, dists_fn = _get_dists_fn(vae) if (dists_names is None) or (dists_fn is None): log.warning(f'cannot run {self.__class__.__name__}, unsupported model type: {type(vae)}, must be {Ae.__name__} or {Vae.__name__}') return From 34523115e0cd0936edbb0a7c23721168cd9bccbf Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Fri, 12 Nov 2021 01:12:10 +0200 Subject: [PATCH 04/30] fix FftKernel to save parameters in a requires_grad=False buffer --- disent/dataset/transform/_augment.py | 6 ++++-- disent/frameworks/helper/reconstructions.py | 20 +++++++++++++------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/disent/dataset/transform/_augment.py b/disent/dataset/transform/_augment.py index 36dfb603..22dfcb72 100644 --- a/disent/dataset/transform/_augment.py +++ b/disent/dataset/transform/_augment.py @@ -185,8 +185,10 @@ class FftKernel(DisentModule): def __init__(self, kernel: Union[torch.Tensor, str], normalize: bool = True): super().__init__() - # load the kernel - self._kernel = torch.nn.Parameter(get_kernel(kernel, normalize=normalize), requires_grad=False) + # load & save the kernel -- no gradients allowed + self._kernel: torch.Tensor + self.register_buffer('_kernel', get_kernel(kernel, normalize=normalize), persistent=True) + self._kernel.requires_grad = False def forward(self, obs): # add or remove batch dim diff --git a/disent/frameworks/helper/reconstructions.py b/disent/frameworks/helper/reconstructions.py index 72d58b6f..18b31cee 100644 --- a/disent/frameworks/helper/reconstructions.py +++ b/disent/frameworks/helper/reconstructions.py @@ -241,7 +241,7 @@ def compute_unreduced_loss(self, x_recon, x_targ): class AugmentedReconLossHandler(ReconLossHandler): - def __init__(self, recon_loss_handler: ReconLossHandler, kernel: Union[str, torch.Tensor], kernel_weight=1.0): + def __init__(self, recon_loss_handler: ReconLossHandler, kernel: Union[str, torch.Tensor], wrap_weight=1.0, aug_weight=1.0): super().__init__(reduction=recon_loss_handler._reduction) # save variables self._recon_loss_handler = recon_loss_handler @@ -251,16 +251,21 @@ def __init__(self, recon_loss_handler: ReconLossHandler, kernel: Union[str, torc # load the kernel self._kernel = FftKernel(kernel=kernel, normalize=True) # kernel weighting - assert 0 <= kernel_weight <= 1, f'kernel weight must be in the range [0, 1] but received: {repr(kernel_weight)}' - self._kernel_weight = kernel_weight + assert 0 <= wrap_weight, f'loss_weight must be in the range [0, inf) but received: {repr(wrap_weight)}' + assert 0 <= aug_weight, f'kern_weight must be in the range [0, inf) but received: {repr(aug_weight)}' + self._wrap_weight = wrap_weight + self._aug_weight = aug_weight + # disable gradients + for param in self.parameters(): + param.requires_grad = False def activate(self, x_partial: torch.Tensor): return self._recon_loss_handler.activate(x_partial) def compute_unreduced_loss(self, x_recon: torch.Tensor, x_targ: torch.Tensor) -> torch.Tensor: - aug_loss = self._recon_loss_handler.compute_unreduced_loss(self._kernel(x_recon), self._kernel(x_targ)) - loss = self._recon_loss_handler.compute_unreduced_loss(x_recon, x_targ) - return (1. - self._kernel_weight) * loss + self._kernel_weight * aug_loss + wrap_loss = self._recon_loss_handler.compute_unreduced_loss(x_recon, x_targ) + aug_loss = self._recon_loss_handler.compute_unreduced_loss(self._kernel(x_recon), self._kernel(x_targ)) + return (self._wrap_weight * wrap_loss) + (self._aug_weight * aug_loss) def compute_unreduced_loss_from_partial(self, x_partial_recon: torch.Tensor, x_targ: torch.Tensor) -> torch.Tensor: return self.compute_unreduced_loss(self.activate(x_partial_recon), x_targ) @@ -276,7 +281,8 @@ def compute_unreduced_loss_from_partial(self, x_partial_recon: torch.Tensor, x_t # (REGEX, EXAMPLE, FACTORY_FUNC) # - factory function takes at min one arg: fn(reduction) with one arg after that per regex capture group # - regex expressions are tested in order, expressions should be mutually exclusive or ordered such that more specialized versions occur first. - (re.compile(r'^([a-z\d]+)_([a-z\d]+_[a-z\d]+)_w(\d+\.\d+)$'), 'mse_xy8_r47_w1.0', lambda reduction, loss, kern, weight: AugmentedReconLossHandler(make_reconstruction_loss(loss, reduction=reduction), kernel=kern, kernel_weight=float(weight))), # pragma: delete-on-release + (re.compile(r'^([a-z\d]+)_([a-z\d]+_[a-z\d]+)_w(\d+\.\d+)$'), 'mse_xy8_r47_w1.0', lambda reduction, loss, kern, weight: AugmentedReconLossHandler(make_reconstruction_loss(loss, reduction=reduction), kernel=kern, wrap_weight=1-float(weight), aug_weight=float(weight))), # pragma: delete-on-release + (re.compile(r'^([a-z\d]+)_([a-z\d]+_[a-z\d]+)_l(\d+\.\d+)_k(\d+\.\d+)$'), 'mse_xy8_r47_l1.0_k1.0', lambda reduction, loss, kern, l_weight, k_weight: AugmentedReconLossHandler(make_reconstruction_loss(loss, reduction=reduction), kernel=kern, wrap_weight=float(l_weight), aug_weight=float(k_weight))), # pragma: delete-on-release ] From 714392efbdedb02d041ca15c3fee45e8e2ec2e47 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Fri, 12 Nov 2021 16:37:36 +0200 Subject: [PATCH 05/30] fix schedule names + run_action=skip --- experiment/config/run_action/skip.yaml | 2 ++ experiment/config/schedule/beta_cyclic_fast.yaml | 2 +- experiment/config/schedule/beta_cyclic_slow.yaml | 2 +- experiment/run.py | 1 + tests/test_experiment.py | 1 + 5 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 experiment/config/run_action/skip.yaml diff --git a/experiment/config/run_action/skip.yaml b/experiment/config/run_action/skip.yaml new file mode 100644 index 00000000..80cf8dcd --- /dev/null +++ b/experiment/config/run_action/skip.yaml @@ -0,0 +1,2 @@ +# @package _global_ +action: skip diff --git a/experiment/config/schedule/beta_cyclic_fast.yaml b/experiment/config/schedule/beta_cyclic_fast.yaml index 5b64fc94..ef8ad3bb 100644 --- a/experiment/config/schedule/beta_cyclic_fast.yaml +++ b/experiment/config/schedule/beta_cyclic_fast.yaml @@ -1,4 +1,4 @@ -name: beta_cyclic +name: beta_cyclic_fast schedule_items: beta: diff --git a/experiment/config/schedule/beta_cyclic_slow.yaml b/experiment/config/schedule/beta_cyclic_slow.yaml index de6f0333..46a5ddc7 100644 --- a/experiment/config/schedule/beta_cyclic_slow.yaml +++ b/experiment/config/schedule/beta_cyclic_slow.yaml @@ -1,4 +1,4 @@ -name: beta_cyclic +name: beta_cyclic_slow schedule_items: beta: diff --git a/experiment/run.py b/experiment/run.py index 51dc0e41..aae9ebe0 100644 --- a/experiment/run.py +++ b/experiment/run.py @@ -402,6 +402,7 @@ def action_train(cfg: DictConfig): ACTIONS = { 'prepare_data': action_prepare_data, 'train': action_train, + 'skip': lambda *args, **kwargs: None, } diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 696a3ced..0ec7593f 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -38,6 +38,7 @@ @pytest.mark.parametrize('args', [ + ['run_action=skip'], ['run_action=prepare_data'], ['run_action=train'], ]) From f5248053fdccf80083132e99bb39688f3dc1e776 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Fri, 12 Nov 2021 18:20:54 +0200 Subject: [PATCH 06/30] fix xysquares dataset mean & std + remove warnings from xyobject --- disent/dataset/data/_groundtruth__xyobject.py | 13 --- disent/dataset/util/stats.py | 107 +++++++++++++----- .../config/dataset/X--xysquares_grey.yaml | 16 +-- .../config/dataset/X--xysquares_rgb.yaml | 16 +-- 4 files changed, 96 insertions(+), 56 deletions(-) diff --git a/disent/dataset/data/_groundtruth__xyobject.py b/disent/dataset/data/_groundtruth__xyobject.py index 9c7db0fe..af3835f5 100644 --- a/disent/dataset/data/_groundtruth__xyobject.py +++ b/disent/dataset/data/_groundtruth__xyobject.py @@ -116,19 +116,7 @@ def __init__( rgb: bool = True, palette: str = 'rainbow_4', transform=None, - warn_: bool = True ): - if warn_: - warnings.warn( - '`XYObjectData` defaults were changed in disent v0.3.0, if you want `approx` <= v0.2.x behavior then use the following parameters. Pallets also changed slightly too.' - '\n\tgrid_size=64' - '\n\tgrid_spacing=1' - '\n\tmin_square_size=3' - '\n\tmax_square_size=9' - '\n\tsquare_size_spacing=2' - '\n\trgb=True' - '\n\tpalette="colors_1"' - ) # generation self._rgb = rgb # check the pallete name @@ -238,7 +226,6 @@ def __init__( rgb=rgb, palette=f'{palette}_1', transform=transform, - warn_=False, ) def _get_observation(self, idx): diff --git a/disent/dataset/util/stats.py b/disent/dataset/util/stats.py index 37389355..6c14422c 100644 --- a/disent/dataset/util/stats.py +++ b/disent/dataset/util/stats.py @@ -96,9 +96,6 @@ def main(progress=False): data.DSpritesData, data.SmallNorbData, data.Shapes3dData, - wrapped_partial(data.Mpi3dData, subset='toy', in_memory=True), - wrapped_partial(data.Mpi3dData, subset='realistic', in_memory=True), - wrapped_partial(data.Mpi3dData, subset='real', in_memory=True), # groundtruth -- impl synthetic data.XYBlocksData, # pragma: delete-on-release data.XYObjectData, @@ -106,64 +103,120 @@ def main(progress=False): data.XYSquaresData, # pragma: delete-on-release data.XYSquaresMinimalData, # pragma: delete-on-release data.XColumnsData, # pragma: delete-on-release + # groundtruth -- increasing overlap # pragma: delete-on-release + (data.XYSquaresData, dict(grid_size=8, grid_spacing=8)), # pragma: delete-on-release + (data.XYSquaresData, dict(grid_size=8, grid_spacing=7)), # pragma: delete-on-release + (data.XYSquaresData, dict(grid_size=8, grid_spacing=6)), # pragma: delete-on-release + (data.XYSquaresData, dict(grid_size=8, grid_spacing=5)), # pragma: delete-on-release + (data.XYSquaresData, dict(grid_size=8, grid_spacing=4)), # pragma: delete-on-release + (data.XYSquaresData, dict(grid_size=8, grid_spacing=3)), # pragma: delete-on-release + (data.XYSquaresData, dict(grid_size=8, grid_spacing=2)), # pragma: delete-on-release + (data.XYSquaresData, dict(grid_size=8, grid_spacing=1)), # pragma: delete-on-release + (data.XYSquaresData, dict(rgb=False)), # pragma: delete-on-release + # large datasets + (data.Mpi3dData, dict(subset='toy', in_memory=True)), + (data.Mpi3dData, dict(subset='realistic', in_memory=True)), + (data.Mpi3dData, dict(subset='real', in_memory=True)), ]: from disent.dataset.transform import ToImgTensorF32 + # get arguments + if isinstance(data_cls, tuple): + data_cls, kwargs = data_cls + else: + data_cls, kwargs = data_cls, {} # Most common standardized way of computing the mean and std over observations # resized to 64px in size of dtype float32 in the range [0, 1]. - data = data_cls(transform=ToImgTensorF32(size=64)) + data = data_cls(transform=ToImgTensorF32(size=64), **kwargs) mean, std = compute_data_mean_std(data, progress=progress) # results! - print(f'{data.__class__.__name__} - {data.name}:\n mean: {mean.tolist()}\n std: {std.tolist()}') + print(f'{data.__class__.__name__} - {data.name} - {kwargs}:\n mean: {mean.tolist()}\n std: {std.tolist()}') # RUN! main() # ========================================================================= # -# RESULTS: 2021-10-12 # +# RESULTS: 2021-11-12 # # ========================================================================= # -# Cars3dData - cars3d: +# Cars3dData - cars3d - {}: # mean: [0.8976676149976628, 0.8891658020067508, 0.885147515814868] # std: [0.22503195531503034, 0.2399461278981261, 0.24792106319684404] -# DSpritesData - dsprites: +# DSpritesData - dsprites - {}: # mean: [0.042494423521889584] # std: [0.19516645880626055] -# SmallNorbData - smallnorb: +# SmallNorbData - smallnorb - {}: # mean: [0.7520918401088603] -# std: [0.09563879016827262] -# Shapes3dData - 3dshapes: +# std: [0.09563879016827263] +# Shapes3dData - 3dshapes - {}: # mean: [0.502584966788819, 0.5787597566089667, 0.6034499731859578] -# std: [0.2940814043555559, 0.3443979087517214, 0.3661685981524748] -# Mpi3dData - mpi3d_toy: -# mean: [0.22681593831231503, 0.22353985202496676, 0.22666059934624702] -# std: [0.07854112062669572, 0.07319301658077378, 0.0790763900050426] -# Mpi3dData - mpi3d_realistic: -# mean: [0.18240164396358813, 0.20723063241107917, 0.1820551008003256] -# std: [0.09511163559287175, 0.10128881101801782, 0.09428244469525177] -# Mpi3dData - mpi3d_real: -# mean: [0.13111154099374112, 0.16746449372488892, 0.14051725201807627] -# std: [0.10137409845578041, 0.10087824338375781, 0.10534121043187629] -# XYBlocksData - xyblocks: +# std: [0.2940814043555559, 0.34439790875172144, 0.3661685981524748] + +# XYBlocksData - xyblocks - {}: # mean: [0.10040509259259259, 0.10040509259259259, 0.10040509259259259] # std: [0.21689087652106678, 0.21689087652106676, 0.21689087652106678] -# XYObjectData - xy_object: +# XYObjectData - xy_object - {}: # mean: [0.009818761549013288, 0.009818761549013288, 0.009818761549013288] # std: [0.052632363725245844, 0.05263236372524584, 0.05263236372524585] -# XYObjectShadedData - xy_object: +# XYObjectShadedData - xy_object - {}: # mean: [0.009818761549013288, 0.009818761549013288, 0.009818761549013288] # std: [0.052632363725245844, 0.05263236372524584, 0.05263236372524585] -# XYSquaresData - xy_squares: +# XYSquaresData - xy_squares - {}: # mean: [0.015625, 0.015625, 0.015625] # std: [0.12403473458920855, 0.12403473458920854, 0.12403473458920854] -# XYSquaresMinimalData - xy_squares: +# XYSquaresMinimalData - xy_squares_minimal - {}: # mean: [0.015625, 0.015625, 0.015625] # std: [0.12403473458920855, 0.12403473458920854, 0.12403473458920854] -# XColumnsData - x_columns: +# XColumnsData - x_columns - {}: # mean: [0.125, 0.125, 0.125] # std: [0.33075929223788925, 0.3307592922378891, 0.3307592922378892] +# XYSquaresData - xy_squares - {'grid_size': 8, 'grid_spacing': 8}: +# mean: [0.015625, 0.015625, 0.015625] +# std: [0.12403473458920855, 0.12403473458920854, 0.12403473458920854] +# overlap between squares for reconstruction loss, 7 < 8 +# XYSquaresData - xy_squares - {'grid_size': 8, 'grid_spacing': 7}: +# mean: [0.015625, 0.015625, 0.015625] +# std: [0.12403473458920854, 0.12403473458920854, 0.12403473458920854] +# overlap between squares for reconstruction loss, 6 < 8 +# XYSquaresData - xy_squares - {'grid_size': 8, 'grid_spacing': 6}: +# mean: [0.015625, 0.015625, 0.015625] +# std: [0.12403473458920854, 0.12403473458920854, 0.12403473458920855] +# overlap between squares for reconstruction loss, 5 < 8 +# XYSquaresData - xy_squares - {'grid_size': 8, 'grid_spacing': 5}: +# mean: [0.015625, 0.015625, 0.015625] +# std: [0.12403473458920855, 0.12403473458920855, 0.12403473458920854] +# overlap between squares for reconstruction loss, 4 < 8 +# XYSquaresData - xy_squares - {'grid_size': 8, 'grid_spacing': 4}: +# mean: [0.015625, 0.015625, 0.015625] +# std: [0.12403473458920855, 0.12403473458920854, 0.12403473458920854] +# overlap between squares for reconstruction loss, 3 < 8 +# XYSquaresData - xy_squares - {'grid_size': 8, 'grid_spacing': 3}: +# mean: [0.015625, 0.015625, 0.015625] +# std: [0.12403473458920854, 0.12403473458920854, 0.12403473458920854] +# overlap between squares for reconstruction loss, 2 < 8 +# XYSquaresData - xy_squares - {'grid_size': 8, 'grid_spacing': 2}: +# mean: [0.015625, 0.015625, 0.015625] +# std: [0.12403473458920854, 0.12403473458920854, 0.12403473458920854] +# overlap between squares for reconstruction loss, 1 < 8 +# XYSquaresData - xy_squares - {'grid_size': 8, 'grid_spacing': 1}: +# mean: [0.015625, 0.015625, 0.015625] +# std: [0.12403473458920855, 0.12403473458920855, 0.12403473458920855] +# XYSquaresData - xy_squares - {'rgb': False}: +# mean: [0.046146392822265625] +# std: [0.2096506119375896] + +# Mpi3dData - mpi3d_toy - {'subset': 'toy', 'in_memory': True}: +# mean: [0.22681593831231503, 0.22353985202496676, 0.22666059934624702] +# std: [0.07854112062669572, 0.07319301658077378, 0.0790763900050426] +# Mpi3dData - mpi3d_realistic - {'subset': 'realistic', 'in_memory': True}: +# mean: [0.18240164396358813, 0.20723063241107917, 0.1820551008003256] +# std: [0.09511163559287175, 0.10128881101801782, 0.09428244469525177] +# Mpi3dData - mpi3d_real - {'subset': 'real', 'in_memory': True}: +# mean: [0.13111154099374112, 0.16746449372488892, 0.14051725201807627] +# std: [0.10137409845578041, 0.10087824338375781, 0.10534121043187629] + # ========================================================================= # # END # diff --git a/experiment/config/dataset/X--xysquares_grey.yaml b/experiment/config/dataset/X--xysquares_grey.yaml index 554f4a98..20088abd 100644 --- a/experiment/config/dataset/X--xysquares_grey.yaml +++ b/experiment/config/dataset/X--xysquares_grey.yaml @@ -5,12 +5,12 @@ name: xysquares_grey data: _target_: disent.dataset.data.XYSquaresData - square_size: 8 - grid_size: 64 - grid_spacing: 8 - num_squares: 3 - rgb: FALSE - max_placements: 8 + square_size: 8 # AFFECTS: mean and std + image_size: 64 # usually ok to adjust + grid_size: 8 # usually ok to adjust + grid_spacing: 8 # usually ok to adjust + num_squares: 3 # AFFECTS: mean and std + rgb: FALSE # AFFECTS: mean and std transform: _target_: disent.dataset.transform.ToImgTensorF32 @@ -19,5 +19,5 @@ transform: meta: x_shape: [1, 64, 64] - vis_mean: "${exit:EXITING... please compute the vis_mean and vis_std}" - vis_std: "${exit:EXITING... please compute the vis_mean and vis_std}" + vis_mean: [0.046146392822265625] + vis_std: [0.2096506119375896] diff --git a/experiment/config/dataset/X--xysquares_rgb.yaml b/experiment/config/dataset/X--xysquares_rgb.yaml index 591380ba..35a45110 100644 --- a/experiment/config/dataset/X--xysquares_rgb.yaml +++ b/experiment/config/dataset/X--xysquares_rgb.yaml @@ -5,12 +5,12 @@ name: xysquares_rgb data: _target_: disent.dataset.data.XYSquaresData - square_size: 8 - grid_size: 64 - grid_spacing: 8 - num_squares: 3 - rgb: TRUE - max_placements: 8 + square_size: 8 # AFFECTS: mean and std + image_size: 64 # usually ok to adjust + grid_size: 8 # usually ok to adjust + grid_spacing: 8 # usually ok to adjust + num_squares: 3 # AFFECTS: mean and std + rgb: TRUE # AFFECTS: mean and std transform: _target_: disent.dataset.transform.ToImgTensorF32 @@ -19,5 +19,5 @@ transform: meta: x_shape: [3, 64, 64] - mean: "${exit:EXITING... please compute the vis_mean and vis_std}" - std: "${exit:EXITING... please compute the vis_mean and vis_std}" + vis_mean: [0.015625, 0.015625, 0.015625] + vis_std: [0.12403473458920855, 0.12403473458920854, 0.12403473458920854] From f64c3bd00b951aed3149d9441324d7a145878002 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Fri, 12 Nov 2021 18:47:49 +0200 Subject: [PATCH 07/30] change hydra_check_cuda to hydra_get_gpus, fixes error --- experiment/run.py | 31 ++++++++++--------- .../run_03_train_disentangle_kernel.py | 6 ++-- .../run_02_gen_adversarial_dataset.py | 6 ++-- .../run_02_gen_adversarial_dataset_approx.py | 6 ++-- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/experiment/run.py b/experiment/run.py index aae9ebe0..bfe57545 100644 --- a/experiment/run.py +++ b/experiment/run.py @@ -64,23 +64,24 @@ # ========================================================================= # -def hydra_check_cuda(cfg): - cuda = cfg.dsettings.trainer.cuda - # set cuda - if cuda in {'try_cuda', None}: - cfg.dsettings.trainer.cuda = torch.cuda.is_available() - if not cuda: +def hydra_get_gpus(cfg) -> int: + use_cuda = cfg.dsettings.trainer.cuda + # check cuda values + if use_cuda in {'try_cuda', None}: + use_cuda = torch.cuda.is_available() + if not use_cuda: log.warning('CUDA was requested, but not found on this system... CUDA has been disabled!') + elif use_cuda: + if not torch.cuda.is_available(): + log.error('`dsettings.trainer.cuda=True` but CUDA is not available on this machine!') + raise RuntimeError('CUDA not available!') else: if not torch.cuda.is_available(): - if cuda: - log.error('trainer.cuda=True but CUDA is not available on this machine!') - raise RuntimeError('CUDA not available!') - else: - log.warning('CUDA is not available on this machine!') + log.info('CUDA is not available on this machine!') else: - if not cuda: - log.warning('CUDA is available but is not being used!') + log.warning('CUDA is available but is not being used!') + # get number of gpus to use + return (1 if use_cuda else 0) def hydra_check_data_paths(cfg): @@ -328,7 +329,7 @@ def action_train(cfg: DictConfig): log.info(f"Orig working directory : {hydra.utils.get_original_cwd()}") # check CUDA setting - hydra_check_cuda(cfg) + gpus = hydra_get_gpus(cfg) # check data preparation hydra_check_data_paths(cfg) @@ -351,7 +352,7 @@ def action_train(cfg: DictConfig): trainer = set_debug_trainer(pl.Trainer( logger=logger, callbacks=callbacks, - gpus=1 if cfg.dsettings.trainer.cuda else 0, + gpus=gpus, # we do this here too so we don't run the final # metrics, even through we check for it manually. terminate_on_nan=True, diff --git a/research/e05_disentangle_kernel/run_03_train_disentangle_kernel.py b/research/e05_disentangle_kernel/run_03_train_disentangle_kernel.py index f8c98573..eb15d491 100644 --- a/research/e05_disentangle_kernel/run_03_train_disentangle_kernel.py +++ b/research/e05_disentangle_kernel/run_03_train_disentangle_kernel.py @@ -48,7 +48,7 @@ from disent.util.seeds import seed from disent.util.strings.fmt import make_box_str from experiment.run import hydra_append_progress_callback -from experiment.run import hydra_check_cuda +from experiment.run import hydra_get_gpus from experiment.run import hydra_make_logger from experiment.util.hydra_utils import make_non_strict @@ -228,7 +228,7 @@ def run_disentangle_dataset_kernel(cfg): # TODO: some of this code is duplicated between this and the main experiment run.py # check CUDA setting cfg.trainer.setdefault('cuda', 'try_cuda') - hydra_check_cuda(cfg) + gpus = hydra_get_gpus(cfg) # CREATE LOGGER logger = hydra_make_logger(cfg) # TRAINER CALLBACKS @@ -265,7 +265,7 @@ def run_disentangle_dataset_kernel(cfg): flush_logs_every_n_steps=cfg.log.setdefault('flush_logs_every_n_steps', 100), logger=logger, callbacks=callbacks, - gpus=1 if cfg.trainer.cuda else 0, + gpus=1 if gpus else 0, max_epochs=cfg.trainer.setdefault('epochs', None), max_steps=cfg.trainer.setdefault('steps', 10000), progress_bar_refresh_rate=0, # ptl 0.9 diff --git a/research/e06_adversarial_data/deprecated/run_02_gen_adversarial_dataset.py b/research/e06_adversarial_data/deprecated/run_02_gen_adversarial_dataset.py index 81fd5a61..1260233b 100644 --- a/research/e06_adversarial_data/deprecated/run_02_gen_adversarial_dataset.py +++ b/research/e06_adversarial_data/deprecated/run_02_gen_adversarial_dataset.py @@ -64,7 +64,7 @@ from disent.util.strings.fmt import make_box_str from disent.util.visualize.vis_util import make_image_grid from experiment.run import hydra_get_callbacks -from experiment.run import hydra_check_cuda +from experiment.run import hydra_get_gpus from experiment.run import hydra_make_logger from experiment.util.hydra_utils import make_non_strict from experiment.util.run_utils import log_error_and_exit @@ -311,7 +311,7 @@ def run_gen_adversarial_dataset(cfg): cfg = make_non_strict(cfg) # - - - - - - - - - - - - - - - # # check CUDA setting - hydra_check_cuda(cfg) + gpus = hydra_get_gpus(cfg) # create logger logger = hydra_make_logger(cfg) # create callbacks @@ -340,7 +340,7 @@ def run_gen_adversarial_dataset(cfg): logger=logger, callbacks=callbacks, # cfg.dsettings.trainer - gpus=1 if cfg.dsettings.trainer.cuda else 0, + gpus=gpus, # cfg.trainer max_epochs=cfg.trainer.max_epochs, max_steps=cfg.trainer.max_steps, diff --git a/research/e06_adversarial_data/run_02_gen_adversarial_dataset_approx.py b/research/e06_adversarial_data/run_02_gen_adversarial_dataset_approx.py index f3687f94..b85f52c5 100644 --- a/research/e06_adversarial_data/run_02_gen_adversarial_dataset_approx.py +++ b/research/e06_adversarial_data/run_02_gen_adversarial_dataset_approx.py @@ -66,7 +66,7 @@ from disent.util.strings.fmt import bytes_to_human from disent.util.strings.fmt import make_box_str from disent.util.visualize.vis_util import make_image_grid -from experiment.run import hydra_check_cuda +from experiment.run import hydra_get_gpus from experiment.run import hydra_get_callbacks from experiment.run import hydra_make_logger from experiment.util.hydra_utils import make_non_strict @@ -498,7 +498,7 @@ def run_gen_adversarial_dataset(cfg): cfg = make_non_strict(cfg) # - - - - - - - - - - - - - - - # # check CUDA setting - hydra_check_cuda(cfg) + gpus = hydra_get_gpus(cfg) # create logger logger = hydra_make_logger(cfg) # create callbacks @@ -527,7 +527,7 @@ def run_gen_adversarial_dataset(cfg): logger=logger, callbacks=callbacks, # cfg.dsettings.trainer - gpus=1 if cfg.dsettings.trainer.cuda else 0, + gpus=gpus, # cfg.trainer max_epochs=cfg.trainer.max_epochs, max_steps=cfg.trainer.max_steps, From 4f0dc152133ea3996cb5043fb56ed96d3f2f48d9 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Fri, 12 Nov 2021 19:38:43 +0200 Subject: [PATCH 08/30] fix clog scripts --- research/clog-batch.sh | 32 +++----------------------------- research/clog-stampede.sh | 31 +++---------------------------- research/helper.sh | 16 ++++++++++++++++ research/working-batch.sh | 19 +++++++++++++++++++ research/working-stampede.sh | 19 +++++++++++++++++++ 5 files changed, 60 insertions(+), 57 deletions(-) create mode 100644 research/working-batch.sh create mode 100644 research/working-stampede.sh diff --git a/research/clog-batch.sh b/research/clog-batch.sh index 171a1675..79602bf8 100644 --- a/research/clog-batch.sh +++ b/research/clog-batch.sh @@ -1,36 +1,11 @@ #!/bin/bash -# -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# - # ========================================================================= # # Settings # # ========================================================================= # -export PROJECT="Clog" +export PROJECT="N/A" +export USERNAME="N/A" export PARTITION="batch" export PARALLELISM=24 @@ -41,5 +16,4 @@ source "$(dirname "$(realpath -s "$0")")/helper.sh" # Experiment # # ========================================================================= # -clog_cudaless_nodes batch 43200 "C-disent" # 12 hours - +clog_cuda_nodes "$PARTITION" 43200 "C-disent" # 12 hours diff --git a/research/clog-stampede.sh b/research/clog-stampede.sh index a45d6fbf..5d53a26e 100644 --- a/research/clog-stampede.sh +++ b/research/clog-stampede.sh @@ -1,36 +1,11 @@ #!/bin/bash -# -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# - # ========================================================================= # # Settings # # ========================================================================= # -export PROJECT="Clog" +export PROJECT="N/A" +export USERNAME="N/A" export PARTITION="stampede" export PARALLELISM=24 @@ -41,4 +16,4 @@ source "$(dirname "$(realpath -s "$0")")/helper.sh" # Experiment # # ========================================================================= # -clog_cudaless_nodes stampede 43200 "C-disent" # 12 hours +clog_cuda_nodes "$PARTITION" 43200 "C-disent" # 12 hours diff --git a/research/helper.sh b/research/helper.sh index 75f270dd..c0635998 100644 --- a/research/helper.sh +++ b/research/helper.sh @@ -102,8 +102,24 @@ function clog_cudaless_nodes() { fi } +function clog_cuda_nodes() { + if [ -z "$1" ]; then echo "partition is not set"; exit 1; fi + if [ -z "$2" ]; then echo wait=120; else wait="$2"; fi + if [ -z "$3" ]; then echo name="HAS-CUDA"; else name="$3"; fi + # clog idle nodes + n=$(num_idle_nodes "$1") + if [ "$n" -lt "1" ]; then + echo -e "\e[93mclogging skipped! no idle nodes found on partition '$1'\e[0m"; + else + echo -e "\e[92mclogging $n nodes on partition '$1' for ${wait}s if cuda is available!\e[0m"; + sbatch --array=1-"$n" --partition="$1" --job-name="$name" --output=/dev/null --error=/dev/null \ + --wrap='python -c "import torch; import time; cuda=torch.cuda.is_available(); print(\"CUDA:\", cuda, flush=True); print(flush=True); time.sleep(5 if not cuda else '"$wait"');"' + fi +} + export num_idle_nodes export clog_cudaless_nodes +export clog_cuda_nodes # ========================================================================= # # End # diff --git a/research/working-batch.sh b/research/working-batch.sh new file mode 100644 index 00000000..53855652 --- /dev/null +++ b/research/working-batch.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# ========================================================================= # +# Settings # +# ========================================================================= # + +export PROJECT="N/A" +export USERNAME="N/A" +export PARTITION="batch" +export PARALLELISM=24 + +# source the helper file +source "$(dirname "$(realpath -s "$0")")/helper.sh" + +# ========================================================================= # +# Experiment # +# ========================================================================= # + +clog_cuda_nodes "$PARTITION" 43200 "W-disent" # 12 hours diff --git a/research/working-stampede.sh b/research/working-stampede.sh new file mode 100644 index 00000000..c4d7170b --- /dev/null +++ b/research/working-stampede.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# ========================================================================= # +# Settings # +# ========================================================================= # + +export PROJECT="N/A" +export USERNAME="N/A" +export PARTITION="stampede" +export PARALLELISM=24 + +# source the helper file +source "$(dirname "$(realpath -s "$0")")/helper.sh" + +# ========================================================================= # +# Experiment # +# ========================================================================= # + +clog_cuda_nodes "$PARTITION" 43200 "W-disent" # 12 hours From a584dd8523c842f1b279dfeea50fad3ec7a62d2b Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Fri, 12 Nov 2021 21:57:29 +0200 Subject: [PATCH 09/30] extra schedules --- experiment/config/schedule/beta_delay.yaml | 10 ++++++++++ experiment/config/schedule/beta_delay_long.yaml | 10 ++++++++++ 2 files changed, 20 insertions(+) create mode 100644 experiment/config/schedule/beta_delay.yaml create mode 100644 experiment/config/schedule/beta_delay_long.yaml diff --git a/experiment/config/schedule/beta_delay.yaml b/experiment/config/schedule/beta_delay.yaml new file mode 100644 index 00000000..a223f278 --- /dev/null +++ b/experiment/config/schedule/beta_delay.yaml @@ -0,0 +1,10 @@ +name: beta_increase + +schedule_items: + beta: + _target_: disent.schedule.Single + start_step: 3600 + end_step: 7200 + r_start: 0.001 + r_end: 1.0 + mode: 'linear' diff --git a/experiment/config/schedule/beta_delay_long.yaml b/experiment/config/schedule/beta_delay_long.yaml new file mode 100644 index 00000000..353e75dc --- /dev/null +++ b/experiment/config/schedule/beta_delay_long.yaml @@ -0,0 +1,10 @@ +name: beta_increase + +schedule_items: + beta: + _target_: disent.schedule.Single + start_step: 7200 + end_step: 14400 + r_start: 0.001 + r_end: 1.0 + mode: 'linear' From 3f21473537410e0d219e0510482b97f67d7e7d14 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Sat, 13 Nov 2021 14:46:34 +0200 Subject: [PATCH 10/30] update requirements and plotting --- disent/util/visualize/plot.py | 8 ++++++-- requirements-research.txt | 7 +++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/disent/util/visualize/plot.py b/disent/util/visualize/plot.py index f4bafb81..826594c2 100644 --- a/disent/util/visualize/plot.py +++ b/disent/util/visualize/plot.py @@ -135,7 +135,10 @@ def plt_subplots( assert isinstance(ncols, int) # check titles if titles is not None: - titles = np.array(titles).reshape([nrows, ncols]) + titles = np.array(titles) + if titles.ndim == 1: + titles = np.array([titles] + ([[None]*ncols] * (nrows-1))) + assert titles.ndim == 2 # get labels if (row_labels is None) or isinstance(row_labels, str): row_labels = [row_labels] * nrows @@ -161,7 +164,8 @@ def plt_subplots( ax.set_ylabel(row_labels[y], fontsize=label_size) # set title if titles is not None: - ax.set_title(titles[y][x], fontsize=titles_size) + if titles[y][x] is not None: + ax.set_title(titles[y][x], fontsize=titles_size) # set title fig.suptitle(title, fontsize=title_size) # done! diff --git a/requirements-research.txt b/requirements-research.txt index 1b895133..e5a6d2bd 100644 --- a/requirements-research.txt +++ b/requirements-research.txt @@ -10,5 +10,12 @@ # psutil # seaborn + ray>=1.6.0 ruck==0.2.4 + +seaborn>=0.11.0 +pandas>=1.3.0 +cachier>=1.5.0 + +statsmodels>=0.13.0 # required for seaborn, to estimate outliers in regression plots From 34240450af85b1d48a16bc722837387eb73c56fb Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Wed, 17 Nov 2021 10:52:03 +0200 Subject: [PATCH 11/30] dsprites imagenet fix for macos multithreading --- disent/dataset/data/_groundtruth__dsprites_imagenet.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/disent/dataset/data/_groundtruth__dsprites_imagenet.py b/disent/dataset/data/_groundtruth__dsprites_imagenet.py index 008aee28..13b80134 100644 --- a/disent/dataset/data/_groundtruth__dsprites_imagenet.py +++ b/disent/dataset/data/_groundtruth__dsprites_imagenet.py @@ -62,9 +62,13 @@ def __getitem__(self, idx): return np.array(img) +def _noop(x): + return x + + def load_imagenet_tiny_data(raw_data_dir): data = NumpyFolder(os.path.join(raw_data_dir, 'train')) - data = DataLoader(data, batch_size=64, num_workers=min(16, psutil.cpu_count(logical=False)), shuffle=False, drop_last=False, collate_fn=lambda x: x) + data = DataLoader(data, batch_size=64, num_workers=min(16, psutil.cpu_count(logical=False)), shuffle=False, drop_last=False, collate_fn=_noop) # load data - this is a bit memory inefficient doing it like this instead of with a loop into a pre-allocated array imgs = np.concatenate(list(tqdm(data, 'loading')), axis=0) assert imgs.shape == (100_000, 64, 64, 3) From 06e5bf5605878727ba3c892c4a15d93c12f2a16d Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Thu, 11 Nov 2021 09:29:55 +0200 Subject: [PATCH 12/30] initial experiments --- .../config_adversarial_dataset_approx.yaml | 17 +++++---- research/e00_tuning/submit_param_tuning.sh | 21 +++++----- .../run_02_adv_dataset_approx.sh | 17 +++++++-- .../run_02_gen_adversarial_dataset_approx.py | 1 + .../submit_overlap_loss.sh | 38 +++++++++++++++++++ 5 files changed, 73 insertions(+), 21 deletions(-) create mode 100644 research/e09_vae_overlap_loss/submit_overlap_loss.sh diff --git a/experiment/config/config_adversarial_dataset_approx.yaml b/experiment/config/config_adversarial_dataset_approx.yaml index 6526c5ab..e984f7e6 100644 --- a/experiment/config/config_adversarial_dataset_approx.yaml +++ b/experiment/config/config_adversarial_dataset_approx.yaml @@ -14,7 +14,8 @@ settings: job: user: 'n_michlo' project: 'DELETE' - name: 'TEST-${adv_system.dataset_name}_${adv_system.adversarial_mode}_${adv_system.samples_sort_mode}_aw${adv_system.loss_adversarial_weight}_${adv_system.sampler_name}_s${trainer.max_steps}_${adv_system.optimizer_name}_lr${adv_system.optimizer_lr}_wd${adv_system.optimizer_kwargs.weight_decay}_b${settings.dataset.batch_size}_${settings.exp.save_dtype}' + name_prefix: 'B32' + name: '${settings.job.name_prefix}-${adv_system.dataset_name}_${adv_system.adversarial_mode}_${adv_system.samples_sort_mode}_aw${adv_system.loss_adversarial_weight}_${adv_system.sampler_name}_s${trainer.max_steps}_${adv_system.optimizer_name}_lr${adv_system.optimizer_lr}_wd${adv_system.optimizer_kwargs.weight_decay}_b${settings.dataset.batch_size}_${settings.exp.save_dtype}' seed: 424242 exp: show_every_n_steps: 1000 @@ -25,14 +26,14 @@ settings: save_data: FALSE save_dtype: float16 dataset: - batch_size: 128 + batch_size: 32 trainer: # same as defaults: - run_length: ... # - 15000 takes 40 mins with batch size 512 (heartofgold, 12 workers) # - 50000 takes 33 mins with batch size 256 (griffin, 16 workers) - max_steps: 20000 - max_epochs: 20000 + max_steps: 15000 + max_epochs: 15000 adv_system: ### IMPORTANT SETTINGS ### @@ -48,17 +49,17 @@ adv_system: # note: sampler_name (adversarial_mode=self) # - close_p_random_n: seems better based on plot of fdists vs overlap (converges better, but loss is higher which makes sense) # - same_k1_close: seems worse based on plot of fdists vs overlap (seems to maintain original shape more, might hinder disentanglement? not actually tested) - sampler_name: 'random_swap_manhattan' # [random_swap_manhattan, close_p_random_n, same_k1_close] - samples_sort_mode: 'none' # [none, swap, sort_inorder, sort_reverse] + sampler_name: 'close_p_random_n' # [random_swap_manhattan, close_p_random_n, same_k1_close] + samples_sort_mode: 'swap' # [none, swap, sort_inorder, sort_reverse] dataset_name: 'smallnorb' # [cars3d, smallnorb, dsprites, shapes3d, xysquares_8x8_mini] adversarial_mode: 'triplet_margin_0.1' # [self, invert_margin_0.05, invert_margin_0.005] invert, invert_unbounded ### OTHER SETTINGS ### # optimizer options optimizer_name: 'adam' - optimizer_lr: 1e-3 + optimizer_lr: 2e-3 optimizer_kwargs: - weight_decay: 1e-6 + weight_decay: 1e-5 # dataset config options dataset_batch_size: ${dataloader.batch_size} # x3 dataset_num_workers: ${dataloader.num_workers} diff --git a/research/e00_tuning/submit_param_tuning.sh b/research/e00_tuning/submit_param_tuning.sh index 87e039d5..6dd63dec 100644 --- a/research/e00_tuning/submit_param_tuning.sh +++ b/research/e00_tuning/submit_param_tuning.sh @@ -5,7 +5,7 @@ # ========================================================================= # export USERNAME="n_michlo" -export PROJECT="final-00__basic-hparam-tuning" +export PROJECT="CVPR-00__basic-hparam-tuning" export PARTITION="stampede" export PARALLELISM=28 @@ -21,34 +21,37 @@ clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours # RUN SWEEP FOR GOOD BETA VALUES # - beta: 0.01, 0.0316 seem good, 0.1 starts getting too strong, 0.00316 is a bit weak # - beta: -# 1 * (2 * 8 * 2 * 4) = 128 +# 1 * (2 * 8 * 2 * 5) = 160 submit_sweep \ +DUMMY.repeat=1 \ +EXTRA.tags='sweep_beta' \ \ - run_length=medium \ - metrics=fast \ + run_length=long \ + metrics=all \ \ - framework=betavae,adavae_os \ settings.framework.beta=0.000316,0.001,0.00316,0.01,0.0316,0.1,0.316,1.0 \ + framework=betavae,adavae_os \ + schedule=none \ settings.model.z_size=9,25 \ \ - dataset=dsprites,shapes3d,cars3d,smallnorb \ + dataset=dsprites,shapes3d,cars3d,smallnorb,X--xysquares \ sampling=default__bb +submit_sweep + # RUN SWEEP FOR GOOD SCHEDULES -# 1 * (4 * 2 * 4 * 5) = 160 +# 1 * (4 * 2 * 4 * 2 * 5) = 320 submit_sweep \ +DUMMY.repeat=1 \ +EXTRA.tags='sweep_schedule' \ \ run_length=long \ - metrics=fast \ + metrics=all \ \ settings.framework.beta=0.0316,0.1,0.316,1.0 \ framework=betavae,adavae_os \ schedule=beta_cyclic,beta_cyclic_slow,beta_cyclic_fast,beta_decrease \ settings.model.z_size=9,25 \ \ - dataset=dsprites,shapes3d,cars3d,smallnorb \ + dataset=dsprites,shapes3d,cars3d,smallnorb,X--xysquares \ sampling=default__bb diff --git a/research/e06_adversarial_data/run_02_adv_dataset_approx.sh b/research/e06_adversarial_data/run_02_adv_dataset_approx.sh index 13ce8c06..75dd8b44 100644 --- a/research/e06_adversarial_data/run_02_adv_dataset_approx.sh +++ b/research/e06_adversarial_data/run_02_adv_dataset_approx.sh @@ -22,11 +22,20 @@ ROOT_DIR="$(dirname "$(dirname "$PARENT_DIR")")" PYTHONPATH="$ROOT_DIR" python3 "$PARENT_DIR/run_02_gen_adversarial_dataset_approx.py" \ -m "$@" \ - settings.dataset.batch_size=128 \ - adv_system.loss_out_of_bounds_weight=1.0 \ \ - adv_system.sampler_name=same_k1_close,close_p_random_n,random_swap_manhattan \ + +meta.tag='unbounded_manhat' \ + settings.job.name_prefix=MANHAT \ + \ + adv_system.sampler_name=same_k_close,random_swap_manhattan,close_p_random_n \ adv_system.samples_sort_mode=swap,sort_reverse,none,sort_inorder \ \ - adv_system.adversarial_mode=triplet_margin_0.1 \ + adv_system.adversarial_mode=triplet_unbounded \ adv_system.dataset_name=smallnorb \ + \ + trainer.max_steps=7500 \ + trainer.max_epochs=7500 \ + \ + adv_system.optimizer_lr=5e-3 \ + settings.exp.show_every_n_steps=500 \ + \ + settings.dataset.batch_size=128 diff --git a/research/e06_adversarial_data/run_02_gen_adversarial_dataset_approx.py b/research/e06_adversarial_data/run_02_gen_adversarial_dataset_approx.py index b85f52c5..0caf72ec 100644 --- a/research/e06_adversarial_data/run_02_gen_adversarial_dataset_approx.py +++ b/research/e06_adversarial_data/run_02_gen_adversarial_dataset_approx.py @@ -348,6 +348,7 @@ def make_train_periodic_callbacks(self, cfg) -> Sequence[BaseCallbackPeriodic]: # dataset transform helper @TempNumpySeed(42) + @torch.no_grad() def make_scale_uint8_transform(): # get scaling values if self.hparams.logging_scale_imgs: diff --git a/research/e09_vae_overlap_loss/submit_overlap_loss.sh b/research/e09_vae_overlap_loss/submit_overlap_loss.sh new file mode 100644 index 00000000..2d6ec40f --- /dev/null +++ b/research/e09_vae_overlap_loss/submit_overlap_loss.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# ========================================================================= # +# Settings # +# ========================================================================= # + +export USERNAME="n_michlo" +export PROJECT="CVPR-09__vae_overlap_loss" +export PARTITION="stampede" +export PARALLELISM=28 + +# source the helper file +source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" + +# ========================================================================= # +# Experiment # +# ========================================================================= # + +clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours + +# RUN SWEEP FOR GOOD BETA VALUES +# - beta: 0.01, 0.0316 seem good, 0.1 starts getting too strong, 0.00316 is a bit weak +# - beta: +# 1 * (2 * 5 * 2 * 6) = 120 +submit_sweep \ + +DUMMY.repeat=1 \ + +EXTRA.tags='sweep_overlap_loss' \ + \ + run_length=long \ + metrics=all \ + \ + framework=betavae,adavae_os \ + settings.framework.beta=0.001,0.00316,0.01,0.0316,0.1 \ + settings.model.z_size=9,25 \ + settings.framework.recon_loss=mse_box_r47_w0.5,mse_box_r45_w0.5,mse_box_r33_w0.5,mse_box_r29_w0.5,mse_box_r25_w0.5,mse \ + \ + dataset=X--xysquares \ + sampling=default__bb From 9f369824f041b98a07d9b4c13e5f166ea19f0051 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Thu, 11 Nov 2021 14:34:06 +0200 Subject: [PATCH 13/30] adjust tuning experiment --- research/e00_tuning/submit_param_tuning.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/research/e00_tuning/submit_param_tuning.sh b/research/e00_tuning/submit_param_tuning.sh index 6dd63dec..6e956bb8 100644 --- a/research/e00_tuning/submit_param_tuning.sh +++ b/research/e00_tuning/submit_param_tuning.sh @@ -37,10 +37,9 @@ submit_sweep \ dataset=dsprites,shapes3d,cars3d,smallnorb,X--xysquares \ sampling=default__bb -submit_sweep # RUN SWEEP FOR GOOD SCHEDULES -# 1 * (4 * 2 * 4 * 2 * 5) = 320 +# 1 * (3 * 2 * 4 * 5) = 120 submit_sweep \ +DUMMY.repeat=1 \ +EXTRA.tags='sweep_schedule' \ @@ -48,10 +47,10 @@ submit_sweep \ run_length=long \ metrics=all \ \ - settings.framework.beta=0.0316,0.1,0.316,1.0 \ + settings.framework.beta=0.1,0.316,1.0 \ framework=betavae,adavae_os \ schedule=beta_cyclic,beta_cyclic_slow,beta_cyclic_fast,beta_decrease \ - settings.model.z_size=9,25 \ + settings.model.z_size=25 \ \ dataset=dsprites,shapes3d,cars3d,smallnorb,X--xysquares \ sampling=default__bb From 83142b12eb91b62b4ac388f63dd6999b3f899314 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Fri, 12 Nov 2021 17:01:04 +0200 Subject: [PATCH 14/30] update experiments --- research/e00_tuning/submit_param_tuning.sh | 30 +++++++++---------- .../{run.sh => submit_incr_overlap.sh} | 23 ++++++++------ .../submit_overlap_loss.sh | 24 ++++++++------- 3 files changed, 42 insertions(+), 35 deletions(-) rename research/e01_incr_overlap/{run.sh => submit_incr_overlap.sh} (78%) diff --git a/research/e00_tuning/submit_param_tuning.sh b/research/e00_tuning/submit_param_tuning.sh index 6e956bb8..129d585c 100644 --- a/research/e00_tuning/submit_param_tuning.sh +++ b/research/e00_tuning/submit_param_tuning.sh @@ -16,7 +16,7 @@ source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" # Experiment # # ========================================================================= # -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours +clog_cudaless_nodes "$PARTITION" 129600 "C-disent" # 36 hours # RUN SWEEP FOR GOOD BETA VALUES # - beta: 0.01, 0.0316 seem good, 0.1 starts getting too strong, 0.00316 is a bit weak @@ -40,17 +40,17 @@ submit_sweep \ # RUN SWEEP FOR GOOD SCHEDULES # 1 * (3 * 2 * 4 * 5) = 120 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_schedule' \ - \ - run_length=long \ - metrics=all \ - \ - settings.framework.beta=0.1,0.316,1.0 \ - framework=betavae,adavae_os \ - schedule=beta_cyclic,beta_cyclic_slow,beta_cyclic_fast,beta_decrease \ - settings.model.z_size=25 \ - \ - dataset=dsprites,shapes3d,cars3d,smallnorb,X--xysquares \ - sampling=default__bb +#submit_sweep \ +# +DUMMY.repeat=1 \ +# +EXTRA.tags='sweep_schedule' \ +# \ +# run_length=long \ +# metrics=all \ +# \ +# settings.framework.beta=0.1,0.316,1.0 \ +# framework=betavae,adavae_os \ +# schedule=beta_cyclic,beta_cyclic_slow,beta_cyclic_fast,beta_decrease \ +# settings.model.z_size=25 \ +# \ +# dataset=dsprites,shapes3d,cars3d,smallnorb,X--xysquares \ +# sampling=default__bb diff --git a/research/e01_incr_overlap/run.sh b/research/e01_incr_overlap/submit_incr_overlap.sh similarity index 78% rename from research/e01_incr_overlap/run.sh rename to research/e01_incr_overlap/submit_incr_overlap.sh index 4b0d74dc..fcb317ee 100644 --- a/research/e01_incr_overlap/run.sh +++ b/research/e01_incr_overlap/submit_incr_overlap.sh @@ -5,7 +5,7 @@ # ========================================================================= # export USERNAME="n_michlo" -export PROJECT="final-01__incr-overlap" +export PROJECT="CVPR-01__incr_overlap" export PARTITION="stampede" export PARALLELISM=28 @@ -20,29 +20,34 @@ source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" # -- original experiment also had dfcvae # 5 * (2*2*8 = 32) = 160 submit_sweep \ - +DUMMY.repeat=5 \ + +DUMMY.repeat=1,2,3,4,5 \ +EXTRA.tags='sweep_xy_squares' \ \ run_length=medium \ - framework=betavae,adavae_os \ + metrics=all \ \ - settings.framework.beta=0.0316 \ - settings.model.z_size=9,25 \ + settings.framework.beta=0.01,0.0316 \ + framework=betavae,adavae_os \ + settings.model.z_size=9 \ \ + sampling=default__bb \ dataset=xysquares \ dataset.data.grid_spacing=8,7,6,5,4,3,2,1 + # background launch traditional datasets # -- original experiment also had dfcvae # 5 * (2*2*4 = 16) = 80 submit_sweep \ - +DUMMY.repeat=5 \ + +DUMMY.repeat=1,2,3,4,5 \ +EXTRA.tags='sweep_other' \ \ run_length=medium \ - framework=betavae,adavae_os \ + metrics=all \ \ - settings.framework.beta=0.0316 \ - settings.model.z_size=9,25 \ + settings.framework.beta=0.01,0.0316 \ + framework=betavae,adavae_os \ + settings.model.z_size=9 \ \ + sampling=default__bb \ dataset=cars3d,shapes3d,dsprites,smallnorb diff --git a/research/e09_vae_overlap_loss/submit_overlap_loss.sh b/research/e09_vae_overlap_loss/submit_overlap_loss.sh index 2d6ec40f..9eb8fc78 100644 --- a/research/e09_vae_overlap_loss/submit_overlap_loss.sh +++ b/research/e09_vae_overlap_loss/submit_overlap_loss.sh @@ -18,21 +18,23 @@ source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours -# RUN SWEEP FOR GOOD BETA VALUES -# - beta: 0.01, 0.0316 seem good, 0.1 starts getting too strong, 0.00316 is a bit weak -# - beta: -# 1 * (2 * 5 * 2 * 6) = 120 -submit_sweep \ +# 1 * (5 * 2*4*2) = 80 +local_sweep \ +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_overlap_loss' \ + +EXTRA.tags='sweep_overlap_boxblur' \ \ - run_length=long \ + +VAR.recon_loss_weight=1.0 \ + +VAR.kernel_loss_weight=3969.0 \ + +VAR.kernel_radius=31 \ + \ + run_length=medium \ metrics=all \ \ + dataset=X--xysquares,dsprites,shapes3d,smallnorb,cars3d \ + \ framework=betavae,adavae_os \ - settings.framework.beta=0.001,0.00316,0.01,0.0316,0.1 \ - settings.model.z_size=9,25 \ - settings.framework.recon_loss=mse_box_r47_w0.5,mse_box_r45_w0.5,mse_box_r33_w0.5,mse_box_r29_w0.5,mse_box_r25_w0.5,mse \ + settings.framework.beta=0.0316,0.316,0.1,0.01 \ + settings.model.z_size=25,9 \ + settings.framework.recon_loss='mse_box_r${VAR.kernel_radius}_l${VAR.recon_loss_weight}_k${VAR.kernel_loss_weight}' \ \ - dataset=X--xysquares \ sampling=default__bb From 15868cfbb7119e8664198b1c773da7479700e91f Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Fri, 12 Nov 2021 18:19:31 +0200 Subject: [PATCH 15/30] update experiment --- .../e01_incr_overlap/submit_incr_overlap.sh | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/research/e01_incr_overlap/submit_incr_overlap.sh b/research/e01_incr_overlap/submit_incr_overlap.sh index fcb317ee..0727898d 100644 --- a/research/e01_incr_overlap/submit_incr_overlap.sh +++ b/research/e01_incr_overlap/submit_incr_overlap.sh @@ -16,6 +16,9 @@ source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" # Experiment # # ========================================================================= # +clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours + + # background launch various xysquares # -- original experiment also had dfcvae # 5 * (2*2*8 = 32) = 160 @@ -31,23 +34,23 @@ submit_sweep \ settings.model.z_size=9 \ \ sampling=default__bb \ - dataset=xysquares \ + dataset=X--xysquares_rgb \ dataset.data.grid_spacing=8,7,6,5,4,3,2,1 # background launch traditional datasets # -- original experiment also had dfcvae # 5 * (2*2*4 = 16) = 80 -submit_sweep \ - +DUMMY.repeat=1,2,3,4,5 \ - +EXTRA.tags='sweep_other' \ - \ - run_length=medium \ - metrics=all \ - \ - settings.framework.beta=0.01,0.0316 \ - framework=betavae,adavae_os \ - settings.model.z_size=9 \ - \ - sampling=default__bb \ - dataset=cars3d,shapes3d,dsprites,smallnorb +#submit_sweep \ +# +DUMMY.repeat=1,2,3,4,5 \ +# +EXTRA.tags='sweep_other' \ +# \ +# run_length=medium \ +# metrics=all \ +# \ +# settings.framework.beta=0.01,0.0316 \ +# framework=betavae,adavae_os \ +# settings.model.z_size=9 \ +# \ +# sampling=default__bb \ +# dataset=cars3d,shapes3d,dsprites,smallnorb From 4b2bab50d83b76cecd9ddf6a4e53d834d933bfa0 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Fri, 12 Nov 2021 19:38:22 +0200 Subject: [PATCH 16/30] fix script --- research/e09_vae_overlap_loss/submit_overlap_loss.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/research/e09_vae_overlap_loss/submit_overlap_loss.sh b/research/e09_vae_overlap_loss/submit_overlap_loss.sh index 9eb8fc78..5e376c21 100644 --- a/research/e09_vae_overlap_loss/submit_overlap_loss.sh +++ b/research/e09_vae_overlap_loss/submit_overlap_loss.sh @@ -19,7 +19,7 @@ source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours # 1 * (5 * 2*4*2) = 80 -local_sweep \ +submit_sweep \ +DUMMY.repeat=1 \ +EXTRA.tags='sweep_overlap_boxblur' \ \ From a691862cd1157e6c42da865fee91315f2aa6dbce Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Fri, 12 Nov 2021 21:57:03 +0200 Subject: [PATCH 17/30] update experiments -- ALL RUN --- research/e00_tuning/submit_param_tuning.sh | 11 ++++++- .../e01_incr_overlap/submit_incr_overlap.sh | 32 ++++++++++++++++--- .../submit_overlap_loss.sh | 1 + 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/research/e00_tuning/submit_param_tuning.sh b/research/e00_tuning/submit_param_tuning.sh index 129d585c..f9f1f647 100644 --- a/research/e00_tuning/submit_param_tuning.sh +++ b/research/e00_tuning/submit_param_tuning.sh @@ -20,11 +20,20 @@ clog_cudaless_nodes "$PARTITION" 129600 "C-disent" # 36 hours # RUN SWEEP FOR GOOD BETA VALUES # - beta: 0.01, 0.0316 seem good, 0.1 starts getting too strong, 0.00316 is a bit weak -# - beta: +# - z_size: higher means you can increase beta, eg. 25: beta=0.1 and 9: beta=0.01 +# - framework: adavae needs lower beta, eg. betavae: 0.1, adavae25: 0.0316, adavae9: 0.00316 +# - xy_squares really struggles to learn when non-overlapping, beta needs to be very low. +# might be worth using a warmup schedule +# betavae with zsize=25 and beta<=0.00316 +# betavae with zsize=09 and beta<=0.000316 +# adavae with zsize=25 does not work +# adavae with zsize=09 and beta<=0.001 (must get very lucky) + # 1 * (2 * 8 * 2 * 5) = 160 submit_sweep \ +DUMMY.repeat=1 \ +EXTRA.tags='sweep_beta' \ + hydra.job.name="vae_hparams" \ \ run_length=long \ metrics=all \ diff --git a/research/e01_incr_overlap/submit_incr_overlap.sh b/research/e01_incr_overlap/submit_incr_overlap.sh index 0727898d..6f114c4a 100644 --- a/research/e01_incr_overlap/submit_incr_overlap.sh +++ b/research/e01_incr_overlap/submit_incr_overlap.sh @@ -21,15 +21,17 @@ clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours # background launch various xysquares # -- original experiment also had dfcvae -# 5 * (2*2*8 = 32) = 160 +# -- beta is too high for adavae +# 3 * (2*2*8 = 32) = 96 submit_sweep \ - +DUMMY.repeat=1,2,3,4,5 \ - +EXTRA.tags='sweep_xy_squares' \ + +DUMMY.repeat=1,2,3 \ + +EXTRA.tags='sweep_xy_squares_overlap' \ + hydra.job.name="incr_ovlp" \ \ run_length=medium \ metrics=all \ \ - settings.framework.beta=0.01,0.0316 \ + settings.framework.beta=0.001,0.00316 \ framework=betavae,adavae_os \ settings.model.z_size=9 \ \ @@ -38,6 +40,28 @@ submit_sweep \ dataset.data.grid_spacing=8,7,6,5,4,3,2,1 +# background launch various xysquares +# - this time we try delay beta, so that it can learn properly... +# 3 * (2*2*8 = 32) = 96 +submit_sweep \ + +DUMMY.repeat=1,2,3 \ + +EXTRA.tags='sweep_xy_squares_overlap_delay' \ + hydra.job.name="schd_incr_ovlp" \ + \ + schedule=beta_delay_long \ + \ + run_length=medium \ + metrics=all \ + \ + settings.framework.beta=0.001 \ + framework=betavae,adavae_os \ + settings.model.z_size=9,25 \ + \ + sampling=default__bb \ + dataset=X--xysquares_rgb \ + dataset.data.grid_spacing=8,7,6,5,4,3,2,1 + + # background launch traditional datasets # -- original experiment also had dfcvae # 5 * (2*2*4 = 16) = 80 diff --git a/research/e09_vae_overlap_loss/submit_overlap_loss.sh b/research/e09_vae_overlap_loss/submit_overlap_loss.sh index 5e376c21..87f444a6 100644 --- a/research/e09_vae_overlap_loss/submit_overlap_loss.sh +++ b/research/e09_vae_overlap_loss/submit_overlap_loss.sh @@ -22,6 +22,7 @@ clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours submit_sweep \ +DUMMY.repeat=1 \ +EXTRA.tags='sweep_overlap_boxblur' \ + hydra.job.name="ovlp_loss" \ \ +VAR.recon_loss_weight=1.0 \ +VAR.kernel_loss_weight=3969.0 \ From 8f777351e0844ae1487edbe93e109240d4cd28f4 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Fri, 12 Nov 2021 22:42:51 +0200 Subject: [PATCH 18/30] update plotting functions --- research/e00_data_traversal/plots/.gitignore | 1 + .../run_02_plot_datasets.py | 107 +++++++++++------- .../e00_data_traversal/run_02_plot_overlap.py | 37 +++--- 3 files changed, 89 insertions(+), 56 deletions(-) create mode 100644 research/e00_data_traversal/plots/.gitignore diff --git a/research/e00_data_traversal/plots/.gitignore b/research/e00_data_traversal/plots/.gitignore new file mode 100644 index 00000000..e33609d2 --- /dev/null +++ b/research/e00_data_traversal/plots/.gitignore @@ -0,0 +1 @@ +*.png diff --git a/research/e00_data_traversal/run_02_plot_datasets.py b/research/e00_data_traversal/run_02_plot_datasets.py index ba47eb3f..479dc961 100644 --- a/research/e00_data_traversal/run_02_plot_datasets.py +++ b/research/e00_data_traversal/run_02_plot_datasets.py @@ -41,6 +41,7 @@ from disent.dataset.data import XYObjectData from disent.dataset.data import XYObjectShadedData from disent.dataset.data import XYSquaresData +from disent.util.function import wrapped_partial from disent.util.seeds import TempNumpySeed @@ -62,6 +63,9 @@ def plot_dataset_overlap( obs_max: Optional[int] = None, obs_spacing: int = 1, rel_path=None, + plot_base: bool = False, + plot_combined: bool = True, + plot_sidebar: bool = False, save=True, seed=777, plt_scale=4.5, @@ -91,43 +95,56 @@ def plot_dataset_overlap( grid[i, j] = np.abs(i_obs - j_obs) # normalize grid /= grid.max() + # make figure factors, frames, _, _, c = grid.shape assert c == 3 + + if plot_base: + # plot + fig, axs = H.plt_subplots_imshow(grid, label_size=18, title_size=24, title=f'{gt_data.name}: {f_name}', subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) + # save figure + if save and (rel_path is not None): + path = H.make_rel_path_add_ext(rel_path, ext='.png') + plt.savefig(path) + print(f'saved: {repr(path)}') + plt.show() + + if plot_combined: + # add obs + if True: + factors += 1 + frames += 1 + # scaled_obs = obs + scaled_obs = obs * 0.5 + 0.25 + # grid = 1 - grid + # grid = grid * 0.5 + 0.25 + grid = np.concatenate([scaled_obs[None, :], grid], axis=0) + add_row = np.concatenate([np.ones_like(obs[0:1]), scaled_obs], axis=0) + grid = np.concatenate([grid, add_row[:, None]], axis=1) + # plot + fig, axs = H.plt_subplots_imshow(grid, label_size=18, title_size=24, row_labels=["traversal"] + (["diff."] * len(obs)), col_labels=(["diff."] * len(obs)) + ["traversal"], title=f'{gt_data.name}: {f_name}', subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) + # save figure + if save and (rel_path is not None): + path = H.make_rel_path_add_ext(rel_path + '__combined', ext='.png') + plt.savefig(path) + print(f'saved: {repr(path)}') + plt.show() + # plot - fig, axs = H.plt_subplots_imshow(grid, label_size=18, title_size=24, title=f'{gt_data.name}: {f_name}', subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) - # save figure - if save and (rel_path is not None): - plt.savefig(H.make_rel_path_add_ext(rel_path, ext='.png')) - plt.show() - # add obs - if True: - factors += 1 - frames += 1 - # scaled_obs = obs - scaled_obs = obs * 0.5 + 0.25 - # grid = 1 - grid - # grid = grid * 0.5 + 0.25 - grid = np.concatenate([scaled_obs[None, :], grid], axis=0) - add_row = np.concatenate([np.ones_like(obs[0:1]), scaled_obs], axis=0) - grid = np.concatenate([grid, add_row[:, None]], axis=1) - # plot - fig, axs = H.plt_subplots_imshow(grid, label_size=18, title_size=24, row_labels=["traversal"] + (["diff."] * len(obs)), col_labels=(["diff."] * len(obs)) + ["traversal"], title=f'{gt_data.name}: {f_name}', subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) - # save figure - if save and (rel_path is not None): - plt.savefig(H.make_rel_path_add_ext(rel_path + '_combined', ext='.png')) - plt.show() - # plot - # fig, axs = H.plt_subplots_imshow(obs[:, None], subplot_padding=None, figsize=(offset + (1/2.54)*1*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) - # if save and (rel_path is not None): - # plt.savefig(H.make_rel_path_add_ext(rel_path + '_v', ext='.png')) - # plt.show() - # fig, axs = H.plt_subplots_imshow(obs[None, :], subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(1+0.45)*plt_scale)) - # if save and (rel_path is not None): - # plt.savefig(H.make_rel_path_add_ext(rel_path + '_h', ext='.png')) - # plt.show() - # done! - return fig, axs + if plot_sidebar: + fig, axs = H.plt_subplots_imshow(obs[:, None], subplot_padding=None, figsize=(offset + (1/2.54)*1*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) + if save and (rel_path is not None): + path = H.make_rel_path_add_ext(rel_path + '__v', ext='.png') + plt.savefig(path) + print(f'saved: {repr(path)}') + plt.show() + fig, axs = H.plt_subplots_imshow(obs[None, :], subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(1+0.45)*plt_scale)) + if save and (rel_path is not None): + path = H.make_rel_path_add_ext(rel_path + '__h', ext='.png') + plt.savefig(path) + print(f'saved: {repr(path)}') + plt.show() # ========================================================================= # @@ -146,14 +163,24 @@ def plot_dataset_overlap( num_cols = 7 seed = 48 - # save images - for i in ([1, 2, 4, 8] if all_squares else [1, 8]): - data = XYSquaresData(grid_spacing=i, grid_size=8, no_warnings=True) - plot_dataset_overlap(data, rel_path=f'plots/overlap__xy-squares-spacing{i}', obs_max=3, obs_spacing=4, seed=seed-40) + for gt_data_cls, name in [ + (wrapped_partial(XYSquaresData, grid_spacing=1, grid_size=8, no_warnings=True), f'xy-squares-spacing1'), + (wrapped_partial(XYSquaresData, grid_spacing=2, grid_size=8, no_warnings=True), f'xy-squares-spacing2'), + (wrapped_partial(XYSquaresData, grid_spacing=4, grid_size=8, no_warnings=True), f'xy-squares-spacing4'), + (wrapped_partial(XYSquaresData, grid_spacing=8, grid_size=8, no_warnings=True), f'xy-squares-spacing8'), + ]: + plot_dataset_overlap(gt_data_cls(), rel_path=f'plots/overlap__{name}', obs_max=3, obs_spacing=4, seed=seed-40) + + for gt_data_cls, name in [ + (DSpritesData, f'dsprites'), + (Shapes3dData, f'shapes3d'), + (Cars3dData, f'cars3d'), + (SmallNorbData, f'smallnorb'), + ]: + gt_data = gt_data_cls() + for f_idx, f_name in enumerate(gt_data.factor_names): + plot_dataset_overlap(gt_data, rel_path=f'plots/overlap__{name}__f{f_idx}-{f_name}', obs_max=3, obs_spacing=4, f_idxs=f_idx, seed=seed) - gt_data = DSpritesData() - for f_idx, f_name in enumerate(gt_data.factor_names): - plot_dataset_overlap(gt_data, rel_path=f'plots/overlap__dsprites__f-{f_name}', obs_max=3, obs_spacing=4, f_idxs=f_idx, seed=seed) # ========================================================================= # # END # diff --git a/research/e00_data_traversal/run_02_plot_overlap.py b/research/e00_data_traversal/run_02_plot_overlap.py index 702c705a..4e63bae0 100644 --- a/research/e00_data_traversal/run_02_plot_overlap.py +++ b/research/e00_data_traversal/run_02_plot_overlap.py @@ -101,7 +101,9 @@ def plot_dataset_traversals( fig, axs = H.plt_subplots_imshow(grid, label_size=18, title_size=24, title=gt_data.name, row_labels=row_labels, subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) # save figure if save and (rel_path is not None): - plt.savefig(H.make_rel_path_add_ext(rel_path, ext='.png')) + path = H.make_rel_path_add_ext(rel_path, ext='.png') + plt.savefig(path) + print(f'saved: {repr(path)}') plt.show() # done! return fig, axs @@ -119,26 +121,29 @@ def plot_dataset_traversals( # options all_squares = True - add_random_traversal = True + add_random_traversal = False num_cols = 7 seed = 47 + # get name + prefix = 'traversal' if add_random_traversal else 'traversal-noran' + # save images for i in ([1, 2, 3, 4, 5, 6, 7, 8] if all_squares else [1, 8]): data = XYSquaresData(grid_spacing=i, grid_size=8, no_warnings=True) - plot_dataset_traversals(data, rel_path=f'plots/xy-squares-traversal-spacing{i}', seed=seed-40, add_random_traversal=add_random_traversal, num_cols=num_cols) - - plot_dataset_traversals(XYObjectData(), rel_path=f'plots/xy-object-traversal', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(XYObjectShadedData(), rel_path=f'plots/xy-object-shaded-traversal', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(XYBlocksData(), rel_path=f'plots/xy-blocks-traversal', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(Shapes3dData(), rel_path=f'plots/shapes3d-traversal', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(DSpritesData(), rel_path=f'plots/dsprites-traversal', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(DSpritesImagenetData(100, 'bg'), rel_path=f'plots/dsprites-imagenet-bg-100-traversal', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(DSpritesImagenetData( 50, 'bg'), rel_path=f'plots/dsprites-imagenet-bg-50-traversal', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(DSpritesImagenetData(100, 'fg'), rel_path=f'plots/dsprites-imagenet-fg-100-traversal', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(DSpritesImagenetData( 50, 'fg'), rel_path=f'plots/dsprites-imagenet-fg-50-traversal', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(SmallNorbData(), rel_path=f'plots/smallnorb-traversal', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(Cars3dData(), rel_path=f'plots/cars3d-traversal', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) + plot_dataset_traversals(data, rel_path=f'plots/{prefix}__xy-squares__spacing{i}', seed=seed-40, add_random_traversal=add_random_traversal, num_cols=num_cols) + + plot_dataset_traversals(XYObjectData(), rel_path=f'plots/{prefix}__xy-object', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) + plot_dataset_traversals(XYObjectShadedData(), rel_path=f'plots/{prefix}__xy-object-shaded', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) + plot_dataset_traversals(XYBlocksData(), rel_path=f'plots/{prefix}__xy-blocks', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) + plot_dataset_traversals(Shapes3dData(), rel_path=f'plots/{prefix}__shapes3d', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) + plot_dataset_traversals(DSpritesData(), rel_path=f'plots/{prefix}__dsprites', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) + plot_dataset_traversals(DSpritesImagenetData(100, 'bg'), rel_path=f'plots/{prefix}__dsprites-imagenet-bg-100', seed=seed-6, add_random_traversal=add_random_traversal, num_cols=num_cols) + plot_dataset_traversals(DSpritesImagenetData( 50, 'bg'), rel_path=f'plots/{prefix}__dsprites-imagenet-bg-50', seed=seed-6, add_random_traversal=add_random_traversal, num_cols=num_cols) + plot_dataset_traversals(DSpritesImagenetData(100, 'fg'), rel_path=f'plots/{prefix}__dsprites-imagenet-fg-100', seed=seed-6, add_random_traversal=add_random_traversal, num_cols=num_cols) + plot_dataset_traversals(DSpritesImagenetData( 50, 'fg'), rel_path=f'plots/{prefix}__dsprites-imagenet-fg-50', seed=seed-6, add_random_traversal=add_random_traversal, num_cols=num_cols) + plot_dataset_traversals(SmallNorbData(), rel_path=f'plots/{prefix}__smallnorb', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) + plot_dataset_traversals(Cars3dData(), rel_path=f'plots/{prefix}__cars3d', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) BASE = os.path.abspath(os.path.join(__file__, '../../../out/adversarial_data_approx')) @@ -159,7 +164,7 @@ def plot_dataset_traversals( ('2021-09-06--05-42-06_INVERT-VSTRONG-cars3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), ('2021-09-06--09-10-59_INVERT-VSTRONG-smallnorb_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), ]: - plot_dataset_traversals(SelfContainedHdf5GroundTruthData(f'{BASE}/{folder}/data.h5'), rel_path=f'plots/{folder}__traversal.png', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) + plot_dataset_traversals(SelfContainedHdf5GroundTruthData(f'{BASE}/{folder}/data.h5'), rel_path=f'plots/{prefix}__{folder}.png', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) # ========================================================================= # From 9daafb83b3f13e0bb231901fa2a5dbfd11d5fbeb Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Fri, 12 Nov 2021 23:41:30 +0200 Subject: [PATCH 19/30] fix plotting --- requirements-research.txt | 2 - .../e00_data_traversal/run_02_plot_overlap.py | 5 +- .../run_plot_global_dists.py | 82 ++++++++-------- .../run_plot_traversal_dists.py | 94 +++++++++++++------ 4 files changed, 110 insertions(+), 73 deletions(-) diff --git a/requirements-research.txt b/requirements-research.txt index e5a6d2bd..93d0e876 100644 --- a/requirements-research.txt +++ b/requirements-research.txt @@ -6,9 +6,7 @@ # github # matplotlib -# pandas # psutil -# seaborn ray>=1.6.0 diff --git a/research/e00_data_traversal/run_02_plot_overlap.py b/research/e00_data_traversal/run_02_plot_overlap.py index 4e63bae0..45da2323 100644 --- a/research/e00_data_traversal/run_02_plot_overlap.py +++ b/research/e00_data_traversal/run_02_plot_overlap.py @@ -121,7 +121,7 @@ def plot_dataset_traversals( # options all_squares = True - add_random_traversal = False + add_random_traversal = True num_cols = 7 seed = 47 @@ -129,9 +129,10 @@ def plot_dataset_traversals( prefix = 'traversal' if add_random_traversal else 'traversal-noran' # save images - for i in ([1, 2, 3, 4, 5, 6, 7, 8] if all_squares else [1, 8]): + for i in ([1, 2, 4, 8] if all_squares else [1, 8]): data = XYSquaresData(grid_spacing=i, grid_size=8, no_warnings=True) plot_dataset_traversals(data, rel_path=f'plots/{prefix}__xy-squares__spacing{i}', seed=seed-40, add_random_traversal=add_random_traversal, num_cols=num_cols) + plot_dataset_traversals(data, rel_path=f'plots/{prefix}__xy-squares__spacing{i}__some', seed=seed-40, add_random_traversal=add_random_traversal, num_cols=num_cols, f_idxs=[0, 3]) plot_dataset_traversals(XYObjectData(), rel_path=f'plots/{prefix}__xy-object', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) plot_dataset_traversals(XYObjectShadedData(), rel_path=f'plots/{prefix}__xy-object-shaded', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) diff --git a/research/e01_visual_overlap/run_plot_global_dists.py b/research/e01_visual_overlap/run_plot_global_dists.py index 3f9028f6..5e10106a 100644 --- a/research/e01_visual_overlap/run_plot_global_dists.py +++ b/research/e01_visual_overlap/run_plot_global_dists.py @@ -36,12 +36,15 @@ from tqdm import tqdm import research.util as H +from disent.dataset import DisentDataset from disent.dataset.data import Cars3dData from disent.dataset.data import DSpritesData +from disent.dataset.data import GroundTruthData from disent.dataset.data import Shapes3dData from disent.dataset.data import XYSquaresData from disent.dataset.transform import ToImgTensorF32 from disent.util import to_numpy +from disent.util.function import wrapped_partial # ========================================================================= # @@ -73,7 +76,7 @@ def plot_overlap(a, b, mode='abs'): # ========================================================================= # -def generate_data(gt_dataset, data_name: str, batch_size=64, samples=100_000, plot_diffs=False, load_cache=True, save_cache=True, overlap_loss: str = 'mse'): +def generate_data(dataset: DisentDataset, data_name: str, batch_size=64, samples=100_000, plot_diffs=False, load_cache=True, save_cache=True, overlap_loss: str = 'mse'): # cache file_path = os.path.join(os.path.dirname(__file__), f'cache/{data_name}_{samples}.pkl') if load_cache: @@ -90,8 +93,8 @@ def generate_data(gt_dataset, data_name: str, batch_size=64, samples=100_000, pl name = 'random' for i in tqdm(range((samples + (batch_size-1) - 1) // (batch_size-1)), desc=f'{data_name}: {name}'): # get random batch of unique elements - idxs = H.sample_unique_batch_indices(num_obs=len(gt_dataset), num_samples=batch_size) - batch = gt_dataset.dataset_batch_from_indices(idxs, mode='input') + idxs = H.sample_unique_batch_indices(num_obs=len(dataset), num_samples=batch_size) + batch = dataset.dataset_batch_from_indices(idxs, mode='input') # plot if plot_diffs and (i == 0): plot_overlap(batch[0], batch[1]) @@ -101,12 +104,12 @@ def generate_data(gt_dataset, data_name: str, batch_size=64, samples=100_000, pl df[False][name].extend(o) # traversal overlaps - for f_idx in range(gt_dataset.num_factors): - name = f'f_{gt_dataset.factor_names[f_idx]}' - for i in tqdm(range((samples + (gt_dataset.factor_sizes[f_idx] - 1) - 1) // (gt_dataset.factor_sizes[f_idx] - 1)), desc=f'{data_name}: {name}'): + for f_idx in range(dataset.gt_data.num_factors): + name = f'f_{dataset.gt_data.factor_names[f_idx]}' + for i in tqdm(range((samples + (dataset.gt_data.factor_sizes[f_idx] - 1) - 1) // (dataset.gt_data.factor_sizes[f_idx] - 1)), desc=f'{data_name}: {name}'): # get random batch that is a factor traversal - factors = gt_dataset.sample_random_factor_traversal(f_idx) - batch = gt_dataset.dataset_batch_from_factors(factors, mode='input') + factors = dataset.gt_data.sample_random_factor_traversal(f_idx) + batch = dataset.dataset_batch_from_factors(factors, mode='input') # shuffle indices idxs = np.arange(len(factors)) np.random.shuffle(idxs) @@ -138,7 +141,7 @@ def generate_data(gt_dataset, data_name: str, batch_size=64, samples=100_000, pl # ========================================================================= # -def dual_plot_from_generated_data(df, data_name: str = None, save_name: str = None, tick_size: float = None, fig_l_pad=1, fig_w=7, fig_h=13): +def dual_plot_from_generated_data(df: pd.DataFrame, data_name: str = None, save_name: str = None, tick_size: float = None, fig_l_pad=1, fig_w=7, fig_h=13): # make subplots cm = 1 / 2.54 fig, (ax0, ax1) = plt.subplots(1, 2, figsize=((fig_l_pad+2*fig_w)*cm, fig_h*cm)) @@ -209,13 +212,12 @@ def all_plot_from_all_generated_data(dfs: dict, ordered=True, save_name: str = N return fig -def plot_all(exp_name, datas, tick_sizes, samples: int, load=True, save=True, show_plt=True, show_dual_plt=False, save_plt=True, hide_extra_legends=False, fig_l_pad=1, fig_w=7, fig_h=13): +def plot_all(exp_name: str, gt_data_classes, tick_sizes: dict, samples: int, load=True, save=True, show_plt=True, show_dual_plt=False, save_plt=True, hide_extra_legends=False, fig_l_pad=1, fig_w=7, fig_h=13): # generate data and plot! dfs = {} - for data_name, make_data_fn in datas.items(): - gt_dataset = GroundTruthDataset(make_data_fn(), transform=ToImgTensorF32()) + for data_name, data_cls in gt_data_classes.items(): df = generate_data( - gt_dataset, + DisentDataset(data_cls(), transform=ToImgTensorF32()), data_name, batch_size=64, samples=samples, @@ -377,15 +379,15 @@ def plot_unique_count(dfs, save_name: str = None, show_plt: bool = True, fig_l_p dfs = plot_all( exp_name='dataset-overlap', - datas={ - # 'XYObject': lambda: XYObjectData(), - # 'XYBlocks': lambda: XYBlocksData(), - 'XYSquares': lambda: XYSquaresData(), - 'DSprites': lambda: DSpritesData(), - 'Shapes3d': lambda: Shapes3dData(), - 'Cars3d': lambda: Cars3dData(), - # 'SmallNorb': lambda: SmallNorbData(), - # 'Mpi3d': lambda: Mpi3dData(), + gt_data_classes={ + # 'XYObject': wrapped_partial(XYObjectData), + # 'XYBlocks': wrapped_partial(XYBlocksData), + 'XYSquares': wrapped_partial(XYSquaresData), + 'DSprites': wrapped_partial(DSpritesData), + 'Shapes3d': wrapped_partial(Shapes3dData), + 'Cars3d': wrapped_partial(Cars3dData), + # 'SmallNorb': wrapped_partial(SmallNorbData), + # 'Mpi3d': wrapped_partial(Mpi3dData), }, hide_extra_legends=False, **SHARED_SETTINGS @@ -395,15 +397,15 @@ def plot_unique_count(dfs, save_name: str = None, show_plt: bool = True, fig_l_p dfs = plot_all( exp_name='increasing-overlap', - datas={ - 'XYSquares-1': lambda: XYSquaresData(grid_spacing=1), - 'XYSquares-2': lambda: XYSquaresData(grid_spacing=2), - 'XYSquares-3': lambda: XYSquaresData(grid_spacing=3), - 'XYSquares-4': lambda: XYSquaresData(grid_spacing=4), - 'XYSquares-5': lambda: XYSquaresData(grid_spacing=5), - 'XYSquares-6': lambda: XYSquaresData(grid_spacing=6), - 'XYSquares-7': lambda: XYSquaresData(grid_spacing=7), - 'XYSquares-8': lambda: XYSquaresData(grid_spacing=8), + gt_data_classes={ + 'XYSquares-1': wrapped_partial(XYSquaresData, grid_spacing=1), + 'XYSquares-2': wrapped_partial(XYSquaresData, grid_spacing=2), + 'XYSquares-3': wrapped_partial(XYSquaresData, grid_spacing=3), + 'XYSquares-4': wrapped_partial(XYSquaresData, grid_spacing=4), + 'XYSquares-5': wrapped_partial(XYSquaresData, grid_spacing=5), + 'XYSquares-6': wrapped_partial(XYSquaresData, grid_spacing=6), + 'XYSquares-7': wrapped_partial(XYSquaresData, grid_spacing=7), + 'XYSquares-8': wrapped_partial(XYSquaresData, grid_spacing=8), }, hide_extra_legends=True, **SHARED_SETTINGS @@ -427,15 +429,15 @@ def plot_unique_count(dfs, save_name: str = None, show_plt: bool = True, fig_l_p dfs = plot_all( exp_name='increasing-overlap-fixed', - datas={ - 'XYSquares-1-8': lambda: XYSquaresData(square_size=8, grid_spacing=1, grid_size=8), - 'XYSquares-2-8': lambda: XYSquaresData(square_size=8, grid_spacing=2, grid_size=8), - 'XYSquares-3-8': lambda: XYSquaresData(square_size=8, grid_spacing=3, grid_size=8), - 'XYSquares-4-8': lambda: XYSquaresData(square_size=8, grid_spacing=4, grid_size=8), - 'XYSquares-5-8': lambda: XYSquaresData(square_size=8, grid_spacing=5, grid_size=8), - 'XYSquares-6-8': lambda: XYSquaresData(square_size=8, grid_spacing=6, grid_size=8), - 'XYSquares-7-8': lambda: XYSquaresData(square_size=8, grid_spacing=7, grid_size=8), - 'XYSquares-8-8': lambda: XYSquaresData(square_size=8, grid_spacing=8, grid_size=8), + gt_data_classes={ + 'XYSquares-1-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=1, grid_size=8), + 'XYSquares-2-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=2, grid_size=8), + 'XYSquares-3-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=3, grid_size=8), + 'XYSquares-4-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=4, grid_size=8), + 'XYSquares-5-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=5, grid_size=8), + 'XYSquares-6-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=6, grid_size=8), + 'XYSquares-7-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=7, grid_size=8), + 'XYSquares-8-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=8, grid_size=8), }, hide_extra_legends=True, **SHARED_SETTINGS diff --git a/research/e01_visual_overlap/run_plot_traversal_dists.py b/research/e01_visual_overlap/run_plot_traversal_dists.py index 422c38c0..8bb1739d 100644 --- a/research/e01_visual_overlap/run_plot_traversal_dists.py +++ b/research/e01_visual_overlap/run_plot_traversal_dists.py @@ -161,8 +161,14 @@ def plot_traversal_stats( f_idxs: Optional[NonNormalisedFactors] = None, circular_distance: bool = False, color='blue', + color_gt_dist='blue', + color_im_dist='purple', suffix: Optional[str] = None, save_path: Optional[str] = None, + plot_freq: bool = True, + plot_title: Union[bool, str] = False, + fig_block_size: float = 4.0, + col_titles: Union[bool, List[str]] = True, ): # - - - - - - - - - - - - - - - - - # @@ -180,31 +186,47 @@ def plot_ax(stats: dict, i: int, f_idx: int): with TempNumpySeed(777): np.random.shuffle(fdists) # subplot! - ax0, ax1, ax2, ax3 = axs[:, i] - - ax0.set_title(f'{gt_data.factor_names[f_idx]} ({gt_data.factor_sizes[f_idx]})') - ax0.violinplot([deltas], vert=False) - ax0.set_xlabel('deltas') - ax0.set_ylabel('proportion') - - ax1.set_title('deltas vs. fdists') - ax1.scatter(x=deltas[:15_000], y=fdists[:15_000], s=20, alpha=0.1, c=c_points) - H.plt_2d_density( - x=deltas[:10_000], xmin=deltas.min(), xmax=deltas.max(), - y=fdists[:10_000], ymin=fdists.min() - 0.5, ymax=fdists.max() + 0.5, - n_bins=100, - ax=ax1, pcolormesh_kwargs=dict(cmap=cmap_density, alpha=0.5), - ) - ax1.set_xlabel('deltas') - ax1.set_ylabel('fdists') - - ax2.set_title('fdists') - ax2.imshow(fdists_matrix, cmap=cmap_img) + if plot_freq: + ax0, ax1, ax2, ax3 = axs[:, i] + else: + (ax0, ax1), (ax2, ax3) = (None, None), axs[:, i] + + # get title + curr_title = None + if isinstance(col_titles, bool): + if col_titles: + curr_title = gt_data.factor_names[f_idx] + else: + curr_title = col_titles[i] + + # set column titles + if curr_title is not None: + (ax0 if plot_freq else ax2).set_title(f'{curr_title}\n', fontsize=24) + + # plot the frequency stuffs + if plot_freq: + ax0.violinplot([deltas], vert=False) + ax0.set_xlabel('deltas') + ax0.set_ylabel('proportion') + + ax1.set_title('deltas vs. fdists') + ax1.scatter(x=deltas[:15_000], y=fdists[:15_000], s=20, alpha=0.1, c=c_points) + H.plt_2d_density( + x=deltas[:10_000], xmin=deltas.min(), xmax=deltas.max(), + y=fdists[:10_000], ymin=fdists.min() - 0.5, ymax=fdists.max() + 0.5, + n_bins=100, + ax=ax1, pcolormesh_kwargs=dict(cmap=cmap_density, alpha=0.5), + ) + ax1.set_xlabel('deltas') + ax1.set_ylabel('fdists') + + # ax2.set_title('fdists') + ax2.imshow(fdists_matrix, cmap=gt_cmap_img) ax2.set_xlabel('f_idx') ax2.set_ylabel('f_idx') - ax3.set_title('divergence') - ax3.imshow(deltas_matrix, cmap=cmap_img) + # ax3.set_title('divergence') + ax3.imshow(deltas_matrix, cmap=im_cmap_img) ax3.set_xlabel('f_idx') ax3.set_ylabel('f_idx') @@ -213,15 +235,27 @@ def plot_ax(stats: dict, i: int, f_idx: int): # initialize gt_data: GroundTruthData = H.make_data(dataset_or_name) if isinstance(dataset_or_name, str) else dataset_or_name f_idxs = gt_data.normalise_factor_idxs(f_idxs) + c_points, cmap_density, cmap_img = _COLORS[color] + im_c_points, im_cmap_density, im_cmap_img = _COLORS[color if (color_im_dist is None) else color_im_dist] + gt_c_points, gt_cmap_density, gt_cmap_img = _COLORS[color if (color_gt_dist is None) else color_gt_dist] + + n = 4 if plot_freq else 2 + + # get additional spacing + title_offset = 0 if (isinstance(col_titles, bool) and not col_titles) else 0.2 # settings - r, c = [4, len(f_idxs)] - h, w = [16, len(f_idxs)*4] + r, c = [n, len(f_idxs)] + h, w = [(n+title_offset)*fig_block_size, len(f_idxs)*fig_block_size] # initialize plot fig, axs = plt.subplots(r, c, figsize=(w, h), squeeze=False) - fig.suptitle(f'{gt_data.name} [circular={circular_distance}]{f" {suffix}" if suffix else ""}\n', fontsize=25) + + if isinstance(plot_title, str): + fig.suptitle(f'{plot_title}\n', fontsize=25) + elif plot_title: + fig.suptitle(f'{gt_data.name} [circular={circular_distance}]{f" {suffix}" if suffix else ""}\n', fontsize=25) # generate plot _collect_stats_for_factors( @@ -264,24 +298,26 @@ def _make_self_contained_dataset(h5_path): plt.style.use(os.path.join(os.path.dirname(__file__), '../gadfly.mplstyle')) CIRCULAR = False + PLOT_FREQ = False def sp(name): prefix = 'CIRCULAR_' if CIRCULAR else 'DIST_' + prefix = prefix + ('FREQ_' if PLOT_FREQ else 'NO-FREQ_') return os.path.join(os.path.dirname(__file__), 'plots', f'{prefix}{name}.png') # plot xysquares with increasing overlap for s in [1, 2, 3, 4, 5, 6, 7, 8]: - plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(f'xysquares_8x8_s{s}'), color='blue', dataset_or_name=f'xysquares_8x8_s{s}', f_idxs=[1]) + plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(f'xysquares_8x8_s{s}'), color='blue', dataset_or_name=f'xysquares_8x8_s{s}', f_idxs=[1], col_titles=['x & y'], plot_freq=PLOT_FREQ) # plot standard datasets for name in ['dsprites', 'shapes3d', 'cars3d', 'smallnorb']: - plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(name), color='blue', dataset_or_name=name) + plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(name), color='blue', dataset_or_name=name, plot_freq=PLOT_FREQ) # plot adversarial dsprites datasets for fg in [True, False]: for vis in [100, 80, 60, 40, 20]: name = f'dsprites_imagenet_{"fg" if fg else "bg"}_{vis}' - plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(name), color='orange', dataset_or_name=name) + plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(name), color='orange', dataset_or_name=name, plot_freq=PLOT_FREQ) # mean, std = compute_data_mean_std(H.make_data(name)) # print(f'{name}\n vis_mean: {mean.tolist()}\n vis_std: {std.tolist()}') @@ -306,7 +342,7 @@ def sp(name): ('red', '2021-09-06--09-10-59_INVERT-VSTRONG-smallnorb_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), ]: data = _make_self_contained_dataset(f'{BASE}/{folder}/data.h5') - plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(folder), color=color, dataset_or_name=data) + plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(folder), color=color, dataset_or_name=data, plot_freq=PLOT_FREQ) # compute and print statistics: # mean, std = compute_data_mean_std(data, progress=True) # print(f'{folder}\n vis_mean: {mean.tolist()}\n vis_std: {std.tolist()}') From f45f5f78c5785f704e44ff9228a5a77f915d4378 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Sat, 13 Nov 2021 14:44:09 +0200 Subject: [PATCH 20/30] fix plotting --- ...atasets.py => run_02_plot_data_overlap.py} | 0 ...t_overlap.py => run_02_plot_traversals.py} | 34 ++++- .../run_plot_global_dists.py | 17 ++- .../run_plot_traversal_dists.py | 120 ++++++++++++++++-- 4 files changed, 153 insertions(+), 18 deletions(-) rename research/e00_data_traversal/{run_02_plot_datasets.py => run_02_plot_data_overlap.py} (100%) rename research/e00_data_traversal/{run_02_plot_overlap.py => run_02_plot_traversals.py} (77%) diff --git a/research/e00_data_traversal/run_02_plot_datasets.py b/research/e00_data_traversal/run_02_plot_data_overlap.py similarity index 100% rename from research/e00_data_traversal/run_02_plot_datasets.py rename to research/e00_data_traversal/run_02_plot_data_overlap.py diff --git a/research/e00_data_traversal/run_02_plot_overlap.py b/research/e00_data_traversal/run_02_plot_traversals.py similarity index 77% rename from research/e00_data_traversal/run_02_plot_overlap.py rename to research/e00_data_traversal/run_02_plot_traversals.py index 45da2323..a054d9a6 100644 --- a/research/e00_data_traversal/run_02_plot_overlap.py +++ b/research/e00_data_traversal/run_02_plot_traversals.py @@ -60,6 +60,7 @@ def plot_dataset_traversals( gt_data: GroundTruthData, f_idxs=None, num_cols: Optional[int] = 8, + take_cols: Optional[int] = None, base_factors=None, add_random_traversal=True, pad=8, @@ -69,8 +70,11 @@ def plot_dataset_traversals( save=True, seed=777, plt_scale=4.5, - offset=0.75 + offset=0.75, + transpose=False, ): + if take_cols is not None: + assert take_cols >= num_cols # convert dataset = DisentDataset(gt_data) f_idxs = gt_data.normalise_factor_idxs(f_idxs) @@ -81,7 +85,7 @@ def plot_dataset_traversals( dataset=dataset, data_mode='raw', factor_names=f_idxs, - num_frames=num_cols, + num_frames=num_cols if (take_cols is None) else take_cols, seed=seed, base_factors=base_factors, traverse_mode='interval', @@ -89,6 +93,8 @@ def plot_dataset_traversals( bg_color=bg_color, border=border, ) + if take_cols is not None: + grid = grid[:, :num_cols, ...] # add random traversal if add_random_traversal: with TempNumpySeed(seed): @@ -98,7 +104,11 @@ def plot_dataset_traversals( # make figure factors, frames, _, _, c = grid.shape assert c == 3 - fig, axs = H.plt_subplots_imshow(grid, label_size=18, title_size=24, title=gt_data.name, row_labels=row_labels, subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) + + if transpose: + fig, axs = H.plt_subplots_imshow(np.swapaxes(grid, 0, 1), label_size=18, title_size=24, title=gt_data.name, col_labels=row_labels, subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)[::-1]) + else: + fig, axs = H.plt_subplots_imshow(grid, label_size=18, title_size=24, title=gt_data.name, row_labels=row_labels, subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) # save figure if save and (rel_path is not None): path = H.make_rel_path_add_ext(rel_path, ext='.png') @@ -123,11 +133,29 @@ def plot_dataset_traversals( all_squares = True add_random_traversal = True num_cols = 7 + mini_cols = 5 + transpose_cols = 3 seed = 47 # get name prefix = 'traversal' if add_random_traversal else 'traversal-noran' + # mini versions + plot_dataset_traversals(XYSquaresData(), rel_path=f'plots/traversal-mini__xy-squares__spacing8', seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) + plot_dataset_traversals(Shapes3dData(), rel_path=f'plots/traversal-mini__shapes3d', seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) + plot_dataset_traversals(DSpritesData(), rel_path=f'plots/traversal-mini__dsprites', seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) + plot_dataset_traversals(SmallNorbData(), rel_path=f'plots/traversal-mini__smallnorb', seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) + plot_dataset_traversals(Cars3dData(), rel_path=f'plots/traversal-mini__cars3d', seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols, take_cols=mini_cols+1) + + # transpose versions + plot_dataset_traversals(XYSquaresData(), rel_path=f'plots/traversal-transpose__xy-squares__spacing8', seed=seed, transpose=True, add_random_traversal=False, num_cols=transpose_cols) + plot_dataset_traversals(Shapes3dData(), rel_path=f'plots/traversal-transpose__shapes3d', seed=seed, transpose=True, add_random_traversal=False, num_cols=transpose_cols) + plot_dataset_traversals(DSpritesData(), rel_path=f'plots/traversal-transpose__dsprites', seed=seed, transpose=True, add_random_traversal=False, num_cols=transpose_cols) + plot_dataset_traversals(SmallNorbData(), rel_path=f'plots/traversal-transpose__smallnorb', seed=seed, transpose=True, add_random_traversal=False, num_cols=transpose_cols) + plot_dataset_traversals(Cars3dData(), rel_path=f'plots/traversal-transpose__cars3d', seed=seed, transpose=True, add_random_traversal=False, num_cols=transpose_cols, take_cols=mini_cols+1) + + exit(1) + # save images for i in ([1, 2, 4, 8] if all_squares else [1, 8]): data = XYSquaresData(grid_spacing=i, grid_size=8, no_warnings=True) diff --git a/research/e01_visual_overlap/run_plot_global_dists.py b/research/e01_visual_overlap/run_plot_global_dists.py index 5e10106a..f9834e74 100644 --- a/research/e01_visual_overlap/run_plot_global_dists.py +++ b/research/e01_visual_overlap/run_plot_global_dists.py @@ -151,12 +151,12 @@ def dual_plot_from_generated_data(df: pd.DataFrame, data_name: str = None, save_ ax1.set_ylim(-0.025, 1.025) # plot ax0.set_title('Ordered Traversals') - sns.ecdfplot(ax=ax0, data=df[df['ordered']==True], x="overlap", hue="samples") + sns.ecdfplot(ax=ax0, data=df[df['ordered']==True], x="distance", hue="samples") ax1.set_title('Shuffled Traversals') - sns.ecdfplot(ax=ax1, data=df[df['ordered']==False], x="overlap", hue="samples") + sns.ecdfplot(ax=ax1, data=df[df['ordered']==False], x="distance", hue="samples") # edit plots - ax0.set_xlabel('Overlap') - ax1.set_xlabel('Overlap') + ax0.set_xlabel('Visual Distance') + ax1.set_xlabel('Visual Distance') if tick_size is not None: ax0.xaxis.set_major_locator(MultipleLocator(base=tick_size)) ax1.xaxis.set_major_locator(MultipleLocator(base=tick_size)) @@ -188,10 +188,10 @@ def all_plot_from_all_generated_data(dfs: dict, ordered=True, save_name: str = N for i, (ax, (data_name, df)) in enumerate(zip(axs, dfs.items())): # plot ax.set_title(data_name) - sns.ecdfplot(ax=ax, data=df[df['ordered']==ordered], x="overlap", hue="samples") + sns.ecdfplot(ax=ax, data=df[df['ordered']==ordered], x="distance", hue="samples") # edit plots ax.set_ylim(-0.025, 1.025) - ax.set_xlabel('Overlap') + ax.set_xlabel('Visual Distance') if (tick_sizes is not None) and (data_name in tick_sizes): ax.xaxis.set_major_locator(MultipleLocator(base=tick_sizes[data_name])) if i == 0: @@ -226,6 +226,9 @@ def plot_all(exp_name: str, gt_data_classes, tick_sizes: dict, samples: int, loa save_cache=save, ) dfs[data_name] = df + # flip overlap + df['distance'] = - df['overlap'] + del df['overlap'] # plot ordered + shuffled fig = dual_plot_from_generated_data( df, @@ -347,7 +350,7 @@ def plot_unique_count(dfs, save_name: str = None, show_plt: bool = True, fig_l_p save_plt=True, show_dual_plt=False, fig_l_pad=1, - fig_w=7, + fig_w=5.5, fig_h=13, tick_sizes={ 'DSprites': 0.05, diff --git a/research/e01_visual_overlap/run_plot_traversal_dists.py b/research/e01_visual_overlap/run_plot_traversal_dists.py index 8bb1739d..093e88a9 100644 --- a/research/e01_visual_overlap/run_plot_traversal_dists.py +++ b/research/e01_visual_overlap/run_plot_traversal_dists.py @@ -169,6 +169,10 @@ def plot_traversal_stats( plot_title: Union[bool, str] = False, fig_block_size: float = 4.0, col_titles: Union[bool, List[str]] = True, + hide_axis: bool = True, + hide_labels: bool = True, + y_size_offset: float = 0.0, + x_size_offset: float = 0.0, ): # - - - - - - - - - - - - - - - - - # @@ -222,13 +226,16 @@ def plot_ax(stats: dict, i: int, f_idx: int): # ax2.set_title('fdists') ax2.imshow(fdists_matrix, cmap=gt_cmap_img) - ax2.set_xlabel('f_idx') - ax2.set_ylabel('f_idx') + if not hide_labels: ax2.set_xlabel('f_idx') + if not hide_labels: ax2.set_ylabel('f_idx') + if hide_axis: H.plt_hide_axis(ax2) # ax3.set_title('divergence') ax3.imshow(deltas_matrix, cmap=im_cmap_img) - ax3.set_xlabel('f_idx') - ax3.set_ylabel('f_idx') + if not hide_labels: ax3.set_xlabel('f_idx') + if not hide_labels: ax3.set_ylabel('f_idx') + if hide_axis: H.plt_hide_axis(ax3) + # - - - - - - - - - - - - - - - - - # @@ -243,11 +250,11 @@ def plot_ax(stats: dict, i: int, f_idx: int): n = 4 if plot_freq else 2 # get additional spacing - title_offset = 0 if (isinstance(col_titles, bool) and not col_titles) else 0.2 + title_offset = 0 if (isinstance(col_titles, bool) and not col_titles) else 0.15 # settings r, c = [n, len(f_idxs)] - h, w = [(n+title_offset)*fig_block_size, len(f_idxs)*fig_block_size] + h, w = [(n+title_offset)*fig_block_size + y_size_offset, len(f_idxs)*fig_block_size + x_size_offset] # initialize plot fig, axs = plt.subplots(r, c, figsize=(w, h), squeeze=False) @@ -268,6 +275,102 @@ def plot_ax(stats: dict, i: int, f_idx: int): ) # finalize plot + fig.tight_layout(pad=1.4 if hide_labels else 1.08) + + # save the path + if save_path is not None: + assert save_path.endswith('.png') + ensure_parent_dir_exists(save_path) + plt.savefig(save_path) + print(f'saved {gt_data.name} to: {save_path}') + + # show it! + plt.show() + + # - - - - - - - - - - - - - - - - - # + return fig + + +# TODO: fix +def plot_traversal_stats( + dataset_or_name: Union[str, GroundTruthData], + num_repeats: int = 256, + f_idxs: Optional[NonNormalisedFactors] = None, + circular_distance: bool = False, + color='blue', + color_gt_dist='blue', + color_im_dist='purple', + suffix: Optional[str] = None, + save_path: Optional[str] = None, + plot_freq: bool = True, + plot_title: Union[bool, str] = False, + fig_block_size: float = 4.0, + col_titles: Union[bool, List[str]] = True, + hide_axis: bool = True, + hide_labels: bool = True, + y_size_offset: float = 0.0, + x_size_offset: float = 0.0, +): + # - - - - - - - - - - - - - - - - - # + + def stats_fn(gt_data, i, f_idx): + return sample_factor_traversal_info_and_distmat( + gt_data=gt_data, f_idx=f_idx, circular_distance=circular_distance + ) + + grid_t = [] + grid_titles = [] + + def plot_ax(stats: dict, i: int, f_idx: int): + fdists_matrix = np.mean(stats['fdists_matrix'], axis=0) + deltas_matrix = np.mean(stats['deltas_matrix'], axis=0) + grid_t.append([fdists_matrix, deltas_matrix]) + # get the title + if isinstance(col_titles, bool): + if col_titles: + grid_titles.append(gt_data.factor_names[f_idx]) + else: + grid_titles.append(col_titles[i]) + + # initialize + gt_data: GroundTruthData = H.make_data(dataset_or_name) if isinstance(dataset_or_name, str) else dataset_or_name + f_idxs = gt_data.normalise_factor_idxs(f_idxs) + + # settings + h, w = [2.07 * fig_block_size, len(f_idxs) * fig_block_size] + + # get title + if isinstance(plot_title, str): + suptitle = f'{plot_title}' + elif plot_title: + suptitle = f'{gt_data.name} [circular={circular_distance}]{f" {suffix}" if suffix else ""}' + else: + suptitle = None + + # generate plot + _collect_stats_for_factors( + gt_data=gt_data, + f_idxs=f_idxs, + stats_fn=stats_fn, + keep_keys=['deltas', 'fdists', 'deltas_matrix', 'fdists_matrix'], + stats_callback=plot_ax, + num_traversal_sample=num_repeats, + ) + + fig, axs = H.plt_subplots_imshow( + grid=list(zip(*grid_t)), + title=suptitle, + titles=grid_titles if grid_titles else None, + titles_size=36, + subplot_padding=None, + figsize=(w, h) + ) + + # recolor axes + for (ax0, ax1) in axs.T: + ax0.images[0].set_cmap('Blues') + ax1.images[0].set_cmap('Purples') + fig.tight_layout() # save the path @@ -317,10 +420,11 @@ def sp(name): for fg in [True, False]: for vis in [100, 80, 60, 40, 20]: name = f'dsprites_imagenet_{"fg" if fg else "bg"}_{vis}' - plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(name), color='orange', dataset_or_name=name, plot_freq=PLOT_FREQ) + plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(name), color='orange', dataset_or_name=name, plot_freq=PLOT_FREQ, x_size_offset=0.4) # mean, std = compute_data_mean_std(H.make_data(name)) # print(f'{name}\n vis_mean: {mean.tolist()}\n vis_std: {std.tolist()}') + exit(1) BASE = os.path.abspath(os.path.join(__file__, '../../../out/adversarial_data_approx')) # plot adversarial datasets @@ -342,7 +446,7 @@ def sp(name): ('red', '2021-09-06--09-10-59_INVERT-VSTRONG-smallnorb_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), ]: data = _make_self_contained_dataset(f'{BASE}/{folder}/data.h5') - plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(folder), color=color, dataset_or_name=data, plot_freq=PLOT_FREQ) + plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(folder), color=color, dataset_or_name=data, plot_freq=PLOT_FREQ, x_size_offset=0.4) # compute and print statistics: # mean, std = compute_data_mean_std(data, progress=True) # print(f'{folder}\n vis_mean: {mean.tolist()}\n vis_std: {std.tolist()}') From db3f8d3a48bfbc41efc0f05b218636aa897b8de6 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Sat, 13 Nov 2021 14:44:48 +0200 Subject: [PATCH 21/30] plot directly from WANDB --- research/e01_visual_overlap/plots/.gitignore | 1 + .../plot_experiments.py | 221 ++++++++++++++++++ .../plot_wandb_experiments/plots/.gitignore | 1 + research/util/_io_util.py | 11 + 4 files changed, 234 insertions(+) create mode 100644 research/e01_visual_overlap/plots/.gitignore create mode 100644 research/plot_wandb_experiments/plot_experiments.py create mode 100644 research/plot_wandb_experiments/plots/.gitignore diff --git a/research/e01_visual_overlap/plots/.gitignore b/research/e01_visual_overlap/plots/.gitignore new file mode 100644 index 00000000..e33609d2 --- /dev/null +++ b/research/e01_visual_overlap/plots/.gitignore @@ -0,0 +1 @@ +*.png diff --git a/research/plot_wandb_experiments/plot_experiments.py b/research/plot_wandb_experiments/plot_experiments.py new file mode 100644 index 00000000..9d342481 --- /dev/null +++ b/research/plot_wandb_experiments/plot_experiments.py @@ -0,0 +1,221 @@ +import os +from typing import List +from typing import Optional + +import pandas as pd +import seaborn as sns +import wandb +from cachier import cachier as _cachier +from matplotlib import pyplot as plt +from tqdm import tqdm + +import research.util as H +from disent.util.function import wrapped_partial + + +# ========================================================================= # +# Helper # +# ========================================================================= # + + +cachier = wrapped_partial(_cachier, cache_dir='./cache') +DF = pd.DataFrame + + +def clear_cache(): + load_runs.clear_cache() + + +# ========================================================================= # +# Load WANDB Data # +# ========================================================================= # + + +@cachier() +def load_runs(project: str) -> pd.DataFrame: + api = wandb.Api() + + runs = api.runs(project) + + info_list, summary_list, config_list, name_list = [], [], [], [] + for run in tqdm(runs, desc=f'loading: {project}'): + info_list.append({ + 'id': run.id, + 'name': run.name, + 'state': run.state, + 'storage_id': run.storage_id, + 'url': run.url, + }) + summary_list.append(run.summary._json_dict) + config_list.append({k: v for k, v in run.config.items() if not k.startswith('_')}) + name_list.append(run.name) + + return pd.DataFrame({ + "info": info_list, + "summary": summary_list, + "config": config_list, + "name": name_list + }) + + +def load_expanded_runs(project: str) -> pd.DataFrame: + # load the data + df_runs: DF = load_runs(project) + # expand the dictionaries + df_info: DF = df_runs['info'].apply(pd.Series) + df_summary: DF = df_runs['summary'].apply(pd.Series) + df_config: DF = df_runs['config'].apply(pd.Series) + # merge the data + df: DF = df_config.join(df_summary).join(df_info) + assert len(df.columns) == len(df_info.columns) + len(df_summary.columns) + len(df_config.columns) + # done! + return df + + +def drop_unhashable(df: pd.DataFrame, inplace: bool = False) -> (pd.DataFrame, List[str]): + dropped = [] + for col in df.columns: + try: + df[col].unique() + except: + dropped.append(col) + df = df.drop(col, inplace=inplace, axis=1) + return df, dropped + + +def drop_non_diverse_cols(df: pd.DataFrame, inplace: bool = False) -> (pd.DataFrame, List[str]): + dropped = [] + for col in df.columns: + if len(df[col].unique()) == 1: + dropped.append(col) + df = df.drop(col, inplace=inplace, axis=1) + return df, dropped + + +# ========================================================================= # +# Prepare Data # +# ========================================================================= # + + +# common keys +K_GROUP = 'Run Group' +K_FRAMEWORK = 'Framework' +K_SPACING = 'Spacing' +K_BETA = 'Beta' +K_REPEAT = 'Repeat' +K_STATE = 'State' +K_MIG = 'MIG Score' +K_DCI = 'DCI Score' + + +def print_common(df: pd.DataFrame): + # print common key values + print('K_GROUP: ', list(df[K_GROUP].unique())) + print('K_FRAMEWORK:', list(df[K_FRAMEWORK].unique())) + print('K_SPACING: ', list(df[K_SPACING].unique())) + print('K_BETA: ', list(df[K_BETA].unique())) + print('K_REPEAT: ', list(df[K_REPEAT].unique())) + print('K_STATE: ', list(df[K_STATE].unique())) + + +def load_general_data(project: str): + # load data + df = load_expanded_runs(project) + # filter out unneeded columns + df, dropped_hash = drop_unhashable(df) + df, dropped_diverse = drop_non_diverse_cols(df) + # rename columns + return df.rename(columns={ + 'EXTRA/tags': K_GROUP, + 'framework/name': K_FRAMEWORK, + 'dataset/data/grid_spacing': K_SPACING, + 'settings/framework/beta': K_BETA, + 'DUMMY/repeat': K_REPEAT, + 'state': K_STATE, + 'final_metric/mig.discrete_score.max': K_MIG, + 'final_metric/dci.disentanglement.max': K_DCI, + }) + + +# ========================================================================= # +# Plot Experiments # +# ========================================================================= # + + +def plot_incr_overlap_exp(rel_path: Optional[str] = None, save: bool = True, show: bool = True): + # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + df = load_general_data(f'{os.environ["WANDB_USER"]}/CVPR-01__incr_overlap') + # select run groups + df = df[df[K_GROUP] == 'sweep_xy_squares_overlap'] + # print common key values + print_common(df) + # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + + BETA = 0.00316 # if grid_spacing < 6 + BETA = 0.001 # if grid_spacing >= 6 + + # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + orig = df + # select runs + # df = df[df[K_STATE] == 'finished'] + # df = df[df[K_REPEAT].isin([1, 2, 3])] + + adavae_selector = (df[K_FRAMEWORK] == 'adavae_os') & (df[K_BETA] == 0.001) + data_adavae = df[adavae_selector] + + betavae_selector_a = (df[K_FRAMEWORK] == 'betavae') & (df[K_BETA] == 0.001) & (df[K_SPACING] >= 3) + betavae_selector_b = (df[K_FRAMEWORK] == 'betavae') & (df[K_BETA] == 0.00316) & (df[K_SPACING] < 3) + data_betavae = df[betavae_selector_a | betavae_selector_b] + # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + + print('ADAGVAE', len(orig), '->', len(data_adavae)) + print('BETAVAE', len(orig), '->', len(data_betavae)) + + # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + fig, axs = plt.subplots(1, 2, figsize=(10, 5)) + (ax0, ax1) = axs + # PLOT: MIG + sns.regplot(ax=ax0, x=K_SPACING, y=K_MIG, data=data_betavae, order=1, robust=False, color='pink') + sns.regplot(ax=ax0, x=K_SPACING, y=K_MIG, data=data_adavae, order=1, robust=False, color='lightblue') + ax0.legend(labels=["Beta-VAE", "Ada-GVAE"]) + ax0.set_ylim([-0.1, 1.1]) + ax0.set_xlim([0.8, 8.2]) + ax0.set_title('Framework Mig Scores') + # PLOT: DCI + sns.regplot(ax=ax1, x=K_SPACING, y=K_DCI, data=data_betavae, order=1, robust=False, color='pink') + sns.regplot(ax=ax1, x=K_SPACING, y=K_DCI, data=data_adavae, order=1, robust=False, color='lightblue') + ax1.legend(labels=["Beta-VAE", "Ada-GVAE"]) + ax1.set_ylim([-0.1, 1.1]) + ax1.set_xlim([0.8, 8.2]) + ax1.set_title('Framework DCI Scores') + # PLOT: + fig.tight_layout() + H.plt_rel_path_savefig(rel_path, save=save, show=show) + # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + + return fig, axs + + +# ========================================================================= # +# Entrypoint # +# ========================================================================= # + + +if __name__ == '__main__': + + assert 'WANDB_USER' in os.environ, 'specify "WANDB_USER" environment variable' + + # matplotlib style + plt.style.use(os.path.join(os.path.dirname(__file__), '../gadfly.mplstyle')) + + # clear_cache() + + def main(): + plot_incr_overlap_exp(rel_path='plots/exp_incr-overlap', show=True) + + main() + + +# ========================================================================= # +# DONE # +# ========================================================================= # diff --git a/research/plot_wandb_experiments/plots/.gitignore b/research/plot_wandb_experiments/plots/.gitignore new file mode 100644 index 00000000..e33609d2 --- /dev/null +++ b/research/plot_wandb_experiments/plots/.gitignore @@ -0,0 +1 @@ +*.png diff --git a/research/util/_io_util.py b/research/util/_io_util.py index 9888a0d6..3c5e8ca8 100644 --- a/research/util/_io_util.py +++ b/research/util/_io_util.py @@ -27,6 +27,7 @@ import inspect import io import os +from typing import Optional from typing import Union import torch @@ -223,6 +224,16 @@ def make_rel_path_add_ext(*path_segments, ext='.png'): return _make_rel_path_add_ext(*path_segments, ext=ext, _calldepth=1) +def plt_rel_path_savefig(rel_path: Optional[str], save: bool = True, show: bool = True, ext='.png', dpi: Optional[int] = None): + import matplotlib.pyplot as plt + if save and (rel_path is not None): + path = _make_rel_path_add_ext(rel_path, ext=ext, _calldepth=2) + plt.savefig(path, dpi=dpi) + print(f'saved: {repr(path)}') + if show: + plt.show() + + # ========================================================================= # # END # # ========================================================================= # From cea43fa749e602da21240b83eff2561a705f1804 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Mon, 15 Nov 2021 14:18:37 +0200 Subject: [PATCH 22/30] fix experiments + more plots --- .../e01_incr_overlap/submit_incr_overlap.sh | 71 +++++- .../submit_overlap_loss.sh | 56 ++++- .../plot_experiments.py | 208 +++++++++++++++--- 3 files changed, 288 insertions(+), 47 deletions(-) diff --git a/research/e01_incr_overlap/submit_incr_overlap.sh b/research/e01_incr_overlap/submit_incr_overlap.sh index 6f114c4a..49c28043 100644 --- a/research/e01_incr_overlap/submit_incr_overlap.sh +++ b/research/e01_incr_overlap/submit_incr_overlap.sh @@ -1,9 +1,26 @@ #!/bin/bash + +# OVERVIEW: +# - this experiment is designed to check how increasing overlap (reducing +# the spacing between square positions on XYSquares) affects learning. + + +# OUTCOMES: +# - increasing overlap improves disentanglement & ability for the +# neural network to learn values. +# - decreasing overlap worsens disentanglement, but it also becomes +# very hard for the neural net to learn specific values needed. The +# average image does not correspond well to individual samples. +# Disentanglement performance is also a result of this fact, as +# the network can't always learn the dataset effectively. + + # ========================================================================= # # Settings # # ========================================================================= # + export USERNAME="n_michlo" export PROJECT="CVPR-01__incr_overlap" export PARTITION="stampede" @@ -12,19 +29,21 @@ export PARALLELISM=28 # source the helper file source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" + # ========================================================================= # # Experiment # # ========================================================================= # + clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours # background launch various xysquares # -- original experiment also had dfcvae # -- beta is too high for adavae -# 3 * (2*2*8 = 32) = 96 +# 5 * (2*2*8 = 32) = 160 submit_sweep \ - +DUMMY.repeat=1,2,3 \ + +DUMMY.repeat=1,2,3,4,5 \ +EXTRA.tags='sweep_xy_squares_overlap' \ hydra.job.name="incr_ovlp" \ \ @@ -41,27 +60,50 @@ submit_sweep \ # background launch various xysquares -# - this time we try delay beta, so that it can learn properly... -# 3 * (2*2*8 = 32) = 96 +# -- original experiment also had dfcvae +# -- beta is too high for adavae +# 5 * (2*8 = 16) = 80 submit_sweep \ - +DUMMY.repeat=1,2,3 \ - +EXTRA.tags='sweep_xy_squares_overlap_delay' \ - hydra.job.name="schd_incr_ovlp" \ - \ - schedule=beta_delay_long \ + +DUMMY.repeat=1,2,3,4,5 \ + +EXTRA.tags='sweep_xy_squares_overlap_small_beta' \ + hydra.job.name="sb_incr_ovlp" \ \ run_length=medium \ metrics=all \ \ - settings.framework.beta=0.001 \ - framework=betavae,adavae_os \ - settings.model.z_size=9,25 \ + settings.framework.beta=0.0001,0.00001 \ + framework=adavae_os \ + settings.model.z_size=9 \ \ sampling=default__bb \ dataset=X--xysquares_rgb \ dataset.data.grid_spacing=8,7,6,5,4,3,2,1 +# background launch various xysquares +# - this time we try delay beta, so that it can learn properly... +# - NOTE: this doesn't actually work, the VAE loss often becomes +# NAN because the values are too small. +# 3 * (2*2*8 = 32) = 96 +# submit_sweep \ +# +DUMMY.repeat=1,2,3 \ +# +EXTRA.tags='sweep_xy_squares_overlap_delay' \ +# hydra.job.name="schd_incr_ovlp" \ +# \ +# schedule=beta_delay_long \ +# \ +# run_length=medium \ +# metrics=all \ +# \ +# settings.framework.beta=0.001 \ +# framework=betavae,adavae_os \ +# settings.model.z_size=9,25 \ +# \ +# sampling=default__bb \ +# dataset=X--xysquares_rgb \ +# dataset.data.grid_spacing=8,7,6,5,4,3,2,1 + + # background launch traditional datasets # -- original experiment also had dfcvae # 5 * (2*2*4 = 16) = 80 @@ -78,3 +120,8 @@ submit_sweep \ # \ # sampling=default__bb \ # dataset=cars3d,shapes3d,dsprites,smallnorb + + +# ========================================================================= # +# DONE # +# ========================================================================= # diff --git a/research/e09_vae_overlap_loss/submit_overlap_loss.sh b/research/e09_vae_overlap_loss/submit_overlap_loss.sh index 87f444a6..9fdbcdb0 100644 --- a/research/e09_vae_overlap_loss/submit_overlap_loss.sh +++ b/research/e09_vae_overlap_loss/submit_overlap_loss.sh @@ -19,10 +19,33 @@ source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours # 1 * (5 * 2*4*2) = 80 +#submit_sweep \ +# +DUMMY.repeat=1 \ +# +EXTRA.tags='sweep_overlap_boxblur' \ +# hydra.job.name="ovlp_loss" \ +# \ +# +VAR.recon_loss_weight=1.0 \ +# +VAR.kernel_loss_weight=3969.0 \ +# +VAR.kernel_radius=31 \ +# \ +# run_length=medium \ +# metrics=all \ +# \ +# dataset=X--xysquares,dsprites,shapes3d,smallnorb,cars3d \ +# \ +# framework=betavae,adavae_os \ +# settings.framework.beta=0.0316,0.316,0.1,0.01 \ +# settings.model.z_size=25,9 \ +# settings.framework.recon_loss='mse_box_r${VAR.kernel_radius}_l${VAR.recon_loss_weight}_k${VAR.kernel_loss_weight}' \ +# \ +# sampling=default__bb + + +# 5 * (2*2*2 = 8) = 40 submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_overlap_boxblur' \ - hydra.job.name="ovlp_loss" \ + +DUMMY.repeat=1,2,3,4,5 \ + +EXTRA.tags='sweep_overlap_boxblur_specific' \ + hydra.job.name="s_ovlp_loss" \ \ +VAR.recon_loss_weight=1.0 \ +VAR.kernel_loss_weight=3969.0 \ @@ -31,11 +54,30 @@ submit_sweep \ run_length=medium \ metrics=all \ \ - dataset=X--xysquares,dsprites,shapes3d,smallnorb,cars3d \ + dataset=X--xysquares \ \ framework=betavae,adavae_os \ - settings.framework.beta=0.0316,0.316,0.1,0.01 \ - settings.model.z_size=25,9 \ - settings.framework.recon_loss='mse_box_r${VAR.kernel_radius}_l${VAR.recon_loss_weight}_k${VAR.kernel_loss_weight}' \ + settings.framework.beta=0.0316,0.0001 \ + settings.model.z_size=25 \ + settings.framework.recon_loss=mse,'mse_box_r${VAR.kernel_radius}_l${VAR.recon_loss_weight}_k${VAR.kernel_loss_weight}' \ \ sampling=default__bb + + +# HPARAM SWEEP -- TODO: update +# 1 * (2 * 8 * 2 * 2) = 160 +#submit_sweep \ +# +DUMMY.repeat=1 \ +# +EXTRA.tags='sweep_beta' \ +# hydra.job.name="vae_hparams" \ +# \ +# run_length=long \ +# metrics=all \ +# \ +# settings.framework.beta=0.000316,0.001,0.00316,0.01,0.0316,0.1,0.316,1.0 \ +# framework=betavae,adavae_os \ +# schedule=none \ +# settings.model.z_size=9,25 \ +# \ +# dataset=X--xysquares \ +# sampling=default__bb diff --git a/research/plot_wandb_experiments/plot_experiments.py b/research/plot_wandb_experiments/plot_experiments.py index 9d342481..6233eb38 100644 --- a/research/plot_wandb_experiments/plot_experiments.py +++ b/research/plot_wandb_experiments/plot_experiments.py @@ -99,25 +99,18 @@ def drop_non_diverse_cols(df: pd.DataFrame, inplace: bool = False) -> (pd.DataFr # common keys K_GROUP = 'Run Group' +K_DATASET = 'Dataset' K_FRAMEWORK = 'Framework' -K_SPACING = 'Spacing' +K_SPACING = 'Grid Spacing' K_BETA = 'Beta' +K_LOSS = 'Recon. Loss' +K_Z_SIZE = 'Latent Dims.' K_REPEAT = 'Repeat' K_STATE = 'State' K_MIG = 'MIG Score' K_DCI = 'DCI Score' -def print_common(df: pd.DataFrame): - # print common key values - print('K_GROUP: ', list(df[K_GROUP].unique())) - print('K_FRAMEWORK:', list(df[K_FRAMEWORK].unique())) - print('K_SPACING: ', list(df[K_SPACING].unique())) - print('K_BETA: ', list(df[K_BETA].unique())) - print('K_REPEAT: ', list(df[K_REPEAT].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - - def load_general_data(project: str): # load data df = load_expanded_runs(project) @@ -127,9 +120,12 @@ def load_general_data(project: str): # rename columns return df.rename(columns={ 'EXTRA/tags': K_GROUP, + 'dataset/name': K_DATASET, 'framework/name': K_FRAMEWORK, 'dataset/data/grid_spacing': K_SPACING, 'settings/framework/beta': K_BETA, + 'settings/framework/recon_loss': K_LOSS, + 'settings/model/z_size': K_Z_SIZE, 'DUMMY/repeat': K_REPEAT, 'state': K_STATE, 'final_metric/mig.discrete_score.max': K_MIG, @@ -141,14 +137,34 @@ def load_general_data(project: str): # Plot Experiments # # ========================================================================= # - -def plot_incr_overlap_exp(rel_path: Optional[str] = None, save: bool = True, show: bool = True): +PINK = '#FE375F' +PURPLE = '#5E5BE5' +BLUE = '#0A83FE' +LBLUE = '#63D2FE' +ORANGE = '#FE9F0A' +GREEN = '#2FD157' + + +def plot_incr_overlap_exp( + rel_path: Optional[str] = None, + save: bool = True, + show: bool = True, + reg_order: int = 4, + color_betavae: str = PINK, + color_adavae: str = ORANGE, + titles: bool = False, +): # ~=~=~=~=~=~=~=~=~=~=~=~=~ # df = load_general_data(f'{os.environ["WANDB_USER"]}/CVPR-01__incr_overlap') # select run groups - df = df[df[K_GROUP] == 'sweep_xy_squares_overlap'] + df = df[df[K_GROUP].isin(['sweep_xy_squares_overlap', 'sweep_xy_squares_overlap_small_beta'])] # print common key values - print_common(df) + print('K_GROUP: ', list(df[K_GROUP].unique())) + print('K_FRAMEWORK:', list(df[K_FRAMEWORK].unique())) + print('K_SPACING: ', list(df[K_SPACING].unique())) + print('K_BETA: ', list(df[K_BETA].unique())) + print('K_REPEAT: ', list(df[K_REPEAT].unique())) + print('K_STATE: ', list(df[K_STATE].unique())) # ~=~=~=~=~=~=~=~=~=~=~=~=~ # BETA = 0.00316 # if grid_spacing < 6 @@ -159,10 +175,10 @@ def plot_incr_overlap_exp(rel_path: Optional[str] = None, save: bool = True, sho # select runs # df = df[df[K_STATE] == 'finished'] # df = df[df[K_REPEAT].isin([1, 2, 3])] - - adavae_selector = (df[K_FRAMEWORK] == 'adavae_os') & (df[K_BETA] == 0.001) - data_adavae = df[adavae_selector] - + # select adavae + adavae_selector = (df[K_FRAMEWORK] == 'adavae_os') & (df[K_BETA] == 0.001) # 0.001, 0.0001 + data_adavae = df[adavae_selector] + # select betavae_selector_a = (df[K_FRAMEWORK] == 'betavae') & (df[K_BETA] == 0.001) & (df[K_SPACING] >= 3) betavae_selector_b = (df[K_FRAMEWORK] == 'betavae') & (df[K_BETA] == 0.00316) & (df[K_SPACING] < 3) data_betavae = df[betavae_selector_a | betavae_selector_b] @@ -175,19 +191,19 @@ def plot_incr_overlap_exp(rel_path: Optional[str] = None, save: bool = True, sho fig, axs = plt.subplots(1, 2, figsize=(10, 5)) (ax0, ax1) = axs # PLOT: MIG - sns.regplot(ax=ax0, x=K_SPACING, y=K_MIG, data=data_betavae, order=1, robust=False, color='pink') - sns.regplot(ax=ax0, x=K_SPACING, y=K_MIG, data=data_adavae, order=1, robust=False, color='lightblue') - ax0.legend(labels=["Beta-VAE", "Ada-GVAE"]) + sns.regplot(ax=ax0, x=K_SPACING, y=K_MIG, data=data_adavae, seed=777, order=reg_order, robust=False, color=color_adavae, marker='o') + sns.regplot(ax=ax0, x=K_SPACING, y=K_MIG, data=data_betavae, seed=777, order=reg_order, robust=False, color=color_betavae, marker='x', line_kws=dict(linestyle='dashed')) + ax0.legend(labels=["Ada-GVAE", "Beta-VAE"], fontsize=14) ax0.set_ylim([-0.1, 1.1]) ax0.set_xlim([0.8, 8.2]) - ax0.set_title('Framework Mig Scores') + if titles: ax0.set_title('Framework Mig Scores') # PLOT: DCI - sns.regplot(ax=ax1, x=K_SPACING, y=K_DCI, data=data_betavae, order=1, robust=False, color='pink') - sns.regplot(ax=ax1, x=K_SPACING, y=K_DCI, data=data_adavae, order=1, robust=False, color='lightblue') - ax1.legend(labels=["Beta-VAE", "Ada-GVAE"]) + sns.regplot(ax=ax1, x=K_SPACING, y=K_DCI, data=data_adavae, seed=777, order=reg_order, robust=False, color=color_adavae, marker='o') + sns.regplot(ax=ax1, x=K_SPACING, y=K_DCI, data=data_betavae, seed=777, order=reg_order, robust=False, color=color_betavae, marker='x', line_kws=dict(linestyle='dashed')) + ax1.legend(labels=["Ada-GVAE", "Beta-VAE"], fontsize=14) ax1.set_ylim([-0.1, 1.1]) ax1.set_xlim([0.8, 8.2]) - ax1.set_title('Framework DCI Scores') + if titles: ax1.set_title('Framework DCI Scores') # PLOT: fig.tight_layout() H.plt_rel_path_savefig(rel_path, save=save, show=show) @@ -196,6 +212,140 @@ def plot_incr_overlap_exp(rel_path: Optional[str] = None, save: bool = True, sho return fig, axs + + + +def plot_hparams_exp( + rel_path: Optional[str] = None, + save: bool = True, + show: bool = True, + color_betavae: str = PINK, + color_adavae: str = ORANGE, +): + # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + df = load_general_data(f'{os.environ["WANDB_USER"]}/CVPR-00__basic-hparam-tuning') + # select run groups + df = df[df[K_GROUP].isin(['sweep_beta'])] + # print common key values + print('K_GROUP: ', list(df[K_GROUP].unique())) + print('K_DATASET: ', list(df[K_DATASET].unique())) + print('K_FRAMEWORK:', list(df[K_FRAMEWORK].unique())) + print('K_BETA: ', list(df[K_BETA].unique())) + print('K_Z_SIZE: ', list(df[K_Z_SIZE].unique())) + print('K_REPEAT: ', list(df[K_REPEAT].unique())) + print('K_STATE: ', list(df[K_STATE].unique())) + # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + + # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + orig = df + # select runs + df = df[df[K_STATE] == 'finished'] + # [1.0, 0.316, 0.1, 0.0316, 0.01, 0.00316, 0.001, 0.000316] + # df = df[(0.000316 < df[K_BETA]) & (df[K_BETA] < 1.0)] + print('NUM', len(orig), '->', len(df)) + # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + + df = df[[K_DATASET, K_FRAMEWORK, K_MIG, K_DCI]] + df[K_DATASET].replace('xysquares_minimal', 'XYSquares', inplace=True) + df[K_DATASET].replace('smallnorb', 'NORB', inplace=True) + df[K_DATASET].replace('cars3d', 'Cars3D', inplace=True) + df[K_DATASET].replace('3dshapes', 'Shapes3D', inplace=True) + df[K_DATASET].replace('dsprites', 'dSprites', inplace=True) + df[K_FRAMEWORK].replace('adavae_os', 'Ada-GVAE', inplace=True) + df[K_FRAMEWORK].replace('betavae', 'Beta-VAE', inplace=True) + PALLETTE = {'Ada-GVAE': color_adavae, 'Beta-VAE': color_betavae} + + # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + fig, axs = plt.subplots(1, 2, figsize=(10, 4)) + (ax0, ax1) = axs + # PLOT: MIG + sns.violinplot(x=K_DATASET, y=K_MIG, hue=K_FRAMEWORK, palette=PALLETTE, split=True, cut=0, width=0.75, data=df, ax=ax0, scale='width', inner='quartile') + ax0.set_ylim([-0.1, 1.1]) + ax0.legend(bbox_to_anchor=(0.425, 0.9), fontsize=13) + sns.violinplot(x=K_DATASET, y=K_DCI, hue=K_FRAMEWORK, palette=PALLETTE, split=True, cut=0, width=0.75, data=df, ax=ax1, scale='width', inner='quartile') + ax1.set_ylim([-0.1, 1.1]) + ax1.get_legend().remove() + # PLOT: + fig.tight_layout() + H.plt_rel_path_savefig(rel_path, save=save, show=show, dpi=300) + # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + + return fig, axs + + + +def plot_overlap_loss_exp( + rel_path: Optional[str] = None, + save: bool = True, + show: bool = True, + color_betavae: str = PINK, + color_adavae: str = ORANGE, + color_mse: str = '#9FD911', + color_mse_overlap: str = '#36CFC8', +): + # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + df = load_general_data(f'{os.environ["WANDB_USER"]}/CVPR-09__vae_overlap_loss') + # select run groups + df = df[df[K_GROUP].isin(['sweep_overlap_boxblur_specific', 'sweep_overlap_boxblur'])] + # print common key values + print('K_GROUP: ', list(df[K_GROUP].unique())) + print() + print('K_DATASET: ', list(df[K_DATASET].unique())) + print('K_FRAMEWORK:', list(df[K_FRAMEWORK].unique())) + print('K_Z_SIZE: ', list(df[K_Z_SIZE].unique())) + print('K_LOSS: ', list(df[K_LOSS].unique())) + print('K_BETA: ', list(df[K_BETA].unique())) + print() + print('K_REPEAT: ', list(df[K_REPEAT].unique())) + print('K_STATE: ', list(df[K_STATE].unique())) + # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + + # # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + orig = df + # select runs + df = df[df[K_STATE] == 'finished'] # TODO: update + df = df[df[K_DATASET] == 'xysquares_minimal'] + df = df[df[K_BETA].isin([0.0001, 0.0316])] + df = df[df[K_Z_SIZE] == 25] + # df = df[df[K_FRAMEWORK] == 'betavae'] # 18 + # df = df[df[K_FRAMEWORK] == 'adavae_os'] # 21 + # df = df[df[K_LOSS] == 'mse'] # 20 + # df = df[df[K_LOSS] != 'mse'] # 19 + # # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + + # TEMP + # df[K_MIG] = df['final_metric/mig.discrete_score.max'] + # df[K_DCI] = df['final_metric/dci.disentanglement.max'] + + print('NUM', len(orig), '->', len(df)) + + df = df[[K_DATASET, K_FRAMEWORK, K_LOSS, K_BETA, K_MIG, K_DCI]] + df[K_DATASET].replace('xysquares_minimal', 'XYSquares', inplace=True) + df[K_FRAMEWORK].replace('adavae_os', 'Ada-GVAE', inplace=True) + df[K_FRAMEWORK].replace('betavae', 'Beta-VAE', inplace=True) + df[K_LOSS].replace('mse_box_r31_l1.0_k3969.0', 'MSE-boxblur', inplace=True) + df[K_LOSS].replace('mse', 'MSE', inplace=True) + PALLETTE = {'Ada-GVAE': color_adavae, 'Beta-VAE': color_betavae, 'MSE': color_mse, 'MSE-boxblur': color_mse_overlap} + + print(df) + + # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + fig, axs = plt.subplots(1, 2, figsize=(10, 4)) + (ax0, ax1) = axs + # PLOT: MIG + sns.violinplot(x=K_FRAMEWORK, y=K_MIG, hue=K_LOSS, palette=PALLETTE, split=True, cut=0, width=0.5, data=df, ax=ax0, scale='width', inner='quartile') + ax0.set_ylim([-0.1, 1.1]) + ax0.legend(fontsize=13) + # ax0.legend(bbox_to_anchor=(0.425, 0.9), fontsize=13) + sns.violinplot(x=K_FRAMEWORK, y=K_DCI, hue=K_LOSS, palette=PALLETTE, split=True, cut=0, width=0.5, data=df, ax=ax1, scale='width', inner='quartile') + ax1.set_ylim([-0.1, 1.1]) + ax1.get_legend().remove() + # PLOT: + fig.tight_layout() + H.plt_rel_path_savefig(rel_path, save=save, show=show, dpi=300) + # ~=~=~=~=~=~=~=~=~=~=~=~=~ # + + # ========================================================================= # # Entrypoint # # ========================================================================= # @@ -211,7 +361,9 @@ def plot_incr_overlap_exp(rel_path: Optional[str] = None, save: bool = True, sho # clear_cache() def main(): - plot_incr_overlap_exp(rel_path='plots/exp_incr-overlap', show=True) + # plot_hparams_exp(rel_path='plots/exp_hparams-exp', show=True) + plot_overlap_loss_exp(rel_path='plots/exp_overlap-loss', show=True) + # plot_incr_overlap_exp(rel_path='plots/exp_incr-overlap', show=True) main() From 96e75f9dd09cb8632a1debfe57a05e67c7190c1a Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Tue, 16 Nov 2021 16:42:43 +0200 Subject: [PATCH 23/30] update experiments and plotting --- .../run_02_plot_data_overlap.py | 2 +- .../run_02_plot_traversals.py | 140 +++++++++---- .../run_plot_global_dists.py | 1 - .../run_plot_traversal_dists.py | 198 ++++++++++++++++-- 4 files changed, 274 insertions(+), 67 deletions(-) diff --git a/research/e00_data_traversal/run_02_plot_data_overlap.py b/research/e00_data_traversal/run_02_plot_data_overlap.py index 479dc961..6712defb 100644 --- a/research/e00_data_traversal/run_02_plot_data_overlap.py +++ b/research/e00_data_traversal/run_02_plot_data_overlap.py @@ -69,7 +69,7 @@ def plot_dataset_overlap( save=True, seed=777, plt_scale=4.5, - offset=0.75 + offset=0.75, ): with TempNumpySeed(seed): # choose an f_idx diff --git a/research/e00_data_traversal/run_02_plot_traversals.py b/research/e00_data_traversal/run_02_plot_traversals.py index a054d9a6..15142441 100644 --- a/research/e00_data_traversal/run_02_plot_traversals.py +++ b/research/e00_data_traversal/run_02_plot_traversals.py @@ -24,6 +24,8 @@ import os from typing import Optional +from typing import Sequence +from typing import Union import numpy as np from matplotlib import pyplot as plt @@ -62,16 +64,20 @@ def plot_dataset_traversals( num_cols: Optional[int] = 8, take_cols: Optional[int] = None, base_factors=None, - add_random_traversal=True, - pad=8, - bg_color=127, - border=False, - rel_path=None, - save=True, - seed=777, - plt_scale=4.5, - offset=0.75, - transpose=False, + add_random_traversal: bool = True, + pad: int = 8, + bg_color: int = 127, + border: bool = False, + rel_path: str = None, + save: bool = True, + seed: int = 777, + plt_scale: float = 4.5, + offset: float = 0.75, + transpose: bool = False, + title: Union[bool, str] = True, + label_size: int = 22, + title_size: int = 26, + labels_at_top: bool = False, ): if take_cols is not None: assert take_cols >= num_cols @@ -105,10 +111,18 @@ def plot_dataset_traversals( factors, frames, _, _, c = grid.shape assert c == 3 + # get title + if isinstance(title, bool): + title = gt_data.name if title else None + if transpose: - fig, axs = H.plt_subplots_imshow(np.swapaxes(grid, 0, 1), label_size=18, title_size=24, title=gt_data.name, col_labels=row_labels, subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)[::-1]) + col_titles = None + if labels_at_top: + col_titles, row_labels = row_labels, None + fig, axs = H.plt_subplots_imshow(np.swapaxes(grid, 0, 1), label_size=label_size, title_size=title_size, title=title, titles=col_titles, titles_size=label_size, col_labels=row_labels, subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)[::-1]) else: - fig, axs = H.plt_subplots_imshow(grid, label_size=18, title_size=24, title=gt_data.name, row_labels=row_labels, subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) + fig, axs = H.plt_subplots_imshow(grid, label_size=label_size, title_size=title_size, title=title, row_labels=row_labels, subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) + # save figure if save and (rel_path is not None): path = H.make_rel_path_add_ext(rel_path, ext='.png') @@ -119,6 +133,37 @@ def plot_dataset_traversals( return fig, axs +def plot_incr_overlap( + rel_path: Optional[str] = None, + spacings: Union[Sequence[int], bool] = False, + seed: int = 777, + fidx: int = 1, + traversal_size: int = 8, + traversal_lim: Optional[int] = None, + save: bool = True, + show: bool = True +): + if isinstance(spacings, bool): + spacings = ([1, 2, 3, 4, 5, 6, 7, 8] if spacings else [1, 4, 8]) + + if traversal_lim is None: + traversal_lim = traversal_size + assert traversal_size >= traversal_lim + + grid = [] + for s in spacings: + data = XYSquaresData(grid_spacing=s, grid_size=8, no_warnings=True) + with TempNumpySeed(seed): + factors, indices, obs = data.sample_random_obs_traversal(f_idx=data.normalise_factor_idx(fidx), num=traversal_size, mode='interval') + grid.append(obs[:traversal_lim]) + + w, h = traversal_lim * 2.54, len(spacings) * 2.54 + fig, axs = H.plt_subplots_imshow(grid, row_labels=[f'Space: {s}px' for s in spacings], figsize=(w, h), label_size=24) + fig.tight_layout() + + H.plt_rel_path_savefig(rel_path=rel_path, save=save, show=show) + + # ========================================================================= # # entrypoint # # ========================================================================= # @@ -130,49 +175,58 @@ def plot_dataset_traversals( plt.style.use(os.path.join(os.path.dirname(__file__), '../gadfly.mplstyle')) # options - all_squares = True - add_random_traversal = True + all_squares = False num_cols = 7 mini_cols = 5 transpose_cols = 3 seed = 47 + INCLUDE_RANDOM_TRAVERSAL = False + TITLE = False + TITLE_MINI = False + TITLE_TRANSPOSE = False + # get name - prefix = 'traversal' if add_random_traversal else 'traversal-noran' + prefix = 'traversal' if INCLUDE_RANDOM_TRAVERSAL else 'traversal-noran' + + # plot increasing levels of overlap + plot_incr_overlap(rel_path=f'plots/traversal-incr-overlap__xy-squares', save=True, show=True, traversal_lim=None) # mini versions - plot_dataset_traversals(XYSquaresData(), rel_path=f'plots/traversal-mini__xy-squares__spacing8', seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(Shapes3dData(), rel_path=f'plots/traversal-mini__shapes3d', seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(DSpritesData(), rel_path=f'plots/traversal-mini__dsprites', seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(SmallNorbData(), rel_path=f'plots/traversal-mini__smallnorb', seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(Cars3dData(), rel_path=f'plots/traversal-mini__cars3d', seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols, take_cols=mini_cols+1) + plot_dataset_traversals(XYSquaresData(), rel_path=f'plots/traversal-mini__xy-squares__spacing8', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) + plot_dataset_traversals(Shapes3dData(), rel_path=f'plots/traversal-mini__shapes3d', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) + plot_dataset_traversals(DSpritesData(), rel_path=f'plots/traversal-mini__dsprites', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) + plot_dataset_traversals(SmallNorbData(), rel_path=f'plots/traversal-mini__smallnorb', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) + plot_dataset_traversals(Cars3dData(), rel_path=f'plots/traversal-mini__cars3d', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols, take_cols=mini_cols+1) # transpose versions - plot_dataset_traversals(XYSquaresData(), rel_path=f'plots/traversal-transpose__xy-squares__spacing8', seed=seed, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(Shapes3dData(), rel_path=f'plots/traversal-transpose__shapes3d', seed=seed, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(DSpritesData(), rel_path=f'plots/traversal-transpose__dsprites', seed=seed, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(SmallNorbData(), rel_path=f'plots/traversal-transpose__smallnorb', seed=seed, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(Cars3dData(), rel_path=f'plots/traversal-transpose__cars3d', seed=seed, transpose=True, add_random_traversal=False, num_cols=transpose_cols, take_cols=mini_cols+1) - - exit(1) + plot_dataset_traversals(XYSquaresData(), rel_path=f'plots/traversal-transpose__xy-squares__spacing8', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) + plot_dataset_traversals(Shapes3dData(), rel_path=f'plots/traversal-transpose__shapes3d', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) + plot_dataset_traversals(DSpritesData(), rel_path=f'plots/traversal-transpose__dsprites', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) + plot_dataset_traversals(SmallNorbData(), rel_path=f'plots/traversal-transpose__smallnorb', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) + plot_dataset_traversals(Cars3dData(), rel_path=f'plots/traversal-transpose__cars3d', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols, take_cols=mini_cols+1) # save images - for i in ([1, 2, 4, 8] if all_squares else [1, 8]): + for i in ([1, 2, 3, 4, 5, 6, 7, 8] if all_squares else [1, 2, 4, 8]): data = XYSquaresData(grid_spacing=i, grid_size=8, no_warnings=True) - plot_dataset_traversals(data, rel_path=f'plots/{prefix}__xy-squares__spacing{i}', seed=seed-40, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(data, rel_path=f'plots/{prefix}__xy-squares__spacing{i}__some', seed=seed-40, add_random_traversal=add_random_traversal, num_cols=num_cols, f_idxs=[0, 3]) - - plot_dataset_traversals(XYObjectData(), rel_path=f'plots/{prefix}__xy-object', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(XYObjectShadedData(), rel_path=f'plots/{prefix}__xy-object-shaded', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(XYBlocksData(), rel_path=f'plots/{prefix}__xy-blocks', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(Shapes3dData(), rel_path=f'plots/{prefix}__shapes3d', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(DSpritesData(), rel_path=f'plots/{prefix}__dsprites', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(DSpritesImagenetData(100, 'bg'), rel_path=f'plots/{prefix}__dsprites-imagenet-bg-100', seed=seed-6, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(DSpritesImagenetData( 50, 'bg'), rel_path=f'plots/{prefix}__dsprites-imagenet-bg-50', seed=seed-6, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(DSpritesImagenetData(100, 'fg'), rel_path=f'plots/{prefix}__dsprites-imagenet-fg-100', seed=seed-6, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(DSpritesImagenetData( 50, 'fg'), rel_path=f'plots/{prefix}__dsprites-imagenet-fg-50', seed=seed-6, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(SmallNorbData(), rel_path=f'plots/{prefix}__smallnorb', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) - plot_dataset_traversals(Cars3dData(), rel_path=f'plots/{prefix}__cars3d', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) + plot_dataset_traversals(data, rel_path=f'plots/{prefix}__xy-squares__spacing{i}', title=TITLE, seed=seed-40, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) + plot_dataset_traversals(data, rel_path=f'plots/{prefix}__xy-squares__spacing{i}__some', title=TITLE, seed=seed-40, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols, f_idxs=[0, 3]) + + plot_dataset_traversals(Shapes3dData(), rel_path=f'plots/{prefix}__shapes3d', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) + plot_dataset_traversals(DSpritesData(), rel_path=f'plots/{prefix}__dsprites', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) + plot_dataset_traversals(SmallNorbData(), rel_path=f'plots/{prefix}__smallnorb', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) + plot_dataset_traversals(Cars3dData(), rel_path=f'plots/{prefix}__cars3d', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) + + exit(1) + + plot_dataset_traversals(XYObjectData(), rel_path=f'plots/{prefix}__xy-object', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) + plot_dataset_traversals(XYObjectShadedData(), rel_path=f'plots/{prefix}__xy-object-shaded', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) + plot_dataset_traversals(XYBlocksData(), rel_path=f'plots/{prefix}__xy-blocks', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) + + plot_dataset_traversals(DSpritesImagenetData(100, 'bg'), rel_path=f'plots/{prefix}__dsprites-imagenet-bg-100', title=TITLE, seed=seed-6, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) + plot_dataset_traversals(DSpritesImagenetData( 50, 'bg'), rel_path=f'plots/{prefix}__dsprites-imagenet-bg-50', title=TITLE, seed=seed-6, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) + plot_dataset_traversals(DSpritesImagenetData(100, 'fg'), rel_path=f'plots/{prefix}__dsprites-imagenet-fg-100', title=TITLE, seed=seed-6, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) + plot_dataset_traversals(DSpritesImagenetData( 50, 'fg'), rel_path=f'plots/{prefix}__dsprites-imagenet-fg-50', title=TITLE, seed=seed-6, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) BASE = os.path.abspath(os.path.join(__file__, '../../../out/adversarial_data_approx')) @@ -193,7 +247,7 @@ def plot_dataset_traversals( ('2021-09-06--05-42-06_INVERT-VSTRONG-cars3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), ('2021-09-06--09-10-59_INVERT-VSTRONG-smallnorb_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), ]: - plot_dataset_traversals(SelfContainedHdf5GroundTruthData(f'{BASE}/{folder}/data.h5'), rel_path=f'plots/{prefix}__{folder}.png', seed=seed, add_random_traversal=add_random_traversal, num_cols=num_cols) + plot_dataset_traversals(SelfContainedHdf5GroundTruthData(f'{BASE}/{folder}/data.h5'), rel_path=f'plots/{prefix}__{folder}.png', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) # ========================================================================= # diff --git a/research/e01_visual_overlap/run_plot_global_dists.py b/research/e01_visual_overlap/run_plot_global_dists.py index f9834e74..08f1c239 100644 --- a/research/e01_visual_overlap/run_plot_global_dists.py +++ b/research/e01_visual_overlap/run_plot_global_dists.py @@ -39,7 +39,6 @@ from disent.dataset import DisentDataset from disent.dataset.data import Cars3dData from disent.dataset.data import DSpritesData -from disent.dataset.data import GroundTruthData from disent.dataset.data import Shapes3dData from disent.dataset.data import XYSquaresData from disent.dataset.transform import ToImgTensorF32 diff --git a/research/e01_visual_overlap/run_plot_traversal_dists.py b/research/e01_visual_overlap/run_plot_traversal_dists.py index 093e88a9..b874f8c7 100644 --- a/research/e01_visual_overlap/run_plot_traversal_dists.py +++ b/research/e01_visual_overlap/run_plot_traversal_dists.py @@ -31,6 +31,7 @@ from typing import Literal from typing import Optional from typing import Sequence +from typing import Tuple from typing import Union import matplotlib.pyplot as plt @@ -44,7 +45,9 @@ from disent.dataset.data import SelfContainedHdf5GroundTruthData from disent.dataset.util.state_space import NonNormalisedFactors from disent.dataset.transform import ToImgTensorF32 +from disent.dataset.util.stats import compute_data_mean_std from disent.util.inout.paths import ensure_parent_dir_exists +from disent.util.profiling import Timer from disent.util.seeds import TempNumpySeed @@ -275,7 +278,7 @@ def plot_ax(stats: dict, i: int, f_idx: int): ) # finalize plot - fig.tight_layout(pad=1.4 if hide_labels else 1.08) + fig.tight_layout() # (pad=1.4 if hide_labels else 1.08) # save the path if save_path is not None: @@ -304,12 +307,15 @@ def plot_traversal_stats( save_path: Optional[str] = None, plot_freq: bool = True, plot_title: Union[bool, str] = False, - fig_block_size: float = 4.0, + plt_scale: float = 6, col_titles: Union[bool, List[str]] = True, hide_axis: bool = True, hide_labels: bool = True, - y_size_offset: float = 0.0, - x_size_offset: float = 0.0, + y_size_offset: float = 0.45, + x_size_offset: float = 0.75, + disable_labels: bool = False, + bottom_labels: bool = False, + label_size: int = 23, ): # - - - - - - - - - - - - - - - - - # @@ -336,9 +342,6 @@ def plot_ax(stats: dict, i: int, f_idx: int): gt_data: GroundTruthData = H.make_data(dataset_or_name) if isinstance(dataset_or_name, str) else dataset_or_name f_idxs = gt_data.normalise_factor_idxs(f_idxs) - # settings - h, w = [2.07 * fig_block_size, len(f_idxs) * fig_block_size] - # get title if isinstance(plot_title, str): suptitle = f'{plot_title}' @@ -357,13 +360,20 @@ def plot_ax(stats: dict, i: int, f_idx: int): num_traversal_sample=num_repeats, ) + labels = None + if (not disable_labels) and grid_titles: + labels = grid_titles + + # settings fig, axs = H.plt_subplots_imshow( grid=list(zip(*grid_t)), title=suptitle, - titles=grid_titles if grid_titles else None, - titles_size=36, + titles=None if bottom_labels else labels, + titles_size=label_size, + col_labels=labels if bottom_labels else None, + label_size=label_size, subplot_padding=None, - figsize=(w, h) + figsize=((1/2.54) * len(f_idxs) * plt_scale + x_size_offset, (1/2.54) * (2) * plt_scale + y_size_offset) ) # recolor axes @@ -388,7 +398,132 @@ def plot_ax(stats: dict, i: int, f_idx: int): # ========================================================================= # -# ENTRY # +# MAIN - DISTS # +# ========================================================================= # + + +@torch.no_grad() +def factor_stats(gt_data: GroundTruthData, f_idxs=None, max_factor_sec: float = 100, factor_repeats: int = 1000, recon_loss: str = 'mse', sample_mode: str = 'random') -> Tuple[Sequence[int], List[np.ndarray]]: + from disent.registry import RECON_LOSSES + recon_loss = RECON_LOSSES[recon_loss](reduction='mean') + + f_dists = [] + f_idxs = gt_data.normalise_factor_idxs(f_idxs) + # for each factor + for f_idx in f_idxs: + dists = [] + with Timer() as t: + # for multiple random factor traversals along the factor + for _ in tqdm(range(factor_repeats), desc=gt_data.factor_names[f_idx]): + # based on: sample_factor_traversal_info(...) # TODO: should add recon loss to that function instead + factors, indices, obs = gt_data.sample_random_obs_traversal(f_idx=f_idx, obs_collect_fn=torch.stack) + # random pairs -- we use this because it does not include [i == i] + idxs_a, idxs_b = H.pair_indices(max_idx=len(indices), mode=sample_mode) + # get distances + d = recon_loss.compute_pairwise_loss(obs[idxs_a], obs[idxs_b]) + # H.plt_subplots_imshow([[np.moveaxis(o.numpy(), 0, -1) for o in obs]]) + # plt.show() + dists.append(d) + # exit early + if t.elapsed > max_factor_sec: + print('max time reached!') + break + # aggregate the average distances + f_dists.append(torch.cat(dists).numpy()) + + return f_idxs, f_dists + + +def get_random_dists(gt_data: GroundTruthData, samples: int = 10000, recon_loss: str = 'mse', max_time_sec: float = 100): + from disent.registry import RECON_LOSSES + from disent.frameworks.helper.reconstructions import ReconLossHandlerMse + recon_loss: ReconLossHandlerMse = RECON_LOSSES[recon_loss](reduction='mean') + + dists = [] + with Timer() as t: + # for multiple random factor traversals along the factor + for _ in tqdm(range(samples), desc=gt_data.name): + # random pair + i, j = np.random.randint(0, len(gt_data), size=2) + # get distance + d = recon_loss.compute_pairwise_loss(gt_data[i][None, ...], gt_data[j][None, ...]) + # plt.show() + dists.append(float(d.flatten())) + # exit early + if t.elapsed > max_time_sec: + print('max time reached!') + break + # done! + return np.array(dists) + + +def print_ave_dists(gt_data: GroundTruthData, samples: int = 25000, recon_loss: str = 'mse', max_time_sec: float = 100): + dists = get_random_dists(gt_data=gt_data, samples=samples, recon_loss=recon_loss, max_time_sec=max_time_sec) + f_mean = np.mean(dists) + f_std = np.std(dists) + print(f'[{gt_data.name}] RANDOM: {f_mean:7.4f} ± {f_std:7.4f}') + + +def print_ave_factor_stats(gt_data: GroundTruthData, f_idxs=None, max_factor_sec: float = 100, factor_repeats: int = 1000, recon_loss: str = 'mse', sample_mode: str = 'random'): + # compute average distances + f_idxs, f_dists = factor_stats(gt_data=gt_data, f_idxs=f_idxs, max_factor_sec=max_factor_sec, factor_repeats=factor_repeats, recon_loss=recon_loss, sample_mode=sample_mode) + # compute dists + f_means = [np.mean(d) for d in f_dists] + f_stds = [np.std(d) for d in f_dists] + # sort in order of importance + order = np.argsort(f_means)[::-1] + # print information + for i in order: + f_idx, f_mean, f_std = f_idxs[i], f_means[i], f_stds[i] + print(f'[{gt_data.name}] {gt_data.factor_names[f_idx]}: {f_mean:7.4f} ± {f_std:7.4f}') + + +def main_compute_dists(factor_repeats: int = 15000, random_samples: int = 100000, max_time_sec: float = 100, recon_loss: str = 'mse', sample_mode: str = 'random'): + # plot standard datasets + for name in ['dsprites', 'shapes3d', 'cars3d', 'smallnorb', 'xysquares_8x8_s8']: + gt_data = H.make_data(name) + if factor_repeats is not None: + print_ave_factor_stats(gt_data, max_factor_sec=max_time_sec, factor_repeats=factor_repeats, recon_loss=recon_loss, sample_mode=sample_mode) + if random_samples is not None: + print_ave_dists(gt_data, samples=random_samples, recon_loss=recon_loss, max_time_sec=max_time_sec) + +# [dsprites] position_y: 0.0564 ± 0.0367 +# [dsprites] scale: 0.0249 ± 0.0146 +# [dsprites] shape: 0.0214 ± 0.0095 +# [dsprites] orientation: 0.0163 ± 0.0104 +# [dsprites] RANDOM: 0.0753 ± 0.0289 + +# [3dshapes] wall_hue: 0.1127 ± 0.0663 +# [3dshapes] floor_hue: 0.1093 ± 0.0620 +# [3dshapes] object_hue: 0.0424 ± 0.0293 +# [3dshapes] shape: 0.0210 ± 0.0167 +# [3dshapes] scale: 0.0181 ± 0.0149 +# [3dshapes] orientation: 0.0116 ± 0.0081 +# [3dshapes] RANDOM: 0.2440 ± 0.0922 + +# [cars3d] azimuth: 0.0358 ± 0.0185 +# [cars3d] object_type: 0.0340 ± 0.0172 +# [cars3d] elevation: 0.0174 ± 0.0100 +# [cars3d] RANDOM: 0.0522 ± 0.0189 + +# [smallnorb] lighting: 0.0533 ± 0.0565 +# [smallnorb] category: 0.0113 ± 0.0066 +# [smallnorb] rotation: 0.0090 ± 0.0068 +# [smallnorb] instance: 0.0069 ± 0.0049 +# [smallnorb] elevation: 0.0035 ± 0.0032 +# [smallnorb] RANDOM: 0.0530 ± 0.0524 + +# [xy_squares] y_B: 0.0104 ± 0.0000 +# [xy_squares] x_B: 0.0104 ± 0.0000 +# [xy_squares] y_G: 0.0104 ± 0.0000 +# [xy_squares] x_G: 0.0104 ± 0.0000 +# [xy_squares] y_R: 0.0104 ± 0.0000 +# [xy_squares] x_R: 0.0104 ± 0.0000 +# [xy_squares] RANDOM: 0.0308 ± 0.0022 + + +# ========================================================================= # +# MAIN - PLOTTING # # ========================================================================= # @@ -396,10 +531,15 @@ def _make_self_contained_dataset(h5_path): return SelfContainedHdf5GroundTruthData(h5_path=h5_path, transform=ToImgTensorF32()) -if __name__ == '__main__': - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../gadfly.mplstyle')) +def _print_data_mean_std(data_or_name, print_mean_std: bool = True): + if print_mean_std: + data = H.make_data(data_or_name) if isinstance(data_or_name, str) else data_or_name + name = data_or_name if isinstance(data_or_name, str) else data.name + mean, std = compute_data_mean_std(data) + print(f'{name}\n vis_mean: {mean.tolist()}\n vis_std: {std.tolist()}') + +def main_plotting(plot_all=False, print_mean_std=False): CIRCULAR = False PLOT_FREQ = False @@ -410,21 +550,24 @@ def sp(name): # plot xysquares with increasing overlap for s in [1, 2, 3, 4, 5, 6, 7, 8]: - plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(f'xysquares_8x8_s{s}'), color='blue', dataset_or_name=f'xysquares_8x8_s{s}', f_idxs=[1], col_titles=['x & y'], plot_freq=PLOT_FREQ) + plot_traversal_stats(circular_distance=CIRCULAR, plt_scale=8, label_size=26, x_size_offset=0, y_size_offset=0.6, save_path=sp(f'xysquares_8x8_s{s}'), color='blue', dataset_or_name=f'xysquares_8x8_s{s}', f_idxs=[1], col_titles=[f'Space: {s}px'], plot_freq=PLOT_FREQ) + _print_data_mean_std(f'xysquares_8x8_s{s}', print_mean_std) # plot standard datasets for name in ['dsprites', 'shapes3d', 'cars3d', 'smallnorb']: - plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(name), color='blue', dataset_or_name=name, plot_freq=PLOT_FREQ) + plot_traversal_stats(circular_distance=CIRCULAR, x_size_offset=0, y_size_offset=0.6, num_repeats=256, disable_labels=False, save_path=sp(name), color='blue', dataset_or_name=name, plot_freq=PLOT_FREQ) + _print_data_mean_std(name, print_mean_std) + + if not plot_all: + return # plot adversarial dsprites datasets for fg in [True, False]: for vis in [100, 80, 60, 40, 20]: name = f'dsprites_imagenet_{"fg" if fg else "bg"}_{vis}' plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(name), color='orange', dataset_or_name=name, plot_freq=PLOT_FREQ, x_size_offset=0.4) - # mean, std = compute_data_mean_std(H.make_data(name)) - # print(f'{name}\n vis_mean: {mean.tolist()}\n vis_std: {std.tolist()}') + _print_data_mean_std(name, print_mean_std) - exit(1) BASE = os.path.abspath(os.path.join(__file__, '../../../out/adversarial_data_approx')) # plot adversarial datasets @@ -447,9 +590,20 @@ def sp(name): ]: data = _make_self_contained_dataset(f'{BASE}/{folder}/data.h5') plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(folder), color=color, dataset_or_name=data, plot_freq=PLOT_FREQ, x_size_offset=0.4) - # compute and print statistics: - # mean, std = compute_data_mean_std(data, progress=True) - # print(f'{folder}\n vis_mean: {mean.tolist()}\n vis_std: {std.tolist()}') + _print_data_mean_std(data, print_mean_std) + + +# ========================================================================= # +# STATS # +# ========================================================================= # + + +if __name__ == '__main__': + # matplotlib style + plt.style.use(os.path.join(os.path.dirname(__file__), '../gadfly.mplstyle')) + # run! + # main_plotting() + main_compute_dists() # ========================================================================= # From 466876af40272156c8c40a61a4b26a99288d99f7 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Tue, 16 Nov 2021 20:01:10 +0200 Subject: [PATCH 24/30] factor dists --- research/e00_tuning/submit_param_tuning.sh | 1 + .../run_plot_traversal_dists.py | 126 +++++++++--------- 2 files changed, 63 insertions(+), 64 deletions(-) diff --git a/research/e00_tuning/submit_param_tuning.sh b/research/e00_tuning/submit_param_tuning.sh index f9f1f647..d7d1fa7b 100644 --- a/research/e00_tuning/submit_param_tuning.sh +++ b/research/e00_tuning/submit_param_tuning.sh @@ -29,6 +29,7 @@ clog_cudaless_nodes "$PARTITION" 129600 "C-disent" # 36 hours # adavae with zsize=25 does not work # adavae with zsize=09 and beta<=0.001 (must get very lucky) +# TODO: I should try lower the learning rate to 1e-4 from 1e-3, this might help with xysquares # 1 * (2 * 8 * 2 * 5) = 160 submit_sweep \ +DUMMY.repeat=1 \ diff --git a/research/e01_visual_overlap/run_plot_traversal_dists.py b/research/e01_visual_overlap/run_plot_traversal_dists.py index b874f8c7..56283d35 100644 --- a/research/e01_visual_overlap/run_plot_traversal_dists.py +++ b/research/e01_visual_overlap/run_plot_traversal_dists.py @@ -403,70 +403,66 @@ def plot_ax(stats: dict, i: int, f_idx: int): @torch.no_grad() -def factor_stats(gt_data: GroundTruthData, f_idxs=None, max_factor_sec: float = 100, factor_repeats: int = 1000, recon_loss: str = 'mse', sample_mode: str = 'random') -> Tuple[Sequence[int], List[np.ndarray]]: +def factor_stats(gt_data: GroundTruthData, f_idxs=None, min_samples: int = 100_000, min_repeats: int = 5000, recon_loss: str = 'mse', sample_mode: str = 'random') -> Tuple[Sequence[int], List[np.ndarray]]: from disent.registry import RECON_LOSSES - recon_loss = RECON_LOSSES[recon_loss](reduction='mean') + from disent.frameworks.helper.reconstructions import ReconLossHandler + recon_loss: ReconLossHandler = RECON_LOSSES[recon_loss](reduction='mean') f_dists = [] f_idxs = gt_data.normalise_factor_idxs(f_idxs) # for each factor for f_idx in f_idxs: dists = [] - with Timer() as t: + with tqdm(desc=gt_data.factor_names[f_idx], total=min_samples) as p: # for multiple random factor traversals along the factor - for _ in tqdm(range(factor_repeats), desc=gt_data.factor_names[f_idx]): + while len(dists) < min_samples or p.n < min_repeats: # based on: sample_factor_traversal_info(...) # TODO: should add recon loss to that function instead factors, indices, obs = gt_data.sample_random_obs_traversal(f_idx=f_idx, obs_collect_fn=torch.stack) # random pairs -- we use this because it does not include [i == i] idxs_a, idxs_b = H.pair_indices(max_idx=len(indices), mode=sample_mode) # get distances d = recon_loss.compute_pairwise_loss(obs[idxs_a], obs[idxs_b]) + d = d.numpy().tolist() # H.plt_subplots_imshow([[np.moveaxis(o.numpy(), 0, -1) for o in obs]]) # plt.show() - dists.append(d) - # exit early - if t.elapsed > max_factor_sec: - print('max time reached!') - break + dists.extend(d) + p.update(len(d)) # aggregate the average distances - f_dists.append(torch.cat(dists).numpy()) + f_dists.append(np.array(dists)[:min_samples]) return f_idxs, f_dists -def get_random_dists(gt_data: GroundTruthData, samples: int = 10000, recon_loss: str = 'mse', max_time_sec: float = 100): +def get_random_dists(gt_data: GroundTruthData, num_samples: int = 100_000, recon_loss: str = 'mse'): from disent.registry import RECON_LOSSES - from disent.frameworks.helper.reconstructions import ReconLossHandlerMse - recon_loss: ReconLossHandlerMse = RECON_LOSSES[recon_loss](reduction='mean') + from disent.frameworks.helper.reconstructions import ReconLossHandler + recon_loss: ReconLossHandler = RECON_LOSSES[recon_loss](reduction='mean') dists = [] - with Timer() as t: + with tqdm(desc=gt_data.name, total=num_samples) as p: # for multiple random factor traversals along the factor - for _ in tqdm(range(samples), desc=gt_data.name): + while len(dists) < num_samples: # random pair i, j = np.random.randint(0, len(gt_data), size=2) # get distance d = recon_loss.compute_pairwise_loss(gt_data[i][None, ...], gt_data[j][None, ...]) # plt.show() dists.append(float(d.flatten())) - # exit early - if t.elapsed > max_time_sec: - print('max time reached!') - break + p.update() # done! return np.array(dists) -def print_ave_dists(gt_data: GroundTruthData, samples: int = 25000, recon_loss: str = 'mse', max_time_sec: float = 100): - dists = get_random_dists(gt_data=gt_data, samples=samples, recon_loss=recon_loss, max_time_sec=max_time_sec) +def print_ave_dists(gt_data: GroundTruthData, num_samples: int = 100_000, recon_loss: str = 'mse'): + dists = get_random_dists(gt_data=gt_data, num_samples=num_samples, recon_loss=recon_loss) f_mean = np.mean(dists) f_std = np.std(dists) - print(f'[{gt_data.name}] RANDOM: {f_mean:7.4f} ± {f_std:7.4f}') + print(f'[{gt_data.name}] RANDOM ({len(gt_data)}, {len(dists)}) - mean: {f_mean:7.4f} std: {f_std:7.4f}') -def print_ave_factor_stats(gt_data: GroundTruthData, f_idxs=None, max_factor_sec: float = 100, factor_repeats: int = 1000, recon_loss: str = 'mse', sample_mode: str = 'random'): +def print_ave_factor_stats(gt_data: GroundTruthData, f_idxs=None, min_samples: int = 100_000, min_repeats: int = 5000, recon_loss: str = 'mse', sample_mode: str = 'random'): # compute average distances - f_idxs, f_dists = factor_stats(gt_data=gt_data, f_idxs=f_idxs, max_factor_sec=max_factor_sec, factor_repeats=factor_repeats, recon_loss=recon_loss, sample_mode=sample_mode) + f_idxs, f_dists = factor_stats(gt_data=gt_data, f_idxs=f_idxs, min_repeats=min_repeats, min_samples=min_samples, recon_loss=recon_loss, sample_mode=sample_mode) # compute dists f_means = [np.mean(d) for d in f_dists] f_stds = [np.std(d) for d in f_dists] @@ -475,52 +471,54 @@ def print_ave_factor_stats(gt_data: GroundTruthData, f_idxs=None, max_factor_sec # print information for i in order: f_idx, f_mean, f_std = f_idxs[i], f_means[i], f_stds[i] - print(f'[{gt_data.name}] {gt_data.factor_names[f_idx]}: {f_mean:7.4f} ± {f_std:7.4f}') + print(f'[{gt_data.name}] {gt_data.factor_names[f_idx]} ({gt_data.factor_sizes[f_idx]}, {len(f_dists[f_idx])}) - mean: {f_mean:7.4f} std: {f_std:7.4f}') -def main_compute_dists(factor_repeats: int = 15000, random_samples: int = 100000, max_time_sec: float = 100, recon_loss: str = 'mse', sample_mode: str = 'random'): +def main_compute_dists(factor_samples: int = 50_000, min_repeats: int = 5000, random_samples: int = 50_000, recon_loss: str = 'mse', sample_mode: str = 'random', seed: int = 777): # plot standard datasets for name in ['dsprites', 'shapes3d', 'cars3d', 'smallnorb', 'xysquares_8x8_s8']: gt_data = H.make_data(name) - if factor_repeats is not None: - print_ave_factor_stats(gt_data, max_factor_sec=max_time_sec, factor_repeats=factor_repeats, recon_loss=recon_loss, sample_mode=sample_mode) + if factor_samples is not None: + with TempNumpySeed(seed): + print_ave_factor_stats(gt_data, min_samples=factor_samples, min_repeats=min_repeats, recon_loss=recon_loss, sample_mode=sample_mode) if random_samples is not None: - print_ave_dists(gt_data, samples=random_samples, recon_loss=recon_loss, max_time_sec=max_time_sec) - -# [dsprites] position_y: 0.0564 ± 0.0367 -# [dsprites] scale: 0.0249 ± 0.0146 -# [dsprites] shape: 0.0214 ± 0.0095 -# [dsprites] orientation: 0.0163 ± 0.0104 -# [dsprites] RANDOM: 0.0753 ± 0.0289 - -# [3dshapes] wall_hue: 0.1127 ± 0.0663 -# [3dshapes] floor_hue: 0.1093 ± 0.0620 -# [3dshapes] object_hue: 0.0424 ± 0.0293 -# [3dshapes] shape: 0.0210 ± 0.0167 -# [3dshapes] scale: 0.0181 ± 0.0149 -# [3dshapes] orientation: 0.0116 ± 0.0081 -# [3dshapes] RANDOM: 0.2440 ± 0.0922 - -# [cars3d] azimuth: 0.0358 ± 0.0185 -# [cars3d] object_type: 0.0340 ± 0.0172 -# [cars3d] elevation: 0.0174 ± 0.0100 -# [cars3d] RANDOM: 0.0522 ± 0.0189 - -# [smallnorb] lighting: 0.0533 ± 0.0565 -# [smallnorb] category: 0.0113 ± 0.0066 -# [smallnorb] rotation: 0.0090 ± 0.0068 -# [smallnorb] instance: 0.0069 ± 0.0049 -# [smallnorb] elevation: 0.0035 ± 0.0032 -# [smallnorb] RANDOM: 0.0530 ± 0.0524 - -# [xy_squares] y_B: 0.0104 ± 0.0000 -# [xy_squares] x_B: 0.0104 ± 0.0000 -# [xy_squares] y_G: 0.0104 ± 0.0000 -# [xy_squares] x_G: 0.0104 ± 0.0000 -# [xy_squares] y_R: 0.0104 ± 0.0000 -# [xy_squares] x_R: 0.0104 ± 0.0000 -# [xy_squares] RANDOM: 0.0308 ± 0.0022 - + with TempNumpySeed(seed): + print_ave_dists(gt_data, num_samples=random_samples, recon_loss=recon_loss) + +# [dsprites] position_y (32, 50000) - mean: 0.0584 std: 0.0378 +# [dsprites] position_x (32, 50000) - mean: 0.0559 std: 0.0363 +# [dsprites] scale (6, 50000) - mean: 0.0250 std: 0.0148 +# [dsprites] shape (3, 50000) - mean: 0.0214 std: 0.0095 +# [dsprites] orientation (40, 50000) - mean: 0.0172 std: 0.0106 +# [dsprites] RANDOM (737280, 50000) - mean: 0.0754 std: 0.0289 + +# [3dshapes] wall_hue (10, 50000) - mean: 0.1122 std: 0.0661 +# [3dshapes] floor_hue (10, 50000) - mean: 0.1086 std: 0.0623 +# [3dshapes] object_hue (10, 50000) - mean: 0.0416 std: 0.0292 +# [3dshapes] shape (4, 50000) - mean: 0.0207 std: 0.0161 +# [3dshapes] scale (8, 50000) - mean: 0.0182 std: 0.0153 +# [3dshapes] orientation (15, 50000) - mean: 0.0116 std: 0.0079 +# [3dshapes] RANDOM (480000, 50000) - mean: 0.2432 std: 0.0918 + +# [cars3d] azimuth (24, 50000) - mean: 0.0355 std: 0.0185 +# [cars3d] object_type (183, 50000) - mean: 0.0349 std: 0.0176 +# [cars3d] elevation (4, 50000) - mean: 0.0174 std: 0.0100 +# [cars3d] RANDOM (17568, 50000) - mean: 0.0519 std: 0.0188 + +# [smallnorb] lighting (6, 50000) - mean: 0.0531 std: 0.0563 +# [smallnorb] category (5, 50000) - mean: 0.0113 std: 0.0066 +# [smallnorb] rotation (18, 50000) - mean: 0.0090 std: 0.0071 +# [smallnorb] instance (5, 50000) - mean: 0.0068 std: 0.0048 +# [smallnorb] elevation (9, 50000) - mean: 0.0034 std: 0.0030 +# [smallnorb] RANDOM (24300, 50000) - mean: 0.0535 std: 0.0529 + +# [xy_squares] y_B (8, 50000) - mean: 0.0104 std: 0.0000 +# [xy_squares] x_B (8, 50000) - mean: 0.0104 std: 0.0000 +# [xy_squares] y_G (8, 50000) - mean: 0.0104 std: 0.0000 +# [xy_squares] x_G (8, 50000) - mean: 0.0104 std: 0.0000 +# [xy_squares] y_R (8, 50000) - mean: 0.0104 std: 0.0000 +# [xy_squares] x_R (8, 50000) - mean: 0.0104 std: 0.0000 +# [xy_squares] RANDOM (262144, 50000) - mean: 0.0308 std: 0.0022 # ========================================================================= # # MAIN - PLOTTING # From cd5f1dc1ea27e8e95647d297f1f58812c4c502dd Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Mon, 22 Nov 2021 11:52:55 +0200 Subject: [PATCH 25/30] vae vs ae experiments --- research/e00_tuning/submit_param_tuning.sh | 21 +++++++++++++ .../submit_overlap_loss.sh | 30 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/research/e00_tuning/submit_param_tuning.sh b/research/e00_tuning/submit_param_tuning.sh index d7d1fa7b..677ba8dd 100644 --- a/research/e00_tuning/submit_param_tuning.sh +++ b/research/e00_tuning/submit_param_tuning.sh @@ -48,7 +48,28 @@ submit_sweep \ sampling=default__bb +# TEST DISTANCES IN AEs VS VAEs +# -- supplementary material +# 3 * (1 * 5 = 2) = 15 +submit_sweep \ + +DUMMY.repeat=1,2,3 \ + +EXTRA.tags='sweep_ae' \ + hydra.job.name="ae_test" \ + \ + run_length=medium \ + metrics=all \ + \ + settings.framework.beta=0.0001 \ + framework=ae \ + schedule=none \ + settings.model.z_size=25 \ + \ + dataset=dsprites,shapes3d,cars3d,smallnorb,X--xysquares \ + sampling=default__bb + + # RUN SWEEP FOR GOOD SCHEDULES +# -- unused # 1 * (3 * 2 * 4 * 5) = 120 #submit_sweep \ # +DUMMY.repeat=1 \ diff --git a/research/e09_vae_overlap_loss/submit_overlap_loss.sh b/research/e09_vae_overlap_loss/submit_overlap_loss.sh index 9fdbcdb0..2c7ed00e 100644 --- a/research/e09_vae_overlap_loss/submit_overlap_loss.sh +++ b/research/e09_vae_overlap_loss/submit_overlap_loss.sh @@ -18,6 +18,8 @@ source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours +# TEST MSE vs BoxBlur MSE (with different beta values over different datasets) +# - mse boxblur weight is too strong, need to lower significantly # 1 * (5 * 2*4*2) = 80 #submit_sweep \ # +DUMMY.repeat=1 \ @@ -41,6 +43,8 @@ clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours # sampling=default__bb +# TEST MSE vs BoxBlur MSE +# - changing the reconstruction loss enables disentanglement # 5 * (2*2*2 = 8) = 40 submit_sweep \ +DUMMY.repeat=1,2,3,4,5 \ @@ -64,7 +68,33 @@ submit_sweep \ sampling=default__bb +# TEST DISTANCES IN AEs VS VAEs +# -- supplementary material +# 3 * (1 * 2 = 2) = 6 +submit_sweep \ + +DUMMY.repeat=1,2,3 \ + +EXTRA.tags='sweep_overlap_boxblur_autoencoders' \ + hydra.job.name="e_ovlp_loss" \ + \ + +VAR.recon_loss_weight=1.0 \ + +VAR.kernel_loss_weight=3969.0 \ + +VAR.kernel_radius=31 \ + \ + run_length=medium \ + metrics=all \ + \ + dataset=X--xysquares \ + \ + framework=ae \ + settings.framework.beta=0.0001 \ + settings.model.z_size=25 \ + settings.framework.recon_loss=mse,'mse_box_r${VAR.kernel_radius}_l${VAR.recon_loss_weight}_k${VAR.kernel_loss_weight}' \ + \ + sampling=default__bb + + # HPARAM SWEEP -- TODO: update +# -- old, unused # 1 * (2 * 8 * 2 * 2) = 160 #submit_sweep \ # +DUMMY.repeat=1 \ From b23ffac61ffe81771e9eb9d1c3db1e9474b2b43a Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Mon, 22 Nov 2021 11:53:04 +0200 Subject: [PATCH 26/30] fix z distance from l1 to l2 --- disent/util/lightning/callbacks/_callbacks_vae.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/disent/util/lightning/callbacks/_callbacks_vae.py b/disent/util/lightning/callbacks/_callbacks_vae.py index 28d5c269..e846b86d 100644 --- a/disent/util/lightning/callbacks/_callbacks_vae.py +++ b/disent/util/lightning/callbacks/_callbacks_vae.py @@ -122,8 +122,8 @@ def _to_dmat( return dmat -_AE_DIST_NAMES = ('x', 'z_l1', 'x_recon') -_VAE_DIST_NAMES = ('x', 'z_l1', 'kl', 'x_recon') +_AE_DIST_NAMES = ('x', 'z', 'x_recon') +_VAE_DIST_NAMES = ('x', 'z', 'kl', 'x_recon') @torch.no_grad() @@ -134,7 +134,7 @@ def _get_dists_ae(ae: Ae, x_a: torch.Tensor, x_b: torch.Tensor): # distances return [ ae.recon_handler.compute_pairwise_loss(x_a, x_b), - torch.norm(z_a - z_b, p=1, dim=-1), # l1 dist + torch.norm(z_a - z_b, p=2, dim=-1), # l2 dist ae.recon_handler.compute_pairwise_loss(r_a, r_b), ] @@ -151,7 +151,7 @@ def _get_dists_vae(vae: Vae, x_a: torch.Tensor, x_b: torch.Tensor): # distances return [ vae.recon_handler.compute_pairwise_loss(x_a, x_b), - torch.norm(z_a - z_b, p=1, dim=-1), # l1 dist + torch.norm(z_a - z_b, p=2, dim=-1), # l2 dist vae.recon_handler._pairwise_reduce(kl_ab), vae.recon_handler.compute_pairwise_loss(r_a, r_b), ] From 13f715823b18ee62597b83a65b4b5412dccb329e Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Mon, 22 Nov 2021 12:08:22 +0200 Subject: [PATCH 27/30] cleanup --- research/e00_data_traversal/run_01_all_shared_data_prepare.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/research/e00_data_traversal/run_01_all_shared_data_prepare.sh b/research/e00_data_traversal/run_01_all_shared_data_prepare.sh index 32afb6c6..bdd2026e 100644 --- a/research/e00_data_traversal/run_01_all_shared_data_prepare.sh +++ b/research/e00_data_traversal/run_01_all_shared_data_prepare.sh @@ -39,12 +39,12 @@ DATASETS=( #X--dsprites-imagenet-bg-40 #X--dsprites-imagenet-bg-60 #X--dsprites-imagenet-bg-80 - X--dsprites-imagenet-bg-100 + #X--dsprites-imagenet-bg-100 #X--dsprites-imagenet-fg-20 #X--dsprites-imagenet-fg-40 #X--dsprites-imagenet-fg-60 #X--dsprites-imagenet-fg-80 - X--dsprites-imagenet-fg-100 + #X--dsprites-imagenet-fg-100 #X--mask-adv-f-cars3d #X--mask-adv-f-dsprites #X--mask-adv-f-shapes3d From 0d44570a3f9125cc1f2606930e679b9fb942c086 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Mon, 22 Nov 2021 14:00:46 +0200 Subject: [PATCH 28/30] update experiment descriptions --- research/e00_tuning/submit_param_tuning.sh | 11 +++++++++++ research/e09_vae_overlap_loss/submit_overlap_loss.sh | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/research/e00_tuning/submit_param_tuning.sh b/research/e00_tuning/submit_param_tuning.sh index 677ba8dd..4f554dfa 100644 --- a/research/e00_tuning/submit_param_tuning.sh +++ b/research/e00_tuning/submit_param_tuning.sh @@ -1,5 +1,16 @@ #!/bin/bash + +# OVERVIEW: +# - this experiment is designed to find the optimal hyper-parameters for disentanglement, as well as investigate the +# effect of the adversarial XYSquares dataset against existing approaches. + + +# OUTCOMES: +# - Existing frameworks fail on the adversarial dataset +# - Much lower beta is required for adversarial dataset + + # ========================================================================= # # Settings # # ========================================================================= # diff --git a/research/e09_vae_overlap_loss/submit_overlap_loss.sh b/research/e09_vae_overlap_loss/submit_overlap_loss.sh index 2c7ed00e..d51572b2 100644 --- a/research/e09_vae_overlap_loss/submit_overlap_loss.sh +++ b/research/e09_vae_overlap_loss/submit_overlap_loss.sh @@ -1,5 +1,17 @@ #!/bin/bash +# OVERVIEW: +# - this experiment is designed to test how changing the reconstruction loss to match the +# ground-truth distances allows datasets to be disentangled. + + +# OUTCOMES: +# - When the reconstruction loss is used as a distance function between observations, and those +# distances match the ground truth, it enables disentanglement. +# - Loss must still be able to reconstruct the inputs correctly. +# - AEs have no incentive to learn the same distances as VAEs + + # ========================================================================= # # Settings # # ========================================================================= # From 9ba4dc1b31caf9754ad7f378bb868bc5afafd5c2 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Mon, 22 Nov 2021 14:09:11 +0200 Subject: [PATCH 29/30] version bump v0.3.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3505105e..a57f1318 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ author="Nathan Juraj Michlo", author_email="NathanJMichlo@gmail.com", - version="0.3.1", + version="0.3.2", python_requires=">=3.8", # we make use of standard library features only in 3.8 packages=setuptools.find_packages(), From cfacba984199f9979e5b829214b3f5813b5d6524 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Mon, 22 Nov 2021 14:10:05 +0200 Subject: [PATCH 30/30] run prepare_release.sh --- ...47-1_s28800_adam_lr0.003_wd0.0001_xy1x1.pt | Bin 36920 -> 0 bytes ...47-1_s28800_adam_lr0.003_wd0.0001_xy8x8.pt | Bin 36920 -> 0 bytes .../r47-1_s28800_adam_lr0.003_wd0.0_xy1x1.pt | Bin 36920 -> 0 bytes .../r47-1_s28800_adam_lr0.003_wd0.0_xy8x8.pt | Bin 36920 -> 0 bytes disent/dataset/data/__init__.py | 5 - disent/dataset/data/_groundtruth__xcolumns.py | 66 -- disent/dataset/data/_groundtruth__xyblocks.py | 160 ----- .../dataset/data/_groundtruth__xysquares.py | 202 ------ disent/dataset/transform/_augment.py | 2 - disent/dataset/util/stats.py | 14 - disent/frameworks/ae/experimental/__init__.py | 32 - .../experimental/_supervised__adaneg_tae.py | 71 -- .../ae/experimental/_unsupervised__dotae.py | 76 -- .../experimental/_weaklysupervised__adaae.py | 81 --- disent/frameworks/helper/reconstructions.py | 2 - .../frameworks/vae/experimental/__init__.py | 41 -- .../experimental/_supervised__adaave_tvae.py | 120 ---- .../experimental/_supervised__adaneg_tvae.py | 118 ---- .../vae/experimental/_supervised__adatvae.py | 328 --------- .../vae/experimental/_supervised__badavae.py | 121 ---- .../vae/experimental/_supervised__gadavae.py | 102 --- .../vae/experimental/_supervised__tbadavae.py | 51 -- .../vae/experimental/_supervised__tgadavae.py | 51 -- .../vae/experimental/_unsupervised__dorvae.py | 169 ----- .../vae/experimental/_unsupervised__dotvae.py | 222 ------ .../_weaklysupervised__augpostriplet.py | 82 --- .../_weaklysupervised__st_adavae.py | 63 -- .../_weaklysupervised__st_betavae.py | 63 -- disent/metrics/__init__.py | 7 - disent/metrics/_flatness.py | 347 ---------- disent/metrics/_flatness_components.py | 412 ----------- disent/registry/__init__.py | 23 - experiment/config/config.yaml | 2 - .../config/config_adversarial_dataset.yaml | 60 -- .../config_adversarial_dataset_approx.yaml | 121 ---- .../config/config_adversarial_kernel.yaml | 50 -- experiment/config/config_test.yaml | 2 - .../dataset/X--adv-cars3d--WARNING.yaml | 20 - .../dataset/X--adv-dsprites--WARNING.yaml | 20 - .../dataset/X--adv-shapes3d--WARNING.yaml | 20 - .../dataset/X--adv-smallnorb--WARNING.yaml | 20 - .../dataset/X--dsprites-imagenet-bg-100.yaml | 22 - .../dataset/X--dsprites-imagenet-bg-20.yaml | 22 - .../dataset/X--dsprites-imagenet-bg-40.yaml | 22 - .../dataset/X--dsprites-imagenet-bg-60.yaml | 22 - .../dataset/X--dsprites-imagenet-bg-80.yaml | 22 - .../dataset/X--dsprites-imagenet-fg-100.yaml | 22 - .../dataset/X--dsprites-imagenet-fg-20.yaml | 22 - .../dataset/X--dsprites-imagenet-fg-40.yaml | 22 - .../dataset/X--dsprites-imagenet-fg-60.yaml | 22 - .../dataset/X--dsprites-imagenet-fg-80.yaml | 22 - .../config/dataset/X--dsprites-imagenet.yaml | 54 -- .../config/dataset/X--mask-adv-f-cars3d.yaml | 28 - .../dataset/X--mask-adv-f-dsprites.yaml | 28 - .../dataset/X--mask-adv-f-shapes3d.yaml | 28 - .../dataset/X--mask-adv-f-smallnorb.yaml | 29 - .../config/dataset/X--mask-adv-r-cars3d.yaml | 28 - .../dataset/X--mask-adv-r-dsprites.yaml | 28 - .../dataset/X--mask-adv-r-shapes3d.yaml | 28 - .../dataset/X--mask-adv-r-smallnorb.yaml | 29 - .../config/dataset/X--mask-dthr-cars3d.yaml | 24 - .../config/dataset/X--mask-dthr-dsprites.yaml | 24 - .../config/dataset/X--mask-dthr-shapes3d.yaml | 24 - .../dataset/X--mask-dthr-smallnorb.yaml | 25 - .../config/dataset/X--mask-ran-cars3d.yaml | 28 - .../config/dataset/X--mask-ran-dsprites.yaml | 28 - .../config/dataset/X--mask-ran-shapes3d.yaml | 28 - .../config/dataset/X--mask-ran-smallnorb.yaml | 29 - experiment/config/dataset/X--xyblocks.yaml | 18 - .../config/dataset/X--xyblocks_grey.yaml | 18 - experiment/config/dataset/X--xysquares.yaml | 17 - .../config/dataset/X--xysquares_grey.yaml | 23 - .../config/dataset/X--xysquares_rgb.yaml | 23 - experiment/config/framework/X--adaae.yaml | 19 - experiment/config/framework/X--adaae_os.yaml | 19 - .../config/framework/X--adaavetvae.yaml | 45 -- experiment/config/framework/X--adanegtae.yaml | 27 - .../config/framework/X--adanegtvae.yaml | 37 - experiment/config/framework/X--adatvae.yaml | 42 -- .../config/framework/X--augpos_tvae_os.yaml | 46 -- experiment/config/framework/X--badavae.yaml | 27 - experiment/config/framework/X--dorvae.yaml | 38 - .../config/framework/X--dorvae_aug.yaml | 43 -- experiment/config/framework/X--dotae.yaml | 35 - experiment/config/framework/X--dotvae.yaml | 45 -- .../config/framework/X--dotvae_aug.yaml | 70 -- experiment/config/framework/X--gadavae.yaml | 29 - experiment/config/framework/X--st-adavae.yaml | 29 - .../config/framework/X--st-betavae.yaml | 25 - experiment/config/framework/X--tbadavae.yaml | 33 - experiment/config/framework/X--tgadavae.yaml | 35 - experiment/config/metrics/all.yaml | 2 - experiment/config/metrics/fast.yaml | 2 - experiment/config/metrics/test.yaml | 4 - experiment/config/run_location/griffin.yaml | 29 - .../config/run_location/heartofgold.yaml | 29 - prepare_release.sh | 82 --- prepare_release_and_commit.sh | 36 - requirements-research-freeze.txt | 121 ---- requirements-research.txt | 19 - research/__init__.py | 23 - research/clog-batch.sh | 19 - research/clog-stampede.sh | 19 - research/e00_data_traversal/plots/.gitignore | 1 - .../run_01_all_shared_data_prepare.sh | 79 --- .../run_02_plot_data_overlap.py | 187 ----- .../run_02_plot_traversals.py | 255 ------- research/e00_tuning/submit_param_tuning.sh | 98 --- research/e01_incr_overlap/run.py | 72 -- .../e01_incr_overlap/submit_incr_overlap.sh | 127 ---- research/e01_visual_overlap/plots/.gitignore | 1 - .../run_01_x_z_recon_dists.sh | 38 - .../run_plot_global_dists.py | 465 ------------- .../run_plot_traversal_dists.py | 654 ------------------ .../util_compute_traversal_dist_pairs.py | 274 -------- .../util_compute_traversal_dists.py | 303 -------- .../submit_01_triplet_hparam_sweep.sh | 59 -- .../submit_02_check_vae_equivalence.sh | 49 -- research/e03_axis_triplet/submit_01.sh | 63 -- research/e03_axis_triplet/submit_02.sh | 48 -- research/e03_axis_triplet/submit_03.sh | 78 --- research/e03_axis_triplet/submit_04.sh | 119 ---- research/e03_axis_triplet/submit_05.sh | 57 -- .../e04_data_overlap_triplet/submit_01.sh | 62 -- .../e04_data_overlap_triplet/submit_02.sh | 66 -- .../submit_03_test_softada_vs_ada.sh | 65 -- .../run_01_sort_loss.py | 80 --- .../run_02_check_aug_gt_dists.py | 168 ----- .../run_03_train_disentangle_kernel.py | 297 -------- research/e05_disentangle_kernel/submit_03.sh | 30 - .../deprecated/run_01_gen_adversarial_disk.py | 497 ------------- .../deprecated/run_02_adv_dataset.sh | 13 - .../run_02_gen_adversarial_dataset.py | 436 ------------ .../deprecated/run_03_check.py | 86 --- .../deprecated/run_04_gen_adversarial_ruck.py | 585 ---------------- .../run_04_gen_adversarial_ruck_dist_pairs.py | 601 ---------------- .../submit_02_train_adversarial_data.sh | 58 -- .../submit_04_train_dsprites_imagenet.sh | 41 -- .../deprecated/submit_04_train_masked_data.sh | 59 -- .../submit_04_train_masked_data_dist_pairs.sh | 43 -- .../run_02_adv_dataset_approx.sh | 41 -- .../run_02_gen_adversarial_dataset_approx.py | 620 ----------------- .../util_eval_adversarial.py | 348 ---------- .../util_eval_adversarial_dist_pairs.py | 291 -------- .../util_gen_adversarial_dataset.py | 446 ------------ .../util_load_adversarial_mask.py | 78 --- research/e07_metric/make_graphs.py | 436 ------------ research/e08_autoencoders/submit_01.sh | 34 - .../submit_overlap_loss.sh | 125 ---- research/gadfly.mplstyle | 627 ----------------- research/helper.sh | 126 ---- .../plot_experiments.py | 373 ---------- .../plot_wandb_experiments/plots/.gitignore | 1 - research/util/__init__.py | 15 - research/util/_data.py | 82 --- research/util/_dataset.py | 457 ------------ research/util/_fn_util.py | 114 --- research/util/_io_util.py | 239 ------- research/util/_loss.py | 160 ----- research/wandb_cli.py | 31 - research/working-batch.sh | 19 - research/working-stampede.sh | 19 - tests/test_data_similarity.py | 14 - tests/test_frameworks.py | 29 - tests/test_metrics.py | 2 - tests/test_registry.py | 11 - 166 files changed, 16136 deletions(-) delete mode 100644 data/adversarial_kernel/r47-1_s28800_adam_lr0.003_wd0.0001_xy1x1.pt delete mode 100644 data/adversarial_kernel/r47-1_s28800_adam_lr0.003_wd0.0001_xy8x8.pt delete mode 100644 data/adversarial_kernel/r47-1_s28800_adam_lr0.003_wd0.0_xy1x1.pt delete mode 100644 data/adversarial_kernel/r47-1_s28800_adam_lr0.003_wd0.0_xy8x8.pt delete mode 100644 disent/dataset/data/_groundtruth__xcolumns.py delete mode 100644 disent/dataset/data/_groundtruth__xyblocks.py delete mode 100644 disent/dataset/data/_groundtruth__xysquares.py delete mode 100644 disent/frameworks/ae/experimental/__init__.py delete mode 100644 disent/frameworks/ae/experimental/_supervised__adaneg_tae.py delete mode 100644 disent/frameworks/ae/experimental/_unsupervised__dotae.py delete mode 100644 disent/frameworks/ae/experimental/_weaklysupervised__adaae.py delete mode 100644 disent/frameworks/vae/experimental/__init__.py delete mode 100644 disent/frameworks/vae/experimental/_supervised__adaave_tvae.py delete mode 100644 disent/frameworks/vae/experimental/_supervised__adaneg_tvae.py delete mode 100644 disent/frameworks/vae/experimental/_supervised__adatvae.py delete mode 100644 disent/frameworks/vae/experimental/_supervised__badavae.py delete mode 100644 disent/frameworks/vae/experimental/_supervised__gadavae.py delete mode 100644 disent/frameworks/vae/experimental/_supervised__tbadavae.py delete mode 100644 disent/frameworks/vae/experimental/_supervised__tgadavae.py delete mode 100644 disent/frameworks/vae/experimental/_unsupervised__dorvae.py delete mode 100644 disent/frameworks/vae/experimental/_unsupervised__dotvae.py delete mode 100644 disent/frameworks/vae/experimental/_weaklysupervised__augpostriplet.py delete mode 100644 disent/frameworks/vae/experimental/_weaklysupervised__st_adavae.py delete mode 100644 disent/frameworks/vae/experimental/_weaklysupervised__st_betavae.py delete mode 100644 disent/metrics/_flatness.py delete mode 100644 disent/metrics/_flatness_components.py delete mode 100644 experiment/config/config_adversarial_dataset.yaml delete mode 100644 experiment/config/config_adversarial_dataset_approx.yaml delete mode 100644 experiment/config/config_adversarial_kernel.yaml delete mode 100644 experiment/config/dataset/X--adv-cars3d--WARNING.yaml delete mode 100644 experiment/config/dataset/X--adv-dsprites--WARNING.yaml delete mode 100644 experiment/config/dataset/X--adv-shapes3d--WARNING.yaml delete mode 100644 experiment/config/dataset/X--adv-smallnorb--WARNING.yaml delete mode 100644 experiment/config/dataset/X--dsprites-imagenet-bg-100.yaml delete mode 100644 experiment/config/dataset/X--dsprites-imagenet-bg-20.yaml delete mode 100644 experiment/config/dataset/X--dsprites-imagenet-bg-40.yaml delete mode 100644 experiment/config/dataset/X--dsprites-imagenet-bg-60.yaml delete mode 100644 experiment/config/dataset/X--dsprites-imagenet-bg-80.yaml delete mode 100644 experiment/config/dataset/X--dsprites-imagenet-fg-100.yaml delete mode 100644 experiment/config/dataset/X--dsprites-imagenet-fg-20.yaml delete mode 100644 experiment/config/dataset/X--dsprites-imagenet-fg-40.yaml delete mode 100644 experiment/config/dataset/X--dsprites-imagenet-fg-60.yaml delete mode 100644 experiment/config/dataset/X--dsprites-imagenet-fg-80.yaml delete mode 100644 experiment/config/dataset/X--dsprites-imagenet.yaml delete mode 100644 experiment/config/dataset/X--mask-adv-f-cars3d.yaml delete mode 100644 experiment/config/dataset/X--mask-adv-f-dsprites.yaml delete mode 100644 experiment/config/dataset/X--mask-adv-f-shapes3d.yaml delete mode 100644 experiment/config/dataset/X--mask-adv-f-smallnorb.yaml delete mode 100644 experiment/config/dataset/X--mask-adv-r-cars3d.yaml delete mode 100644 experiment/config/dataset/X--mask-adv-r-dsprites.yaml delete mode 100644 experiment/config/dataset/X--mask-adv-r-shapes3d.yaml delete mode 100644 experiment/config/dataset/X--mask-adv-r-smallnorb.yaml delete mode 100644 experiment/config/dataset/X--mask-dthr-cars3d.yaml delete mode 100644 experiment/config/dataset/X--mask-dthr-dsprites.yaml delete mode 100644 experiment/config/dataset/X--mask-dthr-shapes3d.yaml delete mode 100644 experiment/config/dataset/X--mask-dthr-smallnorb.yaml delete mode 100644 experiment/config/dataset/X--mask-ran-cars3d.yaml delete mode 100644 experiment/config/dataset/X--mask-ran-dsprites.yaml delete mode 100644 experiment/config/dataset/X--mask-ran-shapes3d.yaml delete mode 100644 experiment/config/dataset/X--mask-ran-smallnorb.yaml delete mode 100644 experiment/config/dataset/X--xyblocks.yaml delete mode 100644 experiment/config/dataset/X--xyblocks_grey.yaml delete mode 100644 experiment/config/dataset/X--xysquares.yaml delete mode 100644 experiment/config/dataset/X--xysquares_grey.yaml delete mode 100644 experiment/config/dataset/X--xysquares_rgb.yaml delete mode 100644 experiment/config/framework/X--adaae.yaml delete mode 100644 experiment/config/framework/X--adaae_os.yaml delete mode 100644 experiment/config/framework/X--adaavetvae.yaml delete mode 100644 experiment/config/framework/X--adanegtae.yaml delete mode 100644 experiment/config/framework/X--adanegtvae.yaml delete mode 100644 experiment/config/framework/X--adatvae.yaml delete mode 100644 experiment/config/framework/X--augpos_tvae_os.yaml delete mode 100644 experiment/config/framework/X--badavae.yaml delete mode 100644 experiment/config/framework/X--dorvae.yaml delete mode 100644 experiment/config/framework/X--dorvae_aug.yaml delete mode 100644 experiment/config/framework/X--dotae.yaml delete mode 100644 experiment/config/framework/X--dotvae.yaml delete mode 100644 experiment/config/framework/X--dotvae_aug.yaml delete mode 100644 experiment/config/framework/X--gadavae.yaml delete mode 100644 experiment/config/framework/X--st-adavae.yaml delete mode 100644 experiment/config/framework/X--st-betavae.yaml delete mode 100644 experiment/config/framework/X--tbadavae.yaml delete mode 100644 experiment/config/framework/X--tgadavae.yaml delete mode 100644 experiment/config/run_location/griffin.yaml delete mode 100644 experiment/config/run_location/heartofgold.yaml delete mode 100755 prepare_release.sh delete mode 100755 prepare_release_and_commit.sh delete mode 100644 requirements-research-freeze.txt delete mode 100644 requirements-research.txt delete mode 100644 research/__init__.py delete mode 100644 research/clog-batch.sh delete mode 100644 research/clog-stampede.sh delete mode 100644 research/e00_data_traversal/plots/.gitignore delete mode 100644 research/e00_data_traversal/run_01_all_shared_data_prepare.sh delete mode 100644 research/e00_data_traversal/run_02_plot_data_overlap.py delete mode 100644 research/e00_data_traversal/run_02_plot_traversals.py delete mode 100644 research/e00_tuning/submit_param_tuning.sh delete mode 100644 research/e01_incr_overlap/run.py delete mode 100644 research/e01_incr_overlap/submit_incr_overlap.sh delete mode 100644 research/e01_visual_overlap/plots/.gitignore delete mode 100644 research/e01_visual_overlap/run_01_x_z_recon_dists.sh delete mode 100644 research/e01_visual_overlap/run_plot_global_dists.py delete mode 100644 research/e01_visual_overlap/run_plot_traversal_dists.py delete mode 100644 research/e01_visual_overlap/util_compute_traversal_dist_pairs.py delete mode 100644 research/e01_visual_overlap/util_compute_traversal_dists.py delete mode 100644 research/e02_naive_triplet/submit_01_triplet_hparam_sweep.sh delete mode 100644 research/e02_naive_triplet/submit_02_check_vae_equivalence.sh delete mode 100644 research/e03_axis_triplet/submit_01.sh delete mode 100644 research/e03_axis_triplet/submit_02.sh delete mode 100644 research/e03_axis_triplet/submit_03.sh delete mode 100644 research/e03_axis_triplet/submit_04.sh delete mode 100644 research/e03_axis_triplet/submit_05.sh delete mode 100644 research/e04_data_overlap_triplet/submit_01.sh delete mode 100644 research/e04_data_overlap_triplet/submit_02.sh delete mode 100644 research/e04_data_overlap_triplet/submit_03_test_softada_vs_ada.sh delete mode 100644 research/e05_disentangle_kernel/run_01_sort_loss.py delete mode 100644 research/e05_disentangle_kernel/run_02_check_aug_gt_dists.py delete mode 100644 research/e05_disentangle_kernel/run_03_train_disentangle_kernel.py delete mode 100644 research/e05_disentangle_kernel/submit_03.sh delete mode 100644 research/e06_adversarial_data/deprecated/run_01_gen_adversarial_disk.py delete mode 100644 research/e06_adversarial_data/deprecated/run_02_adv_dataset.sh delete mode 100644 research/e06_adversarial_data/deprecated/run_02_gen_adversarial_dataset.py delete mode 100644 research/e06_adversarial_data/deprecated/run_03_check.py delete mode 100644 research/e06_adversarial_data/deprecated/run_04_gen_adversarial_ruck.py delete mode 100644 research/e06_adversarial_data/deprecated/run_04_gen_adversarial_ruck_dist_pairs.py delete mode 100644 research/e06_adversarial_data/deprecated/submit_02_train_adversarial_data.sh delete mode 100644 research/e06_adversarial_data/deprecated/submit_04_train_dsprites_imagenet.sh delete mode 100644 research/e06_adversarial_data/deprecated/submit_04_train_masked_data.sh delete mode 100644 research/e06_adversarial_data/deprecated/submit_04_train_masked_data_dist_pairs.sh delete mode 100644 research/e06_adversarial_data/run_02_adv_dataset_approx.sh delete mode 100644 research/e06_adversarial_data/run_02_gen_adversarial_dataset_approx.py delete mode 100644 research/e06_adversarial_data/util_eval_adversarial.py delete mode 100644 research/e06_adversarial_data/util_eval_adversarial_dist_pairs.py delete mode 100644 research/e06_adversarial_data/util_gen_adversarial_dataset.py delete mode 100644 research/e06_adversarial_data/util_load_adversarial_mask.py delete mode 100644 research/e07_metric/make_graphs.py delete mode 100644 research/e08_autoencoders/submit_01.sh delete mode 100644 research/e09_vae_overlap_loss/submit_overlap_loss.sh delete mode 100644 research/gadfly.mplstyle delete mode 100644 research/helper.sh delete mode 100644 research/plot_wandb_experiments/plot_experiments.py delete mode 100644 research/plot_wandb_experiments/plots/.gitignore delete mode 100644 research/util/__init__.py delete mode 100644 research/util/_data.py delete mode 100644 research/util/_dataset.py delete mode 100644 research/util/_fn_util.py delete mode 100644 research/util/_io_util.py delete mode 100644 research/util/_loss.py delete mode 100644 research/wandb_cli.py delete mode 100644 research/working-batch.sh delete mode 100644 research/working-stampede.sh diff --git a/data/adversarial_kernel/r47-1_s28800_adam_lr0.003_wd0.0001_xy1x1.pt b/data/adversarial_kernel/r47-1_s28800_adam_lr0.003_wd0.0001_xy1x1.pt deleted file mode 100644 index a5c2fdacfe885da267e098e4458b8eab5fafa5be..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36920 zcmZ^~byQVf^eqmOBB^4df+98w2AnmI-HBkg0xBjdb|NMS3ew%(%{`Y!N&yA&0}>)A zU4kGe>N{`ze*b*m8}9;xanCq+pMCdUd&Qh{>1`Gh7ZH(`7WuzkiXtOLj$JZ5WpdS6 z)99GlvH7NF&aPV}FS7rCyzUP(G`qmxHUG#JGn2EIWsY1jK7PgItkDtEW0#JdGd456 zBy&;Z|8a|%@%hUaE*-h5C38`9=FGiP{1h(pgB?3*eDVJuRc77U3&+e1{=bVd7sd9< z^Y`d3(q5{&bjjj{OBXHCT69r-uNZ&P(DcehiLEQAnO&6Ju~}s1W>Nm<$mSzv7o}$M zSNUK5ov7JG>1T$97tWqFHZ(K2aQ?E)_De>_myC_pm>8PLTpT`2|Duf9#S!`!WuKn< z-*4oeT$I0nVuDzbZrs_dXn(mz3IY)uM>nzv&RaTnJWw5EFB>{d~1ww(^f^{ zz@BkJN%cv>F(#9RVxL9{V~;2c@4Of%RGTR#^fBn~<3I;k_6o3=WP3)0Esu6F$7UfmicG@C3geD-LP(Dt{oP$g%SFd%uf@Yh2H z;jQtC!lldPgqFuf3a9ld38fv!2^Tx63gZU+#hbeol6Gm*7{>EZt{i zTjlLOj)di^1Xf_^+}E5#mKSobeuB z+e>jf=mWOTD8r;H)lh5gz|;~Ews~+A+c{@Uq{zVDl?a~RtjS)IRSBq0ZC+nxe8_}FlM%GO@t5aXt z*t0+=<@%dbR7vGZf}e7wdHS5<1}ScJjEx|uXspiZ>`N({pPw(aiTjeWcI_3x%ncy| z?K%nWlEGBYL}nFtKxzZGw(&UU>3N8Y=GJq6Cu(r-R>^QnOI`}T9rjN4d^w*UQxx4c z5rrbR3fqoa;LD3w@Ve@armzq!9*RP>OBAL)ii2H1JZ=x9AYq{ZmQoy|-wNPu!NIgR z6J`$Cc;8fp3fTroUKeE#7k$Q`+F}$;&c&^7LMVlE_&isDi(>@Hm>G|Q(#bfkBtXOB zIOrQh;%{Xr`Z6MrRqKZ}0$aQnF~?qwb$AgfffnZonya#r6x}B=DtgldvB!M{x9;k4 zu~u_AS#~Q|JpTx{@yZy^_6EaMC8%=6tr3ER=^28~UoI&JwH66t#yruvJvt_3?z%Yw zeF;~=k;w0Yi`84WA%)vqTHkAKsbfD^eON;%GIU6IO2%1;E9SyRWo_YI=`K!?p2Yp~ zC2k5Q!_|qa3wB<3uUlzi#>}dcBQeK?bXrS?yc8GG`{LOvA@A#JJSoH>*- zHtc!I*YHAtf7?aDUGo(wN~cZ;PAIKQd7O8Xi7)z(F}2=Lk=IkHC~G*hW^BN}&KsyN zehIBmHyp6{#jDei=-U;H=bV6$2bVz*OiZ`5VV{;EcCAupJ%2NHA(6P5{z6f*=NSw?~r z#+3PbMI)Jy4=b6b2~y-AILKT)p-qQYt|H;yUrf`shs?~s`eq7ABpPW)2D?DodOsExbTBy86S@`^1GN-Am!!?g77o3T<(hao_ zWEOa@qb*IxsDJ%)nzr4FVt?4t_yi+TNIg!w6lYV%-|3G%KeG-ZJcm25Ppr5~5k*50EeoU1@jzHg(@9W&_LjYFhdYCXa{^%+WfWCYPqHpm2ATJoXOT$sJHx8O& zanP2GL)GXw*pExb+1DIA-j!fOdKFGM|HGs)br?!{gBd{?ILG@aZLC3u zMxm!85~J3IVX=4==H`SV@>UoQ%nOF)Y+o2xIw1J%GmJ_yhRSuGEVpv)w6R{~KCvR7$&(KhwWM z?twd-+F z6*C0|tG$^C?dz%9$ch}(r zGk#h7A|Najb^C(xWmX*CJ47I5P9z*HW1&|Ti^Qi%_#u~sj%_K(gaGqL=izPDdwe?o z4LehYv5HJRbVAE;d~q&3R8#OWCA`a4& ziSBf{A(WSu|;_|Y(wdq`ty)?4Zd5Yq1T)})YSN<$Xe zOyu|Cl+BV0xk+O`aZlw}2>0BXDRg%W<2IB?a} zVz9^}5p4w=u4-{GJ|#q3Ll#Qs6k^`h|1g2ib-W%*u?NjsF!k#vJZ>+*o)>9wG8f{} zgj6VONki)6c*NgJ#IbKND4Z0F!;0~Uz7PZB6`>HZi^0uNJ}9{Cga*w=c<|pIe9GB| z-4-)Y_-Qo$mWaSEwu0op7gOM~K$1x3_u#LPqI-pud;23L`4`gTtbe5Xq@V7MlSR&S z1t>ZyW8L1#koqKt7Z(3WEWVG_vdZY0p%>jbqD7&AOxTVIpk(63+$|AcsOJq>xMUgNn}JchL-<9li>j1=N=iuYf( z>2biV1nh{9M2buxoc+Bqcib~PG(L}J@qKVI&<3n$Bf(Y)tHwzp&H5{??fOjN)5_^E zE2O{!MKo)$i&{o^k@ls3RH?0kbE`%me~k)4jn#2x^k^J@t%A4m^0+jilb#-_q_$n* z6i{%7_VjVgUnWlQI_?E$d1k6`h0aLfqQVp0sVC)0X-;xf(rH6O%m-pVRZ*ddB&sef zK#{^6ER35EJL5$Vb()FP&?)$}bsUOwM?kc_gEXxAsq}3HwKwFGvie7wYF$WWO+|55D#kt#!k5qEJ5Q&eMMsFOS&8_yCmy!XW6;wQh;mgo`0+7+*6$m5*=&sS zwOcUYya;0X$`C)Kgss;^(bL>cdCQCG)=(MU(`}?>rEPT7SOk>?GFY2A0aue$Q8;ZL zR6OV681GkU`Wy^pPQ+%*k*Hce4CD7_QFF)>+F2q^=Laqe23DAGSNi0IFI%LAYR;#* zs=`OQQY&YY`U`)m%deqgXE{`Pj6#^qY}{VD7#Ak77}=_g4;k8+&^ZedT~l%AloGV& z{h^alT{K3%krH(ZX!Y7sD!p1ok%vT(sVs{Qx3Tz9I1k56x4=Z*1S{Mx<8qA+d=g#Y z{M;WcE5jjJ5r>;=k}ymy2@SnTP>o8)ngAgRrBiYIZ8~aNgs8ZYj)-Ad_;b1pZ)_`( zb*c^7N0eCoH60MI{SUo*ZxFF18#24n5VA#xd9{2_-zfz9Hwl%B2^h0D0Y@7+1gIt= z>}nDgKa0YVssI#>^v2fJHW=Z46-?M^q-3)w=$(s?e^imGsDNLW6tJ(hj+&M?(u?E@ z`jRw2i5^nu+9QU)gQM`zZU#O?&w|OqB{=KMLVp&6iQkvv<7N$*Z<~OXI^z6y=%%c9 ze#CVRqqVwTDgGI2Isae>(MiB50;6WB0TabaKss-R{J4vS}YEx#bKXRGPsvXaCw}9h6~9swNHYQ zQ3~333sL(t6&cUcvEX$!&bSp~^3F=gP3pz-kt12HbDh}xtp=)>i_qYh4*x}2(0rW^ zi)s#T$2nN_C7|?766V&V;9o!re*H^C^wtDy^9sY)z5q0<3C^hA3aw0p~K~}bo=F>S^Z7ZeFY^H`CBXqGXY8f1db|LoK9VB0}#u*c5 z+?Djhoa_kP+8&F7S>aft6oXasVsQOZG!i0X(AX7^$peW{UlWIRRRLz~C_wm^Vyt`I zij&V3*&jQbQ0QF-hh2Hda7%*q*F@NDi^FHPc+_8yhvKCKsK1Pc-@`a0{*A-5JBg6H zAA^()f!Mvw6*VcgDA2rs;Iso6yJ{`!T;}4|!|5n>RlrF5Ix;-aK@09y(4DnuwC{cx z{WT~gM*JI%+1*A@A|zq>pAs(SO+!o3bo{GZfKTbVXx%ysac(LYWg&;u!~du+GN0sH z?J4!Y`Lvju$)Y2c(&l$ikccGKc&H#? z!c-jTpNq3$v*FM_30w6iqkq;virzOsDo*9}V0Q+cToXpxp_Zio$(hVXdC>#gDEh+J zQ)@T0(ZW_q94H@;vUQs9NnMQr%kwDje+;KkXDHtbz{69Ks4t1c#Lv-iO^iU--*A*~ ziAKR-EGi7+u`V|bMW%6BG%5)hm$RWBT8`3)Mm#t!&zesBir}S1$kNP)Z(lO{>ryat zVKip)^?=B=7_@WIDCv(vYFRV_mL)*MH42ONL_l_3Ao9mJ!SwDE%)WRIX501P?lK>) zRw}raDUa=$9rXChTl%xUf{tm0Q{O*NvdMR+LcaEjsf!~6qgtBy?mO9r$U!h5fk3EZ zbG9}@=FG+L7n7meGZCh|-9>-sAX~e))TjM`-j4gpj9oBBpeF0eJ=io!`25Fcq2C$@ zZvPK^!GyAS#^v=@Qjtoh4d1`e;j=%f$94o%P2~}yr2)2Z3QAHH!N`up`;{HU3@@Nz zyJBgLw-d>BTqUmPKCLOWCd004^ytQ8GN0o~7q^FzV!~%SAts5BeExQ1;e0&Wv8}c+T%!z%xm8OuJDVvqM*?xpqhO#l4R?l*-_f!Y|4`_L`=?kEFo4*0j#ooPvL^AekvMXe~FFk~3zJ zRDnJj=~+_p=Et;cVi@hwZlJMe#9{m2NHqVa4w*&Upqp)htxsP;0zQa3;)maN4M%$aF32PKBwf5nN+O* zkv2z(VqTyuG}M*x;_X;mo6qO>*MHLtk1pbVR?^zL)>Iz}3fd8^)A%t}P-GI!NijU$ zt~FjLkW1#$Cr;)pEgmkplqO9xd~ef0H4hWt$RgqS|LEs95lE$XQS^m>% z58GTJOD>31c7n%tM;LoOgKqc*Xc+5ar`0Gt%d4W{L$9ghM-<5@JfcH#w@J}@8Kupc zLIYQ1=%~&|M#ZF#*>_r(dM9qCeACPHv?Z9V6tk)DTQQyc`;THKiDBYIMXcQ}fk)jv zwAa0q7Kp@CahMHVzU0HK$n@6T*}5KbRlzK~Av7m|=@qHv!f(q`&tSSm*vSG*~owIF7z7M_Ex>GK_PD%Z85Jpp;-95V`gj_-r#z#Tl+ zw1cdS9Y(Hj#>@qdc>UELcgDHkKY3?NoZx}QN*>VQToL-z8)tn&fU8j`4@t+H{bwkITaM;owb3+^u_|+PpOddhh z=P({l7=z>gs;KYX3u4?JkhazuvbiytKdV|az4bSFj1zF3EuuXvmZb6n2YNwzad zkxI0p?GWYUU8TL6ain!Kolc9@Q|kCZiVGejn;9R;?PV<;DiYF!1RrXsxJ;`qL@}J? zXo1z+9o(cRdED;qNkSb%1z~iVkSky(acag_1<&W0Gv+Bq^h*0K9k354N7Z+ zrdA3`8iro8kJNgZxB1bp=#>0^N)2mbYC2ytKl@HIZG+30&lNGcY3V__>31r0FLjqN zwZ0|HA|EMIJGX@1ygxyHqo0w*v`qS}C5M>3`|(J^0&Ws7v2M2=k~TW?HNOkQJ?&s< z>x7;hXUNWVgH69XPHDSf=K*)P@EF9PbqFp`%|u#aKa`UAT>eHRcISE{*wGb}Mmgh? zrW^k5aYFuI2W;i>vXu8O5F6tF>UGBEeKz1cUt)uv1p?X*V%LJv7|xZ_u;DHgKGKSu zvXrTP|704cqevr%2AGifDNN!=3#R1EVaB!hzV6|ed}d&K7ZXvbL#qQf(P%L*>PgHX znKf@o&b68z9{xouPk$rHs5bJA&Z2;gFUh8C4+)xrnKlutl#=cZ+`#Ne&SsUoaNn9S z!fh4VT$Sz#PWh^XAht!DY0Y0n=HDMuo2&-~W|UCUvKsocFQM13nJa4UI#=I1viqKVL zV781@O%us1R{~pJZbH@OJ185l#-Vf@nB8>5RU3QUedz=za)jh+I|PQYZ>T)ZdhzTA~ zw_>)mZ=)=in>5uVhJt41(vgpKbZ~b)MGiF3InDQ!`ST?OO6{W0=5y(uwKWs(p^~!y z?N{AYy|aSy?uUXp+t0~1Tc+vgt+HiGyEB>ceKHh3ZxLMx-$V1l9?=?c0SRpf$)s8j z4H@SAnpq>b@)gQ#9Wm~R6LOw9qT{V2%Fj7KnBj)z6|RV^cEZeVS4e93BYrR%hq5!U zclJLBB(rf$EDANt{qSwJKXw~>AxhmH1;6Q-4o z?VLH%P4=R6izGZ+{K@dkGtwDnK&6Z_efOPCwc^&yF#8hbgW&__<0NUFQMX5=y!fb+ z60`j{qm?+BQO%Mh_up%1FyJZG9&@CI+;rOiUo$QE@r@$dddMZWjINA!qfMh9(sA7# zR6XxG^AVjX{!h1XUd;k-`$2KxzGqe3{A)p+snZ-z?DrAD*3N83%y%F89X6-fiZEJa zm`kSrzEIOr5$N-nP0p1JC)$!9PGi<}(~74%;P~JU)Oh>ueqe*L!;ZKd z;fk-0&iJb31p8zMynWz|(#x*c&10Sy+&xgv^B$Evzmi~`futBIHhEtb| z@0{@^+YPe}T(Rn!D~kEpA^q15kN7d&`W$idzAG9&y@GzYEs`Et;mVV}$p0yVt>4qh z{>fvS7_^7ZJ{?b&CW_Eq3puJ8%49stRx>xJTxB%x8|tVvNiy@Eg)njfam*K;X|!$J zDte&hK#N`m)8p23`V-kp>r(njs{eo0Gz9KkCjrG6>e)8dRs-bxeb)|NVk_xGaYW#z*E^!*+6> zXha@+ACj;lhLYlIDZOqYMyQ>_?EBBLWV;ix=QzP;lPey4^S~?%Kf&*^~>Vs(74J4TqLW5DfKwFm|##7K?hqC&de` zjV?HG#|y`AdEneeZ$zAQLui64{IXwQTjW`!Pn&|*JVsf^`BBZUSG51B0quFdfede| z($DapOx%S7%$EQ)CS>wGCM04m6KPn_IK6IS)>$p3q((QoX6a8J%PVP>b|>AC8pY>T zQdo0P0v|*BC{DSUl(q#@{e}DFQ&Yu!veiggo}Dh(9P7e)ytn00B+2P5S;yrcD-=x3 z7tDhSW2Y^ zvCQvP0!B4yA#?6K6`?(3j)|!{$K5Z}Y|I15RkM@xhreU&vbcAxGK|35)o;Ydz0l1tr6= z_dOn78_%ADKBvUd@5kNEuO zptvU_FWDox=q3zSPsU1~i#RA&OnK^&)IZ#r=6t_R_nldaJ${HRbwy~Db_273Yh~=y zqnM4$MpNEmS#l25BZJKkX{u}#E$=C%wljZd=YBzO zAx|VTjf{#x+f7XNrTRXa$MD!##SnSfTFh$n-B+I9v>Ti zCKz@%Ll80|7$(6!*dE}64~9YT(GA2Z%|N&h55Q#}KaHE~f;P+C5v#ggu)-6NSr8!2?3GQEoW#Dum*ffVCJ zu+m`^4*TzbOomViSC37C~u#4%FCq>{N@uuzlf(ycdW4mw29>$EbZ{!tpRF z7LH|+INcbEtql=ae%===`fV^;auXajs3TwPAFT~5rzt@hR5rzx4#_{Es6bwSY-oI+^@rmQir;eLC|zgF4Q4QlpMMcFCyWn#NMR?hZ6#XNTs zm57R6NjQBs1}652n9AeJlTRdIyg-1Hrm4KoiU>yOIDOY zVOl=UI;CO4v3Lm2$D+t63FA@{VY4n40n?LktxSkI=VToH8jspbiBP;3goQe8s1-kn zzw#{B@f=TWk2uaWl+m@y3`*(_qgg+nkm3O&@=sn)o#PHuvhFsDQ8ggj(8pv{8b@m$ z)|0xn1Ws8@fcCJZu$Np7y(wFGY(oz>vz8&`^ep%)D?`3i3NJ?_k=v~8MBB8;q1lvf z*O_3B#(P{e6oIXO4COkc(u$rwl581|vxVA76IqMQ-G&HKJqhFUhWNA97{5!_W82LI zSmLFI4*??BE|Epn@(-!y;6BP=Cen`et7(Fg66NbykX%+Ay*v~{2lm#G%P|>r9vO>! zwu|t7mmXG+yat6tcl=C?!EqiFczP%eIWu{U+V?b^aL9s%SQaD`1n5vq$LU$wDEg6w z=0imgpHTr>#YWsz7GuXuFkt7`NU-XnbEuyqePM) zbRl*BOZ4^KJ~Ff3MSIoxxG7;v%h)%x&T1Gk3>5J0F$?a^I)v;z0qc#2q2PED?_M0l z$=_Sy%Ih+AyqN+K>)%vX&huBJgtYK^AJL2woSUu44j0)6OEnQhTpEq{h0~z?XBqZv z+Kj@gljw`QhRN4&qt4m{Yy2+ZM8F;#_ud4WHXk_}WAIC+gqTUbw64I0`c@n!n=PiK zpZ0*>9I_+bPL7VKmQb|yUpj59f_86BydT(tu_G_y#P%0>DH(}xw}m*ZQGhb>Y&cGN zi%I3hu)kIWt1G$qESC$Hqq+DUn~U0oMX;Iv5xuKFBgwxL^HgP7^&2Nxtr!`0Q%)-m zo7W+?xE#CBzkxXwLcYBaHEZ*sGFXUSr9woO=Aurj7=H~*&?r@ip8qmX!#}^=?iJJ~ zUV+G}UHFZqm~?R}CV!Q}m8EU;cS|MpCMHsJ!YfkJdqSa4FVpW1XIil_o=S{+XtIh7 zdgC>bG{*opeGVag`~{@b4frtkab(ST1lS)#PnH2{LUiy^eg1*aj; zW}D1phv_`WI{%Hh8?+ME@%nh{aU9zu&O<8u0kn&r;r9tE9(#I@N$t;IzU2Xi6iu*L zZ7rVm&BCy?GWa1{L1ue0DDZteS(rr8?9~wzv*{fz@oOc09vl3!Tn$>WOEIv=0IyoE zp}W=^V|MwY=xs87^IFW!=^qfp>pZS+_<;7o|L`Q>14@J5@p#TV+_o=AwDt#d$X8(I ze+?*3Y{g%PKj@k;nl*I4%+8-VmUYefi={J~;k>yX;(yCmEWnz3Y<&#V)&G$xX6yivKn~=9hJmJzitY+ zECF?q5m?wT5v%?#z%y146;_5AdUgZP`s`6y7Jz9U9CloP2eDz5P)w+W)Sa(reqD<- zi#~(<_8C$^HJBSwhxLgKNE`hHM%~|d4!sv=r~Jkg4GA_uZwmWk?-lmlxT)-YO=;HY z*AVUneaC(AufW$jEELw`yk;E^?yW`cwg&vLZNN#RMtEPWMgPaoxOc4z$-ad!c1p$s zu|OCse}&G&4^X%E9NaUuAl;J1cm0`IU8R89Drt;+B?X0jvKTT_!uoNev7E<06ASjD zIrlP{g-;Q%p2wu@+;DZh4{m(&=6OC3e30X7!yT{Cd(#ra9k(!AiMM0%yRey@h-Z)I zv%ah%`(8Q`>HBQalJx2_!o2qfVVbcQa`rp%bkr`WjX#U^AI)(3wHfxleg?~bwxAdVnJ6UJ}Eb1!``p>u=y)OH`YUcelw(Vzv9C37Ifa| zhIQruRDGq{{@k(bWc{t|Mm1%2=`(S5Rm&i1JG*hkq6LzoO)%xxdF{1t=+v&qvKb9L zcKnsEVZTB6su8JUzanEz9U_*OXvh&29k&bZM@f?F%_o2~liq1P{ zFy_HtoX*&VF|YT)$6zN0mK;Urj&mr~e}Wg^ZE#uF6&~N*uqVeK&a?esdo%>w=LF)& za!-t(;DTztmM;5bgQRITaCNamxG)i8Hz~3X?UJm8O(r&wJIq|1u$`~p4F`O2BgP;5 zeg&cAUh&qj`Okm=%t_z7L6|53t(#7AEX8N8lE7 ztl9Ahha#;|b=(DYgZ*REc#z>zm~sP}3@R<5Ot2G?sAvzV z4i~JD3gp)^2>X|XAu}%md(K9p_(UXXRs~^R5T7UC=W%TnKiqHjg|oOHta;tVluQXW zrDGWTVRk;IDTHCe>QERu`(w`25Qtrgz`yf6hgz9{CGS)4!z%_mo+V<^KsaJ}Zu?1d z5cVGRM{5(0y_|f3DrH`0vCSH))2$#_@*Ly7IYEOzcQ1~5VL64sD<=vo-t#){TX`_9 zEkQ_04el@hg1KLs@#R$$s@OI(rnI7MR|~F0weg%(3l26kA?9fdTK2bM=b|pSZ|TR{ zk$qyqkb0NE7z7o5$K%9Nm`Ws#Gy%-R*Lp-Guhwiq)X#H2*mu-ZaY#WYc zwL^VeDCgI|3Cv{~)_36n?`(u$ISZ2jmm+ zG(8>;SSPe!x5JG`FR&oQ7Wam` zfob+b_ib+&ObLN=WjrLOrDOfpT&VWG!?(U#NZ5SFwx`V)3Tp$~*bAeo4h+3%#e>{d zK8J6@!Q3lewZf~t5q%Sy@pV%>uJ7)|ZO=AL)&GVP%{E+k(SfFeO>kXOkI_-3II$uh zSwGUCpqz;OhY<+;5`cl9zUVsVj1dFwI8tJZ8l4x&?{&n*)|WV%?t<&8VHnuyi%Z(k zP`?y|Ql}J1&P>GF1wvla7KJjNOUvRl(Lgw=T>S9DC=eAL!6;eq8aqrx*}L|`SeZF* z5ET%KpzA@%(Fp=89f6Gpcy8`HuSMlGMVoHLxj zkJmamFi8?4@O0avHK?VqvCWYn4TV-uV}|4K?~C38sR?J0*};oRNJ@WgH|)lds>|Kj$5q!T2oC^N!OzpyT9-9#?xDTj+rH zdpyUW;e=y3o^TBg!I-dMWOs){^==%`QF6S^#zA#`B393fgNs8fB#onBZ559DDZyBA zITGg&L?Hk8TYQuhVWsYivQDLMFjpfQj>#d&xD)}W#Zh>;G#U?oB;%t}B7U}}!1upY z_v?P3`r&shH*dp{1MSe{ z{g|d}GXmyxVFs_;Ola-FP)-{(EW4ogs1c`ke#3>quMmV);MnM5JQ7KV%IjoYm=Obq z=@HP%@ejUGddXyKXO=H z$?LmjCt~PDGW`EVLT6qmQes2k@I3;DedA#HI}c})|Dd8`0NDm_F`tb>&Z9`oFb%_p zZQV&YdWo z-;N}y4r~!^g>xV;eRb_9OJ{09uZVvcshm*@1c$w!lf2HCrvl z-qQSqzZ1V>g5-CYA8beNloos!X@~Wpukb(73C!x?Ym!!UcXnWuWIKLlx8PsbSEMx7 zqv1ENc~Hv7sTmo#7n*=$<70UpXAsnbgW!D54P$v8`N$b(=&9e(`a^+!!naM=`rx3)ZXH6j{i2NL1U$DR&h3cTe~@jHOGdw%T`H%4K- zX9y-7=l5UJ9TJ=~F4Z|vqJ#wMK~5;w2d9 zFGEJ{M@VmKf$_8^^lGsz}ax%>wv zyc)y_PJ|t?bvXNKf(kq7unaqShZt);V-P1-_X9uL(DAw*4vX7iWY-FV9qsV*>q6Rn zK9(Hq!F>4+y!LI!d(jT4QX>omwU{=q45zB{VH2DRk4Xuz3yVPi*)Wt<^7@kN9%u-0 zfu4>l#$R&8gfJH<2wd>-KQ})A2td?82)v`C5wSl8idzI|^x?I0#VNR}n*i@6Nyy{< z$%yw+5XMB{pbNhr&4~!?kcp%9``~=?6=AA>oM_gk{A*KPn7z?XfU-OGbNt zGCu1Fu=jil96xZ_y*vTBJl54bBOLN4{7@U=g_+9EST&dDna;h0qOBcPraHqk#u>@c zemFW92*ojBSi|?7$@Ra+MXzGGeJO`)Z#^aiG@|QMJ3j?I*#U$2z+X%LZ?*lD09mM$E9rzs9 zfwL)X5D425-PDB8hmFvG--5=!?bv>^9qij?6!Lp2-q?nvclmvdBX;JvVQq#Zba?DJG0Yi$YrN1sE*R~@BQfi846HUK zVtBd`J$}5MUP;DmE584tp4U9jNW%0pG5B~k3aTb?7?>hJVO}x*8!f`lNET&Z{&SRJ=?pfr-^8eCzmvrsxhF zx9W!O?lz>(ZpZM@4(#Clj>V=9Gz7F`+}2L)@M?yB#SaX;>xNoJKm2Bkuqm4**zR4j z?D@m0tm+vV_FO9;w-)`uuG2qYo!5=Ij_o)W*pBwid`z`%120Ix)mPnk`lAa`UM=`N zrW-x{Gwb1i-qmxA=2u_)nqpLbS((T6#F;d>Ak4`o1;|6j8y z9rquk;GtRyUO4mpOEV)eIL{yETRpI#-Wg=!j3=#bn19s;se@iH*bsmrnP5J5jKvSW zHW=TKjxSsZ%-?*##I{B}&TPi7ab4&f{v9`}+AyHqjfwla5IwdBYx;Y!#j2gRk01CM z)qy9Edk`4a&-XEiut|2}?4?Dr?1>*!*~#tltnOI}HnIOFQauLw7~7AbDL?Q{p$ilD zwnF()2Z{|kpc&YO7{&K^|1#Xbyd7a^01hiY?@TDsbmkJXx(mxHW@1?@p zEgOawshDb(f~c2?P*#b-)5bVRUFCKEpHgwKxeSM^hq3v$hq0PY?=f$o5HfYicwZk6 zQ)8Zk<9lr!>@yH^Egcs=XJPC5OsHcLt+;WPAM}HV(`S_e$<%?%3K0I&ZfxmTb zNE`G;agj4Zm-|5OSSZ$)g`u~AKgV4{Bp77DKK~tNJ6B@mmsX6D>4ITkCpNF{Lf@nx znE0v-sUNyv_w)zKOMB40>O1o5dvSf|PsA(#!0oQz(6JF^duoQWcWx-LyMonOiDOD^ z=wS)g{k<5gHbR8`71)PSWqk-r>Vm?yF3djv9RohSIL!1Sb9py*@MlRjU;yX5dtk|Q z;1iF}oqkpW{q-f7Y?KSfQ3A;FI@-p}5Ukr1f~n=+h~Ddl?_YgjTjPxFuYDkM(;Hv8 zAXxEzbN3SBcpM@LgHZ`sdnOAR=9#>o$>4FlG(6$$-u8DqocTVQTKOc{CdH#8NB~E- za+tV^up%PjZ0Nv8nA@hqb_R!yQhW{dDg`lbQ{nWJ$8ei-VAK8@Hs$&Fdm#_ZmP`~# zWMH&Q3a+`tV1;rp#1nk5{2zY?S9svjDxQzK>xp}=0eIgKgMUl-9;ZPLn@?tA$f6iA z|0-ZL^Bb=8bwlu|5Bn+xp`ST~hs%GWaQ*-UH~R5xaUV>7^5TJ>=utk zM`aN2yFKvbjSo8bGdY6qU z5(0{WB7&lV-O8DJ+1-Vmuif4Hs9>O!BHi8Ha1I>;(h7(mh|)-kiNLqMUp8E>efFA} zyXI~RwAC`<<&lgy-y}?&pNKy77_1nG#MD(Wu=_6od58EsF;a@nEgHl|uBk`B&`b=< zN=3%h6zn~diAS?DQ1UMqw}*4gTkAc7(~2=(u9$OsneeN7i(k{z5cVtyYJH(tt`i8; zAaCv^@xj$%KLi$e;>g?}D5^%_>SG}^H)V03KmpE2e?WNdC(dcM<4eN;p6c~s*`MDS zHd2z^cl(8s4`M7Y<2Nj)iLr6ce_>M8hsp*0*r?Qx$^S_*-6CnG(In4K4;#WJwTxob z`}Nt+AEQ}e$Y3U1tjOkBr|K?p^GR!A#y8l?+SAlGRx_@-82<=jTH&JP(sc=5p^sHpE*qasP5Els_aQ zKQ{?(mN^i-uEMuQSr(%|j2$%WK+C#fOtN^3BDn&X-^qtCz7$)|Rbr;^M@%%S;~L~D z_}!|(|CWEiqrwuDU3-ga?)!7Nlz?8(D7ZX}#@$o=9o!ZU;k*b~Bnxr1CJ7c{X;|xC z1lNa^coNe9wN>4yIxfy8?UZHKS7n%Ay$o|6BgZpM+2ILdtiq68RxO^29j4&*--BgC~7 zg9EFuL4kX}4t&H^er|V6e~;`X`B1o> z1%>?5Xr9i!;zbzxsuU08Y7joY1qNY1(c&S_J`Yu7Nq>g3u|1=ij?HNH z_<ZqX%o&fc}IOF-!+k~jn!Zpn?|tHdsLZcz5>JK!Ax68hFS7{=$)$~+qP1X&6AdA zUd77ntd%ND=^M(l#|&emCuuUZmm}DgH%jb>mLv<9@e8weH{in5YV?>@VDidBgr=wC z^zl^89hQm#p4p(yIhRGz*{JM(kMD~sG57peR1R&1WlI|>_jbWrxf6GKzTxs=j%V!n zg6fS`m>pSxpQ}DYm!GqtlsMB}tIcA+FJo>J>MXUkpX(g^aq{0^)Y*x%^pBEkMBX4K zaX^V}8l=KbyQnbt(?i*^fgwzN<`C9qslY~UlVuq*B-n_+pBP>C6Yq}=z-+b@E74M5 zW*Wm-tfvOM@nbyG8#13Mk6z7UGj_59%j0b0kt^(v)-5(-_7>)=#8~mhxlBDN z#460S*ruql>`0X+`?6;ovy<0m!%mN9CN8=xanlSIDnqR8&IZ=B=^QJp3}O$jo?}&g ztJ%ey3t5ldbmpEjk&S(;&DNS~GUqLt%xCmiCU2?DRQ1NOu~lQ4ZHpee@o)~i-@A}) zlG?~N=H6uUZ7#ExlPB5T@EvT_@Kwxt&|LP|S%)#R(d_xPAQ` z)O7y9+t>Z*y(Pt(k14S0hm~2%Mio~3mEVQ4RoKX}%53jTdDeMGmTlu$Q}5tDEP4MM zhF|_-)IBj~QLe_WH7;R#o5!)Z9x;~0XS{RTm2k2A1Yf;R5PYfUxK0Zqdz){}#f0e&W(-he0WO;VYScbhDuFMK{sIb0j6}B&QB+IDRVB))mG3QYn zAKE6xcIf;=b#gP}kt;~ z+li*FrC7hV2&wl9Fqm^nK1H14Ec$?LrPc6=tV8F`uP_qUU~$O@%s>1dYj{mk)l5ft zd=d)N<4`0WjnI|h7^xQpAC8@V`IvzF;>lR>HVp#dTPVhUMtWvFj!o@B#c(lZBQc0c z>&dYTE-eGE-d6pX<~?EK5!euf0y(zulq9QL!|P3@EYm+O#y^MOvC^~-BHIe=omT*ZwORNRkc#(wCYpXQ z9JP-lpy3>Z*a^XSygmZHe0J*kn2DCnc{pcP4726$;J5w*X3BoViak|)<|)HtR)O(r zi=nf%5Fcva!pf)+6Vi(?_G}9@#D=mdpzV06Jzch zB$$J$I9n6n2P3zBcj<`Gd~6V$3|KANLOp;NsH(&JTCu z`SC9ps#=0lt333cN=7xGZ!dWBnmRTRa<6@GVmF`91KrWZxzqesf4sRC3>}V(&Eqpr z$o@>UzJ3Fp!Ntf6$;GPEMff!}AK_YWVR|wfIeQa1Z<_%r%@q9T&BTknpHar=ab_#c z!pD^%lb@r-_(Z(7kHrt!F!=flG1(v)XSu#b-7g)QQ{Q0o>l{cfO~m|65o{V`@U=e- zPPUw9-r@m+S)7Yq!TH2-T>mKN2IWCMFwYCbTk$a7`^V$s#x&6490bmYkocQOwDas1|!AJ;zG zr=e6m4LgVM8Q^6Snl~kYUEnyy(Lf9t&V6NCoL^YQ^~2##nA**=QDSW&y5@nq`~C2J zW;i;>L;_DmXy5b(8(1-XMpWalSv^{!I`Mt>cdVY+g9&rHP?6Y)^y57klk*$)w|`+z z=1;6z`v>vFc@dXis2S0Zo%~K7q%OyvEmCFkA53B|t}3&ZIkHUij0BU4>BINK|8Vpp z$2+~+kZ`mehxNO#R-E@954tcbyB#Orv}5Y@ZphbmavY=`slJV9TJRYg&b-B^9~tm0 zPe55_I0D}JBYb}#8q(ci)$N6$&Rh?@+lk`}_Bhz-jHlzhA)e!p&pHuE_m4sF77^#U zl5mFKQ@W%1c`Z-EIU^CuZp0%%G7LAnLm~bn0m&ysaORl8sA4f*pCp*c|6@CwW8v)` z3pI{E*goQ!8>3?p@+k=yxi9|ch!kijaxCo|pADoqC&PPp6=5Lmz6^!iPL3(GIOEJQ z2iS->kG9JdJ4QRg>6a@$+IZs4iV!@z5e_OA;pc-in49KftI<119InHZ;#SCfYlVST zC+rq<;pspJGL<^G?yMWNMxETx*#>KlIh1|s!9T7Kc{KGm?ky2#?q*W#xVZwW71w5N zuN2u#4@p)%mCwB2f1+Ic2f|Kwa_+MoE8jI^%&it|EN;i_y{+8m`3=t29q_Jf!V`|A zwdgb>`~7G9IaCV0z4V69Z~S$JnM-2QBd!U zN9mek+~Qo1ROEKn`e`?*{ATc07l}7SB&Qqv4Gs5_h}fZgUXiYI%;6X)HEu z%|=UaE|Q|k5t&kltv8xcbAn^Wa^KM`*^UzjT5)Mo69N}@a-66gfn0NN!@Udjue*8A z@CWT{e?eCNH+FIiYMGBb3zZwooH>8>md|1p+k`+aISwbpk}!txJcP3MTz^v!x$3WwJ@*ZFm$ss+yc4s}ccA)D z3$|6ZpqOV>?2u~Z*c9h;4cahSstdZV-|Y{2N<2Pa8aeb20yG#Vw^an5ML0?9WEf@dk80V`HEBHgzRYuA-x zS86sIeEb1thYboN;_d*EM@?^}}5{8O$?vtAv)Wa&G`SBLbiul7#L2olablfqnd)OTY0I z6^FTI;9?Vg%Je`rpcOf{+fX9WhIgFv^Qvn_2N6pF#NF^Eq|guiw?<|!t?SuU1qlcSKuF;Q8CV0iFn zuzyz+G@7Dta&j?FsP|!K#6P&t$-|+ok*K&Bg63?0xcG%&QE)hZyytvCiI8W(#G;vF zky|$kahLbI?%tueP#*}Zhu&yw_QZ!x_NYH;j{+rI$f(%zp56wt`dtw<-Umrqe%Rs3 zpJlKR`(?NW%RUztSBjv&{ws$6sKh6?0^R6 z66fvuiLq5}2pRPg2TlIK)=H9H=RKOoM=dtrRi0@iNw6dF|8SVsiS7KEO%iLtA-z_7 zc-DyQAfpOYr3zeDHPz~oAv~#ZLJ;-Ztl@B8Of^jP)2%0Bj@ou*e z3fjCs^$dk?P9zrkhoI*|FjQQE&>IyAi$kF>kITcT#r=@z>4(c_o-Y{{jSHH=7_SzB z?c;)Y#%CBdi1DmS&lp&Q#v}560`~H{ZI>Q~t208t4g^B9-5bB$T+uPt4*mR{{Gern z{w`aLuH!z|O^yJ6Hcx;1;bn0+&xa7=%8_)GPtSqnuXlJ_R)@`P^{71Ef~%ZU`bO~BYaRx5@SaSa0Juqve*!8oS0$*mFZw>5wQnh|dB4Y}?OTu=E0 z&;PxHT5d5cnv+qvGY%fDp@=XIMq#5j1gl+8W@C%qpEig(YlWK4-TJ11g4hq5U34z%EDh!ElBjKBshp5d1 z*v0Rh7v)*_ANL1*<-B$i^T&$*K(t;Chl64adbDED{y~VvoOhJePQ3SbPt%Rhy0)0H%O20tULiTt4u5{zU@7m15Aa&N$AQ=49ijNXH5M*B(@Cx%pIf}j zu;O72#yzToYFRT}X1613X*-7US^NaosSR4(0=bQ?a2U?@f`8jktMU!UH+LiGN*5GM zd*S0I!F)a{vA~-;?C$NstZcF*(_Hr(o0f6x!>0{4r7f@%@5DJ#6O^8|pvSlcB|}>A zBb964oEs4JyoGa^jo3Y}0Y2qbaJXEEqZ{AC;$AAI^h9B3O&F?0zFfoS3N>#hO#5h! zuRISx$bZ~nY>S}@c7TEtO27J{m*<`&Ye(P=?^A~Vih)CUEc7pPzV=}(zFS8@=Wie; z@N5UyAV1vWe9LvkP+ZGOhup}1Brca=vlqQV%*hah3xkmKg#W$U0SMsrv3plI4k^VT zU{)N*;p1^FJ`o3|MB=SZ2!d|}z-fjDqMV%IGlpx1_FJR2@)gGHx5R<~EA;j{qxiWi z^wa|2d^`|`-J{`Gl8paU-eB(9G8E^1Mx0LruEaM$u!L))ezt&kE&R8!1!H1%C%ET{y6=f>s0xi zb!K)vYS%`?Ts;=s_+4}3csLY(1>;VK2TC0M5VI^8B3@4$^Ky`JTbw;zAwA)(YhdR%qFmlJXXs(G{7Y(6(3f3&h#_h=^xFB1LV$%lf z<2`<0L??#w&t}cjR(w`!hWxE|DEQaoY;r3eD78RyY$x{Ev_USohv$~|!C6h3J)f(_ za+2mS^W&;)*0TJdlc)RN*jF-HEQ8d?S zTfKy`x(%w#U*nXe7ydpA#H~86Cus~p!@O7o^GsB+4Y7D~CKArdeD39QkoXL)OWW(g zwHsbAc@ql%kPKLVm1HemQf&SD49L&&K%RvY%=x=?X0|I%b3bH+PawSKhr?}eED}#e z;NJcSn0E)j&O88aoF{Z2Z3UCgXDFCqiX%}^pp*Rsem{(0nEwbrDvcqcmw0!dd%N#( z4H4IQPKo0B(8VGwc%BOlw`yeU_>8?;yxxX4<8c?i3v`>ghN2N8tiHk0uo0%(4ajWb z??Z130{{07sVBQJFryb@Gi90K(xFW53$Yc#q3lzg412?Q#T}*XaM{s@*L-IB5>bx> z{9ehDZ-y|h9<$#zpQQP8srN^4O2}`2H|H<^|Q`E(lWQ^Oz3Tx=9Mr+9b{Pei_Ujb4|jyQcI}$yu?*= zGnD4N!rUXy_{XzmM(cay(NJ&HalL`Z2Tz`x<_wc_&S>_13HRql_;lkE1{4lM)$at- zZX7`5mE9bF-p8}iE+JszeaxKq2!rOCqVks&Bu=@aG$Rt8%A7w_%7f$MDwJ}3voxq4 zdsa7Kd~7|6FE-$G8b7zKycaiX!8pTKEK6v`fw*SGDs>`hSs$dOWZ0Bj>P+ObiD~T^ z$zJW{c%-l&9<#pV?6_t;=g%=!wG9iMns96DXN;ZNh*$i5y0yIl&-c_}>WxMiKdwWE zMFled7V->}cv#j1aNf!dt3TLdX}1Yf2cDp+>lQpGoWiS1yRk}VCj`&;K$ZXB9{FyE zXWjSv|RxSsWeA3yIY?_8n4{dC5!Em4+lfuXHt(41w9 zC?&4lspNRFN7sw8LhWgfbuzyvA6FDN7uJb|vTpLXI`T$uw&*4AsEtqqA zCR`H7quOjZ9y^NhjE!zedMpLcKYx8wuv;@iYAi9<8v%z)d^Of4D%i znfn(exZu^3*T`r##|8e*BnBH|@1E-@dS?trt!Oljp3c@@pUh6*e~mj+Rzf9oDc*cA zzni!h3F!w0vNmf z)!2xe|4__zP|0!CPgQpfPZZ7q8DFTvDh z>$soZ5@)S+S=Un)X8dC_j`zvq@dau8STPnC`T^CdHSn-KifI9-;G|}V@XdE{l)Uslp z`Sunbsrkr0P>LaHwdn5d!uBHyOyb*VcA-d#RV?j-iE=G0*S^QBmLdoqXJh^4Tr4}2 zit4Ua;v=-xU974$9yD&<= zjH*vJaQX5j7`6Tn7q6^^lh0ykKb!^yzrW=6N*V#LmSNq8!R-2`Jgf-(M;eKy^kr!# zt&SB(xv4h#m+0ft^fkyCxeG5EE}(eb5o`@Qg1jxuF~&$2Nda2O)DXjpQ7JUH+kxK2 z8B)^Zz2tZ7JWZ)OPf-&+Nx#aMhS=qitZWfEToT8Nc|$S%l0LTgZ$^3010*sJj(czo z=s*&5lQ=HjnvCo6sTlq?8PjBwpt(+jkz13Y7MzNul!;;CZ?NZ3Ir7(hMK9MithCv| zUS~)#jfr1yn!hs{d`8TEoDQR+WRA%RaqmJhzC06Q=LgOQ?Mugt=Seu_83+5>LWJ^N z2NA!Vd1lxX1lSs&MxY0id0M<*7sK7yF8V_$6tl*X?p@>?6Kt20j_q;!(6x+mFFmA_ z6R%U4iVJNW`knr_OAd=wW?=rU^;~;@3WZlrV7S9sxYliijnoPpojnK5Cno}%_#L2~ zLr+zV=|0Z~S$i=LQm0NMNq;+4Ict-UEvDs}?zDU6UuwNM3W|BtVCJzB1+J^`L2dDI?7utBjjebr(LvqJwQik*xlA5@ZG*0kbqqfVmr^|v~ao_W_ zUs>cjt(*3n$RPKI4wRP7$L-X!XxHI$dbS^eAB3PSE)pAh;=!`w5VJoSGUd^j#Iet@ zr3o-?isd=EF&NLeQn{IVd{!(+omU@%g=?9_s(&!s@&O}fzQxs~B)ET0#Oh8yqmPL} z5ZA9;Ka0d|-za$bh2rC$XoQ%@V#@YNysHU9lZOpHt~-lC!{%`vpE{;|S47I88v5~& zXYlzrlVsp461JSDA)|FEJA69rdj!((+em?PZ&RsuFbz_8OD7YhaQ>AxBDAK$XxIv5 zAKik}W*ZPNcqwGnr(r}j&-Ru2N{TOIsLbj##Vt-0w9nd0*B;$NP=70lDg29bK7B%e zj+zvSeKesFspa(ei#X2(9f?h0I&i$s5I1xY4i5n90yUtfpbWEsdUAG1CSUO&5?iy4 z8kbF@hgO5BuULx~DAx-5Yu1zPg@cs1=N9D~S&^|(7=60%p2q2mA!o+~-Ycv}$Zrc| zt@B34xImt-90@185IAoQg-?AblJorWlxuuI^*4BTx( zvTz|YP;JGtB{}H4!m;<)Ja4xtl4srp;lLB_?^?*`>FhAX#|OcU_k5Rff_PRJ*EqfR z!W#!yJj;E7{Iy$9dPWn=mx!ZZ>JweIk0fUePfFK*K$p*MA<;*D>WWjMf=k~7mZk23 z;9=_YbK3}V(fuF&@{J~=E1$`%R}q2v8p!zHOuQV#xHfVIhPqDXSDz2&o$C;e?MGm^v9hD zUr5JzAmoG}-tjqnP(Rl@^WR@k5sWuSc=m#GGHy+L&ovX1*xjS0XyTgPBVnAA_~VbZ zC|~Rn3&7Y0A6Rnyx~;|&qB8-wx+wse`T=+v>V?qVo=}OmL-mXM*k(Q-f$yZS=IdA5 zr4>OYem2z4?*(`B`BZaoDs8=}NrR3`Q9sX--g6>T@V^cP`grl5K+$(U*>u>F`_n99 zA3xBtTzQla8IAEJS_nzjfuMd}+uuD#2y&5X|TsirWx}&-M?r zxIUE9bg$7lr}-p`lBF|?>je**%>`?uS_GofSiw3{r={N~QFqZw`m_8Hm94)>*VVnL zc4R7Tlb68U7G1bWKfr&@FOhrG3HltX9oprEJm!rXn|*LIknaYtaf253f5=C8z)`~s zLM1G&Yghd%bs7>bnJO1aBOT5t)>4PZ&o(Q_?3F+h0$JxCzQ2NfnZg}T3)j0o7An}O2q#|57WzJM5l&VT2&Im1NN{H5 zf~ezX>1nw=ot{}nCEXIZTOo-hdk0}_yd2KD)ROuRA*Ig@A?say--eea&Dvfo_;BRC zz~shWfk1RkaKUH4;9`e|VEtt!x}iUgR1-8Q%;Y+${bx+4lf7u^;BqSR8-kxrS2!Q} z0`K^}`z^!?Cm*<=?>omuQym~~?u2SFC!Sa52D@xGd@*;!FgJHh@(V=Y;uy@CpNGBf z!Iib+F?h$spEdgeSF7C zwFwSL9E4Z$NL(x^qQ&pM$aJj*{S@q?-lMCi(@=qC&if^Ju=Tm%?m0ujj&*YdHKzac zi|cZDeru9om-s|_eP$0COmU<;OEM{cRToK4lfi~Y85DjM$D3W9Bvp_=|9kL^!j`U~ znZIiUt3GW?m~m~Z(0$ZvVXtD8a9&Z3@UUuvaE!uPq4Jf&_>nr}1no~3Q|ob4Dw-Ne zFYZ@RV0$+mROz8vJ9=o};%Ks5=}*hwJ*GnmOK6n80{!>fRAAIDup_c~N? za+^Gvw#=for}yY&vOUeP{YbK3d+4sscZxYtL+#2rBqtw8Gfv$iv#rOe%1tEDSz?)R z@a{+XFDVih?;9<8b4e)dJ#b67cc+|i^>1gvncpku)Z}MmDdkNKmY-?JjBj+J z<1bB`#Ct0FWI~f08T`CPYR#L;T=tJZeUgJfrstLZ{PRlj+e+@}k3F?haHaB^;Nqu` zf|(X=0;NqdHn0k!eY$mIT}b;G!Z#UzRQI0jJlCmGA#gYcGf+fHdNP&na; zw{5Rc^`AYoc3X1|m=)&bIHKaK0~)osFE7m++INj%^!GlBSMB6JWJ$dLFNrFwpVH22 z2dJ@m6Xg!=5%{ky6ReOvEwFffMqpTAp`ZTjas2WZatX1j-TGf<$qQ_a#E1?_kX!0~ z`lisrQqhVQ(WS{QD~GMCERed zSy(PULbUepd*Ste(}idD7$+3287b(h9ZkPCKA=+v?J1@`g-SM-QEFH#jcLuNmz_zZ z+~PnFPn;(MnPK!j)=RL}ZMMj>d zSwE4UD}i2Yzd)f^lSn&53`T;3klV|5jg*+dif0WxxM$71a5lK_!1KX(*`ac{HQLD< zH`{pLD9^kcRbqz>DG$iZ4Te?#&%t^23(61Uu)o?DSDieuZG{cywc26tI7fUrYmMA> zrWilwC0Zs~V?(Yb+;&+*Ue*Hod`>^9Wr#-!J29zP2HrhFn%rzi%e&T)phA~se@YS9 z>Vyc+%H0>(3l<6{ylB<8`#C8=;)+c|<=+GP6#-iXVYh1qDJ$pDW&LNgG{cgXhb7RS z|9B?4!h5=RE}v#J3d#Gm8)?10NqwD*=*06vK}2*!!uOP2!W`LR;e#PVMJ<*?MBX>j zgyA<<3;nNIBt*Lz3vv?HQ0*Om`YYu^fsfK@sP$j6@NcFUtc#vKC?##aTc$SKnjE)Z zBnL+++H*Ts;9=`0$h|f}@GZ<-|DuGiV1jpq;8RJyAh$z}jEWWLgvSv&Uwo35q?ps? zweRV|puu>3=O7fC@4!#m5>~%nXZ?`)4 ze`I)IlzcG0pH4)Y-Y*=wCc^m25JYn=mhmbl=29(S zXcg`j1f6ozcUpB*kUpkDpeP?FXh>3{1dUZR{GlDiw}sRA>tE@(NgJuoYNAH(21*pZ zC%cv2)ED-W&M(_c`kx2UL#?2Mm+$8Yt*;4%O%aNs`dOny<|ng+!^Y|h*S(cZxN$`! z2z_ZnLH$l-ES*X{4rNq5@Q>aqOCjac51z+cM-$UBY48Xqk|@1Q8`BokmZ5=y{F+=r z=(6jA$GwL7OENPAGk0eTXm^1?oqZMLoYEjY(Ifg;`iurQye5rhX_VAD23;m6@V@Xd z9$evmc~5H`J>i1w=C)XT$sU&v*<;au8|3RaV%1$|sJ(VU?M648ec*#>oYRkwOULe) zzwllv1>=^6^4uD4Y&gqzW!1YNG}I039nXOEaloA|wy@`ZuyvOm@G61d^|p2>y!#S6 zZyDlb+;L<$j)ZqvH9al1p@YRn^nLCX8h%=v{?7j=a9mv}SaeV%5dC~9&~7vmg#9l{ z|Gn1-!SN`0+Hrg#N!k2IW*kddGe3nI^Xo`M)JXv!ztKiM8)>{Nr5QZ`R@u~%#^ybw z&R5j}am9l8bS+(B&OoYguE`M5>}%si=QXp1uiSK8MDOi1MBfDiawCR6IWH`f;7FO>gJzoLI^y`y@dYnM=T!-Mtg95>d zOaCL!tFtL7!hpu}oU83RA#`iHh_Q+mZ9oHLz;Lqo3f9VXAP^O7A-pLc~hKV#`B zuGq~pEtm1SdghHA>T`I`G56a9^BF+g%>!?rbB$4X93D@5i|&|zaQyuSj{n8+-9w?6 z!1EvsZhAuA))THTT=C(PGxsAo;rnJ6WRyGMz&{Th8}5v0S+-oqVUD7*iySwdi^`{i z(6^$Tj5O28y4{<6rk|#Sv75*sVmg)8YSV(xy@G;f8^LyQbHVEVCV}K=ZE;rN(hf=j&H7p-iPn z_~g8{NGWQBsPaINFz;SK!b!V10{6M&>Cy^!k~dGHm_w};6fF*eroqtg9D-D#8YHay z=y%_5x=jW2Y(gB}y7!7`#(6TbxkJkuM$@zdlgYbgId$~jpz#Uk$jjD>zSb6z`MwGo zcC&{%r^!L@pdO4=H{qW7W3(7qAz8{5H>SGcn1ly*Z1?0^7gt=X_r(QyUubD_9e}q# z&w=v86hpqhAvqTHBe@sT;2i=Yq}a{lh3I^j%4d!kjGGjQm7KS}{K^Z_j{_iK;0kA< zJ1k87(OAj3Wo<8PxZ}=sBrfPUWs42bhM1_Z2~o54uxOPE;sXBA!NE18Q^-8$wQi%ag3^Dy(#aKC%wDxLt>igRG`;HpMEIeN`xY6XHCMPWt%W*-%TueYld&N zt~heS3!j2HN1DO8MV_m8O5l&^TyHoYF<)xk?eL78GLSiAp^D zt;jY=enw1ZE~cj?gPBBP6Zd5Qy%35Q!T`u055mYjp}6yuXQSl!!niI3!_WGmV}lRs zcRJz0;uko#^)g!Ctw58_6x5v>k4E!<6f~@dCU(Ztr3eq&w9JK!t*+7EQQJsv)DDUc zu%?sE4rJXOKn`7e*OgHx)jgGgCJpP-u|109+rads4+M@ zS_`#u)9_kq9*V5SL!o0BHYCWxMqM0#dh03QDud=YCD5c;4=S4PLhidG$TqQny5p*8 z)>|pORT+*)Z?y1ScOHUFSK{Nc<2bR?7&^R`zH@NLlcHdrnG=qbv61L$ih$LIP^>WE zI{pznU*lCQ0=gqG?`8~sbH35aAPYOL7ogxkBlqgVHfgB2d+(YTDve-6t6rP1> zAb;f)q+im7pOzM+hbyC9R*vUKHqncjvGm`?YgBu>P*6KdBSCo5SU6|caM8$dnxf~< zrb0VK>-fCuqv^2K3wl+bO@;cM^nTV5TwSFF(*`Y+Nle6T-TBDfJ`MA9C&7Kt2-rs| zVMXRw>iS(r!c}E7zBr$neq~ej;XKNf{Ylw}r4hVv5bo*d;FR|a{5D#P!~1uma?d45 z=kt7{7fzV_-3Nx&f!OlDP@csQh9>E7=)8|WNlzr~rty4HzUN14X)M%L6L3#18Q0Ed zLuqw66wANh@lR#udAI?A>q}twE)z)ylDTJ;dxs~-qh2Z!avNe0nHq*kTvz;fAPl{W zBJeJk&jwtp6*bKV_FL?6qs9UcCq976e_Oeyd?oA+CZhek8kX^lc=cOd6kqa{%tu$z zoda>SV|X;}8d^#lDsyOyV;fD_B@4e5W1u>13^q0M9apUjU@Jcxk&`uHDn1H(d6uzp zQy(P=zfgw!OS(T%g&xd*njmI&S7ri^3BWVhAsn1M? z=jV;VeoTSp@fp~(fT7~tJoLpAMypT8!1eL?yLt##H;7|b`EQyxO$rx|@fm<;1MXJo zrE;MJe0L0o{giRAvYd4N|b;fALt(eXS*2*TQZJQ6C!1_Z3FN_G27!wnsoVH3Bovhd`&0Yvux-5q!oJ zF^v~6DtjM}wk`qtFdYrQhQW8d5|r~4Q99%+mE|?ktEH9XZr4HA)PB(AYDwHORD_LS z1TI%ihQG{QJioON^{eJ^t;Q6-FLx3)N)5$-@lud{R79(^-6(a(FM*Ey`}lcbCc=p7 zA)?A~CDG>A7s4SsAH_XkBWP*44_&xjOD;$M(V|I1uvu0Uj}8(R7%oAx>LTQioQ_35 zbun>_4x~(!U|q&DPsfO1iE%TnU*Al9H{0o!RXv^VRD|Zlk(lc>0Sf~K&`Dnb+gqm* zsQMVs^2`uE+z}MvgV{cOM~rPOjP&?tb!;3KU5VwH4AFR!pMX71DKJ}^2qoQQxF1cy zz5H|>*Ura}jw+;>wPAwG5Y{`Z4aPioPB^y+H%rsuz&*v&G!sylkqB@1IGmM;g`0IO zWWU95{xKHeo>7Pp3i0M|D5f9s#(l1tzF=a8xv5vNe%EoV?OG36rMXx}V{kN18Mk?F zJ@a)feF~@{*I|W}yhsvP^Sa1Hf!Bq*sz@&#hJohs+-o)+{YwS-S+xLao8~}N$~lfz zYIxx+1x5cil=k#FwS{e=8>$Nh5@jcZs_RCI68uMsx|1FX%f}s0=(8R|MfYxz){A;d z*A)Z1G88vWN8{5NT}Zmm!p!W)-x-KVF%g*xUs+p$eAE zxlfFHGgNw0pp>7Cd;cY)Wd0Rd0TUJ811T|zYE(S-30V>z! zIh4yr;!n;voap47MB#MYESZ3F6DHvF$PuuI zy;Jl=mLn&M&Ue`fYw{`+ZvO2NT)KXnyo}<>C%&2XPaB~6RnqV_oPb~a3~fs$!%u1~ zmc@?2wn$}oZR{tv&mE-GT}=mri>Z2L5lsv(r2naYCS79*csonrX1^L#_Vnao?t#@@Z}}?B1UEfyK@_zIJ01a}9VX)7-$5Ah zvV$BKx6xrc5&cvCKmp53{4RL6`%uEmyhp;}8KXtY-ufcTRSCia=T(K$3seQO zQ|D6AV?&bt;zb6#OK9aJdCcHEfMek>?xPxkabMJMcJc_UyYP?dLizXC$l-a+A4%q1 z1_>Vd(yS79ac4p&FC`^_XVNXmvKf`GJ;O8GUGZFIKZ#3*uKHt~)5i&;KFyO-d zi3TkgUGM?wT;Dw5Z3^F4n9Oq=gz#OHfQZ;=z&Qq!hDX9=dKA=VaW9iE_lU@ZV~tb* zE^;q}L@v)HHMxt6zYpT2&N}Qh(SyN#6<}-!)r`)es=HD2M8=C$r$&&blQDI=$I;Ef z;k4Q>ors_Foxp7Eq7v%>if<{DD{(#>&J zS!C^&z-NtK+H&bL=|xu1n)G1$`P!Bq3B$>KfK5XMz#+Nbvtp)aQyx{jYAE^KIgV*>l7?_3N56+|qIR!M?w32omkjC$M zzp1BL0{0EtsAC(ieUq~()!2#t9GXP)116F2-mdt))0YZ+6l#PmlbFasXS*o#o}wr! z(^EK{V?9^0^7Q@H-U&_{?WUysKyuiXOj7C@bZ}Qa*&NNJo3Tk0cRQCVoqQ?$z7?Gv zI77qV9-z>$6TC)Gr{lpZNOXxvpzx6520x*7%bh7`a|Mmzv-ZL9Iw;t@5SwnD$JWYc zP-wG(N4+OxE4^?n#vg?m+#kesaBV!_>Pl$@zMlxk)$#~T-xmSe!N z#GC8YxNb9m5x=V8D4)-B!Bg?lJC6Ij!ZF!73hW`@MR_0;#&Kb&yB>shT!Z~>2j3rQ z?u&DrOKf)M`&0HEohVwBJdO@=sl*t&c9!sD(bX)!dsF7Y5Nk*=!P(R&xyQE%pA5rF$A7w7l>s`98lw zJ-0s#hS}YSPu{I0R2XV1e7HkG^zg+FQCa&H(H(tl(U!V2VciA3Q_|icA#eM4{kch1 zf~WP`WOLb&?igB=-6S8Ysvjk%vr3(sQtkOU0 zVJ_JH(_0WbF-GugQ?Vd5;Uv9Ie?$vS!zod(fl5BBOTzPM@F&trGahRUNbCfO>U}j+z;%@ze-rXvU zwttJ#B}q86F&?=-Q8?EVj+xUVFza~;u5$hR9qvmoOYy}EKF`jb$+gzQJ(0QH8C&o3 z-3Qn1<7Dg>j8ss^>=Rv7JS&Rw4&Ef=-0_^Js298{3Kdix&KJ~&j})ZmnsU7AieRi$ zlpt4Kn*Oh|YYmFx%EAMKFwCF=3g`kN8kGo$3Ifx88jNm&Fo=i|cY_LuyiqI@!PRAy zuxJ7SAAl$*@=yeWC_@mY&jn-=sI0U<<|hv8AmZbj5Y9`2<=Ow9h<+jZ;S(|yi; zRrRf|)8F~-KmFZUgBTgRrz?i_c#^;lq@QH1T~4rOdD*Ow|3UVC=xR2j$(8)AJuCTeFLh?&*lEVE z|1tB;K||K-vK{LYUc=m7Ud_CcjWX%AD;?XkOdY?!T(2DQD^Rv6o%r7NEdIg6r~LI4 zCtgrhS}xw0D`id&Fr5x@>{z*y6}*zbpM|bql}CLktqEYiB@~1JJ3uK*!XuDFR(;9BTDzn0<^nP9pg2|o9a1MNjl zz*%U4T1YuNv^#>0KW@r05qV70_|MA0v!48xbGwwsl2iG`!P)$#nvbp@e`CS>-m`Tm zS(wcjEKg&GliQihCSCR~edA2n$M3O!jhN56)y`raUKC231xAEN` zUrVcUs+k_QY}RF(8U+8t6~;Ekf^PLmFb&KFJ&I8-)GC5!KPX_xw-kyg50=Lp0#U=) z!LCdRol6lc6*P8LQU!*dH^H_<1&ZD-khE%WTB=^SF24(rbL%1cXeC6_Z^`~-9^99e z!%V#jkmi)ax6uli^)}&wb+pDPAHNHoM5?yjjA& z-__!{^~@o@*iV257nx$!{2rgeW7tgF&ggz!B>@C;A^syp7zB-Xtb@WN$ znpi0nAFfETx>oV>fVuM6J}orlDN;J)2=eEvaD|b8L>eq2|5_13?hK}pHRlK^)=wd~ z<@?A5XJ^t%eVbzP0*}Uihd+%MpiXWgs>P&YKxZ%p1)V_4kOZ_g`wF+GUP7A>uV7eK z1NM(sqR6iu?W0>Te^(Rc=pb5GC1Ti@ey9i-;q$iFmhZo{f#L>5VB@d>(uF7C$Ko<@ zcTs{kxenYuy$v;+>foBR5f1k>z$;5Co8JYE-bS#kybluj17I>#^gM!AkY#qjmo87? zl$U^OI%L3EZ61f9L;r!GsBUnyZGo&o6$}PYALsXtuzsKkN|xP$zWEK{|Kv73f4Cmf zp4CC@n|hE(-hfW?ayYiG1R|v;Am_0={FwieWg54zM-%jv%Q`hNe!dO1!+Fe~lZ4*M z4Hz`zCQ81j!Q8{&V%JX!T(GwgH+i4NcoP{`%Xi|6NBi(&Yah%|#$jyh1#FjG!N9Bc zF>7K7os;#5Kw?Sc!tJDJB#A6;EGLgzhRD?lC3$3cmS{)Ck!ho!5sSaMk@dPCko1!G z$tr6D^3YzF1hh^k7RLG{Mt3%8=`|uz%6Vk-Y!h-ebS{zSnUdVgv&brsS!CZuZ9*eH zvG@ELjE$LwseUS^yJvvaJQ4@4#T?O23T_Lyd)+m?+OaP9*hP`HEV837juH2(sCL?N`TR44RHfnKO^`~=< zK1(?xi=~`zr4HAueiw9l_R|>EE*7G%GWH3x@$d4BD1Lk!3u-jT=6oaabDl2A@X;Z2 za;B40%0H10b{LZpv6#HNFq?e0Tts?Jr;{%>=n-daLvm-`65`=*OA<8Jkq5!MNq=)J znX&LRX)7Y6arG18Dbo@e)Xa(3NTSSXG<(W~8UlexB>0~+Q{XJ{-y0AT8Wm_06}b1) z(8%2%tziY9{Qt>TJlT75R`6zZ+S_FIl&MV?Z~O0k_e05wEq3PjNh@eQxm+M%cQl9)XvsH>R`WWg@c{_O1nQc^!p*se5aw*aS`<&2 xrl!@D(mbK{TXo@7?bq_=l&pX3mkPDhnyj@>$#jTxJ*H_+-V@SmQvVm-{{p8)S@*i{d(E>|RFacZQIY$o@qgnyzv7avtrz$<4-EU2etyI06)0ytc|pjdt|*+TgZL)zQ;+wf6?MHIDAi zp3a+Hy<9z2J>>o$MZ8?MY}@MT=wq(xAwO)`GG$p4w#h1UUgzrZ|JSN&AGg>1{W_?k8Z;kc&K`LbYI}1 zwtxNqzUUF^p}xRFWAr>*WtI8w=UsX(`YmNV-`&QUl zDqe6^4i*O*j1g&oyjXZEnY_>GQog>2(9`>~Pe1c-}iBm9KJ3=1E}mTe z^`}`ULj)z0y=>@mBh3HHp|Q1q?aR{?=Zx>5bhC8YubL(#*RMhs^*c!Rs77BSFq0P= zqPW*m%v}GT7S;r?{_}Ns!x(FRAoej@3v@(}AZKxat%bPqqoOpNlght8 zQs#>eyNJW*cBOIt2he@jYb30?jIZa+&}-ot!7h9moh*HZBg>>r&)fqqhDq`EW*NjH zH~cWV$nNYg#>A$%d}FOKRI4wt>K&eVzqSkun@dss*&XK>HL$X>r|_>^L|JzdgxkG` zBC6LV3=&`A$?iy8&{d%)Ue03o|5Uigq%{6@W*JWu*K%f~SUT$cIPvH&2hr_;ju;j7 z2+879o?oxdXDW|ob#a}PDp-mSnl;2Wm-iCmd*yhWP>(l>%W?ivieQ|aPA-W*XsN!o zc=k;J^XW9ffc3qhb!H8Fw>wlAsWyt%zmKD##lfOsthun5s=9V~`qH_*2G zUaZwqfoo|G;nyA%W4XS*=)7g6cuWhT$*duyR;tO@=LGWBN*8|eq6-H3?G@iYIYlFV zquDNx!}yec2w!G>U`Lv3ggwfE=?v@>TT`lAEdM>VmcHIc%;kzv9T z9}TY35g}xL-YZ-kQO$zFUEsZa2G0JT%aSb{gb2k$7(X|VtbdjZL80&2JL3p6yC1~( z_1P?BMLeZ@28#+8+%Z$$h5uO^#0My7@ff|K)U=n2|AJSFI>U6u*4jLb_%f8={PYXS zqi3+}MR(|p#ZYm$vZ~0#6%X5>`V2yL zh&ihAl9`u6mDIFrmf#<6LKp1J#iriBDgMk>!O$y2I%lIDGbvNW-X|+@v~m$H6y9dX zXFsAl^P;Hx9!>fbv6NXa*^l*x^>FBui}#mJ+28v=X!3-f;xOM#+PU=r(+hruLDjlk zC;JSd^VG$a8|*|kH4Aa^uN`!}wh*Jj#`4%=Encc+W%Xlbv}oY{f%ab(g$J{{VE6lJ zxbou~leVS^mH)g*MQsDpURY4TNs>w=eQ{#A3%R&Yx_g-QGd39R3rx#myE6RjG@6>P|v8QHxtQzrbAY0A^N@OFMdN zi_*Z~B>!j;J3KobqgK|!AvXk5LI<;JEk=}_6hleMFDUhI4*Qrl31;7xAgs149{uL* z-tAK9tnupvHdjv6{gFs%gSWEJ4@=o{-(L7=;11`!qX_N253M(hv062oPT5YOo`0tc zA18cdQ7UJ#_wz$6|5k?His#wIOSi}?_Z5X*@}ihzJ=|6O4AqM|{9b-KGUmUc6@?>2 za~lKkcaZ~iHV5I|CN;i(@MAcZx(XB2BE^?Vf9XJzF$K71Gu71=Q2x;s6Dqn1I|`=I zra=x!oj!@)z3nEbjx|6_vnz5{buiQSgLK!C24TwXe^|e}Q8>Tkk@T~QHde+#UYE#^LwABkcagzI0RdA7xq{5g&B*6YNYM9aN3%1gXJ}lw z21Q>Oge8t6aeJOD-i?moKl4OPn39NF#ZxeGeh3vMR|)rTJ4&oE(y$CG zgiSw$md6eV%NfX;?|BPe6+QUiI#;1rxhdOZunHw-55siOWJqQokfwx5sHxHl|0Z;y z)(y*8xQRVl-v>j%=^0CC?nqjz>1z79GJvbRO{H8ZR((t_lgKCQ`%5N0i5Sk=cId)(*3rtf|n zFgk;TmEjl{IReX@0!aUwR2bwbl|&gc^m<-~9siyqe`7WD&8{wwseLHxhDQ##UD}69>DjH5e7sXiwDI0H1XSP z;hV{Sh&-KxTh-?gr>hOULobEKol$J(&u!E@_J#D<2ut)meGJWKw_x+)C2Wz;8KIkc zSH8|KO^~x&!KOd;fb+$}(4z6M8DT7ZKedkTUkhPLM_&l)*1BK=(xCl43;!LKV6Uo| zU~)c;+-4e!vlT{QnBik=YI~0-5BK1A-DzR{r34bgMe@#EMJnB&Vda=XJhFEW-m5qS zj|OB3lDs53D#@kj^Lfm9VL3*3d5i^yWr+B0g+3#vOEhDz3QuzP((Wg^aOfR|-baq1 z^~heZNwx?|GlIwcF=%;^EKIrZLP%>NCimqKn;G7~5?XpgvVSTD?lVTr%&{y=A(=K# zHK*qdn}wv7i|pi#L%6trpg1TJU`#$9-X8of@`QZ^n(Mp69WP@&I*95!xSGFqeL_=~#=+-{F- z>}8k>S@~b3MGr$LG<~=bw0AnDc0GexI|K0N#Voe)a+Q#9`VWkpuL-)kZp?ag0Q^rR zBGb_ZcUt-icPau2%F2Sg+6+PS)Nv@jRp2%!<#}{!68h_%Wq%!mgbUA(&;r{cScZJX z1B=&ao4p+?BwZv0J*Sh&y-+$)tVkvwKjOdmp}Z!eC;wOz293fv;p?`uRFZO;+|=Z) zcD5db$BSa@@(jiPCSuo9q68;rOFEU3LEUCdWo6I%VygQ@1pVrRL9Ug|x6_j)|4m~X zhsn{EHU}~Ev6g75)B_Yh%PkDbU~UX4?Gd@%Sy6b)W~jrMGeB-PUs ztUO<5A!VB`&p6YEUwcxIdpU})<99U;Vn5dI z5QZJ!Lvnj>)46|L#l+}Q;tk_$8c+LZ1NPJB_1UCc`;^qzb`^(*^$|}tI*UGimWXb1 zzmr=^6uQ$NQC!d+m ztq;O2$M2-0^og49h0~!8Gc9KnL?A}~5`GGy_-P+R{y%S#_Lbw5>a8QqIF^H8JsrL( zSC@~Un~iy;Wi+49rGMYe1UD-~Y->G^45MUZKOPG^Cy7+?NIP45HC!0xeTQkeM`L+q z4HjO>z^(Lt*r<|JYPh&3pVmIA)P2+dgw|)_)502T?0FKqobR*4CVPda{_>cS(N}t9 zhXtYrC*XsBBD5Eb#)KqIq5s=yv}<@Fh21U0iPc*CK!*-@>y?X9o7PA#TAZety&9?b zdN8T1-V4`;YP_mEj6IJBNrr6SPkl1plAYgg`nKbP5Rmka1)FN)@MTrz*s~ioCPfjI zS5ro%lGyAynY=DN5MnYqg9)$VgA?QLZ)n@&sbeY zQ##zpE9sgf{`3K8`<7!`p&wRQMAGuRFUjg%5t+?DEkwxe!|2X0Xh{ExtN%73b>~O2 zKa@aYk_S+qyix4l8;*n{lOQS_U`zd_LYG&g*@GRU=yvH(X=bH5Zo6*4U>ggJ^Nue& zb-h0+M_$LwkA3OTV!yJvI%Y_Bbi+y=9lVMWq{Z7u()Wv2?An`B!65V$tFxL0T~!Gj z-Y;jK3Yru*{WK+gdq*2@`oS>vJI-AFg=;DMaLD?cQ2M=)awoi?Wq%LSUA`S(msDcl zi6gjnNr|lq*hIhc>Zne+k>nz$2(PACAmHu-RE)pNrh2&ujjyJW=k}xYFT0VxwuTF% zyBbPfJsc{zYa1bWc=e|CZ{F0r>O3{nnTz{h-=xrACk6RhQz;IET^|2~`Me&7Shg6; zhu>tUhqw{?{1$XJ1qeQ0qFIBnJ$^UEVX5*Fm=n(fev-_}#X_Sp&GZfzXM|0|PXsyavvO9cRVsv{X z+HEGoVn#A!H^YVduMOGi>E=}6q9rIQ_CSTY2huN2!Tss-Z1Jpnf^BC9swFMLu|sk! zYS1)9&EAUIZR%)Aua!C;8cW$bt(f~+e_`>_RQBMqHG&_RVCMWz$u8H?)cQ7y`nt)9 z6J3C5Eq5^g<2@XhvksLPt_xw4OG)ihE7?14qgMS0e0x=o8=+}vX4UNS?%wphFot-a zB>LmJja|Qf2(5-^;BM-K9|@t%uIo18xwi^!QQk(oT`HK?84G-SHx7}#+t_@wrR>oV zE9v*TRN+G1ZJOb`K?rqPfr???m?pzGBUuXSUR}VJN5v>gK7hY#A_O`2F>KcFN9^o7 zN5uUof_B+Mco|)XLzp!jye3g&$!PkytzDY^X%X5dG{frEKez`Mp!D?s@;;bBUik;8 zC83WHr4R(2qBkh%`U)W@mf`mBht&3RKTUXjO<4b^g{>0z!n`vEnsFS18x&cCMhZ*F zeIdTmQ#|clGdQre&kyFed@aLQ)~ww+iFO=(F@K zuZ2MUd=@yMKikk_ChUEWVwqVGMlQX_9u6i!`{+K}cm5$&S=(bw-g6|}dX0^)dttMv zyATr^Mzk)A)-Tnfso6z%k}k)4>DO4INJRG0^iO zJopAQnU{hQsnSZ;t@_M=>OG*J#+3$gS#!FQ?YlXO4nbbbGAND z7~h3ku8xDrmlJGzW3;fPtxPcXJ`CZ+XEc0L;AcNP!gYt!SW>kDek1$f+!C;|>=N`! ze2OQYFOYfv0fyYWf}dS#aZddiB zEQZUXVoX!q0hM=tthWBK(EU*$%geQfv+HGqHGYA^q!ujs8Hf#?bD7ak5+<5#CHc7- z=vSn~8y|Jyj{X_Q{+P*V&MjfhtbU}k`Hf`I!e7|s-H)%RQQ>zz&cfkhA&XmXD+C@- z5C*O~h+w<-_!sd7QQhj%=6nPgA;XhF@7WZpv$9Y+jKELjkULk6oVp9xp_YmC=LLAM ztpfIwr=mxl2^s4Bl&t!z!M-W#v#3{*tjYKu(+bI9(-Uj9P)q6-MRwO*+9T}vt27S!1= zMmnft1x#mD!o#luQ$DP~{?b#_i%+GQcbx>iSB5C~myfLa$5_6j7}qWj#fi)Z(y3pT z(bCx6!uRE$+1CVbgl*c5o!@$(L)n8B;!W2Q|RN~4}&ms=5hJO{zt#7yXtHlCEkWHRr+91%8G;9FjeC)XYD{mdq5 zS=xVeL~2hBk#mJ{o^>$!E5o>>ZbN_LOeC(+5lqa@>Cr7$GBj#qlIlZH3yAO&OhR?y3-eHP9A34elJ5vze{NJ{RzJp zorwNb1gDp4A$NQ^Z~w3!;T|QpRP_}~>E98fRfQFC%P~wlhOPU&8~&Qfh^?)IuEs4W z_e;QE_JXx!mkGf)UFq1~VhVdW3jzI$acbsSj8S~aOz&$@LR$_M$=#vG@MC1C{E5l! zn29-864|x8eW|775@m;V6Yp1QiN|g_l2?~FVfOcQ;qa;n6#D8cRhc!@6YHtq$Rqp_&^J~T&6c$V`xH$IVsHx7e0JgEUeC+O0$2fh;J+u#kUrhDL-Nu z?P}l54mOTKXxMocy80wt`u2d*%d2SAP&3L4KEMuX$ncM5A`HyBA*9z+V*As{*kThc z9~CUHnk%gL2nOM@5%h1=v9{;OSRcQif@$2A}X%yacZyuUIY7dJ@Qu`6|gT3QE= z533QjG}*E@8?3N)_3 zNUSBpFnKY&p_}OMETKUivxWWT!Gf0IXmZo|POqh{_byZC$wQRw7AM@y|HEu1Y{Pv$ z9gIKjPD?uL2!9Ib!JCoPu;wI-`F9Xs+Y%5mND~2x&uPZ-Z2FuTNKv^}!oPVo?9duj zv^{^p_J*!uCYyg)4YnCiKC4a(bYm5p8?Oxe{&MiQiL`wBN0B7D9hffr{P;paV&Nx` z#h&W;-t5Zm)EyF{^YsOO%JIJ{CJn8k6#bUWO zo@AxKC;bTi#HXAzu`5%S~s!N|(y^qU~Sjsi9 z_ocSbpAnC|#w3hxo`hv`11UXYG}*pBE$m3V#g-b!pmN7GJlh-(&lEuzadSKAdO1?& z1~=jI=CPQ4M+)2GdK@|sgftn)_U@iD{nx2RZnq_pWtpSU%f(xUZ|t!xTm_HH4oZe^ zc_t`l4l6S>*2V{YFB#Xn9W~x2*rgcFW~oYeKdU6>U_Bg-aejzs--Qo8`j|GSi=^A` z$3n+)KS}gTWoTAyM1A{iY`tTG_qNw1|Msd-P{%R~p;J`U9)!*Q&A4w;0qwF^>3^!>jW!ro6H)gFJWyPyOU1dH=Kbc z|I;DIgI~Tz`uz?(O;X|+c|ABAt;yG|^M|I>0dy^lgw4_nJV|NA+jK?VtEmhB&*3^u z^iDDJ$!@Y3RlsNY7^I*7foeHDUQ+)Xh3$8k@&-HM_*Q*kw_*9VxAUwVh4au>iBK7o$Xm#hNzUgr{>Xf{o8HH2r7!X{{!@rNv_HrAD;< zeuil8I)qAdaDs<(G-V?rp$<9p0(~Rzpo@MiYVtHN((SO7s!}RDcrF%~wQs@w&`Sg! zB=|~}V(5Ec)-8Uv)G_aga53#Z4*%BSarsJox>p)LZ+*$mDfSVPlO_qlc5UqFTzTH_ zpdR-!|A`*WC-7K91#t(Lv*+zG?A-IK2ya&5 zi+__-;~_hgct`VR6e+#MaxUZ0W%`)sl&?_mkrlZzna_PVgxh`9;gMbS`1*T&dCOh~Byhu7!~Y#!$$8?NA&=Ob+QE(G&B0k^Z6Xj*?6?xP-J)yq!ox7-8qLNLm#JaK#2 zejHs?f;F;r>4Jw}aChYeEd70pg%_8wD>i-cGcz7x$7EP!TpwPlugvERHb&NokCLo2 zwUR1m;NA2bd^YdKtt$HP>*ik(*Va=w_-#8gTlt+m3l4#|aWiZVcIEXGmG~c(X52K& zW+&D;NF1Bm*<*(wm~<3F-@FE8Viu;h#9--#{d|n=NuX~I9M;$1e&15m$Opk~a0i=m zGfMK~TpjyQXhQb6t~}dRmAgD@LDq`>u-NyTjZF(?S8Eu^UkQfclmL(n3pD9H!KKyLkoqqb%hP0QuTlAM33-5y=2{GyY9{_ysUz;{ z&=qgD_7G1$eM4{RQ)t(6N18nIpWqV^Md>&9)8*<2`Y~b`?N#qf>t3q~TYJiV|a^gh%6B{YHz2y%o7yV5l!b0ga|$28zd7>{*YdO)0-}A+(8Q$h0)(_ z;naWi5xQXUR|v>iLVLW=QD2K!^l*cQsCY(K+&@HDJg#jf>a5x=zIfs+*16e>tl3Wd zaCMYekSZ^>cUF+sl2G!`swb=Rn{>tb4b7E2r1a03WH-t|R=)v-<*cSwqh`8&_z5j* zzCtzO{?zu|O4#@|lT|!?B{?-IhE65d(xT&1y76BC-B@QMHJdaY+MACc?|eUu9CC@~ zy983GlMRh;@DL`rJ!gJHLlCtu1?s!=5t?a&+MbhG?M7nndrU!}M`73@L}LBFWmsBR z%&t$A@C9j^EadNUBt42oK%p$pE}f32YcDh1!&}&zyBq)X7>^k+L?KSidFWL^O`D} zJfn;F@8Br0YtC%(o`$_>Qs^WqeD@R2`AI~%atkqDb&|M!qlH+qrLXw8@C8MbB~!^Q z8(LKoL1vAy^lhg|3MUK6dtU^N-_(bSr@a)m6_^n2ea}azMUy@GQwutrUCWi$PUC?o239B?r6L#y!<}jTU!EA>U zo4vL>E{yrdx|gqDBlkv0<>q(^M{B-GMy4)?l`-&O;%{~%HnYS$NsDBh8ZG}FO{S6e z$x*e9`8Ef_?S};4(|$<*>+qus)l$+`tfE)1<0(nAO!5pFY*FX?oYy5?{QDi-NJ0%Zvqi=P}RI9mL__xTWth&!KtH`zXQs0s1 z1afZ?l;@5V_V!xB{@PVBnD{nOD7|#HY);TQb~AbaMvOLxn)h866Mfb4-Cn66Dn;vX=Xc8Ft4Vo-%Zg|hVNHa zZ-@Cke`d5&o^~$TLGH@lq(UHrbAB<>V?y)>bXsWKj<+JF{YnMs|z zN?5Ol3)ofZcws|vix8%kE(mOhbe~hW<5hJaft$?B^34|x zy>7y%S37nJNwA(jo6dOsm433QmIOP!WyKkuGA=F)Cq|TDL&iz0e!o>{anWZhqPk+` z2w&Vfdk(AHiqV>N2In6J!`&;8cUia>YMYbL5mtiOD`|L>y#u9{J<%vkWVULHg~rx+ zjJhjD)#Y5wN!S7J_b-^j{A7umCrR6VrjXajH0)09#HopQu%(ADie}0DG8djomk+xl zHJs;PwryS!e1lqW#N`|s9*;-W%hOD&a}cb65cTmof<6qxo;!HTj#Ts&=$rt}bW ze#k+gUOt$wKm0>?V)}$V=rcF~+mGa<|3FQ?KhKr(w~2gW+)wTuSjK1c4&=-7$MVvg zZhYqLulVWx4%-$!!DfMGX?%9vkLOQ37ES0Ffu+L z!Nu1CLUt?h16loL`e`?QVAp>*QZNN}+a>6&AB;_YO%UpPa~~~Zu92(2I~4jzdbZEP z*3unt8gL%Zz1|^sVRx=BJ14yVfaC!uEMHrgpz%bE%r|idk2flD{S6v?v8fvOI^Ts` zv}W-s6T0x4obEgzPn}QyrpWytHbUb<9Q&8#6@$g<=ihx_r-VP9B&XChYb%))Hp9{#IW<2&XJ z<@S0b_^|mJe4X+Uc3Y;Ek7~bwpW=5&uITaB=SKY2lY!i)vKODzR>V~X=<`=Ujktlz zVD4bwi(|=Oq}I!PndQN-GmKA721mDE_@J{-Qxl?s`onXM*Gk>6@pcVBwH(;RIf?d8^{O2YozG?P3 zKG^mpe}8)!?_Ht9J=zu{W?;D0yOqJhB~>Sh$Be2%)-(a11SK>#c8A(EZ}`Nv;8dqN_tX50X~Une@>>R&|LG0eC>Y_Y zY6?c*RpL*L`tdPE|KR*PTN7^q^ylMl=#nqYCPOaj;CmNv*(}sp(be| zJQd?GeQPx$9a@q2tp>Y4pTsfq5We{IDO4=J2){1PNd9>VK09J@p>7_=@ZspwHH2+k z{R`0>d+<%qmH3uZ^_V>PFp7uU;pW?E2zeDKE&KfsvE%#k0cP@ih*=sIS;?^N;-{?m zmmz9i*0Jlmz9ZL0jc-_f54wN$U~Rbuyk@E6*3cQ)RkaSw8d9*|DF$ymRv_%4Ir=4S zhQv4rZ>z53l8h6NxG)ftX3Ud3H$Ei9rade>Y3qzlsZ|tJUI&w;^KC zgQMcy)sf;=wXLGb)xo0b@#AC_e2DT6m(q31cQk*Ayy)26N``ylsdtkPeepj_d|VlA zuzNt)BOlP11&Orf;5*@0eyt>CLN}qqwSgjzm(rl#59m+KMJiLXqb%$05-q*A?0&^5 zshXFem_Fhb9nii*{p!!jIM|_->yqfl<^oDMA4iQQakMGv2Ak`b&Gd#! zEv>_bkj=|SrSz{QfIj-5h8&t-?-*rHTG%ydcZwOBC4mF8zHy zSomx4PjdEGi%?j8nm(F8peId_snNcGs&^)nsxXi3&k3qFs;aqL$JOR?@6alcp!q#Fn>|mGGBB zZEw+N_Xs*OCsD{hkt+=BHlNy;ohADZxs=@zK|u~Wq>|Gj7`ofgf15Vbo49l;IF(0} z3moa?Zb5qEm;$={hp>OA6Uq2a0L{xfN%e~BX=qce^sdzmbWuEmI}X$EL1>^$Ci--@ zt1^A}3KE9-UuE{~0hoNe5E0j8{vDSfmU%+~aXMd^m8A)`hwYJhx+1V`NQf+U`ynJ{ zBEMrW0H)D<5Ir{XvLA8+rn`WFj?K?U9E zXi_&?7gA5}-%LdO;Y27sU5rahuSu^>bfM%makM!-fTp#`^zeqgY}${1Y}oXf(o@ck zre$vx0RZ zGwH_#8wJ{#{9ec`RS9zNc~krX+Bhn9@QBi#)$EzAdQ zDN8WVMhA(8H7xCGEj#X~E~t6rVx?9b7Vle)uzM%jx0?)n_Z|uCy4YtHqt4rOTwn? z#lnG?wg)ZPDEcPR zp7)5YpslErc`vVw>W5)h?z60{W5WKaomK6O#W)<8iFK=k;Wz1};Ba-Vu%fXy z{achHjA(zy($dpWSJQ%?(V_V3mqwi*HVRt*LWQ&;urxQ2L-J=IG)T`PFew-6QYA9G zS;%f#ykc!5jPY^F8e}{^fZ_S!C@9{9r$guPC$^qgy7V9-a}QwCu5~gW?@&BXe92sw zRx`t{e}s`M^D%vR1}-cQka5dDS^2lyRx>T93jGb7gv_yLDXsY+4vl#TJF6^cuKUKu zOgblA)96jHP2R$xLmqJr@r{PDAe9cnrG#hUxuxiXE3%#gg3-&{KJX`Z{ZV*xsBU z8*mBZHy%Vo+yso=YK0tcUl<#>A+p+^18#b~*q3&hf`+jG?nxojy zE%)Is<0y|EErF8bFsP<@vrobIn45*n^F5~oQ=3{*6jF=Eb9 zxb1P7S7v`68v1U>+%pW{{3gOSBMcc^s?pPP4ENal0k;x{V%x0O%;Z`b+woKmN6MX$ zy5>9{ERg4|jhejw(bKpjZ-d@HtfNxE&FkxN?4mY$zW2h%z_oE#Z z=RU*8?hQf)-$wMqBD^{s4|XaX%T|0BJ~#h`ilZXeHGGQoTKPDa;g3Q0oYAQkiZCZL ziIA_%d#xV9gYtB=VGXO=iRhXb53!pz|8ZD>e``yJi_#hl^#9J>u3l!b zH|3BtYbcbrzr$IB2z0lN6*LaC3X=z)6OP*S7S@*qvwLa!XxSZwhHWpQHGLF%{EB9y z^kUf$JB|SzeEtD8T=h4!*2;hEbnN#6Qicz2cKdiO!N zGbRE3a&EF~UB|KFsXg#-Of1ey?x9ul9^5K{ktYhF6F3)t_e{cq4=1qq>pMi`^yFru zEUu3cqwz9NrBNh0`)V5;P+Wvptws3aZGp)LWIDp} zCz$m+2Jh~7hnC?wrtW)-Wp#~3kF}=Q<&}l4Rj1 zORJP^ZhKI{@>(?Mk;5>YTdm7n^tkXS$xl#Tl~U%pH;7%;TFoBsIZYAT(d^Kk8G^l4 zr_ke$I_WjZQ*E_Wxch9Qpjvd0j4YK%&Ci#t71C%%QZ-3)n#pm%Z;Gh55_KD*#hSws zafaJpTK1xuT&GkL6`UcHt{#-5{##gR7fL-hSqtGsg7A4jr=X)ffTG+#3HwHj5z;~< z+1u@LWRU8|5`M)ny+ie7fu1V_pLlbj*u{ud)sDhjbv4rJRwdK#`#|ILSvGt|i{xPX z0!exIMAqMwK(%YOI@IXhnqn@6o6%ManK?1Tboe;-2u;U9&%#9LO`iz`{>#w5w} z<+AS!Gg%iOf8k#4PQiM{I5x<7JUr(FB%I1f{ zO!$tdM5^~cVSaKHdiOYtZr1zpSv&=OSx(;5P=S>N+4!@+QI==(@Y=l^7d>(?XY(oC zn0^S~I}YKlehmDdwhM7xFJsHfBCN>_gSCnoJQv<#la^d!{E;#|i;vT?C&qZ#doK=G zSfRpfG#kIjL+EX`RM0uRl9ecgkWc*)Ns?I=ThUEdGTiuy&=G7(iErD5&G`{*PqXa# zFFp~BpZE%UH@p(GqD|@Z?=h6NAXq3W55b^s!KB-^i84=3qEL;=^jT*Nb(?ix*th-$ z8?|UVKd*e4yT5nG`i0|J;;mxo`lD7t?AJ)?+4p0GUh~a`^YgwT@5WVDp|2o3Xc;2B z4bo(X>zmoQ#bep=VIe|r?h@R!St5NE7%R+Pf1>1*w+4zHyCW*`4KtjyfV4JWVjCWH zN*8TzDa)VO9f5<7LoP&$Bj$E!eqKyD^&gn4xyX(iG%@L)DOhO|kJQhVxc>Y$dQRR- zm6c%#lg-!rzV1c7-$^7-EG| zk>?Lc*LT0dW~`k_`=g2nQLj>n(ISzlf<=HL-o! z2Fz9Av-DZdYUvB#W)_h14YU7@5wfj%2ptZ`rDv5D1f@40rE~tHY=585mb`Ih>j%z6&aYHqk6RF& z{=161<4)jRsusF=d9v*h3g}i)g)) z5=lJP9$fLiC_3+OEWbaFTN#yIlvODWl7@=syiZA`opuT-_0`hW9#%?1g)%Y|QK69W zobw3DD%pevrKwa}X#MWrA1;4!8Rxq1&v}p6hZZUpftPK3u^Wye9Y2NxbF*x!|o*P5IV$9 zc09#}tTz06Schd*3+U#z)d(nkiwhCYU}pLNx09}8&5{_TG)1B6z+Jvm^#{t$I-zj0 z5qF|v@JnY0YR|7hvaC0Dc^=^9JB#pN$3Pks)`u?jO~$CJBQS2i5;8^xV(Bq^(Pom1 z|JKUU%~xx&Pq#(;wGk(UuW{mubpP5&x*=#gUe|JR*d&YOy;X+VBzu&N+zL-W zcUWy*K^OWYW54}aymYd{`+lL=T=f8z)*0C5yAL%dzhYFO1@;;A5WNyll-jmJ)j@@R zjW5Ra5Rfv>^Du0Xg=zQ*cu&p1--T-QL!%L$Tib-cg+do_X$V|oj|d&~(`Z8$oc_JW z^US}3=Osf8s$03+&70VLHw`L(CD5yS1`Cf**fOgFGp&m8dgxNRYH}&o^nZ(4)?K*r z&Y=^=Pip|2%`zrV|*R9|XBG8~7vZC-`PE zg331wKUXlr<8*UOx!xC++XQV=XD#VhQ3k0!Dm2Ew9aA58LD^LvHG`~>kop~-YDcjl zEDKvJYhY{}0ndfA;s0tfd?wsP+nrUku)jO9Yu(WAsy*6g55%~PD$+f)n#fMr1%pRX zbk_HabmZO}gk3n#s>3o_!g(21wnc|1L{1`2i_J-c&KP>2c`yuq-OKen-6eXoa}3;{ zg}_Mh5L)t8p(>pJN9Pw1<-Bv23WLT%C8iY4A~`x+B@GjIs^?CwT#D_ra|utaBtw5E zV%+P#^lE|uZF9JfM<0WDz%vhgUgV1RzlC0LHAj_x7d*}T(B`)ablO{K$zCvc*)bf; zs(}5A=U8*&E%ptr#ri2%5H)xuRguJE&+Z1uy1aqq+*cTStpa@>NpSsiAx;kY$3r(L z(>W@N)MUj^th`i!qA))k8|jK|j^0=;Xga+~dk-4GVr~SRohrjq zs~ebx*Iri9lEnUo)UY@E`_TL6uI2n^X~U}`;^ zFkX=ltMn&!Z-u^CdN-{2CG6OKoQ?RKO;nAN*~cTExdlxgm~JP`?aG6x)$s_-oL)Q93?N-BLiReGB4sfAR!k1`cM{f6X4QZ(^_B7KoBOO4+B zMErvQI$)X#H5KwbnPpu#JjaCk-C08inAy|wUuM!5qwMHwzb+{Cyv3>yB3j^khECaj zh4wvphEB+vNZTG;2xnM^4lNl$4>owx*-1BOzIP_Is_~}YHJ4!HN2%G^>9oOlA&oox zAN{4bo6eAIqN^U8(camYB!j#j;#Ho|WBF=9BepD{H`gwq6UHo~|4AL8Vef`c>-9WJ z*Uqw|hh{IOOO7q1es4tduc-n3QeT4-or{t*zlrpHq6_sNzK^DMyU=-^O9ano8jTk; z&4Ni0c`qd`NpX-Rn${5IF}Ut9-~O$KC&oqVWJUxs*U%{Q%MIw5G4qYwy zS{o*wfyNg#(eX3kY~#Uu#GuQSJ)Il|`I=t*JCKRHf5M5$qd!c>>K?Nn?1}|Sr^&gQ z#z<I>A!)1NlP$zmSi3ytZE$s4S7)cgw*wq zM>g$COU#DS^vwH6kPqR{Ke%JA;dU53@WC^iJV=gx!jepR>bA2lo%Z-I@2?}qrilz1 zJ1fz7_Z4<_zC}ZBBT{3ppy=*$x~F_47(Z(xYj?VmFG6qd zrF^F4)3glX|J8@;$Cu&SL{q$Q)e%W=Sxww_~~x)^aS@JaJ45w2d<*qILc8s0RoroybyPsl>d)M$Zzc ztX@tPZ63j^s}mzj-yvjt8v-{M;C6HdX0>GCdh=uMOXaAzzZ|`^;60A6%*BewXOXq? z82^n<~zPgupp-4vAl1froi2bd~yBfYCK_RO-DpJy%eG`%<3VUC;{49<9LRz-`zuI~ElKsv&!^1O2{! zhidPAo?v_#v!{kZV{-~F^v}ocUr%s7_cdnHdTbcHgwn$eIKSjICiZW^uKrc%ElS0L z=a=ESHWp8gb3U-;JKm^$$HfOVh$xDO?!;XfX|e);p1b4g$^HD_++qauYSZUW-eTM6h5wUcsn@|zjPBtrSozyH(bbve6PiYA4k#Fp$^sTmH5=qgU_XIfKe{~tgMDw zdm_&DY{&N%TM_WQ1Xq?g(zxa*JZigyhS&>uo3k0^Wj1h774A{?S+LQ%M%!1%(ZzN- zNbjDH6FZeLbISl&-!(^G3qhhfa7^eI?7I62TRvxES~WPCr(5_ z?Pw&gYnqJ0&jWB)cm~Fkae`uIpGl#D8b4rC$P<BJl=N1^N^G_#BO?w_T3p6)GYK7 z^Nlg~=|KFmoq~Z*QGDaxi+J4DjnAu_Fy_%=1SN^dr0g@~dV~$MGQ`a3m(X8WScXm2 zmoYAIFn%N)BwpL4P`&0p&N+3F-}!F1+HnzQzPVuXEEW9ir;S(Vf^cyHp_vx5;Q5^3 zsS{vzQXlIE41{>h5`?YHfzyBSG_5C)`XAbZdY`Fevu~MLzC?;WbG#&0Y#PXNMvi6f zzTa^+vXV@jUn}0VUW$!3QzFh^TgYd(Wh6uBzWx|!2Qq1^c${(!`#kbN&XB@^crt$< z{(Je8uw@H*o?$&XoODY3DXuYhQCmMO%e#cv<#{-fzEsd=@9>v#-$}hm4nh4JGWz69 z{M3wq%=c257&TzcmtA~m!dbi=<%PuSei*}}@%(xkCW-R^;a#_F>by~e*O4H;{V*{Cf2Kw z)bB1{R!3m{I~C~2rspoIOB2`FloL^d4UaVaK_;c{g~f0^OmiI}ns_se<=nFq*SI?1 z`Tjj@#UTe`b*q8gj?^cUV%69TpNHaE#X;{BO?jo&oBb-*5;`M{U!R(fO5$RIYmYXT3x zKZz$E`ocB`c(E-Dp0XPTtNDdWUvjsxlZ{py%}vHNvF}YQM1vKI$=RJjBAGot{9ymR zWMt4HwoB;%YdT{>0^SHLx_lp8Z5e^d*-^Z~QxWUT?~$a7m85On7|cuvKym&Z=uavF z{r8%^zkLaZCZ5AwwP3tZk41@84jh{EP(m`0)KD#CnNzXM=^lix7;gDRqaghp0$MMi zH$u1r@J?3WQ-t5L>6miG6AMq7I+WJuuzneWplY(Ty$car>M!!pfS^kZLgd2lqB_rFN?Oo$}x{Jh-7_DfkpjwuW3 zm`G;dn}7>igZRbH0@58l2oIu@N&c@3qLeKQ1f^MowA^~M+2`Xh>V>4#x~cp2Y$ z(SyhK-^DAPW^&`@8vL@?cedBalZBi;iEY~si*4d^*opEBOx;46_-+`A7l&q}UELVz zGr}OXtddmMttDpPlwmEL)56N3IBnlP|Kil)XXQ~g{SY8$YYN!Q*-fbU~y2;Qar&~NabDNmPa zE7Sb{`clQw=5&yAUz!%9ObbI5sMP*$c;0vg)elYZ-du`ngXXip8S*rR3(O~FX__>! z9^EZD7`QA8W+}CJGq06=cq>gO1#8gAoX-gBh(&Y7LNqz8L6hJqR+&zP)j@%~5!i|k z6=66LsgAB8_QZE(4N*wG@a4ytjRcW-qVXAIcrf$!&NOWN)Obi7t-Z2qh-&JE= zsL+`@sY%Z_D$}x4$t-py!=c!642`eH^uFKmGp-Mf2vedP_V%S-I$Nn~kOKWUM~Oam zm8Q`$-*MBu8N2sA5?BT4=va7%CHGUIqra%r87pPzJ^KdS+LV9`W|tA%d<%;%=90|m z%G7AM3C)OBq4l@3@Y;7djAMa<57*JRz!OFb6=?lPRaz@Bjtp2`DC$(OQ>%$HDC%u>Wu!g3Rv-9KA@)m?Yexesr^Y zbEWChHF7k`={+9SrozTf(9nz=q4&WD2h7_n)0ZmH&fQ~#^Po!OpWnvSog* zPowm%GXi21>4BZPuxaxb~W~WdyH^hb^f2{SXhm>#SDKhY)KLH{rMa-f84{+*%GWeznT7c zFG0hc0?7JF5dSD1PyhKqWo}56fmmL(%}u%Yi4-^Z5b6J#G2_rbC_K%=l!Ienv2hbQ z^SP14$wuHwQVx!IG-HAFJru`zqsVqN1RNW7?+n8$Sw%W0Sc0CiErydU_9Q zp_=^k&Nz1KR~NarHwC>mt+;!`4=S?~`8|CLw&ai#Q+~Wolt~nzeb)o2!3kKAbqo6h zUA1h=EAsx_b7GinijD{u6bX7E5BNj;gu;4Wt7?F}zsmU^zR6ysm-7Nrzav6=;LY}$`UY?4wf&vp zbgqd^c~c{zrskxtp_ZTnq(VIZG7O(iMU2o>O~x7Gc=R{fo8^n`f){W(W+q*$ph4@D z&f|bfG)bu~VvbIu_|+MGc#6vb_H11mxzMMa*e#yQs@k>qkBkZYLcdZr<QH^=xNO-25}$cvQ!UWtQ=^4J~b#rK!maQiAxe$`qe#xCwOF1Mg}IhlXbT_3Z1(07wy})#s~Lo_ zh&7m7;(>sc9q4{B7TsH|=+}mcnCi0wr!IKl^(5g;3=leW8n=kiy$8f+hAH#8oPmOW zk+A#mA52Ci5=(Vw_9E^LQ+XlH-QG9wK3{FnC6$VCi;m&P`f$;uffM=F)>FLd!91=O z?ae!ydPtS`5^NpnMnWGe@fr_LzDQ{&KYXW>{g<$W_0^w8woF?j3N&qDlQvuM8^QX# zB7(4>m_ibg6i>EaCyY^Lu3s>gTdF-|(&i!LezGt}FH@z@e=Q*$1C)5)nMmH4^O9?^PMV!}IU6A2t?zwA>DEp?!_l z$lvB}F^+uZpgZFDGs>_GKS>6^h~P$lJb1j2#opJ)nSa)~D*l|k07DKXVt~>dc%(k$ zCPUSEbh;uh3o~QdE9*$Sx)+qp1z%#kKm`5jPloyS#Z-4Ge3B)|PCt#@MPYDoK7~_0 z`dH;KgEj{a74nOoI7=^~r_2M^9Rx?qd&!KyDuQ~xSko+SZk9K;(_*(^G$+m3W zHdP+3slp>|s(ANQiaTL3aPyvz-fefpNw1xSSv-u-z2nV;|Hg9VIv-MR^qcHaJs_TC zvxzH(-sGc>rg1)gD__2E7I&$uVrhPZ_#LMJeoZZ1$i>I;Z97-V#&vLF7{}*%Y4X;W=477UTuF7=X31-JABifxDCsLcCz(2OpF~n&B&qR^ z;1d>H;29Tkx$)l5+^Sqgq95{v>raj3-f@Sx{GxcS+Q#@b`8K}#MXS&yyv)B1eZq#! z98TOLq}U>zCV|~h#D5-o!o@~+xtr5se)nOjNJH~8IkhNNtQDp&$&G!$J(pJUecp-u zmzN%&Bj+l57bziE@3)Yb+ELv3R21)N67uxV!ujbLNBNFx>-j#r0G@HMf={uDgrQC; zxAh6&wWfAlIb}aD{CJuF7?r`Z`$zHoQxV)NznNTkd4p)4^0X|UsK*oMT;=`DE4jt3 zDlWD)kl1XIh?kqRli-l)qF*By^U_cGT;s=6zV3D*-?6h+;D4yIfZ2v5ub`7ToR8xp z*R}GRyytvreJo6hz}{VG`Vq+P5n@788~w|%on7? zyS@s~kKaJD(PnhbHAkB1f2dsCpH#&KC0&eE_CW5tW)vEz(Dpup z)@1XQeEV#O8-?G(%~2dy_j!moY(j{#uN96^7-nixI2-Sm292faS+1(9u40Ud?5k>M6&P15eN{ zDa9tUaHx-1i6uh*XXmXWWY>YeaQdW7^V0j!aPkP%Lca64)(j!jHw%rkn#DK$WvP{d zK8>>xdMP)P1pjjsQdfT_+Si6)&YM=E)9?kgG5u*;WF!7P_JIALzWDl02CtQ-Vw%8R z4|p4gJ$Hq?-T9?>6+Z@ty|+QxEfto{6^P8gh1F+u;XndJL#q4)E=FALGjoC0=PpZc zS*#JfrI)m(Cz0yvZ=mrC18BmICLt%$g#8^&(AfJEZ-?tqjr9X*dCeDusW;$GK`Wwi zq-a=$99aR>uiEPtAkH zzkg^QI+&U-H==%*hS5GdbSbOPqE2;#>Brn*R5oiQUEel{)=ZP4>(lbV8a>b)8A$p( z?N9r=j-t!>XgYhpDm^E83h$c&S3NHdpKrV;Hm42fc}nQH&!eb**%xfck3$1H3CrvY zxM+D4R?BqhjUqETMMIr7mDVHUUL2~^qwsKMCcc;#!FGl$b>AXQ57o7z(61Uv3g7Uc zRf8sc(4j|We8cBYO-S{uM&i?_m^DI6@V}g??#@J7>(ws&4SQ&W&@=R%;zj!yPNM3A zmFO(RZp=Q{hRoD9boOe}IqNj2UcX)xm^{Ojsc%qe)`wPG%F-PR?!k>k;Pkn(=hr(U~>!lwDscDl}OJ zG2MkV@2_LO+*?!@s?rgy%Cur%2WkdB%`v!R0W<4JjM8ex>jXI}F;SsnGv-ILe#k=%m0Vfy)^oB+->{X`Af4;-58#mCi z@u-jsor`CN?Un=MWayQ=5p;RX06OVO6*jy+fl**?V_iE|4rqk$(;qnEqE0KVHRx@Hr_izc1e?+x zG;ZrlUzts&IZHgL{<$1lHMX0Msg0vyf}go)w+8)@o{G5b-Y6^o{&^|U@Yx!jS}j*CL4e)!HypUBc6D?B>^94)HIO zlzDKoFL~VJ4IksHLhiyC&ANH~@;0I0R4|!44Zq5yKJ6pqwiyl$S_J3R1*ly+nN6wO z!@3S%ww#=wN_u-nz;wkpte@0QV*5_FEC?P=pWEHFpnp5b36}{__cKInzdPhkUWw%{ zi`8s+q9t26#Rdc4EreayP>eLPC68P}n5*AJ{&d?0e(6sEKlyhIj&{Uj#MTqo_S=oL z8L9FO=HC4G{3ASI*dnf5q>IsQYw+t}CefMV%jDXpa_uX#c*G)IE`Q2fT%4UsF2rsp z5_?;wzNw9!?zqV`m)VLh+uhAI9x_1e-{He{ZdK)7=eF}FG9LVO$Y>t*M_a6yp@|K7 z(}fix77~Z=(tI?dh#ogi-uQ~4kfo z2yXA+*K+Ty_K!KCMu1N$;P zNW6Q32N~5ahhM7&HgCcaQam_;by&&Lp?i{8(|T8;Bc}wJygnGW+Mf*hd|j-%bp{K5 zGLoF!y9}lVf(CG?hq#KLS{l#R6CM6~nH+L7W(nKY^A|mt{PMLzel~VxwlAHk{&r#5@AIp+Xjdv0mA16=0_z%yEjOJ zJ~~S59G6Ja#p5MED`X{~NFl#=$eW*fUCs4oRB*e@*WCZe3*K@%l}C&1xy~XzUMv8s zpPD!D5$3Ht$>%=TPdLbpt!J`%o1@97`tc;lGJ?}O3HKeI$EUqH$a}2zi(k)OfYrgl zNZ&L9cYhV}J#G8>(9QFBzj|Nxb>LSL=W-fv9*CEMW$ZKv5fWK%qoTDpL75Y|COvdw< zr_rtPiF6knVH$DPe00en-q}>hr$r7#jzbW74d=sF^g$f?;|O0XSHv$pDB|`Zm$}*V zPom7kaB|h$iG6>wkDt0-&Smbl^VRE{d75}5Pfr}mwbQopy!xBGzDt-tSH9;}A8NRl zVOId%=H~`J zd_V-Z{N%~atzR>}(HZ28k2Bsrafj_83phFV=WZ;8=-PYBr|m$QTa8` zLiO7oW^X!)Uhcn>4f&#HDN0*LZf#2wZP<2$HRjrKhb|xPVy?oCq%7br@JJi;o$*N} zk=$GOl(oO{=BtM%@WdI9xU|l5BGAyG_}v6HW6H$~wyfYJA)0Ry?l8a9Z1|Nq!=N#1 zD?TXIlBLF(tmexhZnFI{PjSD*n_PFWfE|9KC#q7+M^=X$_wnXCr<~_wJiYj0?GQft zw+!DHxP||4y3XTuKl4fX{Uz7$YfFr?2T7`LPLOmRErYjRGnSuAMOk_nR!9;sq_Y^O z=DfyNFJVW(!ow)c^h8?aPGmNmf{j8p#*Fxc!f^sCOTGa8Y}<(B{4L_zIT&WMV&Je# znszB0(Dx%1Xi0=IydOxBWSN(ucN^89bU7Vsm+xx-lc520n!IRkpZ?HMrpY7HAfKiIEj1Z({CBm8>@mcmgg$hAfeu}=xfj-+;ZQOW*lohD zgj|_NBrR2fwh290nDtTMYURTCqBkaEI}QolD3Oqb(3R~1U)}<@UV%HIlYkl;fjpNq ztQGwJLC*@|S1zKt_h(XCwNB4C&%CM)YU01Rb|Z@zS^uy;tf1 zPZ{dnPn-5W(xVSde_~9{G~69-gh8KnW7)1UlzD5@YcD5J{fC2Ty=V)XM_(iRgpX4e zYhlvUf+I>p>F^GrV<)33r~n4UV75F$)0X3~UM4Q6xc3PHR8?8oN3gzgChF;`eh^KqJdNCOC^v^0)`sTYV&GsyX-Pb*+Q+q@r zpJ-#ng*F&=s?aaZgJ}PaQq=BND$*KOW9BL~tT--5@*ejWy8PqmycPr6rPqRqo7Nyd zBbL1Umq2>qgu5ag+P!Q7eNn1KofOJoxkJ!eFB+p@IYEI=H{M(5((k(m($0CmVbxHF z>pvxs(z_2=r9#}kZ$$ggF{BQEhtRau18H=_KzfRfqrG07ZNrAsNHiVp2 z*iGI;4hfUP0kdxG^3bAH_ybM$hLrZ&uq(OhY{I&S#8U4XTviKti<_EMsrU(|YZ zdkbch&e93C85p7V05-#g+*Dfw2Jei7i9{D0tWJn7h^~^jf=KiTnzYi4HjEYWH+L)T zk@5Z%an(5?4$ZA(N~KMLX01WLOzcCW9IhhkKqK)n8pUG5Y+1{wX7b1JH~w27oVD+N zkR%&||9qv<7{5ti4kQrv`XPo-QlL9pWa#kYjTrXe8iGHah3~^a+|WIV`V>`aH&2;9 zekDuCo%{g9zaNn|w;M|}1umoF53~s#)HRNl0_fNAk6mCe%Kj3 zlpT%G5ak>H7Abz7dW$gL*5CRMy zE>~}|zEckHb9Ni~kX{$|&U7-K$lMijSAqvs?ujM$qPT1FdVXcI953?q5Oukm!B}2k z8$EEuw!WIM&6+IAzw(Zaj;EsW&e|AaeF)~&-a?nh6ztelA~hBCPeB;*4I78#30`Pl zb^@=a(aKv_v$E^Aj?0&92Pe19#e`Z}1G67L2n_G_9-cX!2%p(WVQ`o4)@w_K)G&iW7 zg;|-g0v|dQ50#e+@B1P$`22m5^sKd_TrVv)H!J{mr@EtP)e@*_ej!Q9=gF0OyUD4+ z7G#h?f4)>~CAmAbn{V0P$Rl5-@p)g5^FY5RQFS)PuS1~O~C8n$nIOa3!jNzV8v zh-&Yf@ruDgtgv|x`IM@Ov^qIR3baIJ{r@l*gHELD{$Wk>^F)CnY3RKkfELv)q&fWz zQ)r#SH~RZo>do9k=JYg?+&oPzH#bc0-qc5j)=W{u4yozJezT|w(CsyFJpDP}}Nx!#761`+K$?BV*_^}(g{N(Hyp0eR8UwKAD@;v<&S6o}pC!Gl4$;yU2VO9`p z3|z((9!=%)@qHynPRK~CP6)e{)|>K&Z(K=k(rEM=-68>>F7uk=RvvaJpU(~uyyEDa z2V18& zMr?dv2D(zKaY%b694|PLg|7G68MB{giY;I>Q#O!0rn9knzaw6n%V5}PdvT@fXZF=F zll;2YMDEp@BJ%!7``@{kufyyfV7E@h`7iO@EZsC0@XFAjQ1#EACnD`e>vM|7qa!*0bd zn7RIdS6MdHx_1j1&o%U5$96Q=<-jED2V7tMg@sQM zUdZ^xz`3lJw1?bbo7YS&N|MTA4ao_Oz5JXzW5WXTncU@h{NRr`zF^{G9;YWInKFE+ zMAN8(n_s!bv-f22&(Y0%)t#UG*;qx%>8a|HHR`sK$h`5~Azh6;Zz>j<4bmeYgJX%! z@)nZaE=JHX8izp}>8d2GQwXR>ncOcW>hV~E!|Y<|@bM?S6NdZp93_Q;_u zFm4(iMpYti&2Lnm&BO1Vo0#KXCtez6$^-9Av+U`=7Ky9xLuTAVJYvpxrIpABPngPI zbz3nHm*FU1#qc$?74<8{XsI^?$$ljsQf$u?V43mcSz{T9m2=hfv}x(id?#H zO@pd^$T#~zsJL<*2bUi~;2mkm^mwq16V`I`r&(gp%cCH~&*Ag04b$vy@Ig= z^9%$5Xl^2R^LE6AW%tneBM<5Nt1-9BCpU5A6n-*g3!gFn5c`}_fGr1mv9bIClFcm8 zxz?ODo7wPxNeB50{f}g_jz8k#17Q7I1o`S!#IMelHQ+lFRk`zPKC!vcHkrg^#0w%_ z89*E@^vF!_JkebtJR_1!<@Y*f;PDj+cBLdEZgC38`yBk~d5Ezpcj2@r7}hl_vt(_% zN!ZLu7|sHrY+H$Me}3SGFw>q274o8Al9}P)xvXpYU1DXO2;0T-ln9(!uV=z8gzg`r z*RM7Cg<&PkcWEM-t9u;|rPB1_3OQ;Toq*w%2YJb?6lR;fomg%>i9eg4K>mIYZhb1o zluvUZ{pRFJ=VQX&wQSsYmyJd5Jn_bVENa#{P@5Q2)W;sjr<++Q7{M@M z!*L9pypQZ{`@}wk93nrBXTfh-F^mTM!nGOAs8aPr%1T=js$9W7PCCHPYbQc^l{DS& zElua0zYeKzG5M2L!k*R&dxSL4iOBOGxa+4w+sYK^qO~#Dd-E3gExn7qoma{vZT>Kq ze~t0B-w{3L5kB|xLxrO`vYi`9e8%HkzZL#y$Y)R*cOO-8S$MlK2_?HF=PWI8 zZP85V20p@_Yo)mTuN>8P8elDaKa;iXaF5N#%fPWjvPK{IB`aXJDh|_Ip5u3y41LLd zpz``TyqTNJeDY2*AMcAq_EHp%&z7Z&><80=D{Xk-D#vW<*YWWshWvWT0wlYMX5qgBmDflgA0F^s6SB_PQ!O?Guds=?gGM$AMP-n1_za3;0?h>^m7*k7(yeIJ~SR{nq<3FU_|kr+P33 z^cTGH@y!^jTm*}~4u}oO5cM_Gin>I<)*lh_|-;fiN#2%NW-;a->%QoZSKqsEI$m1nbGAU|K_s#S2HvT~ z>PUF_QtW`|F zFww{*fm6r+f7d4A0GPisDG46&pVEb<6mP;-3v^LT#eC_TKM8AzP#dk zIdl73OR`3v0ZPK5d%y;9HwF>~$1IYxw1br@8xx6(9m1C-qMThucGg5>>g^*f(@m+B z%Wk6EIuVcOMC1L>YbZ0>0NfZ&=Ge8fxm!|*iQy{EwbSOW2S*sR>1jv6Y6_!!$i##kX@~vw+kJuv+z$>hpd(^VP%b70!y=s`Tw3woCOx-Kb4(ms?tEn zm}aqSkSQM+Jf6(d3})-5-ymxb%!Q1Q!*%$TD4LQsfoJT!!-LF)9pRfNKvTK_epby` zRJsPEs_NO@#o0Vq{{@fPeu6KU>mx7-s^C|h03WGhVzYh-SNS67v(w|b&*c?lRn-}E z9Sy^iEMecPj30UH@4_~n)!^Sn-hBP|8dBFY5jCgGF(&N`ah*AV+%2jVhg7ApbhidB z6FfoE5tGCR746{n;|+veXdY~n$_|mCrm0A)K7hTsQq6-KFLI@;wp``OGPXPG70G+; zh>xcxVTOqbtFf);=k?2YMZ!-0{Ou&d_FO=zbslz)>7m@cqz_g*WFwV)B^lQpt2Qw&9R~pAHOM=DA zUMeHA(iP8*ry%l%jHR=g0pBpL8m`-P`R)G_EZbL4zzVAP;8kLy~m>GN)Y>zHj3{VdxS3xzs?5@pCpkovXj_U_Hf;( zBJR?5ksr7x;)m%wHYcTsi7#33u(fiM5$EpnQ8O;^zYVhd+Ge4z?5B;l7G31gmv$CD zM@sV4{|$fJ;m?gaY(=%gp3iDljF&2h@ocL<|9$K+ADI@-4Z0d~RZ`M$L3IF?&m2Hg zr{Bh*tX#g?WF^;b6*2kgBk|y9G@@MuFZAtXxF36%dv>ohf)ae#6ZI#=sxl0Nib^3r zHWj@!hNyBJVmW-SD&74|-SXk5p>UVIiJYw^5M2{^1L}`SR+%3wI;n&;CjG@5Kbv7o zZwzL>2}8l3aVYzz$+Sx5@GLFHAJE&l7CnIG3Jg~3^h8udEEXHqM)60Hb$r}0KR&(w zFqYIv@ZhX4%PjA;v{nz{;-(I6xJ^nD3@M)G*+yEjRH2|ZhQzK^<(gra_@Myd4p5{Z zu?(Hcn_oU=j-DOtcl<2g7#+s1R^8|M**!ez`znc5%z8=nR4vJ~U-A-NBY~wDnaXd6 zpW$tO9^7F1b^ddMv81iGk0fVyHCN6&%;#Epu{pgHN#903b#AC9m+CV_GR#C>61q2& zzZ#{>{M@~u_l}_|>nQ$uU*N2D6sRF zR*=cRud%{4XE~zlp;E0&mu((E$B$0K$)6W;;)5@8yVMF^p&i7hy*LPs7rAH>?h290 zYHVU9`zAn0&n;h*UIdaojBGr10^A>MD zz4Z)l?!CoN41dA@9O@(axMF~0k>VzaoAY0Ga#=D_I2S^~e?24yLN7{ugW|B_7`#8= zMyBfucPyCmqSHYvQmT>c$vTSwr(C=>35B+&nrPGXBHphzmhbjH%@Sn&F*04y8F!AM z&doms)}8aeA)j8ZQJ!J%{S}pD41vg;)2(ct*+@c5nGT(o+zF59~Lz zq-E(TjZoN*TETp~`mxXqU;gCxAhz|#2BVckwS~o455-NDWw!? zQ6!R_j&Gp-+MmiobbTY zf<2FTaCf~<3@l<<^PWk#>De(pFemy_rP8D#CE!{ng#F#Hmn z#X@8);8unacs4Bnci~&WB{>7$mu-iTk=hWY8;$A5?=v}veO7}OSwYH-a>#0LhBcEJ z#Ko480qG}E<8c;lzIB|Lyov#*KzSP8G?osKY=Y6+i%}`&BHDk-#_X}ltnp*Cpuh7P zCakH4dWX?)F-Mncm+izlTR&E;L12dc3E0tB2r-sJ1pgj_jltuoD9q}WnKCX2h(mQ*!{^p&B9h4z~>%EG0DA@kl)UMtVRPY3_A@Ij+nrhW673b zfp?j$y5MVEW*oS*Zi94<{qWv977E8Wfkow5m_m&p#w(faxj7PBJx79H=0tchd?DPq zl_cOri=kaI103q?p;~1t+BrRDE~#=>t-`;>gev%CXpoUcL4vPQCZd?`A8JcHv( z6;S8G0@$i4tO4t_XrT8E7~YbH;?pGgXx4%byXA5EqRlWttQTA=YX$6kf>pIz2sdl2 zMYknV*scAFtoDk6qien3vU4CAm?4GsGBZH-o-LlW93wKjKLhG*;vvV)6H+e_GJcgC z8y7K{4%;}Nb*$4RDq_BH?doo5Xr2aUTC+r_@@z0R#0D1HO5^rpA;e|qQaD=?2^mVY zI{0!Hy6*4_=`Ja6<~j z)gK1Wi7C(-v;cO=Xu$vpb-~uBKHhK1LBHVXq|T{`xRnow+083pY4IMx4qiMAU84qd zp9}DIyfsFxU&m%2S9zmn0R9#m>fFd}@)*OXqH5 zGMUdox?m8^uaAK+Rc|!R)a2V+jQ9<^ci8&00QeFmI_Fp?oKBR1Hn|PR)H?8u`xCqv zx1Y2fD21Q$p2C%t`Or}Df%LpdWVQ`97`*Q!p1P|JcjC4|l;uv4n(qgZ0m?AN=NK_J zTTJYpWn=ZYKHU8!noa1=DEIw&6zO`a1~#>xK(pt8am9I}R$GUw>yokTj1C(8K9YDu z?SNdh8?f#~5#0PdiEIq`9gWPD_^iGn?DU8r`AY3j_Ev=|$_hHJmlU|7l@qTTZ^0wC zM&LQGQaGYBnd(HUQeD4$mVU9R{O$dA9C%8^KED_Qe%Bmf$G+>N_E7=*bxbgZ9}9v( zPfamPbWN0YUl}X}{Gf|p1sT60muYRjz>avH?1c?Pk2cOY1F|!R@!u2|^N;|_?Usx~jVFq*)-pZkC}HF)sH zyN2=Ww@!lfRy|tv{xe|hcciuH9_o)U;Efvx@wZJqR=t|XLDS+IH0lPy{G>RMaeD~v zQ@)Cu+ogECM*?v-FoUj?;lf#29-)T*qF&d0HmgYw-)frj5vOB$rC|a3z0pPQkE_{} z5=)ZTGXgpX8^V@X$4SK-Rc<)?1D41iMF>k}XGeKJ!;xDsd&dO`>M|r@=M{NrwFQ@G z>%=3yp=6_z3boLkK_k7ML+8?=eA#UezVWF!FC&eX(*pgd$E7lQ(Y=7yYDv>yzxwd_ z6Y?l$p9pUnUxL3u7078#fLm4jS)BM>?3z*snqJk!`NnEUaTesDD1vzP2toh7lC0Z2 z0iwcPXkn!>Xw9o8g@RtPZ7m14`5ZcPL!rj^200|)^aie z?gD;MLk<_``10F=Z5(Ug0=hh-oJPklrLnCC%RjqY@C&njxSZ}#URQS)MA5c%M3)k+ zGZ{;IUVK8&A%weEEAub8S|oaS1*}^33uvXef$i%wHt$R!>Nj-b7kxEukXAyXeI`R# z+As)qoI?ut#WH1c1H5+nFb=TY!nLv`DiY3b=H^>Xxu<_0?q8XSzH`(u`AY%|*&>e9 zFMIRIO$&LZf*Dr{xQ{1Xg4vy&ws7H*H7vVogfV4vxRGoOS2ncfDLyi|oqmATk%X%0 z--N2CM*MJMBu_~YT#y@SgPLI;{UYE$#a&xyZr&oA5ipyZ!(vi&P>~jD+f#3;NmT1u zCj`t-fTDA|VY*f}6Pd_`DS&e+rJY$aOl=0M(A+4IKT zi|F;6O;l3m6ZOx@r3)-|VW{7%Zy{B=viWr_` zCs%C6=H4UA)58QAYNM!-55q1TnGx=;|mTDJCW>EA~G@QOsJ*J9Mf4inV^mYyCpk ztO#0dZ%f2}`2WMCLH(`QWG{CY7h3C&u8srxlfT+`d+l2JQYiOFrTz#12!Hm**E`JH zC^&HCzq$WDF3o>%nhy~BYb^cT#+K%$W>Y3lv9vTXn_@Ec-xdA6i;DlKC^hlF6@8l& zS4U~-iQkAuN*Mk{F7Yk*XL{vZS$WqV8G%oxKlk8US?HoLA5zl&_aua?pZ*Wp{{X0V BnA89O diff --git a/data/adversarial_kernel/r47-1_s28800_adam_lr0.003_wd0.0_xy1x1.pt b/data/adversarial_kernel/r47-1_s28800_adam_lr0.003_wd0.0_xy1x1.pt deleted file mode 100644 index 15aee5c6deac99411f5626fde6df3642ee3bad35..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36920 zcmZ^~XIK<%&^3q(f~c4kQACWW2$EsC4oFUtbIvG1vY;XoB{Kt(lVBD>Q4Ao{h?p^A z&KU#vC}vO$+wboFd3UdGfD5MS>FKGv?o+3#PT5+H5)~1VkP!L*Jtm4w7V(P-SQ8Q- zq#WoM=cg1E7H(o7CF1_S9w{RO;v%`Tlzi63g@ngS`oskJuL}te^ojC|@mm`d7Zf8I zE%JXjB`zo;HZsO1UR5%Bgo1*rICl%N+{OG>2SxwCSCupgkMxUk_z8ZGKNiaQt(wJuuBN`HP_^k_Rv5e3T;+?S7~Ph7OP0(X@A=AMm+ zi9m^JIKhw3L1H z1SMNbaS88#f7o8`eg5ai_Wx+JSt3*Zm(7&__r01JO!)tp&)qORqoXs}sbe}%t>gJ| znU4H;v5tH7AJ~YXyX?=-Gpt|30amV}p6%(|#;Vzuva=;B*qc(@Syi7M?6%b1ti0QP z_V(lkHt5VwHZin@U7MZD3g3IN&W~rZrc2fcr?y{X*0*nA?L=7q1oMv3|JcMp#v)Erwk?h#f z`Rvb#8n*vn9qW>|o838PAM3YrC)=yBm-Sg$&yJj5%ZfP^v+KiaS&8IbY-H>KHY}in zeXn+lwcGU{``cNx<8`KV$NZ-%9q!{DJ94eJbX3f8>R7BcuVa0iXotkDTkM2c``Pcc z#q9RJC{}5kG5hAX3_I3zGOIjoP&oZYzwq~QDb`zAlKs{u$7UNYVNdQbU=w96*;$KB z*|=;Aw(q(gTUe~j*8G~px~;z`TsAaSSjaNWs{QhmJ7+Hq?HGZej1~Ac=mZZ1cQlO; zz_R$Y&@>7IKOqLH`O#?hkHyi_IGEPQV^>oG>K-NH`Au9tTy;7l*orLayjnMm?faVJcu-_Al?Y%MZSRR9W2H|Mj7!I{vp_pbC z1f3pt^j~yB$sHA(+#&*1{tmL0l&6Hrlb9!0<_O1%whLRu#U zd#c@neX6U^#))#rH_5Uuc~a~(ts&uv^B;s8Xa5$C*f@!O?mdTPoyh(^7Pfcui{0_d3|V-rZ5ye6d(qckqDFRriuG>cI_RQ{!pj z-JNa15xSMaWgp#zJJwHY-?MKV^K@$ulVj#hvm*DCYtBE~&^aG--)f<^&IF+gtWcw9 zixm0Jp3t^)hrFE|oQ9krW^IQKFDq=)GsdE4>Ubfu z04=YEXmC_B3AVe?XSu6PXo3i{u3El5HF1(KCo@;L;n#6tP1Y4*=d?S*is@}afA{mk z3)j+x2U@g*5(_T2%ZROS&)6!;Luw80U11T=QM~Fz*XyEox3(N%_KiovuOSOriHkmL zIlqE^y1Sb_`{^Hh`@BkraFKLJ=7f9f%aAfwzr}`K@nr&gc;P-_@2;lywFM=-f5ISU zY*PgzINil;`g@9LI(Ut#w=8GouPS2JmN4JX8(_rt1=VMla*2bm2e@%7<(_7A-R1O zcp^4PIB$=4&2~7H>x_F9F6jQ^j@|n`;osv6o#E9uy(0=z=hD&OSqf#VGz9I4gJ<~~ zJo@f~8CyNEu+RgUx1F&^+XW8e9U-S?k6o{=u|mxX-ZL!`9b$%RMZ&ZxN(j`Q3GJQ_ zBp$w>#LkD%iD4Nsx!=UJUpHYEx*g$dam+ih=*0H+s9y)$^XYB7$djb@n0@Lx2j`{m zfN4x~k}D(qw2Zmi*U2n)eagheUSVPzk1?C7z~qFUVgoLj3S9$)ZGkNT%-_`8%(r!tBzZ=G5^6Q5 zV7C@2x@ggo@VR8YVgfOX`Wa#0PR8e)3-hlbfO))8h1n?*#q2GOWj1fgX6B}LF(y?% znfUVy$Rs$7@`434CFc~)>ic3!5e|$qg84lYB)}47n^&Q($_8_n+2d)B zBVK-SK-3H;cynvespEyKzg8o(EE*E2X*fEm1bw-w_^B6zq8UM`SNBG#ge!s%aX6A- z2fmX%yd7*&p>2i4N0!*L&jL3SOz`Bl0mhaSPG&8~+9T3fko$|I49?O*K{>fz@}@>j zMS8Gih#6lbWHzi8FqiK8GE3g;Gg@zgne@CkW^h3olVz}zd9dX(6Y@ifRz%8@xx+k~ z#M7t10d*=qv6No_9ZQ?<9Al1&c{25JCtKxjCkaJAim}Ee{_LnZO>CN(ct@wtM>Zy_ zh&6n^h;97YBwV3Bru|{z2FAbcKZZ4zqvA!zv?4ixlJ0DzzD?^%<82tV^aK%BI8pag z73y&wPxYI{=AxYK1g$dyHLXhvlyw@krndu?f!jo$rqFa&J5i z3xw|ISd?_6p-#FS8jsR2Zg(6GbG9!((K@0CKyBhZj)Oy<$P zv+o#}Yf()2@W&J3$IFEo-b-2ieWh&0g%51oggfkjVGR2`_pb1V;=6WhX*oH1p}xpUvbvW(}pJ8I#H%6?zaik9s2IX%Q`@ zz_Y59G|`BJZtk>SG>b;9EuqWF&D5@Xofb^`N?GO;@t@Qp$b8X+zMUb`O+9+oO7vJ!)@TU?TT^@zRD!a?-}{5ej(t zLkf12hsZ{=hvtSIq|v6uw8AKZj6)*H_qZjc9#*H}P3m;=hCH?UDpT}ohID@!(*u(L zQj$$09rJw3_bs8wCuJ0Qsglku&nHcTG`eEqOv}s_$o^(OGf)!2WK6rzzGK`y;b(&| zHa7G&J5a;2lV7i74KJh%E$r)gp+i@gvgSo(>*Y@0Ju~V1-5MGZu#4vB?W1D%L-cdg z9=aj5gN~+F(1!R7I@TIVmh)|>ro)cJgUo4Vq!lH-@Fvy9FxvSmn{0hbX-|J6)y+On ze72XIibtUKL+-JgBI zd*zQ`VXHC0HxkJYHpAm;4rWc>j?VsEM98KkSXlg^Bg_ za9uGiI+aQ<&wEgS?0mZS`W&;Z^8)W@Q>t)`F2fq6b+MM;Ti8$f^Vm}o!|f?O2~71= zaat8-MV5;*C|$CIy#DN_vl4BzyYCbgr(C4V4MOTVf0!P>X`s>iWi%L^O;^MdDL5d8 z#LujuQ{TfWA|;hfPv+BCohn*U+C(Gnc9E>XW7;|6Gqv`OLAvoA>`2grzmp+$9$W?0 zX^uF3!vzz{T(M-K8|3zRKt;(5R((ELal;#}mwlnH5`3nbWMtR|wxfiCFxMD_@Gd^x}!U|bO-08GN zaijtIvQ?2VVGi7Wjl*)iFSL8iefsfV2Zg#flC4o4WuuTnR;7}gay%7pT}w0e#8P=j zG^yxklXY_)8R+gKJ*`%9P3WRGo6b?u+fEvNwvG0V+)Zm6N~r%<5*@QMr>j>c(B+H1 z%;cz3vxPMFtd;*!&z?5S2rsOVa7x^DXp5p@3AKDk-IZFBQog zBjpX(=zPL!5-Np_3;DKf9wy*9kMrZD8kRiVKV;f*;O9oB0HUjU9eZHAWO{x+NR)7zaGt@rQeS*8$Q&wy=#zQweohc=ff)4GxSY3 z;rKh9#ZR6vul`dZU9AMNtSF`knFDl>-${E-u2SE+N93IKi25sUP_FS6DwFP{I^_d& zzo3jP4D;w#K_1E4rqL6*3>u(9$~sp`52W|g{<=2WIHiYb-@c~y{i851WCFBQ=Hr)_ z2BiO)Ai~}Ob2VJBeV!{$wzxx8!xJetz0f$)7jxHpbGeW|Mur5y@$wohlUa*$kM;07 zlLqVH0>t$;qU1UsjYeru_FD(v*AbY%F&IT({b1J=fNDv9RHS+1y^S|Es&KLTqB|mQ zxZ~+$XV^ToLEc&uY+a#^&i(VjZ<~OYMh*{k`^Y%9hZJ^pQRJdVs!FXPX7*OvV^~1- zRaNQQhM%3JB|_wSl+j*PlBi@A;6n`dp(^9%pH@aT87VE~VJ` zIAXhXC|T+vb5eS`&g1u=gd5gWv8zSO+4b`8g&N@VHov*f_+B%lpMTa-fMN|~**Kmk4S;*+ae zJ@s*G*Xk*xI?HbQ>Gy(MKm8^)NfP7a<={V60}jf@SbD)4LsMK3a?2H`rQGq{%M)(o zfe8ja&<^&*d_^B9P4LHBn_!#{i@;rrII!`lIGa`g2e~Fp)a0Z2L@JD%;@}<>0)(!C zr==f?PxxT;Vs9*Q_Qn$}Uo4y8g;GOTD7d=8VV)B-qHR!QZ;IA2S{QL$4l!THftSwZ zw##1Algg{~cSeB&hb^Y#*N zbty$!#L`ctrDV4IDx+2$r}J{mZ(&fzc2-tU&-yn?u?uG`YR?bY&Rk*Tsc*XvDHr6^ zf;Bs7py)U)UT}&;IGnfjx1ryrsiWX z>*fdA`RP6#PVAybeN9xMQbFh4(y8t6T1waTpeH+BXw_PK8WrzD%N*TFYRv}v6TSi8QON`d@q5mIh|)meHiS(VBS0ne?))lmg=$DPj5%(pb<*SH(|Kld+H* zUhbp(K{pwn{|XiuRUt@f}YvyA~1GI|q<{>i5n$-Sgx z-bxl%FO$TMA7qy@2@?jFBI_)nFUthqcbZ{ulm%`#SYa^R3OehoAvxCu+n(EEwx~U> zZFGXc9S`)555}rB(eON&j^LwvaV$IofjTj0IqHwYC*0xM>;SbiM_kOb!KV~U6w7k2 zzq$%HH(OxAEE_m4F-5JvDe?vx^juWJk#!R=&-5!zKG#FTL!ROrzbJ3lIv+d_JNr8IAS6X`BELif+Ic{e+D@k8PfDOQ>g7x1hYHPy50A^49jfV&lb+!!%Ai-v0ImV3)ef0Vcw>_ zXO1jVrH95bbRsvC0(&b+ea22|3_U2!Nf2p#fJrgjT4dVF1y zoCMF9Lf7xi>=A#N@#*8}TBJI;JL=JwTzmTbA&o-jm(t1cd&sEi0@;`Uq?fsKG5f&^ z$kq_7F6*K8^-3IYG(@_L33%^~Q1j0KcxD3C*``<^VTFTtZ86f-9SeVj;CxpS`n0#f zNhAr^cLd?t0av`8Y=@}ns~~&W0z5HO{CZ@>#Q~XniwG6u z{hfhq=>BPCD!~|fShtVSH>qa)r^hkt{!3?`7Su5AU0n=+%S*;6 zUxH@8okGtR>Coq7cUr|^W>0nvwMty0yE7*tzjYxjwKXvNxi-9Id63WHVPqgMJCAU4 z4Pn)Ou3}|th*8akD9Sg(_wP1%@ZA*~^dm7MyA0(^VqqfmK>K(OPfnSjVV)6WU+7`d zJqG1-b>Tc!7o|&qM-B`MAM)@nT>}Rr)GG|jIy2z9ixHW2nvr~Xk1;-`NN(kUbl}KF8XhjB z^G7&)CsdNhgi_jKl11l!hf&ceOVSsSp*_zJFppo$@XoXegrDPd*rw7ZcJ%c|_SzvM z_P=Kh!dqj)+xJwaGOYuDm^N)qI+ox~CoaTOw|5Q|ITTRf-(1piPo?r@0pu*EMaz80 z(&V$P%)llUX4+b5WE;s(y~4h6S>;X0`_H z?2n{f$M(?dpwU=yPZq)3m%(<7GMe1fVP>QbmqXe(`cDTMPt@UDp^cgl25w`3tSCLa zZ8pWbzm71T7>tH%0-U)Lgq1Jtpf%G3YRM}hFd_uD@Q_@tg`MeIut?U%nMGQNYEna_ zmMY>SRq$V^B0{Fg;o|~H>~h^t!)Ea`yiSD*cTJ}k{-RVQ$}(3^?qi%(@|Y*X#5hlO zXXY=A=AE6!^7ds;VEQ&}VeV}`&U_k{p!MD=B)`UkcB-e-Mih|FPyzk;R6viFlBmts zokUudXz9~047)gz@qE9ieVT5U(0+*#YpvVJW)9c0Wl0w7?Cqz7Tkr2^FIXGMRQ-C* zEOlH#m))!=GA^3rH)hk->3QUGHkbaTZJ^n=>?yNTiNZQQGeP^RneU;anB#sgc;=%6 zc$VG*UWNP>-rzAU=IOT0OkT$h=3xI$=3EBnL#L3v|4%@F9uQVS-7%M z5ykDw$XKX`lYdmvqpyMPbsES`RmTQ3ElfY51raG8()s|o`CQ%oxE(TG0^s3N1h=+8 zgrBv6Oq~(zzUX3sA`hi!dH7YQfpg!rVUeSODMz%B)~$j6zNz6;&2kLRQG{T479280 zU<1d~e?of3TvI( zShPDbtgtf1njLMilWn!GV518yS*zif!j~hZgiCL#Ftyc>nDcArP?e|!C0nnhtA#04 zH;_S_lhY~lUM!6oG$M)X3+eFhyNu(25p%=kAn%2!4DY`h8J#}AARR&49$xjx>paUk zE9T;)DCVN4fU)Sh!BkKG$c)XOLZ8bGXvvIHQrjbjlp~s96JnNFu|_^UG(12 zfrYXb?lq{Q?wSVNV%4FNt$|i!P0UkP!PdD-T>oSlW-pzCfpMa^a<`I>Yq=0#d@}i& zjv&34cbKC=<;^1?Q}gXDQ=~nY zJ|DKE@iI~5|1pC+PiB(L=_I;r7ekGg%*m@rp8hR+#dPs_%wCOK?R^V6g&Nl_+2gsj z?0m0m_I{Tp8#J>+xN%)wyGd;T(|Gqc^I^j>DqH17wOcmQ-mmG@U%}PtmgmyP>J*CB zwWIiPDkQSx19Pu8m5EXv!Du~O&dWMHnzt!?ozC@VH+5{rPhiYv1TmhUQkhdjyOr`ROSA*MG4J0j4h4?CUct8u= zy|wUirZ(=x>EeNr5%NCTpzvTYUbR-?^6VfeiaQ`K$_VxHz=L->aNewi;>&87a#J0# zS2SR`nA=BH>WDa^h9h;$5xi;%L>J5A$UHIV9IvI#%BB=KVLk;I3^KMpFPUhL3o1UW zVYWBhF)Q6AnEJNmyio}`ZH^nKX}crrJaUNo$Qa)@XZlNaweK|-3jgKlv%holS-nS|?7}cPwyddH zIAf1ryH!R6!cl|70yj~#Bpm!Scg%`)j->6~IGQ2>6ldt8PRA6V z2keotDio>gcC5Y7aF-7Cyh!g1@3R)^}^;_7zRY zj8MhOUrKl;D~BngB_MUVhOWQ2C69e`$hBlFtsV7{+1}H{bj~Vau8oRh`u{Fw1AkmejxqT&Lw4yV(JB-L@pN5b>}Vg;)5f7 z4b`NSou8R|YeJb#0ms^>&uSDtKcU1P$P8lD7foe94vk^uk}8DNlGSY$>#CSkrRgO0 z!IUI**O8k_0V%Jorc{$1bfT}8mNpfUnC=F8pkqVlUreGLlReDRMGP~zbUssLA z;q#T7I#}KgM7Q&B?-+w+(F{KR1QxCYzHHM)5XV2a3|Im8Tp@Ko5KNA4jCb>5O|*km@5ON(S%JOdGO@;l6$ER9tHBCyjllAA5<}t7Gl5icszNymrBk? z(3m5jL*a`k^qUwlJMS}xf1P4F=ImiA27Q>`zk19nS81j-?i=rE$9SeGMwO|aeS-1- zHklr_7}DukNtCprn3jjuQ;)`WT0F9vYG)SHCWUq6xxkJ(?nu#$j0DC)cdU*qQ!5;d zoW}M%oW+h1+6%+Rb_->srwgr*@8zj)>12E)RVX;bjXsHHlPT4Z(&q;19y~zm9R~=@ zx0B|xJaXL;MvX@ok+$eYJ~SoP4P3+0o8mTv}JC>$>&EQ7Pt{v?LHX4#11)U%@MZF0MEDTdPho-%8u8J zL*FUJ_t$o2Pjx2anYV@sD$!vq+Ts{t(aG!~FRG{fvzI4fg%N?zE(YdjWbYjVV zG7#)1&%0IhNhFh28VAzhDpmUZ<2rNx&rx1TiJs8*@(JNROBG?;z8lP|wlTuxP8KJW zJnfmm*a1d4&XCrgO{Jk3)%1SGK{^}M%Eh5}+VzyH?aV(!7H#D;{#Y6v_cJ5CcO%K+ z-vLH2P|xIyThBj|iWO7V#rNI>9BQ5YO!w?Gt zR>C1%9fA|HI9-3hm~uaoTjoRYwR5R&=`}_mbBOVCC}w{Ctz{IsJj+S! z7PIT-J4ViaJOzdFC`HSGjEgoB8C8&_(m{H@uAh{(mq5s+u1Qj+_b{bPfeiB^c|v%-L=HPX|q5UOB@Cpy-U8(4+s2^Ls54_yAW5Zk}Kqh+=&wBN0e z1}AQ$s~I*FBD`xmK z1#~C+0A(w5k!0Ex5~;gFV)9q%;q}w>_(TJ_o-3oF&Jaq|ktK}{XBhD+c}8h8kMYc@ zBTd~|i0zlASzR}o=20?qd!0XJ4U~~VbPFlTpP_XNAJV-oy(A~wNAf%0((A*Is7kks znn&#>iGdWZ$EHQoq$kj+T{oBt-hC#w^$8QJc#qj=EkV1TW>K#MsAabUMV{kwvg;+J z{JoK$#^0cC3&jvzpoYvn<|y-Uzpig5-ve!5-EnxFC!9w3 zAha|9cOOKesUZcg^oudZ>OTac7-Od;W8&lp=p_bXoV_O^Ydr9Uiviaq+_7551BRnK z@t?6fdZpdCSmKDiYwhskvKgLusUd!pILA|O(Aa+mDJ8#%3P;6LdX*E2De95y^5ryh ziUetXy~D`%USr^`-pouM1+4wB+50sVgNLid)>CbPm%%=|ZVNwR+y8kQ&F+~98d@xhB+72;_@MF~0o zX{DnP*J#Jdmn0wXlcKhb#17|?_|9pXw#5ENg3LQK<)4r;*KDP~2_B^OX9hj+7Nf=s zy-d&07v^;HAj5w%lkO&fG+JEf_3s$+Dk~<#r!#mbs@P0HfCjlcXvLNPPg(&gA7}-+| zmE~Jd=dcOO#G-KUVh~PV3&4&>|Npypc)24Od3^zB-w=d1x@(~C>WgWYJkdMO0nP?1 zF*!mSu_wOL#Hm;5jmSZ|v$&KjH4-RmiYM((AfnC%v|zyq3VJq@id|nbJ*!1&^?)>4 zb6S#!(lXkV!&1iid$e-VC$7geM2!zbFy441!uDkLv?0c~0B@`$RS}qPVbG5{Jbn!Y5%8>=ef1 z%Yo6DH|`6$vOQ#GvzHuHQ>k0Pqaa;rvKgL47wlt(wT+ciPNz`IqNN1%E zQet5@c^iMGrIC}+rKpNUIriv!76cdnwOF((2LC3;LXy*e>==rLT~-ttS45-RDh^>W zvA8IgfQY`$_#m|fN$~=F)!2(CbH?gslge{?$KD(!1-ptUda=-#XK zG~tau@uET3?BzH;zzp&&)B>lsrv|9wHLl59r&?Q(f8dzWVi>9l6qMnIE zWUelba8)ThJUSB#tLNf(@O*qaBM+NHb1|i2CNvL8qR8kA-O{*3YgcWj`H_({SnEnL zmn>;Tr~_3@wxKQY3A7_5pRAwO(#-D%srKS6>K+=Pk zVjG0LXDsk>*gE84|v871m zO;rD48%2%1K>eRS(+`0-o-N|~L0xk(&2v5!OXs8Dh8)gaoQ}$;6Y$YN9Lj;esi^b| z*-8B){nslnO8pXA7F>n@L1zS7&WCgBOgLVZ;cB#Vk@HRgBlDH<$zKC`y<9CyRSUmX zt6`Lu5(3`IVTR8{EOz}(i~6q7vJFjSw|55>ay6Q@2e>|sb1m7YG*ZnjmZGHYP~o$W zRC#hVhDJ+6qIL!DuQfxZP9RneZRGNvbjniI9;E!TM?Dh=@```F?4by%-Jg@Bh(! z>07jFE=y*del${{npRd+(@V)p3TdpO#aH$dZ`p0S*F8vgW=lb7hb%UkC?QKj9b@`5 zFlJa28)H<_z|{x;PFFx0$%6@+g+p1>k>NHE9b8YyV9_-UCp4l{-2-xeHL(53a(Lt@ z;d_J{Y7?}fZKaE)I}IUs&m8xUo8b3VV;nrB3svsgca|z8KC5Ais zVyNYQXG@R9@lR5anIw(btK^`hxEOn$tK&xaN-Q;Wf@-oq&oQu`b zIavHQ4`KW~{ISV}Wq1z!@8@AbS{}sjF1avC$%TkY9voG3aKf8g7uh@{#Bmzh*%_Gk zX(Jx(4TeO#Bc6&IVkE1MW3dWYxN!~+-JXVqfC)&>5l3;@Xq-tFg=6SwRF{fkV#8G2 z8k9$GsWPV5X`wVw58k>4aNT7J4KhYlj1lJk=0UGt6KaQ5aCFWJ^i(WI09Rv*5%Bas`a&Q^MNSnz-?gU~FcFFcBx*<}|{` zC$EE{O**VjWnzYA9&T3TBX~>!8Vd?wY|lsU{d~BOC_q_6KJJzkK;{)6&qftO)VLT@ zoG$-;NIi0Y9Kz+}uMq3pg6_n6=%`l0E}$6w)qG6muCsexJ_e5EL%B5{%$`tsheP@n+d~bIk1uBW5Sz4tWDvgl_$VgLE%p#Oj3vgo> zA1&=pn#`0Y2gt1W-TAhs9Dp*6uIBz6m)||7s!SQ;l0(J?dqREvz?~;pjv& zEKIimaa!PI5%v(}G)q@K+`0ab2jcVG@aCuk%x~D>>;+3$=<31og$7RcDZ=KvBCu#V zI*%)1CfARvHqi#J9uNc>B2#J=%ynIm8{-c}?`ZI)QXqCJ3!1ID*rk_`Lti+X|0zJ? zQ9feB_{gi{<43rF)2Z^&ku1P6qe4tvTm;EwTM^|^fu;ZI@cQ>bsOR>e^1}ffLme7S zDscAqR-}j*;p{U3x<(g*5%M9os{o!O_}E>-N6&!*EVL28@SOk_=L(RUl#8#`S=>Es zg5JYO^rZV?^ej%hscDIbJqBpF$l#5;7Vh>cW8y7kgo`P`NL3jLBB~h6)#v2R^pRL- ziu1WPxK+yOAgtZ-=!XjiSvM?u?Sz3YuKv+)fg5uzV4Goz%DJmhx!(qdN3O*^nQEvR zRzb*du1*6N%<8j*X1xWP&srjLoh`O9Tpluun?so9fpl#ztZa41)N*GmwY9~f77OSX z8{*Y+9k2(~Ai7Eg7PHi0SF4K2E$S#8=HWnv9!%4W;Sy#8e?50(i1}mmhiKfqoq{c| zGO^zx4_loIus?#2f1GW+@2{`SOLTr6v5=nR_xC#!yl_{ zsNm*xOnzR%z@!5ZDr`ebcsbhN6l2Jv5DlC7Sb2nxjB}jLiI1`#l*r`I-0Mc(-wj=Sbz%+msdxfddIK!AiVeC%m1gdE4;*K>cruoED&UH}8m zPrt7%!pMmw&?+fMqGlaFi#4Jm=ptx*Bm7IZVK-BWgQCUw(O!tfw*)wz$4B``K3AjQ z{JcVdZ*#U{Ca(~SZt~%5T8P~<_z0VokLEc!xZ|7()z2|-8VJM}XHGZ!!v;oi=8%4B z2s0nx_j)ay>Qd)&Zgsp)Qpdq9YWNe!px?;|3x+JQ)XEkG-L8<4@kGsYPkgI#$G{j@ zNLag~d#62aR$5~gH`5_sYlCwfZa)=`=4K5lac*`c!dv1{aM>GD3+*sq&S|hYoFX?T z{1JCWh0qf(cKD#b!VAkd-_I=ObVh@YP!X|)U5X)UcQJVTLK7=$)zHoPeH>o{n{H{s z=m(%$tcT|&{3%&fq!P3w>+ z?SqMX9MQ7K7Q=;h*t5U|388KnUf_i|KW`k)_QRr1FD@5wN7zkA%=pXgqdWRo{|Ttg z)Py6)&EBukz!!H-%u?Wb6iS4PvidN5!Ob-2Sfiiwr;gEq2uX{Et1uZhMOiqdo&(8Y z4(~bK`O9*+bD;nawH|4+FW^@$=!!UHcA)`=uP-{5)TPH9~HmalC19cODYi z9LVcuVqS7GUh-lfu<=LLE;qPydcn{^Q+!h}g!gnHutXb$jT#s|UJHj(G_Z!_ zlqpqW?wShxwM@XKKi>Ej=Yq>hHgF2IMWl~Ak_J36KHVE*5`3|CtPiUC{cs@B4cBiu zVaFA&HW+7&H7^NMxSn3gPE8#7t$}5FTG)Swi$mWTSROY(hQJJJ-)!*azB|sm_s2Ee zSah#V#iNDUcq*0$*S{Q(wBtiMi4Xt$0<3EkU@RABhej4)S6~5}rx)P!1Oa|bDnhpR zR^&}CL+sZY{Cv9)YC30Nl(!cHAxU(Cnz5d!$? z6+)52p|}NnT;=e8U|s<_nLJeN$ij^$DUe+i2M3J+_-S}RgxmYwFvEV!mFTPGLGFhZ zww_Z*BG*^jI6@8AFKeNQ``mhWINi!J8{9tT2)jTJ+`i<8i8($fI_=Bxd(L+4p7=Au z6-ARB;5gC=jSuaid&m?0cIz={SP7?dmFWB#k2+@$%(&_dTT4ewtaFB6vPY8)zOuOZyg%?uV2;Ri$9X9Hqi?*f8yYh>U89~o_I+?cbA}JT9bS)@KPph2Q-v|Q8&NUQ7xA}U z5!CF;Y3rPEn8S!wAH5-e+8=HOL8u(R8u}c_8L0YTjSr{ib+bp{N;4RQ8RGMhHf|{D zVElg0AGn@eem4(sQo88j8R1E)8J7duL2}3&5v4(p8e9ka18H#0;9{IbKAtP`ae7`M zihpuka&;j>J_^uNF2F1ft1G^7_B&C4v5yKdhs&+Z3QBM)lDp@H+aS`q9|23wW82yN zNE4}pbX*109u?#KgF+|=39yjkIg=jo@mWHEDNX_ywH4xmTp>2r32l5yb%x{fHfohVDy=rGriys+4o-9b=?bY+}g}jb%N|)7sN01!_x7a zkZx3iJE%dpTry-P2VmwYcdWSU3Q-qNbaU4@;_C;&@6|Y}x(5HELQtp{iuVQqIC0Gj z59OROr_=&>gN=~d!u26K7+g531D*k=E$b$n64ytWFPBq=SmKj6H}lu^KQkmDxOZ(m zIwoY`5#^z|i;wo@g~*O5LjJ2l{N=D_RY?)fA1*|Vc_ADwa(U3+LhyTv@Y;DREM!U% zbG{PIS=&)N_W*`YT|$b=0c4%5N8N!cWK1kYgP;fv?+dZ~v;bzIT%73U_KhQ#ugxmL zP$`!i2NuDaJ13UoLQ~WVVC9>OsO~hZdlQf6gCU6X^ugp-2fR9DiTQ3O5M8noW}L>& zNJR&M;#!D*r;Q662}?P=*}mKgjpv*(F5d$VyZrH@HVC%St6|)-8dJvx;zh1M5O zRCb5K<6t-%!oo4tCj0?k7J6Vx33uTt|ge(UIFPNwTRfh8)lg;7!`Q~HH|H}Zm=6^M{AKRSqW#C61AWh)MBDS??`38y_T#>b>0v~XO?J7)`C>TJfn{0Jzo z2-$Kzd2zEAzAzQ8#{b39RYq0SHf;;LunWNs6cx#{ z=R!alq`N^7QIST(qPr0Rkx~&!gAn1MsMrBwAa-|Q=i@i;_ro8uSZA^Bz0ce;SIrX5 z2z(t>j3@c~n8%4leVQMp_g;-p2R&dJz6w7+d1Lx)KWyawvuJGy9vBAWojUvMRRZu! zbshd5_P|aRduZyJqOYnRqUUJA;E5JW#M*Fvp^H}vM)>!`4C33i*j(X;w&?ZHS+yA} zvtr=-IR#nK*-%}X3!`g0FuXe#HnVnOYdGU%b8}$*ItNScXXDw19cX@JGH2eAv!r}FmrmG6$qzTSwK5P(QNqc!(}(K|F4YbP;AlIa5SC?QSkW}firgev#nvMeCCh36RYv8#08&2ESTRg#g;>+Xm2w|{|*a$p5urT ze>X(ttw!*Y0PNWuft12n*uG7}k~?{LvZn;=o)kl8KryN83UjXf>LMZerf%?W$xVcurGN2BRd(Ogs?iai^I)^=44H&hp02=auOD+z)dfhoa-r zW)%MyiGg;JxE~({>E;NWFW&}*s!&wU3uYf!AO=K*AbR2!Xfn<*%K9KeYmVbhPcByN zi^q`AD6A=uz!AoFH>bsMjZZ+#qa>7VWGt(O@2xYbNcB#_(y{Rfq8Mb}+Kg94ept;I z;05MV_dewB*qGHwD)EK~_cL#^88fqv!Zpi8*bm5JZfqy)MEh{RqZZ2+iecH=fVGW> zk-q&PB1YCgCZ-Z+ORF%1@q!~O%b?Y}96`4$;54!V`n_wg(54QybH&)*dIlOb51?x? zLQN(10bIVF#e$wD<{WEbeYP4|?<+7tzZ?a2<*?M?8BCJzEpH_}TAAc=|3FDP&1wQ8&U}4Q7y>0@uuVB0)3+yK4`bKOJXiEAOvSvb$#^?65t|#>|Em>+kyj!y{Awib zjEl#gVY!fVYQ+MlrwC9wfRh1vxUHRyxItWN6L+BXOFr7hmLR5285EzEG`2PAwau=5X%|DuN$CJ@hKiaObh!{S5kC zJcX&JPomEE6n5#gLH@`&s4lq3bHx>GX}yUf10G@LpdZlQH%2Xc=4iEb+_F~u4Vi;%SL-?{CA8%$M>GgJen3D(J-!(X}?g{=SAIC}|7pRJb ze@8T&7e?ZP4ZrtYu~>F10g=;_(SKVi-uz6#VW|4&3?6YaLiowM9n{Xu5A9vSr z#=$%fIJ$eH_K+9+C4KPRDHwyKw?Gsa3DqggVNA@#h=6?PEZmQz=vw$Gw_uiH1Dx6q zvA*&EOhT(+H?0!=EGuzqLM1#Ol|kjda{M=<5}~^)@n}>Hwkg&^OQ{(Ttj{6j^?k%` z8=>a%;~}OEIEUOnO{i=;j1h^|7+z3;bd_=xG4Akq&wh;5EJwFW6_Q#j`Oc|^`0fGz z+{0+NT@R_o7IX$2#{4z=G51?OUX^AsmzIcuM=Dv1>@M>MN0QKYXd)i$iNWuPXnd%R#3<$p z?ij~nKzI)Bm9|2&_8=kyQgGR8GqPd=aZR7^)&aq|D7}gO_FHlMa3qx2zZ#gp9M9rN zbWRKBXBr0G%K>oz&fdQuXRNuvzP)Rv7$$9oG#N7#m0P1G-4V4<+^~L?7xwOE9^5V* z2k*yV+SGJxxSNaIal0`yw*=a=i!pul9@PCT!pO%(SZ`W{B*w;iJ>P|$&V?{2EyCfy z#puZ|W{u50%sEkoSEltS={yUMx=-kT?F{U@>M+@%3J;V@ap+1xGTSZKCJIBtOUAIK_(86hKUB_!p-v?kN0kpCV#_}K_2N5zpdWLHt6;$WOZE+K z$PM*@!c#v?>&<;l?nY>7h2XMO2*ykJBH3^adRjbiTHY3U$)>P1&_kDj4pIi{!j!qi z#09!IRBMdzGgfH-VaIdZO3eA@iy136;*C@c633;%AwL_gojLHFnFp^yc{p)B2MYId zpkb4Tw-a-a@0f!Zk8_~BA_rso=fN&-Cw49_#AdTSxc{gEi_@C1G3Y)PZ4%?EPZf@? z*@I-oLOk)xgS6WY>^r{$=W=(zXh04mLUzDYG8ZL(vXSbVgZSt?EK|+J8Lp8XkvovY z=f3em3`}-};f%f?_t$QSnd$(aznnwz*a%ma>EVy94t}I*BT~@DnH&0uZ#9KefIUVf zu7uyrwGhb$;NhVVwEJ(sh#Nt;))5GUTRzy>%-qNX)=bY^g`6j=FnF3TvO1$7yPy<< zb&3&I9D#xIYcR5l@wZZY?4IF8u%^9Mi>?!#7{LfCG|LCJ8=)KSlZG|ytA%`*{Tmw_AGvLLm6J9O@5 z;-LoP&?mQJ>irD9OEd7FOgbcO;_;X@4C5KwO`7F_9bav*$;lMvyY=zZRSOA!)zBu- z65LzMaQ}cBJmS=``nVo0$eZFomILmbaD$PLC+5!Z!Q--ZaI^G>=hSs5v001oZSEM^ z?u?iHT@W$M9oYfi*tvi)^p--j*c71T`eq#U^h6fpp32ORj8AaHN3jFKl~;1DUWMm& zYalzu3lmB_(UtFplq1eCv$w|D|BMiAro%HSVEI>IADIZxUZ}#{Q$RzP8kCqT)cebM zIAs=ib;bpv=c}>2aswhBN5OnzGG6Nd07c=h<0M^v>kX^l8KTP8MxDs zjt_fM5n#=jbbAKmyE0JSlaAf_{JD(Z%NR4x-VddnThMf37fyZ1hg+Xb=sUy>H~QLQcME%g zS%>%2!w$jgT@lIo9gY8aV97@}42t)_*=lFhF>bVkzmE}X3}ALs9mD1m>rs@UC&xau zzsm5`P(dSW9DcQH@yx?q4A&pUJSXUkS`F8tAUs_Z3B`x;cxaY{qhnH0_%Rhqmg(q- zO=T`I9gVeVa864@!G$zTXZ-jP<7mG8OxAA6#3;Ti#_uUWkWVH4?l}$dzj6fpEWk_F zhz#z?Lak~#XYZtOkDP{nQ`7LcBNc1*rsMGSbhP%Q;XCsK*Xz&pv|I!U6E*0X8bW!t z1+vs^VO!yh2bVpt#=sLgTRj+ecf*Tc&Tu^8h#D7LyuZ!ZeTF^sKDc3zL>OFt?u3DC z9uEHr#psbPc%);C$kA44=w}0EF=r2GIwOSZPl%)^UTj#2wB(f-|IQHyep|zQrYT&9 z>Eo}gI;;a#aKVHB9`<)^TBm}s%w-N>Uti52Eo_N5fR2(0k{@y=3%?hgrU57`3&)f# z@hB-wCsyLx7dwdOn!)$4F8sAMAlxSG9;yllJtYife73_y z1q%$6asP`tq9kn_HKPU5vM9{EsJBPD28}w@#kS zqg>?kWSq@bb_cTmE5Iq9nZE2g#r(m3Nca|D;F=tqoS%vLi!yPUvFO4vDQHhkK^5yD zK53`m#mF?s2#g!|@mCn?~f&r$@kpXEABLznQoXGX?waj=BcU3=u2xM2m?)?U{+Kc>nF zp;H|3^^65V{H*Y-n!PcMKc=a=V!+>E1YFMrjmyJXy$#UOX3zFHOAHuqhMg7`*zIEl zzhjIQpK^k0oEw((;Y^8s{M}HqL7HHNiXTRBWkXcWXv3!isS23P^ZhhUeu>Wb`pb-X0x{uhfA5DMDeqGJpRRkbq!*^soe$jW884phPbd-IW=rADhme{xlrs?@N$x3byjyx*#&>j!spvZ>8G~mfat==% z6yn067!`omjJ100vd0E{*8G1p{Qo?$35J9dlI(ZfsEmOgig?pU3FU*8v79};YgwD~ zLE8jUHWqj{jWZw8oltG$0!0Hp`+p9Qe`bf2ZgV`nV~QKUO`tf#92ztEZs@ZCXXEnl z$}A6ak^}JgjV(U#9YZcg*dkhiT+T1s{U2lW8|+b+<_PPxPFNghj~B*v=;z1TeTqhy z|4|d_&YbhtSD7^xO00!qzRFJl_x0FcRw;tLwL1K3^wI05DFPxL&@$EwLoNrx_h~o` zhR0#4P9hdGq~KOq8m1gcL*DsROk19Yhey--er3*S40C&*6J$1$LPfV)ifgn6JphhXomUEu_P3C4bJMG)QrspL{G8U3zJ-;QF9a-~Li>afT>dkM+eS-FerAv1hK_iV z#dnE>1BP|kqV*8ZuBAq(WUjcimx#56>`4~O!~dlM4)#?5W-1`>j|wIvtHJ6TYlog0 z!?(&7TDhLwgZty5Ry%8Hyjg?BwV~FJwIZx*eVxlSh|g%= zwsfpw{nWE1SqNv`H(*8~XPi~wLiGjwaH@p2O92$Mau95miLO~0Sk#q@wgw{(Ka0X8;E`zIFs!f_ZU-+@nWVv zf=8%h*L49S&MUxpggnx}ErG^idF+0$1d;1S$T+2mA4d!zk#EkKZq_hr=PcB5j%d)c zhh>Ki{H9x@UEKsRR}2u7$Q+WjA!M_;Ctd7|*36waCXtUB@;6mp)s*I%Q(+^=M-3{r{Y+2Dr|Gp zP^p@RK*mjr-=ttAYt)+Na=+-5iUrrwq3^aG^DpLLZF?au|ER|Km`=Pdt$}TK5lr{y z;mn5ZFw#tiXMP%Vn3qk8Nx^tC#-I{Ya6~^1ds&0^;Cd>~{z(M|bM917B92PMp~`v- z1}yYLeU=-J9JOU_urc=C(ni%;5rRJ|!ozL}e!rK6Y>+He`pF^kz#^ELvDe)e*!|oP zvkiF0O|pT1m>r@O99ZLFj~f2Jl%W;w%rrr-2z{iS(?!rvL%3cthn44g6nx9W(w+ju zuULZvUpY6WQw!6!X~Au#9zJ#$!86JN!@VqVsL~c$C+wIjx5BwQ=8zs?%(#m#rey)6 zhb)ESTsat@nvWlP3-D7{7WV3_o13r%qjsob;4pQBhU&xj4P!np9I@rudQ4`lXii}a zb~`6PW@|Ejv5rH5-|w6f=E@JIvSuq4c862opv1jRPb%V=_q$S%j_%kjxOOpj#kgWH z9mGhX8%3WEV(5=zY^}+|U1Oe&ZPIaS7wd0Uq%wb*ih?Dn*vq`E#i2C3xy4-hC+=DM z^1LuB1uv&2U>$P`H|n-xril;YGkIq8V{CpN=Z`3>V`i{2K5dl8_fvB48!C%;0rT+w z(|lM2%3@2H0=@_UvDVV$ju8aTUz+1)hh053&h`-lhnQj5j9x6d#BjX3m@fnBne0qeCm#BgMQ?;>sfB|;DF-CTOGsqX4fX13) zQL{eE{%WIw`Fm5-rC97e50mAm;*XLPrgci;!Ez~hWlG~f{4A{TmSueb*GWe;h&=US zYR>hK{X3h+Y=CU9X!x1OLG6AbJbY5HXif@iy;G1{mI9YM$v9HV`VQGt7;*0t^fm=g zb<(h-IupSWd8jQZW*+4*&MbL?;ckcUE43IZi*hl9y;qHVZ?9xs)h_8|OyA2|WYc6= z4M@g}&&deOOhIvlVp+#W>`CO@tsMLeh!3hMFXt zVhs1j{zR-YPr}^!$(S~Sb=<#`ar}RnLud{fJ&Te4rXCM0zr*8fJz|9ttn|*o&|~RH z-j;%VaT4|}PsG}`1h{NW#Qneo{yrrlgMYrdI}vt25-@*t98z{f;$7S(1aI# z{M1o-WhrJQ%|@$^6w0i};nLg@7#KJZ?XLYHf4(omK1e|QJo`uvjbi^6=L^nUh=LBz z%($h2vpffnOwxwLN*$Co@?GdELfmu_EK}r}(~{$SY&mp{Q-qDb3zQaB;+JzfW`^3J z_q6FaXgU#!Zzf>2%Tx?|J_iGKEW+#>Mcl7e!Ri`i*gso>EqM!ZvW$Hvdxm3mLobLG zF45Nd8lstnRCjPE4fM;W{RSLHWX;~9_*QBOy+LuBALxl$f7EG?!_RwisF-g6YYom^ zbl8YF%thAyjX~bCSX`2bXI*6+h98RM8Wszgp>eQHj7R&JM6^6k#A4Rk47JRGOh^fo zRK<9cHAZcLml%Do6l3Xy}01xKVdNv_QX)TOR^)P<#5}ds=0ap|HW0CValGeXP>wVkkPRjurCR;!r`*P^- z^IUT9E}&3@8ai)um9!uIpiAbXU~4RmlU#2n&1cW9feP;A@+{_{fZbgSptN8%%pXZ( zweJ)RTR0Kj8q;uPvNiPGuW(-BdFb?A2&ad?>D|BA6dm%H>Muzkr+y@UYfMC5m^9|k znGVy?8QgEkVD*&o*s^~Z*0=P~%GKvdeo-++&-AA!PL5PkXF#^AZAj^*866q#L&b7& zH2cp^iY%`qlQUPyQ2Qe(izHFqJ`W2I>0!YlZ#0b#L6-GqY^@B3C~_MjKCnLT#5SBM zIbI>Z)t8o2jyL^ zB)>ah^tVTY78pv?@AngFpWZn7H-0ira?qf6t}E%5dN}!R%%D5tt4L|WMOrNTp4!5P zqc(OS9{jV!2FB?WRx`HT=8w~Lfe5P!#HRWFaON8PE#4o7>jUv(b|7s2Y=C%dI5HXg z{yLO#u`kT$+6+-!D^Z5qncRO?#9+$ztq6V@4EGm-DE0A&wp9>hwEUpp&Aod;Fyx&A zVfn@neRbC316JYkQ)}o7OR?QllJ(O+=_OlP@yAGevzu}PnJq*rSgD~iGUzGTL zphbP}(eKz+y3xCi4zzOanN=Um`(1^IOIwh4rIh?vIC2)`D!Oqyi^{zZljF(``a0wV zH5C6PoBCdGEbE7!wq7VW`I-8E>ZHVxjno#IOL4BYWIuZ*O(@O z3U|c6bH`y>4>Vp~i6rtsNwNnDkFUYMBIYRUH^J?0EJ~a6QNHs(EdH5~U&mt6#=P~K zIDfRZc%zT@Ds=4hMEgC~iS2X8%ULTqgWVJBoZT=$%N3gpT=7ZX9KA27W8m`%Q1|*q z+qPb$0oDg-l3^Bg90;bMrS{}7QjI>hPb7n=exy3{w-7h>mvCock6=7|GEE!pL?GkVi&*PlY(rwhVy-v>hb=|{rT$wR3iZx$tgQ=sFmoKt;o zBZY+Wb3Iu>4o})CPoqDgDirX6J%=riEn!(|gE#rM$Z5308f!Z!J+Z^K(RT1%Z3~~* z_UOg4V#@<}d};B;c8zVwe#Y9VEpH%yg7sCiw;{xqwOjK%VRhe$wJHvHvDF@q(soGf zWrs%>9Pr?X6YluhL;H+1ww$%Z1nA@L*rj;&p)YjDU7=^U57O;>+o`rPjCP8Q$fI0= zvtOmj>uMib+xLo4-*sD%I@%)Cr#}~N&z0Z|L89`aXu994m<&c7r^$JDNRH2XYTsA% zd-p3+Ki^GR`%Y6?Xf5r!nL`H_y3w)IGSqkEB|))d3Qg=i1#fb9la#-ca3pD{@GBISFOLia%Ec>yyLjy5?jm-86^Zuhl2>xvMF#ZwgJETuaug-qMcR zIgpzLB$#pSPi7y>DHF_7FvU9Vix>1a#fF*{`1aZa{u9hF!^(^^z-`bl+Z`=6{`lT6 z5r125;!j)x#=Qx^R_2b%Va>TA=BV&7#qBT?%n+HN*xZnNQB#OajG-H~f^}pDFqYM0 zUk%|y)@)q6`;vYHAE0ZQS){TejE)^JqPg!ElhNf#q_w^`P5p9H_}I5tNd22GOz$Za zuJ^4J=H6%*T7rg>*>+c2P?SJ3w8}`9PLksKi?rj+ZtCl{g81r2i96XKGO0k1b^TBa4RI zucCbyj?$gPV-(Ca%XnZtHH^-sAWgY>2FFqHL6MBO3@Nj?qlE9`sGmhq6-o4 zAc$CN2yc4>JU*?3M~%ApbXX6QVh!NcVFHIp8=RiwfkEdY@$15AYyoSL;yf|F)&{aO zjdA0t9yTTGLS~a5>%;W0ub&>2J9M#hhyl_pbrJVZ3;De?(Ee5)uZ*TZb8;s+Ox;cA zr-##xKqqQ=CQqxsPvY$1;dE!fcR}26Lb&=NSqNOSSMVPhFZ>KE5zgjZ5#ApdON)by z$T?#(^}U@z?xUo4wu%OOTbQ)khl z=4F>J30Ge&Zmyly(mb}nrrD<8k?7R^4k6>k9MYKLPQ5ux-Kl>b?chwzNpmacpw|Iv z-?*E0aem8nV;3q6ktJi*w}PeIcHv}rxZs}PDM;Suy%ZcGsVa41lf@9`;0T%tK zm&q)8K2wWads~y^j|5ur_&A;4Jr-s6WU((%6-!qN*r=|F{Ck>ciq_zaKMh2j(tzHX z<@g=0fpy-jH{{%l%OXoe&2>TQ@l7zYX~e$PO>kv@fXYiVlsOqd=9ebSQZ;#QTaLx@ zTJV0tcS@HQOs{f2X3jErPvN;qRs|6wEQjpw|kqTh3A(1PAn z>h@3g)Y-uq#e0SMJN<;F`XHfGY$w>ijuzyCvV{C~?}gI=OUU?_6PYberK`HTY3+?N zlKfdkcTD!vblE~0UlLDmGuP0}cMB=cS}b^fsuazQo7DWn_C)il6&B4l%V&spXpa^z zi%e`DS0*|-`rdv)Cv6~Id8$V*>_RAFb}H2r1rXbf=!)k|>K!qNJk#rhKKsrKiXMA~Ze2&AXT(Th zpzCo_#rFrI;b%oc#Wri4|^t(RdEuvXshs&$Q^V(&^(+^ZOm%r>hx_eW&P(Apr zaAkxFN#6FR=z?g9mPn=0zbTZzHJwgAilIT34%C*klukGfAe+hpA!@)NVS-L?q1~j9 zu(>8nq%pKN?*$np%rWs5e2n%8KFVdnv7h&Zp9y{GpVBOPkgiWtl(Xpx4S@DnY0N2D zh{p$)U=z=AAs;y>?ScxL^;M9~J_BFo=oGdIIB->f#BLyLoi4gm+4mw|gNd}8{ei5X z7MtPWAAO`fQpY9G@ui<9sm_=Cu(>x?K;G*lF3a8R^wqonZg z^(~QMexcCk{(C|3of0MPbE8Anu@pUjJE z8*}04>-oEzC!CxucD`&W9zP{f{Bf6t_|Uhe=6g$vnq*h13Jv0~LSCgjZ7;Q@Zz&NJ zvoL`+`lgUXWdg|;1yJV_OWKgRh%_X=2#&upgsYy9MP7!RL@~EUi>7ZlC<@X#Eoumt z5u9Tzg!;&2VcxI(g1h5mL5}-S#d{+t;eifS73Gi=`~T}ArEpJMmhsVr@bX)N>CAoE zFHpp*JO%uyQiRkX&Z}Olg3%&^ETwP~5bHeZum1GKcdZ4$Q}h*eTHPlYlsA2OX|5A~&T; z)aBfl?*6?ec&L^OKkpU^pMAZA%6e%*?p}w;b-JOdjeLWu((-E2*SBv)Jr%yfJM%}v zJMm0QjZ}Tmn2c60rt2zigs_c{f_rgN)3?Cp<}1SHuk2reA)>l===ecSH_f3b-xC$XLaly!bd7n(O96sZ$OSljWgB@(5ylWS^V@&TuVy zd{_yKnv}5dCc$pD8eWdm!O&CYkdmvDi(jU zS}s1Hb*Z`1ue?d#Q%`7V`5;JoFQPwvt!UNgP+H471;)%vp-CH3se5TSX}r>-6FrOQ z{<=qu&NmO6@yA((aIOA|zJ`Z1Wbr z%#s%RtqB(WF5V$h*gr?)|M8D#>x$VzcR`j=mH1K!Zjz;5Ox~F;<=sl(<7me5B>K0G z^D%XO=zZi03O%%d+WP$#+75UL`x6tIn+k3ZyH!8{+EA1TwZ1_mX>O3PJh0= z>E6f;!D{Xc;n6b{`f$RL#=Z}y<9`$Bk!Kq1=$A_Ssy378yBmF+qeNCmhSSJOu^_cp zUr&x3WuIu7M{2K5H7k-ruVX{)F)~kJy=me z!=?;G;^wL74w{c~59OGbQb51jrBMHRnGM$j_dVrLi%8JY@EycsW>A) zQPK-@`((nhXB9MVGnZqei}EOS#4YD6P1Y5pUR44)DWiRr5}xyO3?9!pr+t@V?$jlC z{(*7$@~OxV8vutL2WeryKzdcbka$rA)pEx3!JyMZ`@~G)TTGIWs;ny@dX(_6VuWy1 zzFG7qS3^*(Ru<0576?vGBk51Ki03Fby4Da$buvk`es2oB<$CX1yM+!|ThW=Dvh?1# zKdF@m3)<-in?CifXpR}GA@&UPe4$Rk%=hDpdIERVZxudt8tm)tlbmlBdV^ zHl!gJL!N^(>HD=DQg7QyT@~51_+T{kec?kj!*!{|V=^uAZ59@m9#U;GOmBWZNJCs5 zTqS;7eNcSeYoU0}T;1m3zO#kQoR`8}FP<-MxYOh_v6Ol|kEB$0Q$;;ziOwh|KkqWK zui8llD|vSl9I0oGEYA@VlsE35pssyGa5?uxI5M^`4T%{;Plm|Q$R(;&B3(7aH3lgWOX~(NG6$&|xoYpXFHdPZQ4~v=PSI+KHSilg&EY*DeOw zx7Y|3vrTbjj}7l)^+b~7ChYVmK-9M|v`$?G!HIE*L=*VL8A76lvmPy$BVf2T4sl+l z&2LS}v@b{fLJg#H|M^mm^Y*VV0wv2}?}uOXcwRf1jpOV+^X+tXYBX(7FsCnf^k^Rc z?OXaxBIiRtgg;@Qg=sxqf@ETkU^IOw-Hn=0>l$rnMp!Hre#@lRK4mn#cNJYwt|H09 zd+CSMPEtJ0xtnTURL~?()fUeL$)ntV97D6DuB>=-=w5N2{2p;q#w4+Ke6Qvamz{(= zog?V&X%lLh7fYT2+sS-m1vM!)QZI264X8d&gKBEY*t(3i{?4XO)2;L(&z(M1X)tak zM+Z}7X#eM#^gC09!2=t&nlnlyiejFnTy^GajzM>?C7E4m89yRYNFr8RGd>Z7hmp zt>2>sc+qDBf&<=B;)07LIkJv?iuaO}OCptdhErqXI{LiPfR;@WQO)f+B%isECcDbh zx>gZw%(bT*3;axr_=;oVRI%!r#_ak>OEgV7iYL;|)?r;j8 zZzV`HzH3SxH&tvjYmeAGb+U!{0}IedD@=_x z!|Fbk%(GZySqSTM%N=nlko~!iKIkPC3WL!Jh-IH=>CJe2_!WkA`@A8g;SSHgc9_x6 z9>IOA@$8Z%S{7R1?N1B#f|}!&v3 zOoMpOjh%Tm&FI)nL-RLLeq9Jn{}n;Kv$(z-$R(rhU8E`0(L>(#bZ}ocjkwrDvKRZ| z?vhD(Ah7@{w;40|&j2f8tntU$3S-CFU}1y}d=2e5)5wl@soFz_Ie&xG&N$%bj&bW& z;qCi%jJ$`lmxuTJ2-Ucyn22?kx8Q;k>*hbK#^OJ2sPkscV>NqOJMAGUuwQVD4e#*e zoo-3i2*1Eu%R#0Ht1y7uQ8lD_EX04|j3e9jg>S=0>W=NA3D&&VWk(HtKa$6p*(vnZ zGKT)Gk0GDRb!5Ial#)85>5E%BO+UAfj`b<^z*0ohpS=v$ZOSnQ)_dKOz%mpNT7>0&NQn+|u9#zg7*y6eFAv6fuE_}Yb@=8=zrR41v_3E7*MQh8Po4P2Z&ft7C4K0XPfrh}(eon-^d~c(Ci8B+hJ-9?KemhB@{Xv+_02Tc{|4o} zenILfy>LI5ch{Lu$EcMmung72e?Lr7Ho+PLKiXmcYI|Hh>3|L`7wEBGcd5BMs%l)J z{>T#pCwsA;*9W4XLD1hAiF-*Y_z+fwkmD&R+Y^Z%g|>oc?w9;1jT#aYly9f1XVd*PJMGx{vsNy{yp zX!+P0>WD2Mr8Sw9T%Sy}^J3_3P6A0pq|@t)9MZQhqm*Vb4emTg=cZmGm&W_Fcim0; zE!{yL(XF&@cOALS$)VyMp>%w%3axFc7e)rAiem5lX)fBFFRqg+6F1x)DmJ@i*))A; zo^bQsOuApXk}hc`ljnxLbn|8d6-%F`Wg!>nKI6XU_Fkge&Q@}s+&~%m`{;9LCfW3h zCiR#oTA0W@+p+{&-jzt_e&kbI|8m+mznMOLx=0#+Pf0=cAJxy4#CWwCkd#s3Y;HX~ zi#6jcPJ4V~?dL$w&n@s}ZP!3&%w})>EXkG7RN{PzR8NTR@Y$=aM;mLVCE3UNRV58u zTq{_2n2K~`_D|MrfV!VAN_<%3-|UG@U(S@6;)*C)7YttM0t3cA)8;zCI?)=Ic9!@) zj`!s1>tdg|DqH|0nEfSmTlHjFSihxgvoW&b~$K^qI{&+3xgFR7a z?8+J8t_V_gVXrdpWa4~^Rfp}cgfpI5X=9 za`dvru zD_iJppVM?|Rx1rOXeQH3)f8|jn-q40Q@mhF(_03U^}i5dWa_7;pyrw4!-*B*m<+La zuUKE~y(h7`Cn{28qI5;r#5;y2oU@^}fEYTWwSz{DE~5v22Pyb-6%F07pVTLC#=db5 zJ<^Y*$-_2~$95-5cxOtxYnD^fFGH%lVND|KP|{zYOeCL6el?YJF60!o4SGfyr~Bak zg>e|BwFrh6*}HLHA3d$CzdCP@C}|tcX0U?5nqG%)*6`kQ#Pprcc=p{DqdiyR#9qeF zBK*-+8jj`KoWCPgf?h`x(C}nL-! zGih$HqoF@^sCSDM^G!B1R^&$w*JDX3C!ZQWGxx_@fW4ZBN%XaxbdHzOAnBc?w=I(9 zOXN58BeO0Vg zW$AE$A|?DZrS0TNdq-}ffm|DhpROTQzO&;(K9jN8NZy$ti%Wq@_>jk$WV?+JIhOaw z_?qKDrx`piTVgMJwDQl`;&+V$wstunrpy^JULN@Khws~tP-NbYMRG6pfu_cx;=yL5 zS*^jGIQBffcEXN8JIwyc&rxKDa@N9)S20Da$Q-*q7^6nl7@6#)a(2_gm+PE8lR6ik zsUvYUy@#61uaM)%YC8B|9;w?!Qf#9;Rrl!8{w#rl$Hx>`IwOzzCFWA@%q)s&ilm3n9Z6@1A`O1?Qkb7&A&ir_+GKO9yZL5p zu(-^kLp;swoj52ZLu|8OR{VW#VY7MXWKrh1m4bHXDUB3$ z&+uRxAUJZ*r9o1A=g{WlVf6mb2SMTYb-|SPJkilUwLa@>;i?){Ndt zd-GCg=D?#gk@Y9r^d#^~kMn|)7*~`80(7;p&5!4+>4xZ(GKHayDNe?)msQCU`(&(e zY=Jf6Gi=c(-WlH~vgeNPRjD;muyxOYRedD$U4h7+yb98W&d7@3{{FQUM#u6FLrV*s z9b<+rH4|j@F~W&whS(7MFyx!E==dYkr|3@gfKbXENs`1=nL8_rHWWQ)5t-2RZ<#~QI zdOvFzc50FPg$eZi(g)!MKDvh_?0Nogka#=ybmYl%g|)cNN|gs$TCF z4BO5aqdro4V?Gdz^%;^N2gigWvVIeX_;aAUqe*#vrYc2 z2b-H(aDA@IJ5N}n=QKM2$9i=LeSn|!6V8` z2pOdzn74Wgljp4!o_r}4a)#U!_79Py#d3`O%ZX_IS`p2BtxVdX(j+lyBz2dx2?@j3 z3%OSY9=UYbrFq$+Kh5`6dyAuj4~m!7^=}ziJgLR@ShsjXbFMhl-9;=FGD1AxCAV3+ z`d-rmsZvpvvbm7;pkBa#Z9=qqmvHiWt+2Usub|RxExc`#5dM5FQ5{_KUt@i*Q%xQ@ zKby)o{5kq|&-SJXlLMQky=fB#?$Q^6N;(Dq-XqDo#fvn(Dk$XtIy=*#D2^zOFDxjh z2xd_PFGK{3fCz%SGtUD&ibGvFgON))geXf1Vvs`uh(XRKWCB(5U&Ap6crEnZ9N|%uLU`|L?i?u_2F*A=umwH& zBX>9-L7Bh9sP2I3U!G8n#&loGPR{U$^e}(WH>C_Xs-=0jI}*Gp=TTcX0~!Y@2R1PS zT+Jz;V^suvuSee}CDgybEy~Gi@q;VjK2X?4nTM|4;B{ju4BqmD3l0mQZ#iYmc(U+> zuB*158Uqs@UbAynm9q&c8`uSN1#Fe9j=4z1dIg3LL~9?#39mlru06Q6MA@I6t#m!P zS{eEKSY?D~j7YiOfXRwm&$QbgV?x|dF}<31nWvX8GkJFpF*kJAGArZPi!xRwDSL|c z5Zy*2tX#YVFU>lLif1=*iDi5Ve;du1AHMUjVBf&4yiTLIS za(h#ja-gA3ICX}bDE*RwDBGg5Hci)rG+ zuP(cpe z_aXNd8xRjZ)@}=HoTXKCPN*nJ7HKMv*WNolf@D42NovMDAr)a`u;oA~x(pt`!QOMo z%T(ivmG<1&Y5v^QHm=_MHtX2#?UwOu_eF_yN}Gp-YXMvZ)Ar)2c)iR`82 zb`0roCbyq8k%O`ND5;!^VOmZYRxd_Dl_$>9UxJ$#&BN4sE1a-<45kF@;6Cj)8Q#ua!$ zT!yt4MOgh*ffD0vWCK>BHfxQ3*1^QCtC5*}pn#35G6gqj8u-SSf?#wrOb}dyWdBn5 zYEJ=Vwkash6 z5BV`2kny}8Mz>Z#>Ys&RB|i*SkK|xHTMl*YGT84d10AXn8e<}Z7Rp34Ny~+*$&_(n zco_0Be}H1WQt&)^4K{f2xKr?PIZFZ!k`b-h#HJ zIhdB1j|Z$u(2x3pobOwLlLF~Cv0w!T868HwdO7Ni&czgYK4wUZF;9jVU1d_p=EL zjSsbr{btSQtv^05(@$DDs_{OYi>kGq(9v<0^Gu;|#&jp=nKM4`=;J9GeA3bWUH^78 zbWy|}!-m-mRT|I#@^-> zd|dbLJM6Z2my55~upo_2F|JNwt zExbjV-)H%fWlNUn8Z0+lrn6kn&|74SFu&2&%gpf|Mx8&)) z|NBNN!drTSx6C3Ni-}@i_h)bK=YP-tzHI)VHd7Xu{6B2A_A{P5KUVqh-ZlIMor}`k`ln&s-J(ElUb`3<5Ut4TiLcKB#%mih1oPA}wrQphGt_C3Nt`)lZPauzqlmD8HkcF3)*2Q3~$ z{=7HG!3nz%_;yQ_F`}90Cqw(Zdh+rKLsjL^VAi85vW>Tj6sj&IC=9z++X{g zvUZ6O7Aed7uSt-1WuY)koBa>-dX2fZrGi{W$piW#>I*0anDO2^%Jb&l$|B{fCURZ- z{kcz>)3}-MzA}Ga$?|kNEqTX8m3W4CqR9B{+1!A=J9p7pkvqmyA(i{&c)Lezc+-`p z^0v)8PrM3daPl96I7^tpS(K)cFWFOh_v&nTVI6Wj>x)Vx?Dkadv498HXg8a?!;2=C z&!+Q4n=E-@#j-q2@nJ^g&ooYSe*kx&Z6fEr1U^#WFmwij-h`WISEa<^ubyuR`$h3$ScrKbf(4CDy68;i79X_;kRTuKXAT zc6Ik*!{IV$fAf+=AMr$SM~M`VyM190&9<51)UH3$Zr!rRF9v+?bfYC z*8B&v&xVDqc7GwgvH>dQDUfv=+wrX12b_4b4$rPV%ZR?Hf^MJB(4AQY+mtpEYkCn~ zx+Zby`?4|P;3g7WRt?9#eS$vmYKXDFz{ISq$DQdPP`~~z9y>gl+}?2u;=BKXQY?i8 z^Dm6sr5kuWb}F}Jw;$S7`;a+1%V4$j2%N4cf}E3=-=fwj=x4RSq zi|)f{SrGJm+C`0BqVQpS8*VRIft~rUN#p1_xb1ZXlB8YW_^AZyLW@z2+mHW7JLst1 zd9pIy3NqG)fOhTykj+XaKmB?zVaI7~Q*;2greK)2!yZP?8UXLoUQ%;@DYgFemabLK zM9sZpMkjl=!P5D$5N{F+D>Ocnr5BRuzt!FtYQ7oMnhMxGoz_qr8w;76PlKM55WL); zO_j`xG5y6F49Y4oYArYbdn*bder*`Mp5jY%zO~ZQPaKx7-GJ$lCCsVckbJi$jXe~~7UxJscvg!Sp3GiJq0$diyLC;AWxU>ES+udo8 zc_+N^!(a*xnfZYntqX>8Y0(h2d>h=E-f2`Rnt_sjZ|R+=#e84mMg)uu;qRvnU~wb_ zKCQEdwMM6L5g5{KiWYEvsx?gSUJBOf98uN^BIywwY-4o-ZU1P3LhX*^wWkGq`@R>Z zcrSwS!u#a3R|7l#R0%6js^c}IDn@e2a>)Gd1ug!juxVN+SvR6V-9j8t@0tL{Z@9}$ z543|g=`&zfWeT0of=Jz)Od6r@h5gc6n6AEtG)38gfp{9E?sJ1V_7NnxoQKZuw&Rie z%W%7c1iAFw7Mi_cz_Qr}#176T-#S0hX-U$Ukn@>_ou5N`U)jM8<7oIV+y({}#K^tA zrTB1zBmRqDiF%5mC9+^a53y9Ypr;HUr^RXW-J2 z4RC926Ekmt^FPus;ht!^3WFH#iZ|@{3bD6Sdk^_le+imi~#Sp$kT0^VH8aPy~N0lrZ$j!n` zLfUhPHM4`%)&!6!lTIed)`qr?mC-5rm)LsM%cOB!0(4#eBVD<5WJqL$$!kuas$XPq z=x`DhIblyO=$lf3 z<5TX@?|Tm5;&rd-^lD?${}Eu(hzkg->VoF!`NTnajOLvSL))4WiqbR4_`yYR(8mT0 zy%xf`L}l{fqcA3kC}XLRC_YnEA@1`Qf&aJ_817gMl@F(rH_cyZ<)tvZu~rCQjm;ey2njMrQ3-#@$@TZ*pL$Jenrl{6@w|xm&mAI z39Iqw8Z}?OgRU#T!I(Y_B3()z#Q8uDu^mYx`)Y0y%_;>*(pN#T%@3&2?x%fBJFQ<;t z|FY2FuLdT)6C~TVOM{2M8tlF^0mgbJlWk9u=>(nIc-T3CdX8>pLLPh|(k;{BaJmfq z=lz#Cc>EjHN{z;>EsJrmE|V3E8zIrh4Pea!8Mr+Ar_r@%t&bsZ-*~b2rz_wRJ2~9XkNK{NUu2nqGfd3IW@i1QO=P9YPZDzZ4c!@e ziB4YTL`9t2+1WSs==QPu)aIKq?l`9a4-MAQgnyZ+vB(3@&HF$-#pCG8Q&(x;{rB|1 zS0P*x(?yPH`_W9fUYh>Al4dGYQ8$$-xbLDhiuSBVU8fRul43W@dy{B%^w1I}^muFS zQbP$mU=oF8i^9=n?_@ks8cnB6QY03seZ*CAFA1VY=x9bLDump`Jz$J~JO!wlSSk^} zB?R#i{C$3&FYColK&!dkczdxRuI|5BtE1UKLf21&?%3O;?P&}vWE+l&8KRu3Wdstp zT=vD=7LvnFhl`cB$Z}zy+KGQhX+Uik?mQ%d3w|zR!@SRtPj7|boX0uRVJAdisjtFJ zD-JEy!g0r`BQ)sM9&&YF7g=4UNMxo)(acgAy!$W=TPtHQ+Fu-#awTfNE}Ov|;DU_& z7Cc}#3*@lt^Gxx?DJxvjErVtMu@uU#Q_ZP-I|&`8!Z<{n= zbxS_J=#(N~a%#cA?-nGg&H=G6)@aI)69qPYz_%YB(BHcLN-WtO6#7jYXy z7wT|YDs^~AQi@8*XG4qJKL|W?AO6V8k-GeT6izVVKF#Y#@nKn_{;LBfr}E?Q#W^5I z#A!+#hmKx4oGUcqX%vQS_K%?a%TqW)4ujRq9%`cW4mbAA;QG{Uv0|MTxX-Qwb!Qf0 zq$J57*Fwzgy@fujVlehYBb}mU4PW+!fn8THnb@a+er>0)-)Des8kfRTK?x*ni3l7% zD#I3P=wkTr5$u?!jejPbBBGv;N%X*1(ji{T7SA=s@k<#PI8uxgv-jfjJXY3#x|*m~U%YeS-FH>*#2cWA-H6VpJwON?69 zhTydfGi`|6p8ICPL5pC<60O6Ao)GA|^MZ=;@P2s5z#M zcaC|{?**1**fWR3KG{ut_gEX5X~k2450aP>=!OplEO35v@sleeTdvp-mO^cKwz~FNem`^wl(zKcs@_pftWnK9GLJ)SJ6(GSD z7vav>Lzw*J8hHHNOtK6M(B?usE~rSse8>IFTk~|dc(e&dHfBL$RsczzuoAydyMnJD z@$l;|U!pcA3zolV0-Ze>U>E4kB-*6nqpmpYio1*}*KHyf?w7;Elt*y=)I~VD{sr^L zCkXcsJV7nNWGM`m!~nUp7ryR#MYf6*GRbK*R54QrZ?9iQ zf&-)BduS=-dX<3JG+XddRAN3pCHUFDhaUP~P~#ln3m>jkf&8fw*mG|OB(NPdyFA0t z@Wf9lW&3x;BhNwOLlIa%9w1j18`7iQmr=rRCWfaZ5pC0}uuyam)_8M}bEkqd zsf3{t8;x;e$FSBUfLz^q5kA63iC(eR5XVbu4>k3E>MMJilI%Fpq zvd7ysvAe;X_P*#QqRSIuA5#vi02QXbzn--LI6Q!3qFz)^cvL!FT zt4u)%_!3Dgex=}W>3>w)QW%Dw6oGxu8*n$OhBMbxp!CinRNH$O1NF4AI=_o-xRnd0 zKRe)$az1DTz9yUI#nZp$|ESxWSXz4UE$QVhgJ|kQXjzd9YYHEeqcgSeyK_5+*Db`? zSEj>n^D21u<_+u^D25g5M4)R~Kb;evf+s}7=pT1MSk;gPRf3)Ht|l9bC-xH6;3ZVM zZ6Erkeqv{>HGta6i=c70987W(;7LLcnR*lGiA+sg^;DD9xv2p;fw3UBrvw(8q=M&D zEht}W!`eK%K`+;*u+Kv{;$FTG0?I?c*!BVhL{@?Fl3Spl)(&pFMd4@Af8@vLFmaiE zmn6H5FtzJT>845YxK>*jULKzUy2KPRrfdLKRs&voYZ3DXLA1@X#-f=M*qY&I%7=fs+E~p-5K%ku44dW^EqZ!-`*t{byRA>mB9@lTQmvM z55}=R{%Y8|a3VL=axxk=hB0%dsKBWZ2N3Pl2lovgjN+YGRQe#rs zt*{w_zbZjTLkv?as(@!UPvjo#(?Fl;No?~9ajaZtk7kLJRVg~r0uO$Z20$Ny5_%? zsOD0PW5Xp>%4HV!I9HCF9NdcD>R!0AQ3X?0PQi!{IjkvLjkfxdINcx^e{>$fNxnS1 zn9KJwjn?=#_bdvYs6w5ayRkRG64yDsr2n#hvf_4OtOBQl;?Au&UFkbkY^uYnl~Jg@ zRgubNJtEO5k;F0b2fba*qWonwuEXOYPMHt5ZWbXGLqEwm{TpPxWIOFjNyn*1fAFqd z6cSxQoIRRB@^;OHw!2TsaoJH?Wcda29CvWapGQ!&=P!+k>L&Vy>TvbtIg+|i5MwN& z@n+?3RE$f*O)FZdN$YjeH|Za_D90o3#t824@5F-D>fC?{i*YVL>2eZ6B%VFv$IUZn zjx3v`eW>Yvd+qMfo zP1eV0Ph1fs?$M8zelUmE|0PWlqC~@f4z`dh7`E>+UhqAQ@594td|$c!>} zC+tyYw+Pp-aT|?#C!$o(7ovVR6y8`nK~e9)zzSmWnQg*R0RGi&PTcBS8&<; zIVd(DM3(3NBnz|NF`@t3>ES24aA1xO{4Zie#@ z^wT=stL)4NJK2{GiFEMy4E)fr1&6YtaZzm{j!h}YjRud7gWMeh0f_&ME%d5N!D`i%LqYsr|IKmpp#lN9K=3Gy7UCF@2c}&Wu`%@gH^Q zg7Md+#(Xu{XAKgvJ%>(n&PEM`vp8713jZ_CC8ptCaMH~k+EbLsjfHYZ@_NzvZyKrz zM$iaW6d-p}^=kb=fp(&WGdrp!J z=h5G1)?tUY8@`sZK-;4+%)ytvq-EqO>HT-QrX?nmZf;<(aBUz4TGg>F(NSdnnVn>) zyO`OfU&^Y3Gqp;1K@(R!M#)d-$yjUxtZWGavj{$S-(d(=3A$ju3@wL4#u=Y`Go?kGtO!Hm^u95IR z+tH2rMNQ%xS)olw_*`5a{DCC!`6~=1_rb`?DGifs`SHk2Z2}UcsE+XoB4V0Yuan`Fe*rI)kXqIc>9EDUIm)K?0?;r|r zhabO0-h@$sqmVC+?AF{bRQu-+j4drChE7LeW!yzj5Qv9=217*n$x`ZQ-%8!JU(xL8 zpU4WUAh6t#04MeQz+PP%M8}@jD(GwBoEmu}SM|pv2XrC)p&_W4*}|1y!63ae0lpaL zknD^_yvahcyk9f#!%O8jP}V&Jo5mLbHVA_GDm&(#%Ml{qWdSSpTmUsv1>eQWq1`bR z+)rr2rsiMlB%J{!GUhocI~EIbEpLMM!JDwrAq_qyPJr3N9aJe^k{S1(1TWVV!h@1w z2)un8z77JE2+k+QxA`3UTuHj#emOkpZinkze?iZ;x3I~39YotIpx)*(yi%u(Go%&Z z>`jE7VQ*pQA_RK{eaPCflL_dvAriHlnO+?|$S^{=S?UkXzRrOo(pd_LKCWI5!iHp7B~VQ5*M4-tzc!9QgZ zWBBPFHUA_G&J)w%vp)x4j&QKXHUSQwct!?8hZ(i=AKAHHS}=cC3Y=V54ifKfz~wYP z_Pc5hCTIMJptKC*bX*XW)fYfqqYu=1$3vWY2b@ll)gmAGCqIE2mz4l_1FhXnK zgX;yb+I|xrPpO3mV?F?ng$eWp(g|Wn#v^rMCqVx(T zWaSc)>Hti2nJ#p%ZibNb4{+SS4x}`7Az?E|Q=b3AMW&fl*XJ;J?tTsZA>&Xz(grTK zZQ+@UBAe%9f}-awSZyT}5U^^1&epe(q`|?_1$y9HUQHStg6Q;XLS&89A@KdL76kWm z;QQtRY|2^#q>}seMiMeGr93UQfxb4P{Kj*GQVAB8A5f z*@EJiDezg+00#KHLw3P3SY)J2WJRCQxaC^ctg5P6v-2u zfqv|gr{U?&%P2n9Nq@!NC6nzlpwotfkBuCAUmy+DB1nB9%0*7A4K*7i}azAp6GH&jeuu1@pqkBo`cr=`N zQ3Cp932?M7k35fIX->%`&ddKQee~ih`E@Q8JWM!vbtny1s^2BguXa-Lo0qWT&~%hk zts^(L@NsufElhlu0ON%X~nu-h5-Pi;Ze50}Qay$LerHT9B9HkettZHvb z53$?_HC(jvGWx2v;kdUpCfeuFYs%$BW}zq~O%);HWGc!WZNx~4uekGmFdiT4qbWjf zNy~a$sB!2ff*UT;|4!xN#Apd_%qh@q4!9Ar84;a3L$lOYbcv}1lLY}j(eom zW9T+T>h$gw>07D=WaSs49jL(PHA^vR`T}lESQ&2Ku0T&-{7SxC%m#0MKQk>gx}#_< z7Nqs!0qp>^Ql!-TaTt-@r35nqI!UAKZu-qT5z~H3a2Dmc_|P|tI>bCA^Q-tdtC3e^ z?}-<5aY``iO!|s(!k~P&oFq@pp@Ofpk^9_=ftT_yFGU^8EkCj? zkpi_}IuFpq8FNt0rk(Z_MPurF6RciyiK5H`%FN87A@SbWJ8cwaB};Q|@4~o-oyJ^b zh!DqXEyklln=mA29Xd`n!X|?VRLJJX$=h2Ij#QwKUKrY}+>dZ86thZhVA<;LC}LKM zXVzw;LGX6e+TTW1x#RTuqjjj8JA$Wll{v}J6S;G)otPdbif69G5;I65bm@dsl%O~QWhDdhhq!J?3zJ&<>n~Ra5JvjaQFy4*6hEjEQ*j^aKlwA`Amo33W zWw9yVUoFG=jl^=+b7i>YHm0~&Du|qO5{2LAk1+8Q?C`|Yt0|Ks66?mCANm+yb(~qZE0zR(n1H_h5Ane{F^;!InakY# z1=HlrQMWOe>RfN5d-gaYLoQ-{LkTuDeZ#ObRk&C)4CS8eLvRbj!IA4YSFat{%SBKu!7jgTn2`LHCcm3aCmt!Ft2B-9T3$VRC0@o^f6n5vrQd002#?YD zT1C{D2>RPP1ef`XaX-5sqN7$mZrwc0j_sHWL9H_IQ96O|oE~xtdBJrO-(Lvo8DD?C^4k%TjA>V%)R`-$OaA!z_O)n7?zR^`Xe~@}k4@^5l$=;poky-Eo zz4`b?&?^+tJDCQ3kb=wKwc!Nev#btz=r?{J8+s;k?~_9CjZOqT-Cs>yjf5GOjGy%8 z;$+mCbr}om>hVX(9ot4njG(dxA*1X(r>qLJW~!0O6urUpDMb1SOl-$ zF-1!}j}}h_IE9Kp&e%(wYiRGFUXk7GvA}GjD!mrQf2uB-v|$M&JQhVCuP?$SF$=It z$Avy0o=#d0c9AK(Yh(yyXy3mxxWFg^^KY&|m+ggY=%-2WeVPu)PpTxQ+I_Tr!$VxR zAs=I}@$iX$4LLP@1dix$1j#Eg%-Dzd*lH)hHGRK^dlp@yh7paR6pxF z+K7@nJTcGhGFp|FQaQ!75X#>Ra$1}qBz!v!-IRja86up_y%ZchXiJB_JHfqeso>^o z2FBGz)MftiK&iC?+^Nm;a86+iFfes za2?z^z5*VlsxbAJ1Mz%c1?^=t;Fd!Z#4HrxJ+yiS~WYV7zI5w1;kRQk_N1- zBh^|dux`ObUcn|MUa{H~9;aIbp>cQE@^ne=uWdN3t2_iR@fSp$ljc>_{)CTwPEt8C zi})Nm0ykFX5av}h?3@1!YClQycJZHksUCuemqD~sU5c}~U%}=Z@^c1uQ+TQw(|Nn~ zCi1LBlA-W#4Ou$3lol?zLdNDsgW_TVo|?sE-Vb;Wwx0rEbB8u0e2F5#wKkC5#==I= z0a*WM7`z8>!oEodVMnnu=va1<5XU3%>U#y4&hLbO%FkiK_9l3$N#V`)T0q`2QmMZL zqMz-ASSAWiKgtK!-eHhr*6`%7RqzCtW$~6MOy)VIUWL!EA|M4;gV-t!kkb1|=6~}d zn}65CczF(#u00QoRuL#|;pe96L!kNPEGT#wNzCllL&nqna9bq_)Z&{#>Td&de2WJQ z`RByQGn@HvUm7H=?P2Fg9ejE?4ts(-K$ATJ-RhOhz?NBfb(Rh>UmXGir@DXIw8{uN96#Og6W!60JCU>ObL2{%4N;WjZW1}R{QQilt z-`7FOo6|&Xts2y{gn<3^EQnDq2J?<{;Be&lTd{q(y4r(xt<-&79Bo8C4Xwej30r-a90)i ze5{{jdJDoSi9j-FE>2&1ex;&H!o+2UEO{p&3P&7yux5%0JWsD9zIU@}eyI)~C-Ky? z>C$&&c_eHj@j_Ja4{V&Mhd13zaR zBt6;>scg`HC{<8RyH_`mfz$}ttdp0%2@#jav#G}#&RY!Yx)y&Cp%9Q7FVWDL`S z36|*-i$yW?UVktaHtKMnQ$um|a5OzWZ7w-)eTqn@IWa4Z8tK8Cb8uIE0vhkwj+6Vu z@sEHR>m+DIsvqoSKKTk`2!x|~#t;rzdg6d~Exq)uj`=WFPFglCV+>Bt#X7GL{JXy! z$8Y7Ll;Jd-G<1L!SUi`A31yhn#*591;iF|Md~BWF0{J$Ea>pEEo#g?qTM4pXYV z@TIW_ws=Nk+9nPi=FH%3f0W{W#y&yUB?Tz_DH#)vUPCAvM#DMkoZ@p4PABpi68Tho ze8v`uiwR!k|DUgYm*5Pe*KpE3v$#3V0^ITHK(xAEKw}#Y&@8(4M67VrAZ#G1-i^!%Lf)T+oB z>+0X((VN%#IrShEa8$(r%m3)Zu`xPBemh3od4Q80o3MXhI@T=I!VR}XXxGkVEc2$5 zp2$f=&3)3`>!A0z`E&(_^k1c}cN)k7vXJZ-`$&bA!f}JR2Det~3l7_R%ufIo&<&Ix4?@hgXO+*Cm;RQrA1^t?fpv=S(5GZtg5JiA%?W zqPOwy?+55$xdR^x383Q6K5CSwhBHqDVdL~lY`^>-n_E^Q`*fJLJ@2Elj%nhzZ#GC5 zS7ArQG%h_cn;SZ4Om(`2iKw&yxxO)SDJ$zlF?>&;KQff3~$x1xmDU1O>f~ci$AZeVt9PT@AfTbPnWV@dyRXH4wQ@t#4 z%fCDt{7Vg@4O8HGMlhJYI!>f6jnn3rb$Gwq3wLG=7#WT@LXt!_7%3-$_`XQe8gK@+ z{A$o|_hFonU0|eHdJ;BS6obT#a5&|)i2S;=17}#Z;F7`}=qjyBlD0-bqIDsB7C8zN zqeL0`R(~8jRL$24^YvM`B8ZVt0?1sr2ylNp)PF29>fdq*XSdYhyd)uHC; zdt`EgFg%^d7kxPOF#pXt%WggSgx*g1&5rKTfa<($3jxR*bg}H2p;&xE})em`B<#>|g;~*ar1G3-GQQe&)+>BeoIJDjX zTB3Sj@XtivuGl_k@!143HXdRP^g>AFS#!3hXEUr@*az=RCh~H>bwf)!zzh^YfoYj!3S7d^#5385e7vg#4OnY@+7I2X<&gc;2AsA25g(H<*sl@> zb7yx#=k7o7XQlve)#5tH%vA<2v1`Qfr9Lbviv+WZaVR%a;%(hEiRZm29a@&~h@lMM z{{}RW^BR1-_DYTyt2&#vL}MoJoWKnbj!k10x@mAN+fGyc=n&|05#=4+#IGIsEXpf% zO@JT&yhuq`0Ho&mkwtQe;PqRS=kG9`H}9SZZ~xz5xGfV(&CjZHUj{EQ+y7JmUYf}( zbkpSRYL(-OF35&2mz#*DXg6JbXBko0p9ZUCC3*cG(|M&1<8bTN8F;r{m#>2Zoz!;;l4T;1FzF`5oNsyCJ(FJOXAu{|rh}Q+QK*`rx|oLD&@WgCxz}M&w?HlLD7i_&EC&oT3BZ zXK)olpZI_o;{@)Z6{KXFC@8o@fs1Js{GNOh(tOI{Q*<6o6?p`Uk7kjB>>wHG(}O}6 zSFo=<3$ycXLHhS;yt8g4yl1-CA>wWn>2s7K=LIJcF6AUiYc(JOvmKf6`8Mq74iPZg zY((DNaway$Rph#{GQgfTvZ5f4y%f?!x5{Z(e}?spti=;j`@|BqemV?SKMs+m@oJiK zRtuXdhN-XK5L-WMHz=hR!JC{^@LAMIcBPA>-P%G_ezy|mc}9|XZn0px?>fvGyaKC%3;eD?fS z8pxbr4I~=Lrx_|BKlcXN`e7>(_0c2mODDnu7YkN4#e=U)TS#Pr!^xtb@+9HhB*Gm@ zB;gJ>;G}UT*Bh$E{rY|vYwnyzhwFgNoq~AtWgL~7DT=F`8u3AQGup4u$I8#4_-FA# z{KW~O>QxQAG?a>w`gu64R*5$xLb3e#WDL?vrLA|a(8pCrF!^T>mUq9z<$<>_T-*?s zEJ5~Ly>G3YRW;Rr=Wx1;NukhLJNL)F1p0*C0WgK**=+^Od*y(r|s|LF0cHex| zHgQ6l+rnCupC{%=MM+f8bkt0GfO4nJxf6pA&@M<3E#w8rlZbS3zwHS-f9-Z$Uz?AP z{5YYq@GdI2uE&MPmFZB$C1$DUIjT1$3g1_~#F$4i+{v09*fyn>9+q;dWkb5z2cL|v zZ$bkKp6bKL6+L(~@CaJTmC%6pOzPt-i_23dkXRpmC|ti1r3}uQv?@IUW8F~ zp}6zZWxh711v@r1;AJD__%ab~sX675bHy)4gu0`RM{q5Mia0KmsPT+oT{f#}( zKJsfC&*0L27xe#e1@FK6h0d2Ias4UJaCg@T$`)s#o`?sYo_QbL_DOOpgk`xIynbx{ z5r-sV0&W%FfU8_<(Eg?fm%K%Td+aikvs^ZWeKozbQ12(iNo-`ao+YBYnlR^Eq{5jl zlj9mC?jX-o5%Z}Gw{=VbHMClB)k9e>*Ho2zB`(Zeop=`8G*&SZGk$_dUMH1H8A8{g znH)~m;JPhDIPv;wjLzMFd1_*KQz!v1Z0*PVmol8Bx+HhEu>%w052Cy51vFVTg5I;9 zV@2&GPJi|vkSX+-6or1)=_r3C7Ei1=h_-P` zC?m_#E$l5C?yw#W_}ZT-Q*Yo{+GU)&cop_`1ksV1o9K#}kE!A>O?+V8gns{3pi`DF zE_)!1{>gl7IM_zz{MKVzVLom~2S#k@5Gw9jhaQv5Xz{P3>^k%5)YN4f8qB>r6#)R1^l8@VWnq#wm2TjchqdZ4Z9Fsqc z<~u8JCh~C`AD4)jJ*L$fwp5@dgjy63EaAU1^R27U>fBZQGGjejJKm&b{wzJhPRHq~ z2XSFGsUjfRB)uGaLAyim-mfBT+q0Lf; zxaeUh)=jU+$NLqzwCYsuUkBI`E!Lalo!ye(ruzDtD3Oe4sfo{8rt*3sZ&Ze+;7 z0fw)6gT96|w9PN1k#-7rKh6%-8dPa&dktA~Ne|{|D1e9VDq=k23|%%OmKxs|!lm+w z%;2|T^6X+KIo;q-_Vzn5JER&}e=8ZPumx+KJ!Og9at(5Js|ev;&Sf+V@|jV|DdcCd z3`rhK;@xbP@b$ETf^HpH`bz{R8d2hFlml_+VxXfS9R@-#!V8aF@bWti ztL4-{$@U_@2tXf#k63`Qcmm|Qb1==g9wzd6Z1;UP$?GJGwoV!ieXa}l$y&?n7S zQ2FXEC|&;!G^P{cDtE%1)*UwCeTTdU_h9I>4d}R+Qj0~x zoU+w_Y{R+~s89UH*GP!+Qro*>LC-Px6;jUBELBAFUNgE3$udk_BG_5rFAKf^u20x&+K2SYcx?2hwPMdH%Tw7G(a$ zlZf*gY*%Omvo(?HFLwTdS05$$J|@HPDkL6Wt5=hZm{^EysU-=D@$jy97`7@*{2@ZO{g4YRg_ zo>e;N#XbQK!`JXguL0KmXoiFzqmXEz#`Tw{!cp^$uxLjJEZva{N9zXQf`=`Cwr30f zros)r(?u2j<_+bLKAAi3ueXKC_w}LNRUQOCH4@9&O(3u?8JZ3z!~1h7u+%CO?A`~! z6uu#BiFrw?*EvIp^C9>e5D#k}Aza23xNc_@Rr2IIblbc1H@3T{Uv-vjcL}G28(z z-fDq4Bf}7X!XlFVN)0fQrqNM^q)L@j;_d8H$;!r~6(@^^Bq zkS6T==u6lGLxjeJa^r+XaNtJ2e^$pqY2E=y*yRk1d|r_6Ervh@1L5E_3e@4{qkxducA@+}jAHD|}%V!|`Q{RrqegH~5R1&7uBg0-;xT5J4h9 z0#E7^(atI4@|XZwX_Lmx$@$B(zCX-+AMZs3f8~%ReQ!ziuQH;e<-{N3=ai^nyuhkQA|R;F4BkD zT>tUC9KKEA<`|`0!EtW{9KYiWU*w9&tnw*1Depcy%u~UbAU)!IXfLegc(>qt=q?Bz9~O5$>14>u3B>>8%=@+TMmK-j3LK_!J{O z?=We+Ucyv7-GP0M%_w<86^kAxV`3T)U0zM(%^R#Hbv_G;LYEpov98CQCF@wbqIx{@ zQx)%7%^{^LtH_BH0!&>k^M`IS+5g9tm`%n)HB~ip08g;8tnhW&Ea!dxxO&=$zeOz zzd(fboH&Q(Ui1Vt^G{^V?ifClHD;H^$Fjdu9a!b=yBI6wM{FjYg5!7JlemwGnDo|) zeJ>%$u29fr$1NhUIrca4aE^f&7qnps_jl$lTQ?92Vq zta$2943r+_I5u6h9LUAG(8;Xd`Q@x>t|-n<{a~)JxdY_JF23Zsi3p+QN%y8>bx~3vgYHEW74XCmPwupy$J*RL?Gh zm{{l&)f`C-yyJ(p7u48Tj*(jQ*8#h}x-$7%N69Lqd?wo35u0v2!J7#x=rt=DW6hS~ zZOMBkR?F+i+Lrkwt3VAS1nco)<2p7iTjGwSk&;xxnk0IUvNIL;PUor#z*P4lo*_~ zvJ(|uWLdMN1-L5ZAIE zWG$Rd{ASJ|c_qTcz9p8Kxloqq?fS>ldwqm#7P&#TmUfbZZYAWAU?49t=nuW~sGoVc zU?Wp_vYf1SFovEOV-WiKlB}^+q>gLMQD{O7y{zm&1DY4W3N{o{?`{Pt;u3ut znT?kEviK)y4mlEP2Pa}8;gFsm)QS|6nZ}wpu<;(+yQ!n~`6=Ytw>{9w)i-0uk3#=c zA9A)@1-k^o(PgJ9_Uj)bW+N_8-5d{HTlYfYLp74h@t!VH_fSWD3hHpV#JgvF;Nr3< zFihA2?X6nm@jW@zugb#L)1v5U%q1>y+u*cxC=?%92vsSoXnE#+?yO~wDJK=_v$A%w zZ{s{jUBfXthGnE@>>Ts?+ZeMvcbpEq+RrZCpng9sYEyO9H7{=6=J{s0^tj7kdWj8v6J-4Gh=O76SkOK z>~VodYun)J<{_BX-vYw9_8?cIMCa_1XKkDWh-^kOj8y)DKSw9<&op&`<@X>^oEAf3 zRmJekGDC7s^BBx?eG1nO48vmoI&iDm0Z*1`z>Uwon{tg6@?l^cp zj9}HY0MLJ#0g}&Bz@4i-)7PfJcd;i>V%13YtdoHjZv)VBSPurj ze4x#v9G?AF=Icq{;NOyK;a6<8<*%JM0G?-ap=oO{+_G2?8|h>yjhzH%E{(#=g-^hC zqygHeyaHpFC$RE%8uWkN3`dx0kasExzADDRpiU!ve%}qoYM#pDD!hu2qcaDgsG{L{A-JcA=fM(`qv=&%zgj+byKoy@l|MiH4Kw?Nb+SR ze!*ex`4yTN%)FVS!Sescki{#wGtnyrKKF3&?dFN|zi|E3BOdagn(0FY%}$Vzmtl}1 zCcu9XD#I7+8vuuv7?`?o7erbJgSoUW7%=(pQSTGH?dXBNw~a8+9Su@Dk|4--8h2*f z1{1{7fFUhl7WNwU6xPEpXa?2TQKIt)g2U?i+ zt{C{_KLodxMfvM{UVzZJ7x<**^5XSXvHRyf-kkpZ@P6q7n8)!|&o~Z>y=e~XAHL`6 z?I>QXOg#~r775#L*1#-r1U4%IJ|D0GhYllH`z)DAFV%pkyB9#PBn?t>QlQ=>9(eOY zL7$rkeMqk*>cZl1C2Ii;rf-1f=G@%#aRtn+m*?+#c$x30qD1CuX%a`NG~ULw*Ld*{ zCy>B12!zJ+{u>a)DQ7Ry#b+bw zagP-occKhpkBp&JshEtCnRKP;MhvM`#5Ubr`XxaIE-pF&6WMJr{g)GQNw1*?vlCD{ zhhSs(kcsPF6Zq`x2aiLKf$xL!L9SF}4iq&}# zlRIGYv2H28TbY0{ZVl9Q$9_`(#2)x=?jX#QgChyq)JQ`a6E<%`!NVH#mt;Qq*3Hea zf|S7G`7+XeDxB(S1=G(T1#tSNccAil2P)Icc&+U_jQUKS>mfqR#wHb?aNfzm1;GpDG`91wwPeM!Y}$IHkO*t z57Knuk-kDJwA>rMHQD5X!TeSM-tA{ zx}qjpFJ{H{Ag#zPgx*6KAR5 zSdB6*#305|u9OCzvcz>axjv@$PF@yd<08H_%7s6s$D@7dNi`9C;dc@@^_QStdIK68 z>Y|G6bK34;MF*mNX|AOiuAh{NnqP0>)Q3fwIn5fKu9wqme1<0JspAjof=#hu*d`p$ zvEspa;f(?=s24!LS2Hl@%NZnbG04x(!G7~<{C8dm=Vx4{@2rRE64}N0a*jVPs;)}fya0H<96=svShp+pYAYXS<4>WJ2w*dHizJ4 z;fH8z^c@c>iLv7afADzNXWXvCY|BX~&v91Vv<&!oW-k`hLTvfmDeSGca%`{1UHpAh36I#zvc*IhkA~mHF&{a$_~sP0u}+wk+H?hV zCR_6yE)T)!q!+Zkb^u?m&|){~PGc98i?HwSRG_o(CfuGn0WW+FN6DMr=vpAl1~XEu z8+WF=@!ACiT$1s+&o`8H?ZDR9iLA+v5uDRifXBw2@JiDKe02Q_)@>Ta{Hh78uJc!X zF#jP^{}2p&9f7Hby75V{5qkyNam%zj7+&U&ss;8~5bcBcKl;#T!GCOs{WyDNY65#< zuO2(~+e1uRbrHWLF2kJq1+E1E}bX zb;52us3ynmPk4g`tAkN+=QX;{B$k)IU!De+TVc-bhZw)0gC5?IjjR9d_+O5LNx&!B29bRA-GKuH!g;XYU|Dr_SLGt$a>I zKUl-bWDjs7KS;xaTh#I?_c@&YlOAY)#aN5(hrLU0fQCpMv=y*KLQ(?PEzQP9e@t=k z?aK+eICtaome;@nG8MYB_xLDfb)!! zaH*Rk*32#=jZMk0zJP_HBQcPi;zw@#T4P_(0P0HEpncD4va&57rc2&}OZ{Q+OX3oV z3pc@yA=&>wJ4>J5KukGSp_f^YBc3y`z%K&-+21?LH18yH|s; zRV!)in8fSVmN4m^I;rj-6zlYX~4{q5)%Hxfwj%Q7o!W4BZ-hYv2emn|Bj|=c;G-&d7D~t0- zR~NuovNZMGq{Wt6-=lus$6+|%iI*zj#_yE6e7r~c!H?Z8l z8&-uDlWoP4uwvvAq&%tMGRG`P*A@c9_1Hb#o@2T<=i^$4 z8=1sU$jRfUe!Ro?;jQJb&i(_7LrdXmUkvCCIl_%wlOV4|1m-Hd1owW9>)ziAHl3g0 zmwXSbv@Qk@)s4`4@gHfG3IMgV04Vz23SRnupvp#oFOX9WrcUax`3}T#1-;6p;Q2{F7%U@xO3&>fGgi zut3G1RxF&xE({9gJ+9{ffE!c!pLXf;o5SS!n%1e%_^ghc^m;+(k8L6oisNCPffRqz zGIjot|37ezxWIiE7(q)$0ZD392k-VO_!lR@w|(&k)_HUMO;!N(+55ri4S&h^Ra`!G z{3f_T55(>H0`1FN;Zo=w$jEDl=6WFzve$$J&Q~OJoa3G}i(ygdDC7#Put-iJ$2sJjJr_V7W%O%B|o63B4VNf2y>%gc z)(Ubc{uepvpaDy10@OWh0-@1|aLOwl);2d2v9e9fd&djJMp7LV0vaJAP>jE$vK^FV zoxtO}CGXGHIk-jT3X|4k57)fjL-(_BVG*q@9gZ(%O$Z--r-oltv7j6dVh3n(ol@L#bA-s=};GZd4l zXjLe93kmSuo8?XcQIjBhf~2-~~Vp@4J0 zwU&7>%XU5{<3n-qbooPgn*JEdKgWYhjwZNU43VRgR+7>28F1=nBm|k{!lJMOcqEnz zH$zT9fZ1^ns)#27X)>T&#{<2TO~C(l3FM|BoVq2)Pml5CZ{50z@+3aXtdY z9YQw5p^6Z<_$$5K30W2t&C6$6G3XF$w!bP)t!2cdGhQ zJM|01_T3UVp}GNd=~Ht2??Jj^ek9J{y%xXixlW@WOoN5{V<1ex57x{(MchJ!@YAzu z48HA#x#F43#l8p>@dewIbWSk zGSlqQNaHTbpV@(@Qf89fITyH0M=o6L_JTFu(ma*_JTP>)0xt*#qnBj}5%7tED>GB! zo3bNlJeK1bHy%TUv(;F6NCYp=I!n4nd?2+e8tjC0;Pb6ij(N2~+mcAsR(GLXW1d)t zZ-mSP9FN)PODUI>@uj%3a^$TM(9zUmnxk9wL z;}dUs??!Us*;+ECI-1wHK!|)6{mm0OXHCyvPG$dQtFe0~-a>D+^=KwG9W}Fh=ys0* zD*b*LX1n!a^wm=Q_}m+p_i3VC#s|82_%{`^wZ~;!o}k2*JLqJ26@wm3$MmQPw6b9- zVg(winYR#XE!_H-~66eWqX&h)2F@vng{2G27K0?;i|f zz8$_pdSkRlhc=dRz}|1u^DJfmW-WtpGfh3U#19lYHf z&$-s_V8@FmxH`$6bHx0m%D20zl!PXxuJggI!DT2q)`x>vmgDp8Zxr|S&=;Q5aiQ@> z^i(Oss}IyzGp|%O{ar6>FzpoUZ>`Mk&8osDDXy4y>^P)3wj3^O;#lrD`zax;DG_=Qj4untt59Cnt|m&vSAZso-MLVQ5G;|Jlo|(o;j%UFoRY{Q?hJ_~QfZSy<<#&s$-ZN32)n z@`@IeQT5^@xXr2`^JN23?~Wj9B=ULR4b90f)fG(8J#mx~cEklu5vbkCImzDFQpUrc zr#aP^JT{#}4t@!xdQ1Sq5hGT(*$ekz6?Cz04eztwO|sW@2JevJEX)z} z$Kfk4aeqfT?mw-9IBIrO3#b$Q&6~bYuYozy-E1A8^ zg_vIPAwSk6@{UhVV^-feO=XwPfQDVZq+m`JS&kn#Pqi?NrMi(*zvU@k`Wvl(CPMW5 zl*uJ+VQ|Pbg<}g>z>lYOL|rkHrkyuNjpoZV{Ix%?&0qzn28Du>^(mNXUPW$smD8Oi zTrOzdXBu+YmmCXpg6_z8@VpZQjgoa_i&q6*ZrFpG)1J{~bN7>^s>5($Z5X`lbpw%m zWyCpL0yEeChu(%VILEVyR6jii3nIe6t!E$Cr*$SH`E6A9L^t}lF*NyaHF4Q}0X}J7 zh5gSCz;K?>Ds$^3dk1AoRRh3B6r9Ab;x< zX{bC(WIoiusi|o|l#}2_S2n=UWVopw0M6_*a4QTUty*j0!GA}<+x;>OFK>j!d+$Jd zL^zB|b`ZglBwmG=9CRPw0{eZcId9HCc;xy7q_3QUv8?+%@i<*<)LcTwPjI=vT`wSa zbRz%J`mgZscMKf77FoG#iX3ZKzKQ-@ybUV)?}PjIaX6;h3R8dE0rSIzyb|pt8~7%i zLv<_1t3QQp75`w0@O@aq&E4C%d}3h<#~+&3RkpRp0DI;ugcMEW$J(}oe1adx5iB9A zEp^bdtC(lZa{0T^78t1g2l9ujz*J)s2%5@(#L@&_pyz9%>K_hYV(MU;SR+gzmq9Y^ zKR7C61Io>Tq}pp5+|lrd=6}ghcHsu}WhTO}mh0etts1si|3^lic9XmQ)4<`CIc$6% z2$G3hf89obFF=y{{dOgM0|yKK6#h4u-s~9zmX|`#K2MnSrm%&8-b!(vivJIhaoL73{|Ezymc6|HDvgv5+k6` zx`DS$AYE7?#ZGa(!{m)*1OJU8U*1lgKkxele$3|>`0%NgoGZ+yyME>oweAoY5B~?z zZxr}mD|&&q>J;QB%!D?XFrw3A3(>17l#6_XvP0iNx}^qmbX?&~@lGiH%eg7vpMir$ zrEtiaV_(+1hU0E`fQ&PM*_9wU>lInpGY=%RICg@2&oM1oV3+?L6cfz(FN;g~;}2E& zW|@N=Tb&DKzT7Pn@uM>Yvrcm|eE%7G2_Rh(Zc3a-0!lh%hKSXnDZKe|;AgOMAM_Oh3AW>rI1 z)^hlrx1QI~^gs5tl!mOG1|DXO@MhUR=+J%#zg!MN?kp3$cD5N?#+Tu(yE5FItP%>1 zzd~479TXf{1me?OY4L6q`mX5^E#JKeihtF?hy7ncLz@9}MLBTvHANYzCn&prDf;a= z1lzRULZqJn|0mM`>8q!KU11shJ;?|E<}Kjexy}a=<_z~2&{S>t$x5?&~FHGA3 z5&Ra&c||8hLgDQya9@xOQcu0$#LIMYsW7ebslO&2@X~=mqcFJPejRvL5nyE#1hZGF zz^`r7$kY)T63)uNl7RV8ZoUc3R=dD-t6cE19E7s}RQX58#o1p;P4v#GR%XlHJd>2( zW?sSax#X?YWS-^vi&SQP4hoIV#6^M*^n{5L8I*iMeonqgOkT^=DJMd3L^KF9&j7kk zC}$RmNWpz!W0>%?gw&6AP_4*T)CkGK=<{ZHX}E-}dwv@1{I)=ObQCWmmaA(j{-NYR zF>d&Fopzm^3|>ALV7ZP9NX!~#I&A8&)Q#hTA7!GGcrHC)XU?53IiCWv8%)!zY30{g z)O;br9vFzh0n63&snBka8;u2Tqt&4MESKJ%kcs>@XH@g4#jnD@ndrU$L9@~YsIt%i z9m5nlFR=t2Zwj#Whv(wyTYkI?r#C~&8&8lpS4^DdE=TpHIT)GTf`6}d(JfMCLyNN& zaiU-y=B>Vt_k-QFlOm%vUhwZWcW@6xvN1>vRO}2*G`-zk$#xn=R<>e4K zBfy$m^Jb^M4qz23m$OSPgxOA!Je;-69^J*Q&~!Er-|Y)PW=$y0Onrd*FG_HOTp-TP zb;cr1e>A3LSlKa%mg@?TKj#_-`Z;hO`v+9(lQ)gESdHZxKhV2GolRB}XTP0#iYrY; z(I+~Dq@BCOtMM_y4TUeThpYd&rUo0W9f`9Z+A(9t3Q6?(-DHcwO#I-}fd?zUZYq~~&c|sphVZ{T<)|O$kN1@i z{W(<6?>aTQzYGs#hT?CrChGH158Dnq(;yF-O1sH}j4x>8;7}$;O}@`D z=QB|3R{{-9tRTxPMPOg792pi<#@7eyFzD5HT>Xpl_iyi{tr^`!ef9>>U)({4xV-=T zEgAS_(L~mkcNyF357QS%o{@{vli`xj1@i9UO{y{GfgMh-@bE<&tQ?z4CkNG#UoE;I zRrr(q)ls4fiZ`+S&|LP=%6vRkFH0Lt`^nS=dcfDLBl~vf(AzQAc+~6_>QYakxgWa2e}TB-z#6xk=h7mHPI-ByIcGL+Qp&^8NBVT3+CbZyUa2 z@Qpf@aZ<)BOT>wVzX&)R6q7UD3_ws*7d<8Jql#!I=d{;EZ?TWelT|8}dqPf7cTZjJ zS!<@Af?*grwgQ*!PNwFoW>RKR61`h{1f#pgIA^CUi!cVFHX^V_yqnkzio&O>Z5&Hp z!gIwy+Vn&MUAY}2*7KC%ME4S?`r!y7It!t1mM-TbeoaFb8e?siJQ=(^K;D;o!oBHS zf706>zI9wALYk_0?u<1akN2l)ITa)%A`-0Z88~q+0`y2NiS$y%%;a?JRd`AxT3X11 zka*CQtbz{)v5=NJNE!l*X_@szHYA~vT2z&ivs|A?$15M2)FVLBv6Eyk9-;5o9l_Sp zf7Cs>i(D@Xg_VyBfKG`7xxqr>G_{I03rn%zI9G`Kya6)BJRZ*3Re^F=930)zOy*yG zN~dOL;Haf4PBW_|H(SDiZepO?HVT~m%E+|^KGc2Oipwu^{><&6L~-shI6gHF5_fsQ zV)aj?w_1#FyA3e@{tG&3x&g8ATMW-V9l`GF64W6HN|fRfUOFn;DAI2gW#_a8jq z$KGiAr)d~}HdESHV+3Y1s^O8rXXvrH3tEbcpyZ@Auk3+25$~SL^G-8@J-T-xO`hX8 z3ajCm<9rC=GHS!0hEa;kS6vzQfJV!Auxp9{fA*wDF#Q(ivWYZgrWCBe8HZe$+jh%f z)`U7R>;C{N8#!J)#Rw9Amy@~40hFAUCI?O)gAd0UaC*!FDy2aBdP|rq`zXDwzOieQ>F=b>0`AAr*wR7p-7+YY;@++=QF0g8Xlz z$M}+m4Ea=Aj^FV2G04sghrqOB5X2Zj&srhyU8c&rU2~em+*kvJvMHbxUjh3HilG=| zzH4?!buc*9eQArAq&Yzl%VL53U(aHNA(^> z6qdDtY2zQkiu{4(q$aqRvJ@m9O`tMrTk-Y3ndG+w4@#37;QspWa55!}bL~iiD;n^u zUfrcO@gktOD;^S;R)NK}D)8Z0nD`@ah+g|o-gAv_OmO9N5ZxRD4X=x#t*9K+1TuGV}(j+yx(Q@I=8NFT$yF_z?wa3p*=co#fh#6gJ)#&7C%^qDV9*1x|B4fh&hWkEQI*jSN6Mn^f((=HS%^}#Tk=fsqoTR8N$LgTGC z*mK06nuK39Zuo<_k!nQD-De;PJamotRRoMjYq*MGK%OwRU#miA=$ zav~gNaQlGV`E%(d$GcQd`yI`T93bab`@*Lck+4$w1PF@B!oOu2OwrTnD0VQ<u<;W?T#bkGCZTdBj98_aLR0^Xk`&csIa0+Ft3 zWNz1Z<3mlrISGmM*lYbtd()R>-GUcnG%Jf2J+v0bz4l_ZzBz8{SxmPX3M+kl3vl_Mq#6ULh060mTGu^hAFjY;6?G?X-Yq~cw ziDm~uvM?I@-miw??~$}AHw;U8VgJ={!TLudJXuv6 zNISI~emtCC>FjX<4_?hgU5|Wp8P-Ed2{E$G`!5NP`OOopxlfh1IpLZdV_YZUghRK2 z=#t$6jArN>8k*8Y4-XNXUTT3kF<+?Sk`iV|oF(J)F_KQ;b}`s1ZpG3Sp}3=yV=qLC zG2_g1_ED}ptLpFo({+xa+O8?svsxY}pdy+CTj3chiNni$@Yb7CxQ?4i2Xj9KW4GQ0meJn7^NrON6R%o@%FoFoOSLJ&ReWW_w}@s zWnv+uH)D_*H&^1#6m3>c=?Ol4!N=saW&~FKBmKKe$lcUNT3r=~@1_6XpT-b$zBf)S zloLo+ycXnoJ}34+{!#tb@A#u&2fH4A;Nl~H=$Bt@2cihQGE& z)3&RPjC@oCbw4GCU)Qfgmy=PXy=aIz@4b>auv&m8Bsi6cN)y87&px<7_zD``38Ymi zS9v*S`^npR(}1(g6Vs-4YTuWEtFvz5WR7?Fr7cIbDyKo!vC}ZaX*#^JR-$p^QOMu& z4F>{u;Jyj5~hHxryZy_8pEC~ zM|jSX5qPpdnq4#+hSm2s^O{!}!7%{`m^oPkT%V`%K6a?!ecuVJ;2dqd?-RhNFOh)n zi#CA5hFRR6o@f&nV@G^E^bL0oT)}%L^7OQ)7F2xj1or4OXnQ9`)htY~W^V-^U6p}q z6JOFfCf=krV>;*+4v>?(6L=@z1kmSj1?8lD@HEGz&Hbvvn4Mh83-i@y1Z>5q$>~V4 zG+-T?x^uIfoJe{xynuOV9!1%Vl_)bW7iYQ`(TP(Zaei(g__Ff@c`xvR{0_TIPIH~o zTlK}%VzL5W_YwuuEo$I-cNMIz?bl~Qo7HQKEMy1EL_)|@d*~)Z~+%#{9o)QDq za$7*^#0+A;RU0>Y1)%DZ8+3(h4)J}?W%-VmK?OJC4>@>*DE`qyYu_SVNr$P+=mSzD z9tHbvmcR+k5QzJBm#iHT#+m+KQTysB^}nA_e2-lKp(E+gy*3aMNfn8&lfgZ^_hNXK zG`2)Dq@*(df^Vk5z#gtouTx68+XV54<2O7ZCX5zt_sLs{5U`Fef|2A%$ldgm{1z3! zE22f*PL&yW`_vKQ+~^NI-lcHxO&Elh=8^;JZ_p-gcI16yDz0!aByruFpg%JfI#>9B zi*_gRIx5ce39iG|907b|r%LYbmAE|g%ux#VTYXpN?*T8Yi%cxn*K-R z-{?>B!|en4`c#PXkvxEEuUZK5E{FSl9Lrtl12-+_!h_!xWXiHghCHIQp!6`aX0{IJ z1&M`R-8}gEBNRR#(gv%o`pk)!I+$p0L1(FVkY2?|*ri+ya>}V7b7&z1-TA}pJ#YlO zh4ty_@t5S2W*Xd^$@v{+Qb49l2edbO7Tf|q8#Umy z8sN>(> zJ1$%OoZN8GLpSG#xJ%y2Vf6B30SI}X z3@1#Va#=XeRzKcOJjaab_6BF{iW8vco-BeEj>o2zg^tQ0&#HVvJQB7i!(1`DyTM_%DA3oiP(ji5VhG4=K3UoQ%MD^o?Z(h z0)|BW8s{Ie&j;Zl?wRx43k{pzkj>{)c#$7&(V3gMT_5ax(sVuq1RL@pUpNmUKG?wd znl~n^SUx)3d`*o;l`6BNj={?AayXU9G3J{bLDGg3z*z^OW!YD%udPJ3Yem2>-PfR* zmjh+$AIV<T)wRyLf}_Z%Bbw>rVJ`I0YuYJWK8!KZk+;f-$E! z0CnGRy~zpHU@`m&WcC-p>WoNIS1=W|4rO7$$P&6u_C1j_y8-tSJ0WOl1hhPuOpH^1 z(m7E9c<9CY+dk?$p-(ywwxpGV)Y231GotKvozY9GzQ+_l_So>Ya{FC#-19(Vc|PcS zgn;lR1EQa3fVBMu)$#`DX_b% z2T9A-kfJFGPo9Eb^!V0$1L^h|SLW%Xj1?RS7OF(p_v%?>L94$!uO08lV$0M42W_1x#m(dI3cXlcig zvp$&7{(*OpyJs~Kt)LiG1a?*HNa?Cg8ha9JJ)Iw@OtvHN52+`9Mu)ykA)3169-6Nxf9%`$bgrT6-I?S;j>rLIAC~%vh@>8A}$^xBTF*KC*6(Ym$)lk zesB&tH!nb?$(QM&dP6cnO&4U6R3JT0g8E4N<2Gg$o)zZf^j=lwyp$Cf>T%A40Y#`% zpMf(}A7RMAb@XmCK$)v*Fv*KTLG5)gZ}26nWcDFjEyFI8DdWDc6PYdcC!s~`4#?a} zhn^|RiPwLPsK@2(l8th3&x2a#t7!;i-gp9we|12d(f22d$pF7a_pC=6Mif5EL_GJ1K8TPBj zK3pv8Pgb124(~=OlyF>Bh|g?1|MMEsxCi)Tn;y<+Q~}@0OQ2F}4Msg5D{aS@BHpt= z{b*lo1xY&gPz46_^dRS)85s_}L}kt{M*U7*oH*!8*POE=i;dDr#lS%#*T1ze|rNW^4;tuZ2Hewfz zP)y!5nf5{|_~ws-LGTkW&Xgjmx8I}E#pSG6KsPRr6eAizO^|SY6hebiA@Z37o$6PG zng@(HH2Dt3=ZiwY{AXa--42RMCt&4)cA7QNi%DNJS$CIp7|HPfDqJ7v;)zOdm@
vH|IguP1B%I-&dAn={N>5Ed$B#EK_mvag+Vq}uRxhI~x{^@y z$`O?Q*-Lkbgi~9YQkuivQ|}uhDB{sXeuui!r?qeCpXdtOe!rZ4kdjA#?wVf|n`5<8 z9#h+Km+>xNVIsOsmv_Hob7jB21Qu}pXFvS_9C#sx9~&>yQ<`eTF0+&DTzrshKHyC+ zy$nDxy()AaH^q{A5qf<>EcvlP7`_f(BNtC>VkGZz=bw@`jOY6RuUHpA}OB!S-X`;clv=sTX~IBGwZoll$?}6m=Ewl14a+g)u0~9Ln=R9l3n^ zy%D1pkfLu@+aL*#DU`T08=!{aI(?LRKA?L%F5d!bdf|rTvbsI3c%DQjylA1( zS>5E+2@7(u6`2XuCm1}QE}5BJNSikCwpF_^sA6U<(N2ycl_TqkPR1Y*SGP0nBO>U8 z93$+T$6#dPRVp=wx5+ukfn%f=#F@V%+pfh+Mn){aSvzB~cVh>A`%52FUn~b}b0>n? z@eWemx``3)NTA(uRT#u3&?jR@lOiiAcsExC3YBF+q9`JhR!qf2w@iF$JO$r7_cLh* z@(}7g8P+Y7f;+oMk^Z@2YOhj-^`qiwm`ODAjkFUV?@2IY_9zhZ@s5(=?NnqLi!U4J z;T*eoiO%_UVjZdjla1xzNKco(ZD$3YmAM!n++KT z+jx_@-%!TXrDH*De+7A*#>XlbW|42Oh5VYnheU?mX6`(&p_+k3^u(4NiAhcl5!ep{ z<$pgXEPt-dGy9n_H8C_+SpiS{c8FGtT}NKXFMz|>=YfHS5Ymh?7`wh~nw)QfSNB|@ zPj+aM`GyO@YwSi?9WohS_SZ44rnjhBwl{wEc}HJ=(IdO!0IJK@!Ts^PJ=jo_%od1n z6OXs6o%4n6O`S>vtTt?JUInddwP0`b6jJp`fQttyVeC60o>`?r0vSzscEkp}PA-7V zwn;=R>ZYAdJU_)7A-aUjBsZM2AZDuzr2Wi?Fxy!Z3+WabFk}-lmb{N+S_2bSGX*@q zt%q0AT2R%XP5hgY##=4J2TM}uBE2F;wnYFvkLG~Zu=z0lm@CQZNTegUF(`L5fv(c* zVCsF!2$TAPloX7BY?%{8>vOK8E$%E8tF5M!mt15b0z=8=JzGgqM>dfYBom8;m87kd z(DMPyVTipYT=riC9=FO#T}&9+5_elt#~q@&DLlW0yel!aH-pwP4=A}d8>U~VCeMqD zC0bSd{g0njP;OfxvqDD~p3U?Hf8OR~vy$hvZ<|8X*&i^dM2H*18yKZ#8|X8N1{?k^ zb#_J|8Lg8+b+dg?``R?@s<9=x-aIa+cM|V!bqCR|-Gn?@iq+0eC}woA|5qW|`pgF6 zOJg8VYy&@pYLV_Wt+Z&19G0iPrV4863E}x{w*wZy*SI+?me6W(Wf!W`EXP;$nD zgvJ7T#?Q! zpKXg;n&nuDHPm;U9}y~rgK0z_RLJnyEIZfeSM->oxyEVCKafceTJb!m4m^&d(*aP4 zS_R2RFG(7|+T-FI+i})~!!)({15qjsg@WO6ptgA}JRDx4FMTZ;LkeC{gZG;B>%GlH zfj^rLN)GUrkIVX+y#`z+?!%3;4VZF4Op*_5;jz|3ATZ4m{C`U$PRvazk@3V;$4{Yw zRyU#h7+ckt;pwkNV-B zWQ=nGwEJ~}o5E@MdG=jN;!o)qr!L~g-rt7HdYeiAkaN&~W9rgKl({nJ{4j)js;R<_YDp)ij$>KZTx<3V%Cj0X#*mpu zlsL^|Z_ZsygZupKFmWpy&pwG+$u0*u)@;Z}CQNEP_mKwnLaFYUiWzcdK z&UV9zsyga&M8K^~HRPnA7wuz(P*yHvXHS%4{cpxYjq(%f+i((nh&*T5n}>#`QlRL@ z+X*~Mpvq$;7{rgl=;dB0UZ09n`^Te-Z8Gt-H-`mB*1)HRM8d8SNJYr4T1qX$TF7q`-7)h=3!M3CAu%j0gE90rOm#j_lT)T~ zZ5rp%uAmlkKV*F;(g>dDcr8%iYCHCC0LRLpFojss~ z@IYoY*T=^}cWP#FAhDoI<$Kxg&Pui=^fcRRUJPFsrEu;m`?<9EBrZKyfjpH-Wc?iP zvEH-tS-%-3aJx5`+gbC8E26nvR){GCE9SFXJnpe;dlOh&ds!lmOyYhq>*w4@q;PY! zi+Mby)9ltoRqUx(d)Z-U&1rUY7H4?u4QCTz$LY!O{zkWSc0Da*<-RnDWXH?+yvFIa&WzIk7>X)*C9oUd3y~@fy$bTpERLFB%7YMW~>t%vxIvmA=$E`{dwQfnXq8 zTZje=1k%!i|4Fg}1A(jWdQUIE4fFgq`0nu9w$;Rt5v=+DllJg|(eI{jv>U{aHju3? zgagH&<9BV!TK9r)4`jK44}VL4%*Neyr|aD9o4x<3{nvF({F|nZQ1Itk2DInt=`7M& zv|zEG?xOj6TDt%2=&w_h{aZ(gasTb;`=(f1NJ-84E~FyT{8L=?z4k|0`Ms@h;P(oz bH*}y@|Gmvmk>8IYQUmWq{4*f`yX=1e|FoLK diff --git a/disent/dataset/data/__init__.py b/disent/dataset/data/__init__.py index 178aa183..1a870327 100644 --- a/disent/dataset/data/__init__.py +++ b/disent/dataset/data/__init__.py @@ -44,15 +44,10 @@ # groundtruth -- impl from disent.dataset.data._groundtruth__cars3d import Cars3dData from disent.dataset.data._groundtruth__dsprites import DSpritesData -from disent.dataset.data._groundtruth__dsprites_imagenet import DSpritesImagenetData # pragma: delete-on-release from disent.dataset.data._groundtruth__mpi3d import Mpi3dData from disent.dataset.data._groundtruth__norb import SmallNorbData from disent.dataset.data._groundtruth__shapes3d import Shapes3dData # groundtruth -- impl synthetic -from disent.dataset.data._groundtruth__xyblocks import XYBlocksData # pragma: delete-on-release from disent.dataset.data._groundtruth__xyobject import XYObjectData from disent.dataset.data._groundtruth__xyobject import XYObjectShadedData -from disent.dataset.data._groundtruth__xysquares import XYSquaresData # pragma: delete-on-release -from disent.dataset.data._groundtruth__xysquares import XYSquaresMinimalData # pragma: delete-on-release -from disent.dataset.data._groundtruth__xcolumns import XColumnsData # pragma: delete-on-release diff --git a/disent/dataset/data/_groundtruth__xcolumns.py b/disent/dataset/data/_groundtruth__xcolumns.py deleted file mode 100644 index b50503ce..00000000 --- a/disent/dataset/data/_groundtruth__xcolumns.py +++ /dev/null @@ -1,66 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from typing import Tuple - -import numpy as np - -from disent.dataset.data._groundtruth__xysquares import XYSquaresData - - -# ========================================================================= # -# xy multi grid data # -# ========================================================================= # - - -class XColumnsData(XYSquaresData): - - name = 'x_columns' - - @property - def factor_names(self) -> Tuple[str, ...]: - return ('x_R', 'x_G', 'x_B')[:self._num_squares] - - @property - def factor_sizes(self) -> Tuple[int, ...]: - return (self._placements,) * self._num_squares - - def _get_observation(self, idx): - # get factors - factors = self.idx_to_pos(idx) - offset, space, size = self._offset, self._spacing, self._square_size - # GENERATE - obs = np.zeros(self.img_shape, dtype=self._dtype) - for i, fx in enumerate(factors): - x = offset + space * fx - if self._rgb: - obs[:, x:x+size, i] = self._fill_value - else: - obs[:, x:x+size, :] = self._fill_value - return obs - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/dataset/data/_groundtruth__xyblocks.py b/disent/dataset/data/_groundtruth__xyblocks.py deleted file mode 100644 index efeae411..00000000 --- a/disent/dataset/data/_groundtruth__xyblocks.py +++ /dev/null @@ -1,160 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -from typing import Tuple - -import numpy as np - -from disent.dataset.data._groundtruth import GroundTruthData - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# xy squares data # -# ========================================================================= # - - -class XYBlocksData(GroundTruthData): - - """ - Dataset that generates all possible permutations of xor'd squares of - different scales moving across the grid. - - This dataset is designed not to overlap in the reconstruction loss space, but xor'ing may be too - complex to learn efficiently, and some sizes of factors may be too small (eg. biggest - square moving only has two positions) - """ - - COLOR_PALETTES_1 = { - 'white': [ - [255], - ], - 'greys_halves': [ - [128], - [255], - ], - 'greys_quarters': [ - [64], - [128], - [192], - [255], - ], - # alias for white, so that we can just set `rgb=False` - 'rgb': [ - [255], - ] - } - - COLOR_PALETTES_3 = { - 'white': [ - [255, 255, 255], - ], - # THIS IS IDEAL. - 'rgb': [ - [255, 000, 000], - [000, 255, 000], - [000, 000, 255], - ], - 'colors': [ - [255, 000, 000], [000, 255, 000], [000, 000, 255], - [255, 255, 000], [000, 255, 255], [255, 000, 255], - [255, 255, 255], - ], - } - - @property - def factor_names(self) -> Tuple[str, ...]: - return self._factor_names - - @property - def factor_sizes(self) -> Tuple[int, ...]: - return self._factor_sizes - - @property - def img_shape(self) -> Tuple[int, ...]: - return self._img_shape - - def __init__( - self, - grid_size: int = 64, - grid_levels: Tuple[int, ...] = (1, 2, 3), - rgb: bool = True, - palette: str = 'rgb', - invert_bg: bool = False, - transform=None, - ): - # colors - self._rgb = rgb - if palette != 'rgb': - log.warning('rgb palette is not being used, might overlap for the reconstruction loss.') - if rgb: - assert palette in XYBlocksData.COLOR_PALETTES_3, f'{palette=} must be one of {list(XYBlocksData.COLOR_PALETTES_3.keys())}' - self._colors = np.array(XYBlocksData.COLOR_PALETTES_3[palette]) - else: - assert palette in XYBlocksData.COLOR_PALETTES_1, f'{palette=} must be one of {list(XYBlocksData.COLOR_PALETTES_1.keys())}' - self._colors = np.array(XYBlocksData.COLOR_PALETTES_1[palette]) - - # bg colors - self._bg_color = 255 if invert_bg else 0 # we dont need rgb for this - assert not np.any([np.all(self._bg_color == color) for color in self._colors]), f'Color conflict with background: {self._bg_color} ({invert_bg=}) in {self._colors}' - - # grid - grid_levels = np.arange(1, grid_levels+1) if isinstance(grid_levels, int) else np.array(grid_levels) - assert np.all(grid_size % (2 ** grid_levels) == 0), f'{grid_size=} is not divisible by pow(2, {grid_levels=})' - assert np.all(grid_levels[:-1] <= grid_levels[1:]) - self._grid_size = grid_size - self._grid_levels = grid_levels - self._grid_dims = len(grid_levels) - - # axis sizes - self._axis_divisions = 2 ** self._grid_levels - assert len(self._axis_divisions) == self._grid_dims and np.all(grid_size % self._axis_divisions) == 0, 'This should never happen' - self._axis_division_sizes = grid_size // self._axis_divisions - - # info - self._factor_names = tuple([f'{prefix}-{d}' for prefix in ['color', 'x', 'y'] for d in self._axis_divisions]) - self._factor_sizes = tuple([len(self._colors)] * self._grid_dims + list(self._axis_divisions) * 2) - self._img_shape = (grid_size, grid_size, 3 if self._rgb else 1) - - # initialise - super().__init__(transform=transform) - - def _get_observation(self, idx): - positions = self.idx_to_pos(idx) - cs, xs, ys = positions[:self._grid_dims*1], positions[self._grid_dims*1:self._grid_dims*2], positions[self._grid_dims*2:] - assert len(xs) == len(ys) == len(cs) - # GENERATE - obs = np.full(self.img_shape, self._bg_color, dtype=np.uint8) - for i, (x, y, s, c) in enumerate(zip(xs, ys, self._axis_division_sizes, cs)): - obs[y*s:(y+1)*s, x*s:(x+1)*s, :] = self._colors[c] if np.any(obs[y*s, x*s, :] != self._colors[c]) else self._bg_color - # RETURN - return obs - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/dataset/data/_groundtruth__xysquares.py b/disent/dataset/data/_groundtruth__xysquares.py deleted file mode 100644 index 01c7e4d6..00000000 --- a/disent/dataset/data/_groundtruth__xysquares.py +++ /dev/null @@ -1,202 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -from typing import Optional -from typing import Tuple -from typing import Union - -import numpy as np - -from disent.dataset.data._groundtruth import GroundTruthData -from disent.util.iters import iter_chunks - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# xy multi grid data # -# ========================================================================= # - - -class XYSquaresMinimalData(GroundTruthData): - """ - Dataset that generates all possible permutations of 3 (R, G, B) coloured - squares placed on a square grid. This dataset is designed to not overlap - in the reconstruction loss space. - - If you use this in your work, please cite: https://github.com/nmichlo/disent - - NOTE: Unlike XYSquaresData, XYSquaresMinimalData is the bare-minimum class - to generate the same results as the default values for XYSquaresData, - this class is a fair bit faster (~0.8x)! - - All 3 squares are returned, in RGB, each square is size 8, with - non-overlapping grid spacing set to 8 pixels, in total leaving - 8*8*8*8*8*8 factors. Images are uint8 with fill values 0 (bg) - and 255 (fg). - """ - - name = 'xy_squares_minimal' - - @property - def factor_names(self) -> Tuple[str, ...]: - return 'x_R', 'y_R', 'x_G', 'y_G', 'x_B', 'y_B' - - @property - def factor_sizes(self) -> Tuple[int, ...]: - return 8, 8, 8, 8, 8, 8 # R, G, B squares - - @property - def img_shape(self) -> Tuple[int, ...]: - return 64, 64, 3 - - def _get_observation(self, idx): - # get factors - factors = np.reshape(np.unravel_index(idx, self.factor_sizes), (-1, 2)) - # GENERATE - obs = np.zeros(self.img_shape, dtype=np.uint8) - for i, (fx, fy) in enumerate(factors): - x, y = 8 * fx, 8 * fy - obs[y:y+8, x:x+8, i] = 255 - return obs - - -# ========================================================================= # -# xy multi grid data # -# ========================================================================= # - - -class XYSquaresData(GroundTruthData): - - """ - Dataset that generates all possible permutations of 3 (R, G, B) coloured - squares placed on a square grid. This dataset is designed to not overlap - in the reconstruction loss space. (if the spacing is set correctly.) - - If you use this in your work, please cite: https://github.com/nmichlo/disent - - NOTE: Unlike XYSquaresMinimalData, XYSquaresData allows adjusting various aspects - of the data that is generated, but the generation process is slower (~1.25x). - """ - - name = 'xy_squares' - - @property - def factor_names(self) -> Tuple[str, ...]: - return ('x_R', 'y_R', 'x_G', 'y_G', 'x_B', 'y_B')[:self._num_squares*2] - - @property - def factor_sizes(self) -> Tuple[int, ...]: - return (self._placements, self._placements) * self._num_squares # R, G, B squares - - @property - def img_shape(self) -> Tuple[int, ...]: - return self._width, self._width, (3 if self._rgb else 1) - - def __init__( - self, - square_size: int = 8, - image_size: int = 64, - grid_size: Optional[int] = None, - grid_spacing: Optional[int] = None, - num_squares: int = 3, - rgb: bool = True, - fill_value: Optional[Union[float, int]] = None, - dtype: Union[np.dtype, str] = np.uint8, - no_warnings: bool = False, - transform=None, - ): - """ - :param square_size: the size of the individual squares in pixels - :param image_size: the image size in pixels - :param grid_spacing: the step size between square positions on the grid. By - default this is set to square_size which results in non-overlapping - data if `grid_spacing >= square_size` Reducing this value such that - `grid_spacing < square_size` results in overlapping data. - :param num_squares: The number of squares drawn. `1 <= num_squares <= 3` - :param rgb: Image has 3 channels if True, otherwise it is greyscale with 1 channel. - :param no_warnings: If warnings should be disabled if overlapping. - :param fill_value: The foreground value to use for filling squares, the default background value is 0. - :param grid_size: The number of grid positions available for the square to be placed in. The square is centered if this is less than - :param dtype: - """ - if grid_spacing is None: - grid_spacing = square_size - if (grid_spacing < square_size) and not no_warnings: - log.warning(f'overlap between squares for reconstruction loss, {grid_spacing} < {square_size}') - # color - self._rgb = rgb - self._dtype = np.dtype(dtype) - # check fill values - if self._dtype.kind == 'u': - self._fill_value = 255 if (fill_value is None) else fill_value - assert isinstance(self._fill_value, int) - assert 0 < self._fill_value <= 255, f'0 < {self._fill_value} <= 255' - elif self._dtype.kind == 'f': - self._fill_value = 1.0 if (fill_value is None) else fill_value - assert isinstance(self._fill_value, (int, float)) - assert 0.0 < self._fill_value <= 1.0, f'0.0 < {self._fill_value} <= 1.0' - else: - raise TypeError(f'invalid dtype: {self._dtype}, must be float or unsigned integer') - # image sizes - self._width = image_size - # number of squares - self._num_squares = num_squares - assert 1 <= num_squares <= 3, 'Only 1, 2 or 3 squares are supported!' - # square scales - self._square_size = square_size - # x, y - self._spacing = grid_spacing - self._placements = (self._width - self._square_size) // grid_spacing + 1 - # maximum placements - if grid_size is not None: - assert isinstance(grid_size, int) - assert grid_size > 0 - if (grid_size > self._placements) and not no_warnings: - log.warning(f'number of possible placements: {self._placements} is less than the given grid size: {grid_size}, reduced grid size from: {grid_size} -> {self._placements}') - self._placements = min(self._placements, grid_size) - # center elements - self._offset = (self._width - (self._square_size + (self._placements-1)*self._spacing)) // 2 - # initialise parents -- they depend on self.factors - super().__init__(transform=transform) - - def _get_observation(self, idx): - # get factors - factors = self.idx_to_pos(idx) - offset, space, size = self._offset, self._spacing, self._square_size - # GENERATE - obs = np.zeros(self.img_shape, dtype=self._dtype) - for i, (fx, fy) in enumerate(iter_chunks(factors, 2)): - x, y = offset + space * fx, offset + space * fy - if self._rgb: - obs[y:y+size, x:x+size, i] = self._fill_value - else: - obs[y:y+size, x:x+size, :] = self._fill_value - return obs - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/dataset/transform/_augment.py b/disent/dataset/transform/_augment.py index 22dfcb72..69082a1d 100644 --- a/disent/dataset/transform/_augment.py +++ b/disent/dataset/transform/_augment.py @@ -237,8 +237,6 @@ def _check_kernel(kernel: torch.Tensor) -> torch.Tensor: # (REGEX, EXAMPLE, FACTORY_FUNC) # - factory function takes at min one arg: fn(reduction) with one arg after that per regex capture group # - regex expressions are tested in order, expressions should be mutually exclusive or ordered such that more specialized versions occur first. - (re.compile(r'^(xy8)_r(47)$'), 'xy8_r47', lambda kern, radius: torch.load(os.path.abspath(os.path.join(disent.__file__, '../../../data/adversarial_kernel', 'r47-1_s28800_adam_lr0.003_wd0.0_xy8x8.pt')))), # pragma: delete-on-release - (re.compile(r'^(xy1)_r(47)$'), 'xy1_r47', lambda kern, radius: torch.load(os.path.abspath(os.path.join(disent.__file__, '../../../data/adversarial_kernel', 'r47-1_s28800_adam_lr0.003_wd0.0_xy1x1.pt')))), # pragma: delete-on-release (re.compile(r'^(box)_r(\d+)$'), 'box_r31', lambda kern, radius: torch_box_kernel_2d(radius=int(radius))[None, ...]), (re.compile(r'^(gau)_r(\d+)$'), 'gau_r31', lambda kern, radius: torch_gaussian_kernel_2d(sigma=int(radius) / 4.0, truncate=4.0)[None, None, ...]), ] diff --git a/disent/dataset/util/stats.py b/disent/dataset/util/stats.py index 6c14422c..08be5c04 100644 --- a/disent/dataset/util/stats.py +++ b/disent/dataset/util/stats.py @@ -97,22 +97,8 @@ def main(progress=False): data.SmallNorbData, data.Shapes3dData, # groundtruth -- impl synthetic - data.XYBlocksData, # pragma: delete-on-release data.XYObjectData, data.XYObjectShadedData, - data.XYSquaresData, # pragma: delete-on-release - data.XYSquaresMinimalData, # pragma: delete-on-release - data.XColumnsData, # pragma: delete-on-release - # groundtruth -- increasing overlap # pragma: delete-on-release - (data.XYSquaresData, dict(grid_size=8, grid_spacing=8)), # pragma: delete-on-release - (data.XYSquaresData, dict(grid_size=8, grid_spacing=7)), # pragma: delete-on-release - (data.XYSquaresData, dict(grid_size=8, grid_spacing=6)), # pragma: delete-on-release - (data.XYSquaresData, dict(grid_size=8, grid_spacing=5)), # pragma: delete-on-release - (data.XYSquaresData, dict(grid_size=8, grid_spacing=4)), # pragma: delete-on-release - (data.XYSquaresData, dict(grid_size=8, grid_spacing=3)), # pragma: delete-on-release - (data.XYSquaresData, dict(grid_size=8, grid_spacing=2)), # pragma: delete-on-release - (data.XYSquaresData, dict(grid_size=8, grid_spacing=1)), # pragma: delete-on-release - (data.XYSquaresData, dict(rgb=False)), # pragma: delete-on-release # large datasets (data.Mpi3dData, dict(subset='toy', in_memory=True)), (data.Mpi3dData, dict(subset='realistic', in_memory=True)), diff --git a/disent/frameworks/ae/experimental/__init__.py b/disent/frameworks/ae/experimental/__init__.py deleted file mode 100644 index 6055d82a..00000000 --- a/disent/frameworks/ae/experimental/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -# supervised frameworks -from disent.frameworks.ae.experimental._supervised__adaneg_tae import AdaNegTripletAe - -# unsupervised frameworks -from disent.frameworks.ae.experimental._unsupervised__dotae import DataOverlapTripletAe - -# weakly supervised frameworks -from disent.frameworks.ae.experimental._weaklysupervised__adaae import AdaAe diff --git a/disent/frameworks/ae/experimental/_supervised__adaneg_tae.py b/disent/frameworks/ae/experimental/_supervised__adaneg_tae.py deleted file mode 100644 index 647adda6..00000000 --- a/disent/frameworks/ae/experimental/_supervised__adaneg_tae.py +++ /dev/null @@ -1,71 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -from dataclasses import dataclass -from numbers import Number -from typing import Any -from typing import Dict -from typing import Sequence -from typing import Tuple -from typing import Union - -import torch - -from disent.frameworks.ae._supervised__tae import TripletAe -from disent.frameworks.ae.experimental._weaklysupervised__adaae import AdaAe -from disent.frameworks.vae.experimental._supervised__adaneg_tvae import AdaNegTripletVae - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Guided Ada Vae # -# ========================================================================= # - - -class AdaNegTripletAe(TripletAe): - """ - This is a condensed version of the ada_tvae and adaave_tvae, - using approximately the best settings and loss... - """ - - REQUIRED_OBS = 3 - - @dataclass - class cfg(TripletAe.cfg, AdaAe.cfg): - # ada_tvae - loss - adat_triplet_share_scale: float = 0.95 - - def hook_ae_compute_ave_aug_loss(self, zs: Sequence[torch.Tensor], xs_partial_recon: Sequence[torch.Tensor], xs_targ: Sequence[torch.Tensor]) -> Tuple[Union[torch.Tensor, Number], Dict[str, Any]]: - return AdaNegTripletVae.estimate_ada_triplet_loss_from_zs( - zs=zs, - cfg=self.cfg, - ) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/frameworks/ae/experimental/_unsupervised__dotae.py b/disent/frameworks/ae/experimental/_unsupervised__dotae.py deleted file mode 100644 index 569e29f2..00000000 --- a/disent/frameworks/ae/experimental/_unsupervised__dotae.py +++ /dev/null @@ -1,76 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -from dataclasses import dataclass -from typing import Any -from typing import Dict -from typing import Sequence -from typing import Tuple - -import torch - -from disent.frameworks.ae.experimental._supervised__adaneg_tae import AdaNegTripletAe -from disent.frameworks.vae.experimental._supervised__adaneg_tvae import AdaNegTripletVae -from disent.frameworks.vae.experimental._unsupervised__dotvae import DataOverlapMixin - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Data Overlap Triplet AE # -# ========================================================================= # - - -class DataOverlapTripletAe(AdaNegTripletAe, DataOverlapMixin): - - REQUIRED_OBS = 1 - - @dataclass - class cfg(AdaNegTripletAe.cfg, DataOverlapMixin.cfg): - pass - - def __init__(self, model: 'AutoEncoder', cfg: cfg = None, batch_augment=None): - super().__init__(model=model, cfg=cfg, batch_augment=batch_augment) - # initialise mixin - self.init_data_overlap_mixin() - - def hook_ae_compute_ave_aug_loss(self, zs: Sequence[torch.Tensor], xs_partial_recon: Sequence[torch.Tensor], xs_targ: Sequence[torch.Tensor]) -> Tuple[torch.Tensor, Dict[str, Any]]: - [z], [x_targ_orig] = zs, xs_targ - # 1. randomly generate and mine triplets using augmented versions of the inputs - a_idxs, p_idxs, n_idxs = self.random_mined_triplets(x_targ_orig=x_targ_orig) - # 2. compute triplet loss - loss, loss_log = AdaNegTripletVae.estimate_ada_triplet_loss_from_zs( - zs=[z[idxs] for idxs in (a_idxs, p_idxs, n_idxs)], - cfg=self.cfg, - ) - return loss, { - **loss_log, - } - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/frameworks/ae/experimental/_weaklysupervised__adaae.py b/disent/frameworks/ae/experimental/_weaklysupervised__adaae.py deleted file mode 100644 index f38690a5..00000000 --- a/disent/frameworks/ae/experimental/_weaklysupervised__adaae.py +++ /dev/null @@ -1,81 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from typing import Any -from typing import Dict -from typing import Sequence -from typing import Tuple - -import torch -from dataclasses import dataclass - -from disent.frameworks.ae._unsupervised__ae import Ae -from disent.frameworks.vae._weaklysupervised__adavae import AdaVae - - -# ========================================================================= # -# Ada-GVAE # -# ========================================================================= # - - -class AdaAe(Ae): - """ - Custom implementation, removing Variational Auto-Encoder components of: - Weakly Supervised Disentanglement Learning Without Compromises: https://arxiv.org/abs/2002.02886 - - MODIFICATION: - - L1 distance for deltas instead of KL divergence - - adjustable threshold value - """ - - REQUIRED_OBS = 2 - - @dataclass - class cfg(Ae.cfg): - ada_thresh_ratio: float = 0.5 - - def hook_ae_intercept_zs(self, zs: Sequence[torch.Tensor]) -> Tuple[Sequence[torch.Tensor], Dict[str, Any]]: - """ - Adaptive VAE Glue Method, putting the various components together - 1. find differences between deltas - 2. estimate a threshold for differences - 3. compute a shared mask from this threshold - 4. average together elements that should be considered shared - - TODO: the methods used in this function should probably be moved here - TODO: this function could be turned into a torch.nn.Module! - """ - z0, z1 = zs - # shared elements that need to be averaged, computed per pair in the batch. - share_mask = AdaVae.compute_shared_mask_from_zs(z0, z1, ratio=self.cfg.ada_thresh_ratio) - # compute average posteriors - new_zs = AdaVae.make_shared_zs(z0, z1, share_mask) - # return new args & generate logs - return new_zs, { - 'shared': share_mask.sum(dim=1).float().mean() - } - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/frameworks/helper/reconstructions.py b/disent/frameworks/helper/reconstructions.py index 18b31cee..723d764f 100644 --- a/disent/frameworks/helper/reconstructions.py +++ b/disent/frameworks/helper/reconstructions.py @@ -281,8 +281,6 @@ def compute_unreduced_loss_from_partial(self, x_partial_recon: torch.Tensor, x_t # (REGEX, EXAMPLE, FACTORY_FUNC) # - factory function takes at min one arg: fn(reduction) with one arg after that per regex capture group # - regex expressions are tested in order, expressions should be mutually exclusive or ordered such that more specialized versions occur first. - (re.compile(r'^([a-z\d]+)_([a-z\d]+_[a-z\d]+)_w(\d+\.\d+)$'), 'mse_xy8_r47_w1.0', lambda reduction, loss, kern, weight: AugmentedReconLossHandler(make_reconstruction_loss(loss, reduction=reduction), kernel=kern, wrap_weight=1-float(weight), aug_weight=float(weight))), # pragma: delete-on-release - (re.compile(r'^([a-z\d]+)_([a-z\d]+_[a-z\d]+)_l(\d+\.\d+)_k(\d+\.\d+)$'), 'mse_xy8_r47_l1.0_k1.0', lambda reduction, loss, kern, l_weight, k_weight: AugmentedReconLossHandler(make_reconstruction_loss(loss, reduction=reduction), kernel=kern, wrap_weight=float(l_weight), aug_weight=float(k_weight))), # pragma: delete-on-release ] diff --git a/disent/frameworks/vae/experimental/__init__.py b/disent/frameworks/vae/experimental/__init__.py deleted file mode 100644 index cb2006fd..00000000 --- a/disent/frameworks/vae/experimental/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -# supervised frameworks -from disent.frameworks.vae.experimental._supervised__adaave_tvae import AdaAveTripletVae -from disent.frameworks.vae.experimental._supervised__adaneg_tvae import AdaNegTripletVae -from disent.frameworks.vae.experimental._supervised__adatvae import AdaTripletVae -from disent.frameworks.vae.experimental._supervised__badavae import BoundedAdaVae -from disent.frameworks.vae.experimental._supervised__gadavae import GuidedAdaVae -from disent.frameworks.vae.experimental._supervised__tbadavae import TripletBoundedAdaVae -from disent.frameworks.vae.experimental._supervised__tgadavae import TripletGuidedAdaVae - -# unsupervised frameworks -from disent.frameworks.vae.experimental._unsupervised__dorvae import DataOverlapRankVae -from disent.frameworks.vae.experimental._unsupervised__dotvae import DataOverlapTripletVae - -# weakly supervised frameworks -from disent.frameworks.vae.experimental._weaklysupervised__augpostriplet import AugPosTripletVae -from disent.frameworks.vae.experimental._weaklysupervised__st_adavae import SwappedTargetAdaVae -from disent.frameworks.vae.experimental._weaklysupervised__st_betavae import SwappedTargetBetaVae diff --git a/disent/frameworks/vae/experimental/_supervised__adaave_tvae.py b/disent/frameworks/vae/experimental/_supervised__adaave_tvae.py deleted file mode 100644 index f1c93cfe..00000000 --- a/disent/frameworks/vae/experimental/_supervised__adaave_tvae.py +++ /dev/null @@ -1,120 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -import warnings -from dataclasses import dataclass -from typing import Any -from typing import Dict -from typing import Sequence -from typing import Tuple - -from disent.util.deprecate import deprecated -from torch.distributions import Distribution -from torch.distributions import Normal - -from disent.frameworks.vae.experimental._supervised__adatvae import AdaTripletVae -from disent.frameworks.vae.experimental._supervised__adatvae import compute_ave_shared_distributions -from disent.frameworks.vae.experimental._supervised__adatvae import compute_triplet_shared_masks - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Guided Ada Vae # -# ========================================================================= # - - -@deprecated('Rather use the AdaNegTripletVae') -class AdaAveTripletVae(AdaTripletVae): - """ - This was a more general attempt of the ada-tvae, - that also averages representations passed to the decoder. - - just averaging in this way without augmenting the loss with - triplet, or ada_triplet is too weak of a supervision signal. - """ - - REQUIRED_OBS = 3 - - @dataclass - class cfg(AdaTripletVae.cfg): - # adavae - ada_thresh_mode: str = 'dist' # RESET OVERRIDEN VALUE - # adaave_tvae - adaave_augment_orig: bool = True # triplet over original OR averaged embeddings - adaave_decode_orig: bool = True # decode & regularize original OR averaged embeddings - - def __init__(self, model: 'AutoEncoder', cfg: cfg = None, batch_augment=None): - super().__init__(model=model, cfg=cfg, batch_augment=batch_augment) - # checks - if self.cfg.ada_thresh_mode != 'dist': - warnings.warn(f'cfg.ada_thresh_mode == {repr(self.cfg.ada_thresh_mode)}. Modes other than "dist" do not work well!') - val = (self.cfg.adat_triplet_loss != 'triplet_hard_ave_all') - if self.cfg.adaave_augment_orig == val: - warnings.warn(f'cfg.adaave_augment_orig == {repr(self.cfg.adaave_augment_orig)}. Modes other than {repr(val)} do not work well!') - if self.cfg.adaave_decode_orig == False: - warnings.warn(f'cfg.adaave_decode_orig == {repr(self.cfg.adaave_decode_orig)}. Modes other than True do not work well!') - - def hook_intercept_ds(self, ds_posterior: Sequence[Normal], ds_prior: Sequence[Normal]) -> Tuple[Sequence[Distribution], Sequence[Distribution], Dict[str, Any]]: - # triplet vae intercept -- in case detached - ds_posterior, ds_prior, intercept_logs = super().hook_intercept_ds(ds_posterior, ds_prior) - - # compute shared masks, shared embeddings & averages over shared embeddings - share_masks, share_logs = compute_triplet_shared_masks(ds_posterior, cfg=self.cfg) - ds_posterior_shared, ds_posterior_shared_ave = compute_ave_shared_distributions(ds_posterior, share_masks, cfg=self.cfg) - - # DIFFERENCE FROM ADAVAE | get return values - # adavae: adaave_augment_orig == True, adaave_decode_orig == False - ds_posterior_augment = (ds_posterior if self.cfg.adaave_augment_orig else ds_posterior_shared_ave) - ds_posterior_return = (ds_posterior if self.cfg.adaave_decode_orig else ds_posterior_shared_ave) - - # save params for aug_loss hook step - self._curr_ada_loss_kwargs = dict( - share_masks=share_masks, - zs=[d.mean for d in ds_posterior], - zs_shared=[d.mean for d in ds_posterior_shared], - zs_shared_ave=[d.mean for d in ds_posterior_augment], # USUALLY: zs_params_shared_ave - ) - - return ds_posterior_return, ds_prior, { - **intercept_logs, - **share_logs, - } - - def hook_compute_ave_aug_loss(self, ds_posterior, ds_prior, zs_sampled, xs_partial_recon, xs_targ): - """ - NOTE: we don't use input parameters here, this function will only work - if called as part of training_step or do_training_step - """ - # compute triplet loss - result = AdaTripletVae.compute_ada_triplet_loss(**self._curr_ada_loss_kwargs, cfg=self.cfg) - # cleanup temporary variables - del self._curr_ada_loss_kwargs - # we are done - return result - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/frameworks/vae/experimental/_supervised__adaneg_tvae.py b/disent/frameworks/vae/experimental/_supervised__adaneg_tvae.py deleted file mode 100644 index 469b4099..00000000 --- a/disent/frameworks/vae/experimental/_supervised__adaneg_tvae.py +++ /dev/null @@ -1,118 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -from dataclasses import dataclass -from typing import Sequence - -import torch -from torch.distributions import Normal - -from disent.nn.loss.triplet import configured_dist_triplet -from disent.nn.loss.triplet import configured_triplet -from disent.frameworks.vae._supervised__tvae import TripletVae -from disent.frameworks.vae.experimental._supervised__adatvae import compute_triplet_shared_masks -from disent.frameworks.vae.experimental._supervised__adatvae import compute_triplet_shared_masks_from_zs -from disent.frameworks.vae._weaklysupervised__adavae import AdaVae - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Guided Ada Vae # -# ========================================================================= # - - -class AdaNegTripletVae(TripletVae): - - """ - This is a condensed version of the ada_tvae and adaave_tvae, - using approximately the best settings and loss... - """ - - REQUIRED_OBS = 3 - - @dataclass - class cfg(TripletVae.cfg, AdaVae.cfg): - # adavae - ada_thresh_mode: str = 'dist' # only works for: adat_share_mask_mode == "posterior" - # ada_tvae - loss - adat_triplet_share_scale: float = 0.95 - # ada_tvae - averaging - adat_share_mask_mode: str = 'posterior' - - def hook_compute_ave_aug_loss(self, ds_posterior: Sequence[Normal], ds_prior: Sequence[Normal], zs_sampled: Sequence[torch.Tensor], xs_partial_recon: Sequence[torch.Tensor], xs_targ: Sequence[torch.Tensor]): - return self.estimate_ada_triplet_loss( - ds_posterior=ds_posterior, - cfg=self.cfg, - ) - - @staticmethod - def estimate_ada_triplet_loss_from_zs(zs: Sequence[torch.Tensor], cfg: cfg): - # compute shared masks, shared embeddings & averages over shared embeddings - share_masks, share_logs = compute_triplet_shared_masks_from_zs(zs=zs, cfg=cfg) - # compute loss - ada_triplet_loss, ada_triplet_logs = AdaNegTripletVae.compute_ada_triplet_loss(share_masks=share_masks, zs=zs, cfg=cfg) - # merge logs & return loss - return ada_triplet_loss, { - **ada_triplet_logs, - **share_logs, - } - - @staticmethod - def estimate_ada_triplet_loss(ds_posterior: Sequence[Normal], cfg: cfg): - # compute shared masks, shared embeddings & averages over shared embeddings - share_masks, share_logs = compute_triplet_shared_masks(ds_posterior, cfg=cfg) - # compute loss - ada_triplet_loss, ada_triplet_logs = AdaNegTripletVae.compute_ada_triplet_loss(share_masks=share_masks, zs=(d.mean for d in ds_posterior), cfg=cfg) - # merge logs & return loss - return ada_triplet_loss, { - **ada_triplet_logs, - **share_logs, - } - - @staticmethod - def compute_ada_triplet_loss(share_masks, zs, cfg: cfg): - # Normal Triplet Loss - (a_z, p_z, n_z) = zs - trip_loss = configured_triplet(a_z, p_z, n_z, cfg=cfg) - - # Soft Scaled Negative Triplet - (ap_share_mask, an_share_mask, pn_share_mask) = share_masks - triplet_hard_neg_ave_scaled = configured_dist_triplet( - pos_delta=a_z - p_z, - neg_delta=torch.where(an_share_mask, cfg.adat_triplet_share_scale * (a_z - n_z), (a_z - n_z)), - cfg=cfg, - ) - - return triplet_hard_neg_ave_scaled, { - 'triplet': trip_loss, - 'triplet_chosen': triplet_hard_neg_ave_scaled, - } - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/frameworks/vae/experimental/_supervised__adatvae.py b/disent/frameworks/vae/experimental/_supervised__adatvae.py deleted file mode 100644 index 4fa02ba9..00000000 --- a/disent/frameworks/vae/experimental/_supervised__adatvae.py +++ /dev/null @@ -1,328 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -from dataclasses import dataclass -from typing import Sequence -from typing import Tuple - -import torch -from disent.util.deprecate import deprecated -from torch.distributions import Distribution -from torch.distributions import Normal - -from disent.nn.loss.triplet import configured_dist_triplet -from disent.nn.loss.triplet import configured_triplet -from disent.frameworks.vae._supervised__tvae import TripletVae -from disent.frameworks.vae._weaklysupervised__adavae import AdaVae - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Guided Ada Vae # -# ========================================================================= # - - -@deprecated('Rather use the AdaNegTripletVae') -class AdaTripletVae(TripletVae): - - REQUIRED_OBS = 3 - - @dataclass - class cfg(TripletVae.cfg, AdaVae.cfg): - # adavae - ada_thresh_mode: str = 'dist' # only works for: adat_share_mask_mode == "posterior" - # ada_tvae - loss - adat_triplet_loss: str = 'triplet_hard_neg_ave' # should be used with a schedule! - adat_triplet_ratio: float = 1.0 - adat_triplet_soft_scale: float = 1.0 - adat_triplet_pull_weight: float = 0.1 # only works for: adat_triplet_loss == "triplet_hard_neg_ave_pull" - adat_triplet_share_scale: float = 0.95 # only works for: adat_triplet_loss == "triplet_hard_neg_ave_scaled" - # ada_tvae - averaging - adat_share_mask_mode: str = 'posterior' - adat_share_ave_mode: str = 'all' # only works for: adat_triplet_loss == "triplet_hard_ave_all" - - def hook_compute_ave_aug_loss(self, ds_posterior: Sequence[Normal], ds_prior: Sequence[Normal], zs_sampled: Sequence[torch.Tensor], xs_partial_recon: Sequence[torch.Tensor], xs_targ: Sequence[torch.Tensor]): - return self.estimate_ada_triplet_loss( - ds_posterior=ds_posterior, - cfg=self.cfg, - ) - - @staticmethod - def estimate_ada_triplet_loss(ds_posterior: Sequence[Normal], cfg: cfg): - """ - zs_params and ds_posterior are convenience variables here. - - they should contain the same values - - in practice we only need one of them and can compute the other! - """ - # compute shared masks, shared embeddings & averages over shared embeddings - share_masks, share_logs = compute_triplet_shared_masks(ds_posterior, cfg=cfg) - ds_posterior_shared, ds_posterior_shared_ave = compute_ave_shared_distributions(ds_posterior, share_masks, cfg=cfg) - - # compute loss - ada_triplet_loss, ada_triplet_logs = AdaTripletVae.compute_ada_triplet_loss( - share_masks=share_masks, - zs=[d.mean for d in ds_posterior], - zs_shared=[d.mean for d in ds_posterior_shared], - zs_shared_ave=[d.mean for d in ds_posterior_shared_ave], - cfg=cfg, - ) - - return ada_triplet_loss, { - **ada_triplet_logs, - **share_logs, - } - - @staticmethod - def compute_ada_triplet_loss(share_masks: Sequence[torch.Tensor], zs: Sequence[Normal], zs_shared: Sequence[Normal], zs_shared_ave: Sequence[Normal], cfg: cfg): - - # Normal Triplet Loss - (a_z, p_z, n_z) = zs - trip_loss = configured_triplet(a_z, p_z, n_z, cfg=cfg) - - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # Hard Losses - zs_shared - # TODO: implement triplet over KL divergence rather than l1/l2 distance? - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # Hard Averaging Before Triplet - (ap_a_ave, ap_p_ave, an_a_ave, an_n_ave, pn_p_ave, pn_n_ave) = zs_shared - triplet_hard_ave = configured_dist_triplet(pos_delta=ap_a_ave - ap_p_ave, neg_delta=an_a_ave - an_n_ave, cfg=cfg) - triplet_hard_ave_neg = configured_dist_triplet(pos_delta=a_z - p_z, neg_delta=an_a_ave - an_n_ave, cfg=cfg) - - # Hard Averaging Before Triplet - PULLING PUSHING - (ap_share_mask, an_share_mask, pn_share_mask) = share_masks - neg_delta_push = torch.where(~an_share_mask, a_z - n_z, torch.zeros_like(a_z)) # this is the same as: an_a_ave - an_n_ave - neg_delta_pull = torch.where( an_share_mask, a_z - n_z, torch.zeros_like(a_z)) - triplet_hard_ave_neg_pull = configured_dist_push_pull_triplet(pos_delta=a_z - p_z, neg_delta=neg_delta_push, neg_delta_pull=neg_delta_pull, cfg=cfg) - - # Hard All Averaging Before Triplet - (a_ave, p_ave, n_ave) = zs_shared_ave - triplet_all_hard_ave = configured_dist_triplet(pos_delta=a_ave-p_ave, neg_delta=a_ave-n_ave, cfg=cfg) - - # Soft Scaled Negative Triplet - triplet_hard_neg_ave_scaled = configured_dist_triplet( - pos_delta=a_z - p_z, - neg_delta=torch.where(an_share_mask, cfg.adat_triplet_share_scale * (a_z - n_z), (a_z - n_z)), - cfg=cfg, - ) - - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # Soft Losses - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # Individual Pair Averaging Losses - _soft_ap_loss = configured_soft_ave_loss(share_mask=ap_share_mask, delta=a_z - p_z, cfg=cfg) - _soft_an_loss = configured_soft_ave_loss(share_mask=an_share_mask, delta=a_z - n_z, cfg=cfg) - _soft_pn_loss = configured_soft_ave_loss(share_mask=pn_share_mask, delta=p_z - n_z, cfg=cfg) - - # soft losses - soft_loss_an = (_soft_an_loss) - soft_loss_an_ap = (_soft_an_loss + _soft_ap_loss) / 2 - soft_loss_an_ap_pn = (_soft_an_loss + _soft_ap_loss + _soft_pn_loss) / 3 - - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # Return - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - losses = { - 'triplet': trip_loss, - # soft ave - 'triplet_soft_ave_neg': trip_loss + soft_loss_an, - 'triplet_soft_ave_p_n': trip_loss + soft_loss_an_ap, - 'triplet_soft_ave_all': trip_loss + soft_loss_an_ap_pn, - # hard ave - 'triplet_hard_ave': torch.lerp(trip_loss, triplet_hard_ave, weight=cfg.adat_triplet_ratio), - 'triplet_hard_neg_ave': torch.lerp(trip_loss, triplet_hard_ave_neg, weight=cfg.adat_triplet_ratio), - 'triplet_hard_neg_ave_pull': torch.lerp(trip_loss, triplet_hard_ave_neg_pull, weight=cfg.adat_triplet_ratio), - 'triplet_hard_ave_all': torch.lerp(trip_loss, triplet_all_hard_ave, weight=cfg.adat_triplet_ratio), - # scaled - 'triplet_hard_neg_ave_scaled': torch.lerp(trip_loss, triplet_hard_neg_ave_scaled, weight=cfg.adat_triplet_ratio), - } - - return losses[cfg.adat_triplet_loss], { - 'triplet': trip_loss, - 'triplet_chosen': losses[cfg.adat_triplet_loss], - } - - -# ========================================================================= # -# Ada-TVae # -# ========================================================================= # - - -def dist_push_pull_triplet(pos_delta, neg_delta, neg_delta_pull, margin_max=1., p=1, pull_weight=1.): - """ - Pushing Pulling Triplet Loss - - should match standard triplet loss if pull_weight=0. - """ - p_dist = torch.norm(pos_delta, p=p, dim=-1) - n_dist = torch.norm(neg_delta, p=p, dim=-1) - n_dist_pull = torch.norm(neg_delta_pull, p=p, dim=-1) - loss = torch.clamp_min(p_dist - n_dist + margin_max + pull_weight * n_dist_pull, 0) - return loss.mean() - - -def configured_dist_push_pull_triplet(pos_delta, neg_delta, neg_delta_pull, cfg: AdaTripletVae.cfg): - """ - required config params: - - cfg.triplet_margin_max: (0, inf) - - cfg.triplet_p: 1 or 2 - - cfg.triplet_scale: [0, inf) - - cfg.adat_triplet_pull_weight: [0, 1] - """ - return dist_push_pull_triplet( - pos_delta=pos_delta, neg_delta=neg_delta, neg_delta_pull=neg_delta_pull, - margin_max=cfg.triplet_margin_max, p=cfg.triplet_p, pull_weight=cfg.adat_triplet_pull_weight, - ) * cfg.triplet_scale - - -def soft_ave_loss(share_mask, delta): - return torch.norm(torch.where(share_mask, delta, torch.zeros_like(delta)), p=2, dim=-1).mean() - - -def configured_soft_ave_loss(share_mask, delta, cfg: AdaTripletVae.cfg): - """ - required config params: - - cfg.triplet_scale: [0, inf) - - cfg.adat_triplet_soft_scale: [0, inf) - """ - return soft_ave_loss(share_mask=share_mask, delta=delta) * (cfg.adat_triplet_soft_scale * cfg.triplet_scale) - - -# ========================================================================= # -# AveAda-TVAE # -# ========================================================================= # - - -def compute_triplet_shared_masks_from_zs(zs: Sequence[torch.Tensor], cfg): - """ - required config params: - - cfg.ada_thresh_ratio: - """ - a_z, p_z, n_z = zs - # shared elements that need to be averaged, computed per pair in the batch. - ap_share_mask = AdaVae.compute_shared_mask_from_zs(a_z, p_z, ratio=cfg.ada_thresh_ratio) - an_share_mask = AdaVae.compute_shared_mask_from_zs(a_z, n_z, ratio=cfg.ada_thresh_ratio) - pn_share_mask = AdaVae.compute_shared_mask_from_zs(p_z, n_z, ratio=cfg.ada_thresh_ratio) - # return values - share_masks = (ap_share_mask, an_share_mask, pn_share_mask) - return share_masks, { - 'ap_shared': ap_share_mask.sum(dim=1).float().mean(), - 'an_shared': an_share_mask.sum(dim=1).float().mean(), - 'pn_shared': pn_share_mask.sum(dim=1).float().mean(), - } - - -def compute_triplet_shared_masks(ds_posterior: Sequence[Distribution], cfg: AdaTripletVae.cfg): - """ - required config params: - - cfg.ada_thresh_ratio: - - cfg.ada_thresh_mode: "kl", "symmetric_kl", "dist", "sampled_dist" - : only applies if cfg.ada_share_mask_mode=="posterior" - - cfg.adat_share_mask_mode: "posterior", "sample", "sample_each" - """ - a_posterior, p_posterior, n_posterior = ds_posterior - - # shared elements that need to be averaged, computed per pair in the batch. - if cfg.adat_share_mask_mode == 'posterior': - ap_share_mask = AdaVae.compute_shared_mask_from_posteriors(a_posterior, p_posterior, thresh_mode=cfg.ada_thresh_mode, ratio=cfg.ada_thresh_ratio) - an_share_mask = AdaVae.compute_shared_mask_from_posteriors(a_posterior, n_posterior, thresh_mode=cfg.ada_thresh_mode, ratio=cfg.ada_thresh_ratio) - pn_share_mask = AdaVae.compute_shared_mask_from_posteriors(p_posterior, n_posterior, thresh_mode=cfg.ada_thresh_mode, ratio=cfg.ada_thresh_ratio) - elif cfg.adat_share_mask_mode == 'sample': - a_z_sample, p_z_sample, n_z_sample = a_posterior.rsample(), p_posterior.rsample(), n_posterior.rsample() - ap_share_mask = AdaVae.compute_shared_mask_from_zs(a_z_sample, p_z_sample, ratio=cfg.ada_thresh_ratio) - an_share_mask = AdaVae.compute_shared_mask_from_zs(a_z_sample, n_z_sample, ratio=cfg.ada_thresh_ratio) - pn_share_mask = AdaVae.compute_shared_mask_from_zs(p_z_sample, n_z_sample, ratio=cfg.ada_thresh_ratio) - elif cfg.adat_share_mask_mode == 'sample_each': - ap_share_mask = AdaVae.compute_shared_mask_from_zs(a_posterior.rsample(), p_posterior.rsample(), ratio=cfg.ada_thresh_ratio) - an_share_mask = AdaVae.compute_shared_mask_from_zs(a_posterior.rsample(), n_posterior.rsample(), ratio=cfg.ada_thresh_ratio) - pn_share_mask = AdaVae.compute_shared_mask_from_zs(p_posterior.rsample(), n_posterior.rsample(), ratio=cfg.ada_thresh_ratio) - else: - raise KeyError(f'Invalid cfg.adat_share_mask_mode={repr(cfg.adat_share_mask_mode)}') - - # return values - share_masks = (ap_share_mask, an_share_mask, pn_share_mask) - return share_masks, { - 'ap_shared': ap_share_mask.sum(dim=1).float().mean(), - 'an_shared': an_share_mask.sum(dim=1).float().mean(), - 'pn_shared': pn_share_mask.sum(dim=1).float().mean(), - } - - -def compute_ave_shared_distributions(ds_posterior: Sequence[Normal], share_masks: Sequence[torch.Tensor], cfg: AdaTripletVae.cfg) -> Tuple[Sequence[Normal], Sequence[Normal]]: - """ - required config params: - - cfg.ada_average_mode: "gvae", "ml-vae" - - cfg.adat_share_ave_mode: "all", "pos_neg", "pos", "neg" - """ - a_posterior, p_posterior, n_posterior = ds_posterior - ap_share_mask, an_share_mask, pn_share_mask = share_masks - - # compute shared embeddings - ave_ap_a_posterior, ave_ap_p_posterior = AdaVae.make_shared_posteriors(a_posterior, p_posterior, ap_share_mask, average_mode=cfg.ada_average_mode) - ave_an_a_posterior, ave_an_n_posterior = AdaVae.make_shared_posteriors(a_posterior, n_posterior, an_share_mask, average_mode=cfg.ada_average_mode) - ave_pn_p_posterior, ave_pn_n_posterior = AdaVae.make_shared_posteriors(p_posterior, n_posterior, pn_share_mask, average_mode=cfg.ada_average_mode) - - # compute averaged shared embeddings - if cfg.adat_share_ave_mode == 'all': - ave_a_posterior = AdaVae.compute_average_distribution(ave_ap_a_posterior, ave_an_a_posterior, average_mode=cfg.ada_average_mode) - ave_p_posterior = AdaVae.compute_average_distribution(ave_ap_p_posterior, ave_pn_p_posterior, average_mode=cfg.ada_average_mode) - ave_n_posterior = AdaVae.compute_average_distribution(ave_an_n_posterior, ave_pn_n_posterior, average_mode=cfg.ada_average_mode) - elif cfg.adat_share_ave_mode == 'pos_neg': - ave_a_posterior = AdaVae.compute_average_distribution(ave_ap_a_posterior, ave_an_a_posterior, average_mode=cfg.ada_average_mode) - ave_p_posterior = ave_ap_p_posterior - ave_n_posterior = ave_an_n_posterior - elif cfg.adat_share_ave_mode == 'pos': - ave_a_posterior = ave_ap_a_posterior - ave_p_posterior = ave_ap_p_posterior - ave_n_posterior = n_posterior - elif cfg.adat_share_ave_mode == 'neg': - ave_a_posterior = ave_an_a_posterior - ave_p_posterior = p_posterior - ave_n_posterior = ave_an_n_posterior - else: - raise KeyError(f'Invalid cfg.adat_share_ave_mode={repr(cfg.adat_share_ave_mode)}') - - ds_posterior_shared = ( - ave_ap_a_posterior, ave_ap_p_posterior, # a & p - ave_an_a_posterior, ave_an_n_posterior, # a & n - ave_pn_p_posterior, ave_pn_n_posterior, # p & n - ) - - ds_posterior_shared_ave = ( - ave_a_posterior, - ave_p_posterior, - ave_n_posterior - ) - - # return values - return ds_posterior_shared, ds_posterior_shared_ave - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/frameworks/vae/experimental/_supervised__badavae.py b/disent/frameworks/vae/experimental/_supervised__badavae.py deleted file mode 100644 index ccc77a54..00000000 --- a/disent/frameworks/vae/experimental/_supervised__badavae.py +++ /dev/null @@ -1,121 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from dataclasses import dataclass -from typing import Any -from typing import Dict -from typing import Sequence -from typing import Tuple - -import torch -from torch.distributions import Distribution - -from disent.frameworks.vae._weaklysupervised__adavae import AdaVae - - -# ========================================================================= # -# Guided Ada Vae # -# ========================================================================= # - - -class BoundedAdaVae(AdaVae): - - REQUIRED_OBS = 3 - - @dataclass - class cfg(AdaVae.cfg): - pass - - def hook_intercept_ds(self, ds_posterior: Sequence[Distribution], ds_prior: Sequence[Distribution]) -> Tuple[Sequence[Distribution], Sequence[Distribution], Dict[str, Any]]: - a_posterior, p_posterior, n_posterior = ds_posterior - - # get deltas - a_p_deltas = AdaVae.compute_deltas_from_posteriors(a_posterior, p_posterior, thresh_mode=self.cfg.ada_thresh_mode) - a_n_deltas = AdaVae.compute_deltas_from_posteriors(a_posterior, n_posterior, thresh_mode=self.cfg.ada_thresh_mode) - - # shared elements that need to be averaged, computed per pair in the batch. - old_p_shared_mask = AdaVae.estimate_shared_mask(a_p_deltas, ratio=self.cfg.ada_thresh_ratio) - old_n_shared_mask = AdaVae.estimate_shared_mask(a_n_deltas, ratio=self.cfg.ada_thresh_ratio) - - # modify threshold based on criterion and recompute if necessary - # CORE of this approach! - p_shared_mask, n_shared_mask = compute_constrained_masks(a_p_deltas, old_p_shared_mask, a_n_deltas, old_n_shared_mask) - - # make averaged variables - # TODO: this will probably be better if it is the negative involed - # TODO: this can be merged with the gadavae/badavae - ave_ap_a_posterior, ave_ap_p_posterior = AdaVae.make_shared_posteriors(a_posterior, p_posterior, p_shared_mask, average_mode=self.cfg.ada_average_mode) - - # TODO: n_z_params should not be here! this does not match the original version - # number of loss elements is not 2 like the original - # - recons gets 2 items, p & a only - # - reg gets 2 items, p & a only - new_ds_posterior = (ave_ap_a_posterior, ave_ap_p_posterior, n_posterior) - - # return new args & generate logs - # -- we only return 2 parameters a & p, not n - return new_ds_posterior, ds_prior, { - 'p_shared_before': old_p_shared_mask.sum(dim=1).float().mean(), - 'p_shared_after': p_shared_mask.sum(dim=1).float().mean(), - 'n_shared_before': old_n_shared_mask.sum(dim=1).float().mean(), - 'n_shared_after': n_shared_mask.sum(dim=1).float().mean(), - } - - -# ========================================================================= # -# HELPER # -# ========================================================================= # - - -def compute_constrained_masks(p_kl_deltas, p_shared_mask, n_kl_deltas, n_shared_mask): - # number of changed factors - p_shared_num = torch.sum(p_shared_mask, dim=1, keepdim=True) - n_shared_num = torch.sum(n_shared_mask, dim=1, keepdim=True) - - # POSITIVE SHARED MASK - # order from smallest to largest - p_sort_indices = torch.argsort(p_kl_deltas, dim=1) - # p_shared should be at least n_shared - new_p_shared_num = torch.max(p_shared_num, n_shared_num) - - # NEGATIVE SHARED MASK - # order from smallest to largest - n_sort_indices = torch.argsort(n_kl_deltas, dim=1) - # n_shared should be at most p_shared - new_n_shared_num = torch.min(p_shared_num, n_shared_num) - - # COMPUTE NEW MASKS - new_p_shared_mask = torch.zeros_like(p_shared_mask) - new_n_shared_mask = torch.zeros_like(n_shared_mask) - for i, (new_shared_p, new_shared_n) in enumerate(zip(new_p_shared_num, new_n_shared_num)): - new_p_shared_mask[i, p_sort_indices[i, :new_shared_p]] = True - new_n_shared_mask[i, n_sort_indices[i, :new_shared_n]] = True - - # return masks - return new_p_shared_mask, new_n_shared_mask - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/frameworks/vae/experimental/_supervised__gadavae.py b/disent/frameworks/vae/experimental/_supervised__gadavae.py deleted file mode 100644 index a7e7f381..00000000 --- a/disent/frameworks/vae/experimental/_supervised__gadavae.py +++ /dev/null @@ -1,102 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from dataclasses import dataclass -from typing import Any -from typing import Dict -from typing import Sequence -from typing import Tuple - -from torch.distributions import Distribution - -from disent.frameworks.vae._weaklysupervised__adavae import AdaVae -from disent.frameworks.vae.experimental._supervised__badavae import compute_constrained_masks - - -# ========================================================================= # -# Guided Ada Vae # -# ========================================================================= # - - -class GuidedAdaVae(AdaVae): - - REQUIRED_OBS = 3 - - @dataclass - class cfg(AdaVae.cfg): - gada_anchor_ave_mode: str = 'average' - - def __init__(self, model: 'AutoEncoder', cfg: cfg = None, batch_augment=None): - super().__init__(model=model, cfg=cfg, batch_augment=batch_augment) - # how the anchor is averaged - assert cfg.gada_anchor_ave_mode in {'thresh', 'average'} - - def hook_intercept_ds(self, ds_posterior: Sequence[Distribution], ds_prior: Sequence[Distribution]) -> Tuple[Sequence[Distribution], Sequence[Distribution], Dict[str, Any]]: - """ - *NB* arguments must satisfy: d(l, l2) < d(l, l3) [positive dist < negative dist] - - This function assumes that the distance between labels l, l2, l3 - corresponding to z, z2, z3 satisfy the criteria d(l, l2) < d(l, l3) - ie. l2 is the positive sample, l3 is the negative sample - """ - a_posterior, p_posterior, n_posterior = ds_posterior - - # get deltas - a_p_deltas = AdaVae.compute_deltas_from_posteriors(a_posterior, p_posterior, thresh_mode=self.cfg.ada_thresh_mode) - a_n_deltas = AdaVae.compute_deltas_from_posteriors(a_posterior, n_posterior, thresh_mode=self.cfg.ada_thresh_mode) - - # shared elements that need to be averaged, computed per pair in the batch. - old_p_shared_mask = AdaVae.estimate_shared_mask(a_p_deltas, ratio=self.cfg.ada_thresh_ratio) - old_n_shared_mask = AdaVae.estimate_shared_mask(a_n_deltas, ratio=self.cfg.ada_thresh_ratio) - - # modify threshold based on criterion and recompute if necessary - # CORE of this approach! - p_shared_mask, n_shared_mask = compute_constrained_masks(a_p_deltas, old_p_shared_mask, a_n_deltas, old_n_shared_mask) - - # make averaged variables - # TODO: this can be merged with the gadavae/badavae - ave_ap_a_posterior, ave_ap_p_posterior = AdaVae.make_shared_posteriors(a_posterior, p_posterior, p_shared_mask, average_mode=self.cfg.ada_average_mode) - ave_an_a_posterior, ave_an_n_posterior = AdaVae.make_shared_posteriors(a_posterior, n_posterior, n_shared_mask, average_mode=self.cfg.ada_average_mode) - ave_a_posterior = AdaVae.compute_average_distribution(ave_ap_a_posterior, ave_an_a_posterior, average_mode=self.cfg.ada_average_mode) - - # compute anchor average using the adaptive threshold | TODO: this doesn't really make sense - anchor_ave_logs = {} - if self.cfg.gada_anchor_ave_mode == 'thresh': - ave_shared_mask = p_shared_mask * n_shared_mask - ave_params, _ = AdaVae.make_shared_posteriors(a_posterior, ave_a_posterior, ave_shared_mask, average_mode=self.cfg.ada_average_mode) - anchor_ave_logs['ave_shared'] = ave_shared_mask.sum(dim=1).float().mean() - - new_ds_posterior = ave_a_posterior, ave_ap_p_posterior, ave_an_n_posterior - - return new_ds_posterior, ds_prior, { - 'p_shared_before': old_p_shared_mask.sum(dim=1).float().mean(), - 'p_shared_after': p_shared_mask.sum(dim=1).float().mean(), - 'n_shared_before': old_n_shared_mask.sum(dim=1).float().mean(), - 'n_shared_after': n_shared_mask.sum(dim=1).float().mean(), - **anchor_ave_logs, - } - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/frameworks/vae/experimental/_supervised__tbadavae.py b/disent/frameworks/vae/experimental/_supervised__tbadavae.py deleted file mode 100644 index 9e5caf8d..00000000 --- a/disent/frameworks/vae/experimental/_supervised__tbadavae.py +++ /dev/null @@ -1,51 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from dataclasses import dataclass - -from disent.frameworks.vae.experimental._supervised__badavae import BoundedAdaVae -from disent.nn.loss.triplet import compute_triplet_loss -from disent.nn.loss.triplet import TripletLossConfig - - -# ========================================================================= # -# tbadavae # -# ========================================================================= # - - -class TripletBoundedAdaVae(BoundedAdaVae): - - REQUIRED_OBS = 3 - - @dataclass - class cfg(BoundedAdaVae.cfg, TripletLossConfig): - pass - - def hook_compute_ave_aug_loss(self, ds_posterior, ds_prior, zs_sampled, xs_partial_recon, xs_targ): - return compute_triplet_loss(zs=[d.mean for d in ds_posterior], cfg=self.cfg) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/frameworks/vae/experimental/_supervised__tgadavae.py b/disent/frameworks/vae/experimental/_supervised__tgadavae.py deleted file mode 100644 index 0739e751..00000000 --- a/disent/frameworks/vae/experimental/_supervised__tgadavae.py +++ /dev/null @@ -1,51 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from dataclasses import dataclass - -from disent.frameworks.vae.experimental._supervised__gadavae import GuidedAdaVae -from disent.nn.loss.triplet import compute_triplet_loss -from disent.nn.loss.triplet import TripletLossConfig - - -# ========================================================================= # -# tgadavae # -# ========================================================================= # - - -class TripletGuidedAdaVae(GuidedAdaVae): - - REQUIRED_OBS = 3 - - @dataclass - class cfg(GuidedAdaVae.cfg, TripletLossConfig): - pass - - def hook_compute_ave_aug_loss(self, ds_posterior, ds_prior, zs_sampled, xs_partial_recon, xs_targ): - return compute_triplet_loss(zs=[d.mean for d in ds_posterior], cfg=self.cfg) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/frameworks/vae/experimental/_unsupervised__dorvae.py b/disent/frameworks/vae/experimental/_unsupervised__dorvae.py deleted file mode 100644 index d8b139f4..00000000 --- a/disent/frameworks/vae/experimental/_unsupervised__dorvae.py +++ /dev/null @@ -1,169 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from dataclasses import dataclass -from typing import final -from typing import Optional -from typing import Sequence - -import torch -from torch.distributions import Normal - -from disent.frameworks.helper.reconstructions import make_reconstruction_loss -from disent.frameworks.helper.reconstructions import ReconLossHandler -from disent.frameworks.vae._supervised__tvae import TripletVae -from disent.frameworks.vae._weaklysupervised__adavae import AdaVae -from disent.nn.loss.softsort import torch_mse_rank_loss -from disent.nn.loss.softsort import spearman_rank_loss - - -# ========================================================================= # -# tvae # -# ========================================================================= # - - -class DataOverlapRankVae(TripletVae): - """ - This converges really well! - - but doesn't introduce axis alignment as well if there is no additional - inward pressure term like triplet to move representations closer together - """ - - REQUIRED_OBS = 1 - - @dataclass - class cfg(TripletVae.cfg): - # compatibility - ada_thresh_mode: str = 'dist' # kl, symmetric_kl, dist, sampled_dist - ada_thresh_ratio: float = 0.5 - adat_triplet_share_scale: float = 0.95 - # OVERLAP VAE - overlap_loss: Optional[str] = None - overlap_num: int = 1024 - # AUGMENT - overlap_augment_mode: str = 'none' - overlap_augment: Optional[dict] = None - # REPRESENTATIONS - overlap_repr: str = 'deterministic' # deterministic, stochastic - overlap_rank_mode: str = 'spearman_rank' # spearman_rank, mse_rank - overlap_inward_pressure_masked: bool = False - overlap_inward_pressure_scale: float = 0.1 - - def __init__(self, model: 'AutoEncoder', cfg: cfg = None, batch_augment=None): - # TODO: duplicate code - super().__init__(model=model, cfg=cfg, batch_augment=batch_augment) - # initialise - if self.cfg.overlap_augment_mode != 'none': - assert self.cfg.overlap_augment is not None, 'if cfg.overlap_augment_mode is not "none", then cfg.overlap_augment must be defined.' - # set augment and instantiate if needed - self._augment = None - if isinstance(self._augment, dict): - import hydra - self._augment = hydra.utils.instantiate(self.cfg.overlap_augment) - assert callable(self._augment), f'augment is not callable: {repr(self._augment)}' - # get overlap loss - overlap_loss = self.cfg.overlap_loss if (self.cfg.overlap_loss is not None) else self.cfg.recon_loss - self.__overlap_handler: ReconLossHandler = make_reconstruction_loss(overlap_loss, reduction='mean') - - @final - @property - def overlap_handler(self) -> ReconLossHandler: - return self.__overlap_handler - - def hook_compute_ave_aug_loss(self, ds_posterior: Sequence[Normal], ds_prior, zs_sampled, xs_partial_recon, xs_targ: Sequence[torch.Tensor]): - # ++++++++++++++++++++++++++++++++++++++++++ # - # 1. augment batch - (x_targ_orig,) = xs_targ - with torch.no_grad(): - (x_targ,) = self.augment_triplet_targets(xs_targ) - (d_posterior,) = ds_posterior - (z_sampled,) = zs_sampled - # 2. generate random pairs -- this does not generate unique pairs - a_idxs, p_idxs = torch.randint(len(x_targ), size=(2, self.cfg.overlap_num), device=x_targ.device) - # ++++++++++++++++++++++++++++++++++++++++++ # - # compute image distances - with torch.no_grad(): - ap_recon_dists = self.overlap_handler.compute_pairwise_loss(x_targ[a_idxs], x_targ[p_idxs]) - # ++++++++++++++++++++++++++++++++++++++++++ # - # get representations - if self.cfg.overlap_repr == 'deterministic': - a_z, p_z = d_posterior.loc[a_idxs], d_posterior.loc[p_idxs] - elif self.cfg.overlap_repr == 'stochastic': - a_z, p_z = z_sampled[a_idxs], z_sampled[p_idxs] - else: - raise KeyError(f'invalid overlap_repr mode: {repr(self.cfg.overlap_repr)}') - # DISENTANGLE! - # compute adaptive mask & weight deltas - a_posterior = Normal(d_posterior.loc[a_idxs], d_posterior.scale[a_idxs]) - p_posterior = Normal(d_posterior.loc[p_idxs], d_posterior.scale[p_idxs]) - share_mask = AdaVae.compute_shared_mask_from_posteriors(a_posterior, p_posterior, thresh_mode=self.cfg.ada_thresh_mode, ratio=self.cfg.ada_thresh_ratio) - deltas = torch.where(share_mask, self.cfg.adat_triplet_share_scale * (a_z - p_z), (a_z - p_z)) - # compute representation distances - ap_repr_dists = torch.abs(deltas).sum(dim=-1) - # ++++++++++++++++++++++++++++++++++++++++++ # - if self.cfg.overlap_rank_mode == 'mse_rank': - loss = torch_mse_rank_loss(ap_repr_dists, ap_recon_dists.detach(), dims=-1, reduction='mean') - loss_logs = {'mse_rank_loss': loss} - elif self.cfg.overlap_rank_mode == 'spearman_rank': - loss = - spearman_rank_loss(ap_repr_dists, ap_recon_dists.detach(), nan_to_num=True) - loss_logs = {'spearman_rank_loss': loss} - else: - raise KeyError(f'invalid overlap_rank_mode: {repr(self.cfg.overlap_repr)}') - # ++++++++++++++++++++++++++++++++++++++++++ # - # inward pressure - if self.cfg.overlap_inward_pressure_masked: - in_deltas = torch.abs(deltas) * share_mask - else: - in_deltas = torch.abs(deltas) - # compute inward pressure - inward_pressure = self.cfg.overlap_inward_pressure_scale * in_deltas.mean() - loss += inward_pressure - # ++++++++++++++++++++++++++++++++++++++++++ # - # return the loss - return loss, { - **loss_logs, - 'inward_pressure': inward_pressure, - } - - def augment_triplet_targets(self, xs_targ): - # TODO: duplicate code - if self.cfg.overlap_augment_mode == 'none': - aug_xs_targ = xs_targ - elif (self.cfg.overlap_augment_mode == 'augment') or (self.cfg.overlap_augment_mode == 'augment_each'): - # recreate augment each time - if self.cfg.overlap_augment_mode == 'augment_each': - import hydra - self._augment = hydra.utils.instantiate(self.cfg.overlap_augment) - # augment on correct device - aug_xs_targ = [self._augment(x_targ) for x_targ in xs_targ] - # checks - assert all(a.shape == b.shape for a, b in zip(xs_targ, aug_xs_targ)) - else: - raise KeyError(f'invalid cfg.overlap_augment_mode={repr(self.cfg.overlap_augment_mode)}') - return aug_xs_targ - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/frameworks/vae/experimental/_unsupervised__dotvae.py b/disent/frameworks/vae/experimental/_unsupervised__dotvae.py deleted file mode 100644 index 4ca79e19..00000000 --- a/disent/frameworks/vae/experimental/_unsupervised__dotvae.py +++ /dev/null @@ -1,222 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -from dataclasses import dataclass -from typing import final -from typing import Optional -from typing import Sequence - -import torch -from torch.distributions import Normal - -from disent.frameworks.helper.reconstructions import make_reconstruction_loss -from disent.frameworks.helper.reconstructions import ReconLossHandler -from disent.frameworks.vae.experimental._supervised__adaneg_tvae import AdaNegTripletVae -from disent.nn.loss.triplet_mining import configured_idx_mine - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Mixin # -# ========================================================================= # - - -class DataOverlapMixin(object): - - # should be inherited by the config on the child class - @dataclass - class cfg: - # override from AE - recon_loss: str = 'mse' - # OVERLAP VAE - overlap_loss: Optional[str] = None # if None, use the value from recon_loss - overlap_num: int = 1024 - overlap_mine_ratio: float = 0.1 - overlap_mine_triplet_mode: str = 'none' - # AUGMENT - overlap_augment_mode: str = 'none' - overlap_augment: Optional[dict] = None - - # private properties - # - since this class does not have a constructor, it - # provides the `init_data_overlap_mixin` method, which - # should be called inside the constructor of the child class - _augment: callable - _overlap_handler: ReconLossHandler - _init: bool - - def init_data_overlap_mixin(self): - if hasattr(self, '_init'): - raise RuntimeError(f'{DataOverlapMixin.__name__} on {self.__class__.__name__} was initialised more than once!') - self._init = True - # initialise - if self.cfg.overlap_augment_mode != 'none': - assert self.cfg.overlap_augment is not None, 'if cfg.overlap_augment_mode is not "none", then cfg.overlap_augment must be defined.' - # set augment and instantiate if needed - self._augment = None - if isinstance(self._augment, dict): - import hydra - self._augment = hydra.utils.instantiate(self.cfg.overlap_augment) - assert callable(self._augment), f'augment is not callable: {repr(self._augment)}' - # get overlap loss - overlap_loss = self.cfg.overlap_loss if (self.cfg.overlap_loss is not None) else self.cfg.recon_loss - self._overlap_handler: ReconLossHandler = make_reconstruction_loss(overlap_loss, reduction='mean') - # delete this property, we only ever want to be able to call this once! - - @final - @property - def overlap_handler(self) -> ReconLossHandler: - return self._overlap_handler - - def overlap_swap_triplet_idxs(self, x_targ, a_idxs, p_idxs, n_idxs): - xs_targ = [x_targ[idxs] for idxs in (a_idxs, p_idxs, n_idxs)] - # CORE: order the latent variables for triplet - swap_mask = self.overlap_swap_mask(xs_targ=xs_targ) - # swap all idxs - swapped_a_idxs = a_idxs - swapped_p_idxs = torch.where(swap_mask, n_idxs, p_idxs) - swapped_n_idxs = torch.where(swap_mask, p_idxs, n_idxs) - # return values - return swapped_a_idxs, swapped_p_idxs, swapped_n_idxs - - def overlap_swap_mask(self, xs_targ: Sequence[torch.Tensor]) -> torch.Tensor: - # get variables - a_x_targ_OLD, p_x_targ_OLD, n_x_targ_OLD = xs_targ - # CORE OF THIS APPROACH - # ++++++++++++++++++++++++++++++++++++++++++ # - # calculate which are wrong! - # TODO: add more loss functions, like perceptual & others - with torch.no_grad(): - a_p_losses = self.overlap_handler.compute_pairwise_loss(a_x_targ_OLD, p_x_targ_OLD) # (B, C, H, W) -> (B,) - a_n_losses = self.overlap_handler.compute_pairwise_loss(a_x_targ_OLD, n_x_targ_OLD) # (B, C, H, W) -> (B,) - swap_mask = (a_p_losses > a_n_losses) # (B,) - # ++++++++++++++++++++++++++++++++++++++++++ # - return swap_mask - - @torch.no_grad() - def augment_batch(self, x_targ): - if self.cfg.overlap_augment_mode == 'none': - aug_x_targ = x_targ - elif self.cfg.overlap_augment_mode in ('augment', 'augment_each'): - # recreate augment each time - if self.cfg.overlap_augment_mode == 'augment_each': - self._augment = instantiate_recursive(self.cfg.overlap_augment) - # augment on correct device - aug_x_targ = self._augment(x_targ) - else: - raise KeyError(f'invalid cfg.overlap_augment_mode={repr(self.cfg.overlap_augment_mode)}') - # checks - assert x_targ.shape == aug_x_targ.shape - return aug_x_targ - - def mine_triplets(self, x_targ, a_idxs, p_idxs, n_idxs): - return configured_idx_mine( - x_targ=x_targ, - a_idxs=a_idxs, - p_idxs=p_idxs, - n_idxs=n_idxs, - cfg=self.cfg, - pairwise_loss_fn=self.overlap_handler.compute_pairwise_loss, - ) - - def random_mined_triplets(self, x_targ_orig: torch.Tensor): - # ++++++++++++++++++++++++++++++++++++++++++ # - # 1. augment batch - aug_x_targ = self.augment_batch(x_targ_orig) - # 2. generate random triples -- this does not generate unique pairs - a_idxs, p_idxs, n_idxs = torch.randint(len(aug_x_targ), size=(3, min(self.cfg.overlap_num, len(aug_x_targ)**3)), device=aug_x_targ.device) - # ++++++++++++++++++++++++++++++++++++++++++ # - # self.debug(x_targ_orig, x_targ, a_idxs, p_idxs, n_idxs) - # ++++++++++++++++++++++++++++++++++++++++++ # - # TODO: this can be merged into a single function -- inefficient currently with deltas computed twice - # 3. reorder random triples - a_idxs, p_idxs, n_idxs = self.overlap_swap_triplet_idxs(aug_x_targ, a_idxs, p_idxs, n_idxs) - # 4. mine random triples - a_idxs, p_idxs, n_idxs = self.mine_triplets(aug_x_targ, a_idxs, p_idxs, n_idxs) - # ++++++++++++++++++++++++++++++++++++++++++ # - return a_idxs, p_idxs, n_idxs - - # def debug(self, x_targ_orig, x_targ, a_idxs, p_idxs, n_idxs): - # a_p_overlap_orig = - self.recon_handler.compute_unreduced_loss(x_targ_orig[a_idxs], x_targ_orig[p_idxs]).mean(dim=(-3, -2, -1)) # (B, C, H, W) -> (B,) - # a_n_overlap_orig = - self.recon_handler.compute_unreduced_loss(x_targ_orig[a_idxs], x_targ_orig[n_idxs]).mean(dim=(-3, -2, -1)) # (B, C, H, W) -> (B,) - # a_p_overlap = - self.recon_handler.compute_unreduced_loss(x_targ[a_idxs], x_targ[p_idxs]).mean(dim=(-3, -2, -1)) # (B, C, H, W) -> (B,) - # a_n_overlap = - self.recon_handler.compute_unreduced_loss(x_targ[a_idxs], x_targ[n_idxs]).mean(dim=(-3, -2, -1)) # (B, C, H, W) -> (B,) - # a_p_overlap_mul = - (a_p_overlap_orig * a_p_overlap) - # a_n_overlap_mul = - (a_n_overlap_orig * a_n_overlap) - # # check number of things - # (up_values_orig, up_counts_orig) = torch.unique(a_p_overlap_orig, sorted=True, return_inverse=False, return_counts=True) - # (un_values_orig, un_counts_orig) = torch.unique(a_n_overlap_orig, sorted=True, return_inverse=False, return_counts=True) - # (up_values, up_counts) = torch.unique(a_p_overlap, sorted=True, return_inverse=False, return_counts=True) - # (un_values, un_counts) = torch.unique(a_n_overlap, sorted=True, return_inverse=False, return_counts=True) - # (up_values_mul, up_counts_mul) = torch.unique(a_p_overlap_mul, sorted=True, return_inverse=False, return_counts=True) - # (un_values_mul, un_counts_mul) = torch.unique(a_n_overlap_mul, sorted=True, return_inverse=False, return_counts=True) - # # plot! - # plt.scatter(up_values_orig.detach().cpu(), torch.cumsum(up_counts_orig, dim=-1).detach().cpu()) - # plt.scatter(un_values_orig.detach().cpu(), torch.cumsum(un_counts_orig, dim=-1).detach().cpu()) - # plt.scatter(up_values.detach().cpu(), torch.cumsum(up_counts, dim=-1).detach().cpu()) - # plt.scatter(un_values.detach().cpu(), torch.cumsum(un_counts, dim=-1).detach().cpu()) - # plt.scatter(up_values_mul.detach().cpu(), torch.cumsum(up_counts_mul, dim=-1).detach().cpu()) - # plt.scatter(un_values_mul.detach().cpu(), torch.cumsum(un_counts_mul, dim=-1).detach().cpu()) - # plt.show() - # time.sleep(10) - - -# ========================================================================= # -# Data Overlap Triplet VAE # -# ========================================================================= # - - -class DataOverlapTripletVae(AdaNegTripletVae, DataOverlapMixin): - - REQUIRED_OBS = 1 - - @dataclass - class cfg(AdaNegTripletVae.cfg, DataOverlapMixin.cfg): - pass - - def __init__(self, model: 'AutoEncoder', cfg: cfg = None, batch_augment=None): - super().__init__(model=model, cfg=cfg, batch_augment=batch_augment) - # initialise mixin - self.init_data_overlap_mixin() - - def hook_compute_ave_aug_loss(self, ds_posterior: Sequence[Normal], ds_prior, zs_sampled, xs_partial_recon, xs_targ: Sequence[torch.Tensor]): - [d_posterior], [x_targ_orig] = ds_posterior, xs_targ - # 1. randomly generate and mine triplets using augmented versions of the inputs - a_idxs, p_idxs, n_idxs = self.random_mined_triplets(x_targ_orig=x_targ_orig) - # 2. compute triplet loss - loss, loss_log = AdaNegTripletVae.estimate_ada_triplet_loss( - ds_posterior=[Normal(d_posterior.loc[idxs], d_posterior.scale[idxs]) for idxs in (a_idxs, p_idxs, n_idxs)], - cfg=self.cfg, - ) - return loss, { - **loss_log, - } - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/frameworks/vae/experimental/_weaklysupervised__augpostriplet.py b/disent/frameworks/vae/experimental/_weaklysupervised__augpostriplet.py deleted file mode 100644 index bad6354a..00000000 --- a/disent/frameworks/vae/experimental/_weaklysupervised__augpostriplet.py +++ /dev/null @@ -1,82 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -import warnings -from dataclasses import dataclass -from typing import Union - -import torch - -from disent.frameworks.vae._supervised__tvae import TripletVae - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Guided Ada Vae # -# ========================================================================= # - - -class AugPosTripletVae(TripletVae): - - REQUIRED_OBS = 2 # third obs is generated from augmentations - - @dataclass - class cfg(TripletVae.cfg): - overlap_augment: Union[dict, callable] = None - - def __init__(self, model: 'AutoEncoder', cfg: cfg = None, batch_augment=None): - super().__init__(model=model, cfg=cfg, batch_augment=batch_augment) - # set augment and instantiate if needed - self._augment = self.cfg.overlap_augment - if isinstance(self._augment, dict): - import hydra - self._augment = hydra.utils.instantiate(self._augment) - # get default if needed - if self._augment is None: - self._augment = torch.nn.Identity() - warnings.warn(f'{self.__class__.__name__}, no overlap_augment was specified, defaulting to nn.Identity which WILL break things!') - # checks! - assert callable(self._augment), f'augment is not callable: {repr(self._augment)}' - - def do_training_step(self, batch, batch_idx): - (a_x, n_x), (a_x_targ, n_x_targ) = self._get_xs_and_targs(batch, batch_idx) - - # generate augmented items - with torch.no_grad(): - p_x_targ = a_x_targ - p_x = self._augment(a_x) - # a_x = self._aug(a_x) - # n_x = self._aug(n_x) - - batch['x'], batch['x_targ'] = (a_x, p_x, n_x), (a_x_targ, p_x_targ, n_x_targ) - # compute! - return super().do_training_step(batch, batch_idx) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/frameworks/vae/experimental/_weaklysupervised__st_adavae.py b/disent/frameworks/vae/experimental/_weaklysupervised__st_adavae.py deleted file mode 100644 index deddced4..00000000 --- a/disent/frameworks/vae/experimental/_weaklysupervised__st_adavae.py +++ /dev/null @@ -1,63 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from dataclasses import dataclass - -import numpy as np -from disent.frameworks.vae._weaklysupervised__adavae import AdaVae - - -# ========================================================================= # -# Swapped Target AdaVae # -# ========================================================================= # - - -class SwappedTargetAdaVae(AdaVae): - - REQUIRED_OBS = 2 - - @dataclass - class cfg(AdaVae.cfg): - swap_chance: float = 0.1 - - def __init__(self, model: 'AutoEncoder', cfg: cfg = None, batch_augment=None): - super().__init__(model=model, cfg=cfg, batch_augment=batch_augment) - assert cfg.swap_chance >= 0 - - def do_training_step(self, batch, batch_idx): - (x0, x1), (x0_targ, x1_targ) = self._get_xs_and_targs(batch, batch_idx) - - # random change for the target not to be equal to the input - if np.random.random() < self.cfg.swap_chance: - x0_targ, x1_targ = x1_targ, x0_targ - - return super(SwappedTargetAdaVae, self).do_training_step({ - 'x': (x0, x1), - 'x_targ': (x0_targ, x1_targ), - }, batch_idx) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/frameworks/vae/experimental/_weaklysupervised__st_betavae.py b/disent/frameworks/vae/experimental/_weaklysupervised__st_betavae.py deleted file mode 100644 index c3042059..00000000 --- a/disent/frameworks/vae/experimental/_weaklysupervised__st_betavae.py +++ /dev/null @@ -1,63 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from dataclasses import dataclass - -import numpy as np -from disent.frameworks.vae._unsupervised__betavae import BetaVae - - -# ========================================================================= # -# Swapped Target BetaVAE # -# ========================================================================= # - - -class SwappedTargetBetaVae(BetaVae): - - REQUIRED_OBS = 2 - - @dataclass - class cfg(BetaVae.cfg): - swap_chance: float = 0.1 - - def __init__(self, model: 'AutoEncoder', cfg: cfg = None, batch_augment=None): - super().__init__(model=model, cfg=cfg, batch_augment=batch_augment) - assert cfg.swap_chance >= 0 - - def do_training_step(self, batch, batch_idx): - (x0, x1), (x0_targ, x1_targ) = self._get_xs_and_targs(batch, batch_idx) - - # random change for the target not to be equal to the input - if np.random.random() < self.cfg.swap_chance: - x0_targ, x1_targ = x1_targ, x0_targ - - return super(SwappedTargetBetaVae, self).do_training_step({ - 'x': (x0,), - 'x_targ': (x0_targ,), - }, batch_idx) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/disent/metrics/__init__.py b/disent/metrics/__init__.py index e4fb6784..98099e76 100644 --- a/disent/metrics/__init__.py +++ b/disent/metrics/__init__.py @@ -28,9 +28,6 @@ from ._mig import metric_mig from ._sap import metric_sap from ._unsupervised import metric_unsupervised -# Nathan Michlo et. al # pragma: delete-on-release -from ._flatness import metric_flatness # pragma: delete-on-release -from ._flatness_components import metric_flatness_components # pragma: delete-on-release # ========================================================================= # @@ -45,8 +42,6 @@ FAST_METRICS = { 'dci': _wrapped_partial(metric_dci, num_train=1000, num_test=500, boost_mode='sklearn'), 'factor_vae': _wrapped_partial(metric_factor_vae, num_train=700, num_eval=350, num_variance_estimate=1000), # may not be accurate, but it just takes waay too long otherwise 20+ seconds - 'flatness': _wrapped_partial(metric_flatness, factor_repeats=128), # pragma: delete-on-release - 'flatness_components': _wrapped_partial(metric_flatness_components, factor_repeats=128), # pragma: delete-on-release 'mig': _wrapped_partial(metric_mig, num_train=2000), 'sap': _wrapped_partial(metric_sap, num_train=2000, num_test=1000), 'unsupervised': _wrapped_partial(metric_unsupervised, num_train=2000), @@ -55,8 +50,6 @@ DEFAULT_METRICS = { 'dci': metric_dci, 'factor_vae': metric_factor_vae, - 'flatness': metric_flatness, # pragma: delete-on-release - 'flatness_components': metric_flatness_components, # pragma: delete-on-release 'mig': metric_mig, 'sap': metric_sap, 'unsupervised': metric_unsupervised, diff --git a/disent/metrics/_flatness.py b/disent/metrics/_flatness.py deleted file mode 100644 index 1bdf05e4..00000000 --- a/disent/metrics/_flatness.py +++ /dev/null @@ -1,347 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -""" -Flatness Metric -- Nathan Michlo 2021 (Unpublished) -- Cite disent -""" - -import logging -import math -from typing import Iterable -from typing import Union - -import torch -from disent.util.deprecate import deprecated -from torch.utils.data.dataloader import default_collate - -from disent.dataset import DisentDataset -from disent.util.iters import iter_chunks - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# flatness # -# ========================================================================= # - - -@deprecated('flatness metric is deprecated in favour of flatness_components, this metric still gives useful alternative info however.') -def metric_flatness( - dataset: DisentDataset, - representation_function: callable, - factor_repeats: int = 1024, - batch_size: int = 64, -): - """ - Computes the flatness metric: - approximately equal to: total_dim_width / (ave_point_dist_along_dim * num_points_along_dim) - - Complexity of this metric is: - O(num_factors * ave_factor_size * repeats) - eg. 9 factors * 64 indices on ave * 128 repeats = 73728 observations loaded from the dataset - - factor_repeats: - - can go all the way down to about 64 and still get decent results. - - 64 is accurate to about +- 0.01 - - 128 is accurate to about +- 0.003 - - 1024 is accurate to about +- 0.001 - - Args: - dataset: DisentDataset to be sampled from. - representation_function: Function that takes observations as input and outputs a dim_representation sized representation for each observation. - factor_repeats: how many times to repeat a traversal along each factors, these are then averaged together. - batch_size: Batch size to process at any time while generating representations, should not effect metric results. - p: how to calculate distances in the latent space, see torch.norm - Returns: - Dictionary with average disentanglement score, completeness and - informativeness (train and test). - """ - p_fs_measures = aggregate_measure_distances_along_all_factors(dataset, representation_function, repeats=factor_repeats, batch_size=batch_size, ps=(1, 2)) - # get info - factor_sizes = dataset.gt_data.factor_sizes - # aggregate data - results = { - 'flatness.ave_flatness': compute_flatness(widths=p_fs_measures[2]['fs_ave_widths'], lengths=p_fs_measures[1]['fs_ave_lengths'], factor_sizes=factor_sizes), - 'flatness.ave_flatness_l1': compute_flatness(widths=p_fs_measures[1]['fs_ave_widths'], lengths=p_fs_measures[1]['fs_ave_lengths'], factor_sizes=factor_sizes), - 'flatness.ave_flatness_l2': compute_flatness(widths=p_fs_measures[2]['fs_ave_widths'], lengths=p_fs_measures[2]['fs_ave_lengths'], factor_sizes=factor_sizes), - # distances - 'flatness.ave_width_l1': torch.mean(filter_inactive_factors(p_fs_measures[1]['fs_ave_widths'], factor_sizes=factor_sizes)), - 'flatness.ave_width_l2': torch.mean(filter_inactive_factors(p_fs_measures[2]['fs_ave_widths'], factor_sizes=factor_sizes)), - 'flatness.ave_length_l1': torch.mean(filter_inactive_factors(p_fs_measures[1]['fs_ave_lengths'], factor_sizes=factor_sizes)), - 'flatness.ave_length_l2': torch.mean(filter_inactive_factors(p_fs_measures[2]['fs_ave_lengths'], factor_sizes=factor_sizes)), - # angles - 'flatness.cosine_angles': (1 / math.pi) * torch.mean(filter_inactive_factors(p_fs_measures[1]['fs_ave_angles'], factor_sizes=factor_sizes)), - } - # convert values from torch - return {k: float(v) for k, v in results.items()} - - -def compute_flatness(widths, lengths, factor_sizes): - widths = filter_inactive_factors(widths, factor_sizes) - lengths = filter_inactive_factors(lengths, factor_sizes) - # checks - assert torch.all(widths >= 0) - assert torch.all(lengths >= 0) - assert torch.all(torch.eq(widths == 0, lengths == 0)) - # update scores - widths[lengths == 0] = 0 - lengths[lengths == 0] = 1 - # compute flatness - return (widths / lengths).mean() - - -def filter_inactive_factors(tensor, factor_sizes): - factor_sizes = torch.tensor(factor_sizes, device=tensor.device) - assert torch.all(factor_sizes >= 1) - # remove - active_factors = torch.nonzero(factor_sizes-1, as_tuple=True) - return tensor[active_factors] - - -def aggregate_measure_distances_along_all_factors( - dataset: DisentDataset, - representation_function, - repeats: int, - batch_size: int, - ps: Iterable[Union[str, int]] = (1, 2), -) -> dict: - # COMPUTE AGGREGATES FOR EACH FACTOR - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - fs_p_measures = [ - aggregate_measure_distances_along_factor(dataset, representation_function, f_idx=f_idx, repeats=repeats, batch_size=batch_size, ps=ps) - for f_idx in range(dataset.gt_data.num_factors) - ] - - # FINALIZE FOR EACH FACTOR - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - p_fs_measures = {} - for p, fs_measures in default_collate(fs_p_measures).items(): - fs_ave_widths = fs_measures['ave_width'] - # get number of spaces deltas (number of points minus 1) - # compute length: estimated version of factors_ave_width = factors_num_deltas * factors_ave_delta - _fs_num_deltas = torch.as_tensor(dataset.gt_data.factor_sizes, device=fs_ave_widths.device) - 1 - _fs_ave_deltas = fs_measures['ave_delta'] - fs_ave_lengths = _fs_num_deltas * _fs_ave_deltas - # angles - fs_ave_angles = fs_measures['ave_angle'] - # update - p_fs_measures[p] = {'fs_ave_widths': fs_ave_widths, 'fs_ave_lengths': fs_ave_lengths, 'fs_ave_angles': fs_ave_angles} - return p_fs_measures - - -def aggregate_measure_distances_along_factor( - dataset: DisentDataset, - representation_function, - f_idx: int, - repeats: int, - batch_size: int, - ps: Iterable[Union[str, int]] = (1, 2), - cycle_fail: bool = False, -) -> dict: - f_size = dataset.gt_data.factor_sizes[f_idx] - - if f_size == 1: - if cycle_fail: - raise ValueError(f'dataset factor size is too small for flatness metric with cycle_normalize enabled! size={f_size} < 2') - zero = torch.as_tensor(0., device=get_device(dataset, representation_function)) - return {p: {'ave_width': zero.clone(), 'ave_delta': zero.clone(), 'ave_angle': zero.clone()} for p in ps} - - # FEED FORWARD, COMPUTE ALL DELTAS & WIDTHS - For each distance measure - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - p_measures: list = [{} for _ in range(repeats)] - for measures in p_measures: - # generate repeated factors, varying one factor over the entire range - zs_traversal = encode_all_along_factor(dataset, representation_function, f_idx=f_idx, batch_size=batch_size) - # for each distance measure compute everything - # - width: calculate the distance between the furthest two points - # - deltas: calculating the distances of their representations to the next values. - # - cycle_normalize: we cant get the ave next dist directly because of cycles, so we remove the largest dist - for p in ps: - deltas_next = torch.norm(torch.roll(zs_traversal, -1, dims=0) - zs_traversal, dim=-1, p=p) # next | shape: (factor_size, z_size) - deltas_prev = torch.norm(torch.roll(zs_traversal, 1, dims=0) - zs_traversal, dim=-1, p=p) # prev | shape: (factor_size, z_size) - # values needed for flatness - width = knn(x=zs_traversal, y=zs_traversal, k=1, largest=True, p=p).values.max() # shape: (,) - min_deltas = torch.topk(deltas_next, k=f_size-1, dim=-1, largest=False, sorted=False) # shape: (factor_size-1, z_size) - # values needed for cosine angles - # TODO: this should not be calculated per p - # TODO: should we filter the cyclic value? - # a. if the point is an endpoint we set its value to pi indicating that it is flat - # b. [THIS] we do not allow less than 3 points, ie. a factor_size of at least 3, otherwise - # we set the angle to pi (considered flat) and filter the factor from the metric - angles = angles_between(deltas_next, deltas_prev, dim=-1, nan_to_angle=0) # shape: (factor_size,) - # TODO: other measures can be added: - # 1. multivariate skewness - # 2. normality measure - # 3. independence - # 4. menger curvature (Cayley-Menger Determinant?) - # save variables - measures[p] = {'widths': width, 'deltas': min_deltas.values, 'angles': angles} - - # AGGREGATE DATA - For each distance measure - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - return { - p: { - 'ave_width': measures['widths'].mean(dim=0), # shape: (repeats,) -> () - 'ave_delta': measures['deltas'].mean(dim=[0, 1]), # shape: (repeats, factor_size - 1) -> () - 'ave_angle': measures['angles'].mean(dim=0), # shape: (repeats,) -> () - } for p, measures in default_collate(p_measures).items() - } - - -# ========================================================================= # -# ENCODE # -# ========================================================================= # - - -def encode_all_along_factor(dataset: DisentDataset, representation_function, f_idx: int, batch_size: int): - # generate repeated factors, varying one factor over a range (f_size, f_dims) - factors = dataset.gt_data.sample_random_factor_traversal(f_idx=f_idx) - # get the representations of all the factors (f_size, z_size) - sequential_zs = encode_all_factors(dataset, representation_function, factors=factors, batch_size=batch_size) - return sequential_zs - - -def encode_all_factors(dataset: DisentDataset, representation_function, factors, batch_size: int) -> torch.Tensor: - zs = [] - with torch.no_grad(): - for batch_factors in iter_chunks(factors, chunk_size=batch_size): - batch = dataset.dataset_batch_from_factors(batch_factors, mode='input') - z = representation_function(batch) - zs.append(z) - return torch.cat(zs, dim=0) - - -def get_device(dataset: DisentDataset, representation_function): - # this is a hack... - return representation_function(dataset.dataset_sample_batch(1, mode='input')).device - - -# ========================================================================= # -# DISTANCES # -# ========================================================================= # - - -def knn(x, y, k: int = None, largest=False, p='fro'): - assert 0 < k <= y.shape[0] - # check input vectors, must be array of vectors - assert 2 == x.ndim == y.ndim - assert x.shape[1:] == y.shape[1:] - # compute distances between each and every pair - dist_mat = x[:, None, ...] - y[None, :, ...] - dist_mat = torch.norm(dist_mat, dim=-1, p=p) - # return closest distances - return torch.topk(dist_mat, k=k, dim=-1, largest=largest, sorted=True) - - -# ========================================================================= # -# ANGLES # -# ========================================================================= # - - -def angles_between(a, b, dim=-1, nan_to_angle=None): - a = a / torch.norm(a, dim=dim, keepdim=True) - b = b / torch.norm(b, dim=dim, keepdim=True) - dot = torch.sum(a * b, dim=dim) - angles = torch.acos(torch.clamp(dot, -1.0, 1.0)) - if nan_to_angle is not None: - return torch.where(torch.isnan(angles), torch.full_like(angles, fill_value=nan_to_angle), angles) - return angles - - -# ========================================================================= # -# END # -# ========================================================================= # - - -# if __name__ == '__main__': -# import pytorch_lightning as pl -# from torch.optim import Adam -# from torch.utils.data import DataLoader -# from disent.data.groundtruth import XYObjectData, XYSquaresData -# from disent.dataset.groundtruth import GroundTruthDataset, GroundTruthDatasetPairs -# from disent.frameworks.vae import BetaVae -# from disent.frameworks.vae import AdaVae -# from disent.model.ae import EncoderConv64, DecoderConv64, AutoEncoder -# from disent.transform import ToImgTensorF32 -# from disent.util import colors -# from disent.util import Timer -# -# def get_str(r): -# return ', '.join(f'{k}={v:6.4f}' for k, v in r.items()) -# -# def print_r(name, steps, result, clr=colors.lYLW, t: Timer = None): -# print(f'{clr}{name:<13} ({steps:>04}){f" {colors.GRY}[{t.pretty}]{clr}" if t else ""}: {get_str(result)}{colors.RST}') -# -# def calculate(name, steps, dataset, get_repr): -# global aggregate_measure_distances_along_factor -# with Timer() as t: -# r = metric_flatness(dataset, get_repr, factor_repeats=64, batch_size=64) -# results.append((name, steps, r)) -# print_r(name, steps, r, colors.lRED, t=t) -# print(colors.GRY, '='*100, colors.RST, sep='') -# return r -# -# class XYOverlapData(XYSquaresData): -# def __init__(self, square_size=8, image_size=64, grid_spacing=None, num_squares=3, rgb=True): -# if grid_spacing is None: -# grid_spacing = (square_size+1) // 2 -# super().__init__(square_size=square_size, image_size=image_size, grid_spacing=grid_spacing, num_squares=num_squares, rgb=rgb) -# -# # datasets = [XYObjectData(rgb=False, palette='white'), XYSquaresData(), XYOverlapData(), XYObjectData()] -# datasets = [XYObjectData()] -# -# results = [] -# for data in datasets: -# dataset = GroundTruthDatasetPairs(data, transform=ToImgTensorF32()) -# dataloader = DataLoader(dataset=dataset, batch_size=32, shuffle=True, pin_memory=True) -# module = AdaVae( -# model=AutoEncoder( -# encoder=EncoderConv64(x_shape=data.x_shape, z_size=6, z_multiplier=2), -# decoder=DecoderConv64(x_shape=data.x_shape, z_size=6), -# ), -# cfg=AdaVae.cfg(beta=0.001, loss_reduction='mean', optimizer=torch.optim.Adam, optimizer_kwargs=dict(lr=5e-4)) -# ) -# # we cannot guarantee which device the representation is on -# get_repr = lambda x: module.encode(x.to(module.device)) -# # PHASE 1, UNTRAINED -# pl.Trainer(logger=False, checkpoint_callback=False, fast_dev_run=True, gpus=1, weights_summary=None).fit(module, dataloader) -# module = module.to('cuda') -# calculate(data.__class__.__name__, 0, dataset, get_repr) -# # PHASE 2, LITTLE TRAINING -# pl.Trainer(logger=False, checkpoint_callback=False, max_steps=256, gpus=1, weights_summary=None).fit(module, dataloader) -# calculate(data.__class__.__name__, 256, dataset, get_repr) -# # PHASE 3, MORE TRAINING -# pl.Trainer(logger=False, checkpoint_callback=False, max_steps=2048, gpus=1, weights_summary=None).fit(module, dataloader) -# calculate(data.__class__.__name__, 256+2048, dataset, get_repr) -# results.append(None) -# -# for result in results: -# if result is None: -# print() -# continue -# (name, steps, result) = result -# print_r(name, steps, result, colors.lYLW) diff --git a/disent/metrics/_flatness_components.py b/disent/metrics/_flatness_components.py deleted file mode 100644 index 37798319..00000000 --- a/disent/metrics/_flatness_components.py +++ /dev/null @@ -1,412 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -""" -Flatness Metric Components -- Nathan Michlo 2021 (Unpublished) -- Cite disent -""" - -import logging - -import numpy as np -import torch -from torch.utils.data.dataloader import default_collate - -from disent.dataset import DisentDataset -from disent.metrics._flatness import encode_all_along_factor -from disent.metrics._flatness import encode_all_factors -from disent.metrics._flatness import filter_inactive_factors -from disent.util.iters import iter_chunks -from disent.util import to_numpy -from disent.nn.functional import torch_mean_generalized -from disent.nn.functional import torch_pca - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# flatness # -# ========================================================================= # - - -def metric_flatness_components( - dataset: DisentDataset, - representation_function: callable, - factor_repeats: int = 1024, - batch_size: int = 64, -): - """ - Computes the flatness metric components (ordering, linearity & axis alignment): - global_swap_ratio: how swapped embeddings are compared to ground truth factors - factor_swap_ratio_near: how swapped embeddings are compared to ground truth factors - factor_swap_ratio: how swapped embeddings are compared to ground truth factors - axis_ratio: largest singular values over sum of singular values - ave_axis_ratio: largest singular values over sum of singular values - linear_ratio: largest std/variance over sum of std/variance - ave_linear_ratio: largest std/variance over sum of std/variance - axis_alignment: axis ratio is bounded by linear ratio - compute: axis / linear - ave_axis_alignment: axis ratio is bounded by linear ratio - compute: axis / linear - - Args: - dataset: DisentDataset to be sampled from. - representation_function: Function that takes observations as input and outputs a dim_representation sized representation for each observation. - factor_repeats: how many times to repeat a traversal along each factors, these are then averaged together. - batch_size: Batch size to process at any time while generating representations, should not effect metric results. - Returns: - Dictionary with metrics - """ - fs_measures, ran_measures = aggregate_measure_distances_along_all_factors(dataset, representation_function, repeats=factor_repeats, batch_size=batch_size) - - results = {} - for k, v in fs_measures.items(): - results[f'flatness_components.{k}'] = float(filtered_mean(v, p='geometric', factor_sizes=dataset.gt_data.factor_sizes)) - for k, v in ran_measures.items(): - results[f'flatness_components.{k}'] = float(v.mean(dim=0)) - - # convert values from torch - return results - - -def filtered_mean(values, p, factor_sizes): - # increase precision - values = values.to(torch.float64) - # check size - assert values.shape == (len(factor_sizes),) - # filter - values = filter_inactive_factors(values, factor_sizes) - # compute mean - mean = torch_mean_generalized(values, dim=0, p=p) - # return decreased precision - return to_numpy(mean.to(torch.float32)) - - -def aggregate_measure_distances_along_all_factors( - dataset: DisentDataset, - representation_function, - repeats: int, - batch_size: int, -) -> (dict, dict): - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - # COMPUTE AGGREGATES FOR EACH FACTOR - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - fs_measures = default_collate([ - aggregate_measure_distances_along_factor(dataset, representation_function, f_idx=f_idx, repeats=repeats, batch_size=batch_size) - for f_idx in range(dataset.gt_data.num_factors) - ]) - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - # COMPUTE RANDOM SWAP RATIO - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - values = [] - num_samples = int(np.mean(dataset.gt_data.factor_sizes) * repeats) - for idxs in iter_chunks(range(num_samples), batch_size): - # encode factors - factors = dataset.gt_data.sample_factors(size=len(idxs)) - zs = encode_all_factors(dataset, representation_function, factors, batch_size=batch_size) - # get random triplets from factors - rai, rpi, rni = np.random.randint(0, len(factors), size=(3, len(factors) * 4)) - rai, rpi, rni = reorder_by_factor_dist(factors, rai, rpi, rni) - # check differences - swap_ratio_l1, swap_ratio_l2 = compute_swap_ratios(zs[rai], zs[rpi], zs[rni]) - values.append({ - 'global_swap_ratio.l1': swap_ratio_l1, - 'global_swap_ratio.l2': swap_ratio_l2, - }) - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - # RETURN - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - swap_measures = default_collate(values) - return fs_measures, swap_measures - - -# ========================================================================= # -# HELPER # -# ========================================================================= # - - -def reorder_by_factor_dist(factors, rai, rpi, rni): - a_fs, p_fs, n_fs = factors[rai], factors[rpi], factors[rni] - # sort all - d_ap = np.linalg.norm(a_fs - p_fs, ord=1, axis=-1) - d_an = np.linalg.norm(a_fs - n_fs, ord=1, axis=-1) - # swap - swap_mask = d_ap <= d_an - rpi_NEW = np.where(swap_mask, rpi, rni) - rni_NEW = np.where(swap_mask, rni, rpi) - # return new - return rai, rpi_NEW, rni_NEW - - -def compute_swap_ratios(a_zs, p_zs, n_zs): - ap_delta_l1, an_delta_l1 = torch.norm(a_zs - p_zs, dim=-1, p=1), torch.norm(a_zs - n_zs, dim=-1, p=1) - ap_delta_l2, an_delta_l2 = torch.norm(a_zs - p_zs, dim=-1, p=2), torch.norm(a_zs - n_zs, dim=-1, p=2) - swap_ratio_l1 = (ap_delta_l1 <= an_delta_l1).to(torch.float32).mean() - swap_ratio_l2 = (ap_delta_l2 <= an_delta_l2).to(torch.float32).mean() - return swap_ratio_l1, swap_ratio_l2 - - -# ========================================================================= # -# CORE # -# -- using variance instead of standard deviation makes it easier to # -# obtain high scores. # -# ========================================================================= # - - -def compute_unsorted_axis_values(zs_traversal, use_std: bool = True): - # CORRELATIONS -- SORTED IN DESCENDING ORDER: - # correlation with standard basis (1, 0, 0, ...), (0, 1, 0, ...), ... - axis_values = torch.var(zs_traversal, dim=0) # (z_size,) - if use_std: - axis_values = torch.sqrt(axis_values) - return axis_values - - -def compute_unsorted_linear_values(zs_traversal, use_std: bool = True): - # CORRELATIONS -- SORTED IN DESCENDING ORDER: - # correlation along arbitrary orthogonal basis - _, linear_values = torch_pca(zs_traversal, center=True, mode='svd') # svd: (min(z_size, factor_size),) | eig: (z_size,) - if use_std: - linear_values = torch.sqrt(linear_values) - return linear_values - - -def _score_from_sorted(sorted_vars: torch.Tensor, use_max: bool = False, norm: bool = True) -> torch.Tensor: - if use_max: - # use two max values - n = 2 - r = sorted_vars[0] / (sorted_vars[0] + torch.max(sorted_vars[1:])) - else: - # sum all values - n = len(sorted_vars) - r = sorted_vars[0] / torch.sum(sorted_vars) - # get norm if needed - if norm: - # for: x/(x+a) - # normalised = (x/(x+a) - (1/n)) / (1 - (1/n)) - # normalised = (x - 1/(n-1) * a) / (x + a) - r = (r - (1/n)) / (1 - (1/n)) - # done! - return r - - -def score_from_unsorted(unsorted_values: torch.Tensor, use_max: bool = False, norm: bool = True): - # sort in descending order - sorted_values = torch.sort(unsorted_values, descending=True).values - # compute score - return _score_from_sorted(sorted_values, use_max=use_max, norm=norm) - - -def compute_axis_score(zs_traversal: torch.Tensor, use_std: bool = True, use_max: bool = False, norm: bool = True): - return score_from_unsorted(compute_unsorted_axis_values(zs_traversal, use_std=use_std), use_max=use_max, norm=norm) - - -def compute_linear_score(zs_traversal: torch.Tensor, use_std: bool = True, use_max: bool = False, norm: bool = True): - return score_from_unsorted(compute_unsorted_linear_values(zs_traversal, use_std=use_std), use_max=use_max, norm=norm) - - -# ========================================================================= # -# TRAVERSAL FLATNESS # -# ========================================================================= # - - -def aggregate_measure_distances_along_factor( - ground_truth_dataset: DisentDataset, - representation_function, - f_idx: int, - repeats: int, - batch_size: int, -) -> dict: - # NOTE: this returns nan for all values if the factor size is 1 - - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - # FEED FORWARD, COMPUTE ALL - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - measures = [] - for i in range(repeats): - # ENCODE TRAVERSAL: - # generate repeated factors, varying one factor over the entire range - zs_traversal = encode_all_along_factor(ground_truth_dataset, representation_function, f_idx=f_idx, batch_size=batch_size) - - # SWAP RATIO: - idxs_a, idxs_p_OLD, idxs_n_OLD = torch.randint(0, len(zs_traversal), size=(3, len(zs_traversal)*2)) - idx_mask = torch.abs(idxs_a - idxs_p_OLD) <= torch.abs(idxs_a - idxs_n_OLD) - idxs_p = torch.where(idx_mask, idxs_p_OLD, idxs_n_OLD) - idxs_n = torch.where(idx_mask, idxs_n_OLD, idxs_p_OLD) - # check the number of swapped elements along a factor - near_swap_ratio_l1, near_swap_ratio_l2 = compute_swap_ratios(zs_traversal[:-2], zs_traversal[1:-1], zs_traversal[2:]) - factor_swap_ratio_l1, factor_swap_ratio_l2 = compute_swap_ratios(zs_traversal[idxs_a, :], zs_traversal[idxs_p, :], zs_traversal[idxs_n, :]) - - # AXIS ALIGNMENT & LINEAR SCORES - # correlation with standard basis (1, 0, 0, ...), (0, 1, 0, ...), ... - axis_values_std = compute_unsorted_axis_values(zs_traversal, use_std=True) - axis_values_var = compute_unsorted_axis_values(zs_traversal, use_std=False) - # correlation along arbitrary orthogonal basis - linear_values_std = compute_unsorted_linear_values(zs_traversal, use_std=True) - linear_values_var = compute_unsorted_linear_values(zs_traversal, use_std=False) - - # compute scores - axis_ratio_std = score_from_unsorted(axis_values_std, use_max=False, norm=True) - axis_ratio_var = score_from_unsorted(axis_values_var, use_max=False, norm=True) - linear_ratio_std = score_from_unsorted(linear_values_std, use_max=False, norm=True) - linear_ratio_var = score_from_unsorted(linear_values_var, use_max=False, norm=True) - - # save variables - measures.append({ - 'factor_swap_ratio_near.l1': near_swap_ratio_l1, - 'factor_swap_ratio_near.l2': near_swap_ratio_l2, - 'factor_swap_ratio.l1': factor_swap_ratio_l1, - 'factor_swap_ratio.l2': factor_swap_ratio_l2, - # axis ratios - '_axis_values.std': axis_values_std, - '_axis_values.var': axis_values_var, - 'axis_ratio.std': axis_ratio_std, - 'axis_ratio.var': axis_ratio_var, - # linear ratios - '_linear_values.std': linear_values_std, - '_linear_values.var': linear_values_var, - 'linear_ratio.std': linear_ratio_std, - 'linear_ratio.var': linear_ratio_var, - # normalised axis alignment scores (axis_ratio is bounded by linear_ratio) - 'axis_alignment.std': axis_ratio_std / (linear_ratio_std + 1e-20), - 'axis_alignment.var': axis_ratio_var / (linear_ratio_var + 1e-20), - }) - - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - # AGGREGATE DATA - For each distance measure - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - measures = default_collate(measures) - - # aggregate over first dimension - results = {k: v.mean(dim=0) for k, v in measures.items()} - - # compute average scores & remove keys - results['ave_axis_ratio.std'] = score_from_unsorted(results.pop('_axis_values.std'), use_max=False, norm=True) - results['ave_axis_ratio.var'] = score_from_unsorted(results.pop('_axis_values.var'), use_max=False, norm=True) - results['ave_linear_ratio.std'] = score_from_unsorted(results.pop('_linear_values.std'), use_max=False, norm=True) - results['ave_linear_ratio.var'] = score_from_unsorted(results.pop('_linear_values.var'), use_max=False, norm=True) - # ave normalised axis alignment scores (axis_ratio is bounded by linear_ratio) - results['ave_axis_alignment.std'] = results['ave_axis_ratio.std'] / (results['ave_linear_ratio.std'] + 1e-20) - results['ave_axis_alignment.var'] = results['ave_axis_ratio.var'] / (results['ave_linear_ratio.var'] + 1e-20) - - return results - - -# ========================================================================= # -# END # -# ========================================================================= # - - -# if __name__ == '__main__': -# from disent.metrics import metric_flatness -# from sklearn import linear_model -# from disent.dataset.groundtruth import GroundTruthDatasetTriples -# from disent.dataset.groundtruth import GroundTruthDistDataset -# from disent.metrics._flatness import get_device -# import pytorch_lightning as pl -# from torch.optim import Adam -# from torch.utils.data import DataLoader -# from disent.data.groundtruth import XYObjectData, XYSquaresData -# from disent.dataset.groundtruth import GroundTruthDataset, GroundTruthDatasetPairs -# from disent.frameworks.vae import BetaVae -# from disent.frameworks.vae import AdaVae -# from disent.frameworks.vae import TripletVae -# from disent.model.ae import EncoderConv64, DecoderConv64, AutoEncoder -# from disent.transform import ToImgTensorF32 -# from disent.util import colors -# from disent.util import Timer -# -# def get_str(r): -# return ', '.join(f'{k}={v:6.4f}' for k, v in r.items()) -# -# def print_r(name, steps, result, clr=colors.lYLW, t: Timer = None): -# print(f'{clr}{name:<13} ({steps:>04}){f" {colors.GRY}[{t.pretty}]{clr}" if t else ""}: {get_str(result)}{colors.RST}') -# -# def calculate(name, steps, dataset, get_repr): -# global aggregate_measure_distances_along_factor -# with Timer() as t: -# r = { -# **metric_flatness_components(dataset, get_repr, factor_repeats=64, batch_size=64), -# **metric_flatness(dataset, get_repr, factor_repeats=64, batch_size=64), -# } -# results.append((name, steps, r)) -# print_r(name, steps, r, colors.lRED, t=t) -# print(colors.GRY, '='*100, colors.RST, sep='') -# return r -# -# class XYOverlapData(XYSquaresData): -# def __init__(self, square_size=8, image_size=64, grid_spacing=None, num_squares=3, rgb=True): -# if grid_spacing is None: -# grid_spacing = (square_size+1) // 2 -# super().__init__(square_size=square_size, image_size=image_size, grid_spacing=grid_spacing, num_squares=num_squares, rgb=rgb) -# -# # datasets = [XYObjectData(rgb=False, palette='white'), XYSquaresData(), XYOverlapData(), XYObjectData()] -# datasets = [XYObjectData()] -# -# # TODO: fix for dead dimensions -# # datasets = [XYObjectData(rgb=False, palette='white')] -# -# results = [] -# for data in datasets: -# -# # dataset = GroundTruthDistDataset(data, transform=ToImgTensorF32(), num_samples=2, triplet_sample_mode='manhattan') -# # dataloader = DataLoader(dataset=dataset, batch_size=32, shuffle=True, pin_memory=True) -# # module = AdaVae( -# # model=AutoEncoder( -# # encoder=EncoderConv64(x_shape=data.x_shape, z_size=6, z_multiplier=2), -# # decoder=DecoderConv64(x_shape=data.x_shape, z_size=6), -# # ), -# # cfg=AdaVae.cfg(beta=0.001, loss_reduction='mean', optimizer=torch.optim.Adam, optimizer_kwargs=dict(lr=5e-4)) -# # ) -# -# dataset = GroundTruthDistDataset(data, transform=ToImgTensorF32(), num_samples=3, triplet_sample_mode='manhattan') -# dataloader = DataLoader(dataset=dataset, batch_size=32, shuffle=True, pin_memory=True) -# module = TripletVae( -# model=AutoEncoder( -# encoder=EncoderConv64(x_shape=data.x_shape, z_size=6, z_multiplier=2), -# decoder=DecoderConv64(x_shape=data.x_shape, z_size=6), -# ), -# cfg=TripletVae.cfg(beta=0.003, loss_reduction='mean', triplet_p=1, triplet_margin_max=10.0, triplet_scale=10.0, optimizer=torch.optim.Adam, optimizer_kwargs=dict(lr=5e-4)) -# ) -# -# # we cannot guarantee which device the representation is on -# get_repr = lambda x: module.encode(x.to(module.device)) -# # PHASE 1, UNTRAINED -# pl.Trainer(logger=False, checkpoint_callback=False, fast_dev_run=True, gpus=1, weights_summary=None).fit(module, dataloader) -# module = module.to('cuda') -# calculate(data.__class__.__name__, 0, dataset, get_repr) -# # PHASE 2, LITTLE TRAINING -# pl.Trainer(logger=False, checkpoint_callback=False, max_steps=256, gpus=1, weights_summary=None).fit(module, dataloader) -# calculate(data.__class__.__name__, 256, dataset, get_repr) -# # PHASE 3, MORE TRAINING -# pl.Trainer(logger=False, checkpoint_callback=False, max_steps=2048, gpus=1, weights_summary=None).fit(module, dataloader) -# calculate(data.__class__.__name__, 256+2048, dataset, get_repr) -# results.append(None) -# -# for result in results: -# if result is None: -# print() -# continue -# (name, steps, result) = result -# print_r(name, steps, result, colors.lYLW) diff --git a/disent/registry/__init__.py b/disent/registry/__init__.py index d3fd9c9e..82814540 100644 --- a/disent/registry/__init__.py +++ b/disent/registry/__init__.py @@ -52,11 +52,7 @@ DATASETS['smallnorb'] = _LazyImport('disent.dataset.data._groundtruth__norb') DATASETS['shapes3d'] = _LazyImport('disent.dataset.data._groundtruth__shapes3d') # groundtruth -- impl synthetic -DATASETS['xyblocks'] = _LazyImport('disent.dataset.data._groundtruth__xyblocks') # pragma: delete-on-release DATASETS['xyobject'] = _LazyImport('disent.dataset.data._groundtruth__xyobject') -DATASETS['xysquares'] = _LazyImport('disent.dataset.data._groundtruth__xysquares') # pragma: delete-on-release -DATASETS['xysquares_minimal'] = _LazyImport('disent.dataset.data._groundtruth__xysquares') # pragma: delete-on-release -DATASETS['xcolumns'] = _LazyImport('disent.dataset.data._groundtruth__xcolumns') # pragma: delete-on-release # ========================================================================= # @@ -104,23 +100,6 @@ FRAMEWORKS['info_vae'] = _LazyImport('disent.frameworks.vae._unsupervised__infovae.InfoVae') FRAMEWORKS['vae'] = _LazyImport('disent.frameworks.vae._unsupervised__vae.Vae') FRAMEWORKS['ada_vae'] = _LazyImport('disent.frameworks.vae._weaklysupervised__adavae.AdaVae') -# [AE - EXPERIMENTAL] # pragma: delete-on-release -FRAMEWORKS['x__adaneg_tae'] = _LazyImport('disent.frameworks.ae.experimental._supervised__adaneg_tae.AdaNegTripletAe') # pragma: delete-on-release -FRAMEWORKS['x__dot_ae'] = _LazyImport('disent.frameworks.ae.experimental._unsupervised__dotae.DataOverlapTripletAe') # pragma: delete-on-release -FRAMEWORKS['x__ada_ae'] = _LazyImport('disent.frameworks.ae.experimental._weaklysupervised__adaae.AdaAe') # pragma: delete-on-release -# [VAE - EXPERIMENTAL] # pragma: delete-on-release -FRAMEWORKS['x__adaave_tvae'] = _LazyImport('disent.frameworks.vae.experimental._supervised__adaave_tvae.AdaAveTripletVae') # pragma: delete-on-release -FRAMEWORKS['x__adaneg_tvae'] = _LazyImport('disent.frameworks.vae.experimental._supervised__adaneg_tvae.AdaNegTripletVae') # pragma: delete-on-release -FRAMEWORKS['x__ada_tvae'] = _LazyImport('disent.frameworks.vae.experimental._supervised__adatvae.AdaTripletVae') # pragma: delete-on-release -FRAMEWORKS['x__bada_vae'] = _LazyImport('disent.frameworks.vae.experimental._supervised__badavae.BoundedAdaVae') # pragma: delete-on-release -FRAMEWORKS['x__gada_vae'] = _LazyImport('disent.frameworks.vae.experimental._supervised__gadavae.GuidedAdaVae') # pragma: delete-on-release -FRAMEWORKS['x__tbada_vae'] = _LazyImport('disent.frameworks.vae.experimental._supervised__tbadavae.TripletBoundedAdaVae') # pragma: delete-on-release -FRAMEWORKS['x__tgada_vae'] = _LazyImport('disent.frameworks.vae.experimental._supervised__tgadavae.TripletGuidedAdaVae') # pragma: delete-on-release -FRAMEWORKS['x__dor_vae'] = _LazyImport('disent.frameworks.vae.experimental._unsupervised__dorvae.DataOverlapRankVae') # pragma: delete-on-release -FRAMEWORKS['x__dot_vae'] = _LazyImport('disent.frameworks.vae.experimental._unsupervised__dotvae.DataOverlapTripletVae') # pragma: delete-on-release -FRAMEWORKS['x__augpos_tvae'] = _LazyImport('disent.frameworks.vae.experimental._weaklysupervised__augpostriplet.AugPosTripletVae') # pragma: delete-on-release -FRAMEWORKS['x__st_ada_vae'] = _LazyImport('disent.frameworks.vae.experimental._weaklysupervised__st_adavae.SwappedTargetAdaVae') # pragma: delete-on-release -FRAMEWORKS['x__st_beta_vae'] = _LazyImport('disent.frameworks.vae.experimental._weaklysupervised__st_betavae.SwappedTargetBetaVae') # pragma: delete-on-release # ========================================================================= # @@ -206,8 +185,6 @@ METRICS = _Registry('METRICS') METRICS['dci'] = _LazyImport('disent.metrics._dci.metric_dci') METRICS['factor_vae'] = _LazyImport('disent.metrics._factor_vae.metric_factor_vae') -METRICS['flatness'] = _LazyImport('disent.metrics._flatness.metric_flatness') # pragma: delete-on-release -METRICS['flatness_components'] = _LazyImport('disent.metrics._flatness_components.metric_flatness_components') # pragma: delete-on-release METRICS['mig'] = _LazyImport('disent.metrics._mig.metric_mig') METRICS['sap'] = _LazyImport('disent.metrics._sap.metric_sap') METRICS['unsupervised'] = _LazyImport('disent.metrics._unsupervised.metric_unsupervised') diff --git a/experiment/config/config.yaml b/experiment/config/config.yaml index fc2947cc..5f25a949 100644 --- a/experiment/config/config.yaml +++ b/experiment/config/config.yaml @@ -35,8 +35,6 @@ settings: framework_opt: latent_distribution: normal # only used by VAEs - overlap_loss: NULL # only used for experimental dotvae and dorvae # pragma: delete-on-release - usage_ratio: 0.5 # only used by adversarial masked datasets # pragma: delete-on-release model: z_size: 25 diff --git a/experiment/config/config_adversarial_dataset.yaml b/experiment/config/config_adversarial_dataset.yaml deleted file mode 100644 index f3f3ad23..00000000 --- a/experiment/config/config_adversarial_dataset.yaml +++ /dev/null @@ -1,60 +0,0 @@ - -# ========================================================================= # -# CONFIG # -# ========================================================================= # - - -defaults: - - run_logging: wandb_fast - - run_location: griffin - - run_launcher: local - # entries in this file override entries from default lists - - _self_ - -settings: - job: - user: 'n_michlo' - project: 'DELETE' # exp-disentangle-dataset - name: 'no-name' # TEST-${framework.dataset_name}_${framework.adversarial_mode}_${framework.sampler_name}_s${trainer.max_steps}_${framework.optimizer_name}_lr${framework.optimizer_lr} # _wd${framework.optimizer_kwargs.weight_decay} - seed: 777 - exp: - show_every_n_steps: 500 - # saving - rel_save_dir: 'out/adversarial_data/' - save_prefix: 'PREFIX' - save_data: TRUE - dataset: - batch_size: 32 - -trainer: - # same as defaults: - run_length: ... - max_steps: 30001 - max_epochs: 30001 - -adv_system: - ### IMPORTANT SETTINGS ### - dataset_name: 'dsprites' # [cars3d, smallnorb, dsprites, shapes3d, xysquares_8x8_mini] - adversarial_mode: 'self' # [self, invert_margin_0.005] invert, invert_unbounded - sampler_name: 'close_p_random_n' # [close_p_random_n, same_k1_close] - - ### OTHER SETTINGS ### - # optimizer options - optimizer_name: 'Adam' - optimizer_lr: 1e-1 - optimizer_kwargs: NULL - # dataset config options - # | dataset_name: 'cars3d' # cars3d, smallnorb, xysquares_8x8_mini - dataset_batch_size: 2048 # x3 - dataset_num_workers: ${dataloader.num_workers} - data_root: ${dsettings.storage.data_root} - # adversarial loss options - # | adversarial_mode: 'invert_margin_0.005' # [self, invert_margin_0.005] invert, invert_unbounded - adversarial_swapped: FALSE - adversarial_masking: FALSE # can produce weird artefacts that look like they might go against the training process, eg. improve disentanglement on dsprites, not actually checked by trianing model on this. - adversarial_top_k: NULL # NULL or range(1, batch_size) - pixel_loss_mode: 'mse' - # sampling config - # | sampler_name: 'close_p_random_n' # [close_p_random_n] (see notes above) -- close_p_random_n, close_p_random_n_bb, same_k, same_k_close, same_k1_close, same_k (might be wrong!), same_k_close, same_k1_close, close_far, close_factor_far_random, close_far_same_factor, same_factor, random_bb, random_swap_manhat, random_swap_manhat_norm - # train options - train_batch_optimizer: TRUE - train_dataset_fp16: TRUE diff --git a/experiment/config/config_adversarial_dataset_approx.yaml b/experiment/config/config_adversarial_dataset_approx.yaml deleted file mode 100644 index e984f7e6..00000000 --- a/experiment/config/config_adversarial_dataset_approx.yaml +++ /dev/null @@ -1,121 +0,0 @@ - -# ========================================================================= # -# CONFIG # -# ========================================================================= # - -defaults: - - run_logging: wandb_fast - - run_location: griffin - - run_launcher: local - # entries in this file override entries from default lists - - _self_ - -settings: - job: - user: 'n_michlo' - project: 'DELETE' - name_prefix: 'B32' - name: '${settings.job.name_prefix}-${adv_system.dataset_name}_${adv_system.adversarial_mode}_${adv_system.samples_sort_mode}_aw${adv_system.loss_adversarial_weight}_${adv_system.sampler_name}_s${trainer.max_steps}_${adv_system.optimizer_name}_lr${adv_system.optimizer_lr}_wd${adv_system.optimizer_kwargs.weight_decay}_b${settings.dataset.batch_size}_${settings.exp.save_dtype}' - seed: 424242 - exp: - show_every_n_steps: 1000 - # saving - rel_save_dir: 'out/adversarial_data_approx/' - save_prefix: 'PREFIX' - save_model: FALSE - save_data: FALSE - save_dtype: float16 - dataset: - batch_size: 32 - -trainer: - # same as defaults: - run_length: ... - # - 15000 takes 40 mins with batch size 512 (heartofgold, 12 workers) - # - 50000 takes 33 mins with batch size 256 (griffin, 16 workers) - max_steps: 15000 - max_epochs: 15000 - -adv_system: - ### IMPORTANT SETTINGS ### - # best: - # - close_p_random_n - # note: sampler_name (adversarial_mode=invert_margin_0.005) - # - random_swap_manhattan: worst [no inversion before 5k] (probability of encountering close is too low, don't use! ++easiest to implement) - # - close_p_random_n: good [inversion before 5k] (easier to implement) - # - close_p_random_n_bb: good [inversion before 5k] (hard to implement, but pretty much the same as close_p_random_n) - # - same_k: bad [no inversion before 5k] (probability of encountering close is too low, don't use! --harder to implement, better guarantees than random_swap_manhattan) - # - same_k_close: ok [almost inversion before 5k] (harder to implement) - # - same_k1_close: best [inversion well before 5 k] (easier to implement) - # note: sampler_name (adversarial_mode=self) - # - close_p_random_n: seems better based on plot of fdists vs overlap (converges better, but loss is higher which makes sense) - # - same_k1_close: seems worse based on plot of fdists vs overlap (seems to maintain original shape more, might hinder disentanglement? not actually tested) - sampler_name: 'close_p_random_n' # [random_swap_manhattan, close_p_random_n, same_k1_close] - samples_sort_mode: 'swap' # [none, swap, sort_inorder, sort_reverse] - dataset_name: 'smallnorb' # [cars3d, smallnorb, dsprites, shapes3d, xysquares_8x8_mini] - adversarial_mode: 'triplet_margin_0.1' # [self, invert_margin_0.05, invert_margin_0.005] invert, invert_unbounded - - ### OTHER SETTINGS ### - # optimizer options - optimizer_name: 'adam' - optimizer_lr: 2e-3 - optimizer_kwargs: - weight_decay: 1e-5 - # dataset config options - dataset_batch_size: ${dataloader.batch_size} # x3 - dataset_num_workers: ${dataloader.num_workers} - data_root: ${dsettings.storage.data_root} - data_load_into_memory: FALSE # I don't think this is truly multi-threaded, possible lock on array access? - # adversarial loss options - adversarial_swapped: FALSE - adversarial_masking: FALSE # can produce weird artefacts that look like they might go against the training process, eg. improve disentanglement on dsprites, not actually checked by trianing model on this. - adversarial_top_k: NULL # NULL or range(1, batch_size) - pixel_loss_mode: 'mse' - # loss extras - loss_adversarial_weight: 10.0 - loss_out_of_bounds_weight: 1.0 # not really needed -- if this is too high it struggles to "invert" - loss_same_stats_weight: 0.0 # not really needed - loss_similarity_weight: 1.0 # important - # model settings - model_type: 'ae_conv64' # ae_conv64, ae_linear, ae_conv64norm - model_mask_mode: 'none' # std, diff, none - model_weight_init: 'xavier_normal' # [xavier_normal, default] - # logging settings - logging_scale_imgs: FALSE - - -# ========================================================================= # -# OLD EXPERIMENTS # -# ========================================================================= # - - -# EXPERIMENT SWEEP: -# -m framework.sampler_name=close_p_random_n framework.adversarial_mode=self,invert_margin_0.005 framework.dataset_name=dsprites,shapes3d,cars3d,smallnorb -# -m framework.loss_adversarial_weight=100.0 framework.sampler_name=same_k1_close framework.adversarial_mode=self2,self framework.dataset_name=dsprites,shapes3d,cars3d,smallnorb - -# EXPERIMENT INDIVIDUALS: -# framework.sampler_name=close_p_random_n framework.adversarial_mode=self framework.dataset_name=dsprites -# framework.sampler_name=close_p_random_n framework.adversarial_mode=self framework.dataset_name=shapes3d -# framework.sampler_name=close_p_random_n framework.adversarial_mode=self framework.dataset_name=cars3d -# framework.sampler_name=close_p_random_n framework.adversarial_mode=self framework.dataset_name=smallnorb - -# framework.sampler_name=close_p_random_n framework.adversarial_mode=invert_margin_0.005 framework.dataset_name=dsprites -# framework.sampler_name=close_p_random_n framework.adversarial_mode=invert_margin_0.005 framework.dataset_name=shapes3d -# framework.sampler_name=close_p_random_n framework.adversarial_mode=invert_margin_0.005 framework.dataset_name=cars3d -# framework.sampler_name=close_p_random_n framework.adversarial_mode=invert_margin_0.005 framework.dataset_name=smallnorb -# -# # 3dshapes does not seem to want to invert... -# framework.sampler_name=close_p_random_n framework.adversarial_mode=invert_margin_0.01 framework.dataset_name=shapes3d -# framework.sampler_name=close_p_random_n framework.adversarial_mode=invert_margin_0.10 framework.dataset_name=shapes3d - -# NEW EXPERIMENT: -# -m framework.sampler_name=same_k1_close,close_p_random_n framework.adversarial_mode=invert_margin_0.05 framework.dataset_name=dsprites,shapes3d,smallnorb,cars3d -# - continue -# DONE: -m framework.sampler_name=same_k1_close,close_p_random_n framework.adversarial_mode=invert_margin_0.05 framework.dataset_name=smallnorb,cars3d -# DOING: -m framework.sampler_name=close_p_random_n framework.adversarial_mode=invert_margin_0.05 framework.dataset_name=smallnorb,cars3d -# TODO: -m framework.sampler_name=close_p_random_n framework.adversarial_mode=invert_margin_0.05 framework.dataset_name=cars3d,smallnorb - -# NEW EXPERIMENT 2: -# -m framework.sampler_name=same_k1_close,close_p_random_n framework.adversarial_mode=invert_margin_0.05 framework.loss_out_of_bounds_weight=1000.0 framework.dataset_name=dsprites,shapes3d,smallnorb,cars3d - -# NEW EXPERIMENT 3: -# -m framework.sampler_name=same_k1_close framework.adversarial_mode=invert_margin_0.05 framework.loss_out_of_bounds_weight=10000.0 framework.dataset_name=shapes3d,dsprites,cars3d,smallnorb diff --git a/experiment/config/config_adversarial_kernel.yaml b/experiment/config/config_adversarial_kernel.yaml deleted file mode 100644 index ea4020ec..00000000 --- a/experiment/config/config_adversarial_kernel.yaml +++ /dev/null @@ -1,50 +0,0 @@ -defaults: - # runtime - - run_length: short - - run_logging: wandb - - run_location: stampede_tmp - - run_launcher: slurm - # plugins - - hydra/job_logging: colorlog - - hydra/hydra_logging: colorlog - - hydra/launcher: submitit_slurm - -job: - user: 'n_michlo' - project: 'exp-disentangle-kernel' - name: r${kernel.radius}-${kernel.channels}_s${trainer.max_steps}_${optimizer.name}_lr${settings.optimizer.lr}_wd${optimizer.weight_decay}_${data.name} - -optimizer: - name: adam - lr: 3e-3 - weight_decay: 0.0 - -data: - name: 'xysquares_8x8' - -kernel: - radius: 63 - channels: 1 - disentangle_factors: NULL - # training - regularize_symmetric: TRUE - regularize_norm: FALSE # these don't work - regularize_nonneg: FALSE # these don't work - -train: - pairs_ratio: 8.0 - loss: mse - -exp: - seed: 777 - rel_save_dir: data/adversarial_kernel - save_name: ${job.name}.pt - show_every_n_steps: 1000 - -# OVERRIDE run_logging: wandb -- too fast otherwise -logging: - flush_logs_every_n_steps: 500 - -# OVERRIDE run_location: -dataset: - batch_size: 128 diff --git a/experiment/config/config_test.yaml b/experiment/config/config_test.yaml index e01a9353..63a66c8a 100644 --- a/experiment/config/config_test.yaml +++ b/experiment/config/config_test.yaml @@ -35,8 +35,6 @@ settings: framework_opt: latent_distribution: normal # only used by VAEs - overlap_loss: NULL # only used for experimental dotvae and dorvae # pragma: delete-on-release - usage_ratio: 0.5 # only used by adversarial masked datasets # pragma: delete-on-release model: z_size: 25 diff --git a/experiment/config/dataset/X--adv-cars3d--WARNING.yaml b/experiment/config/dataset/X--adv-cars3d--WARNING.yaml deleted file mode 100644 index ac8d26a5..00000000 --- a/experiment/config/dataset/X--adv-cars3d--WARNING.yaml +++ /dev/null @@ -1,20 +0,0 @@ -defaults: - - _data_type_: gt - -name: adv_cars3d - -data: - _target_: disent.dataset.data.SelfContainedHdf5GroundTruthData - h5_path: '${oc.env:HOME}/workspace/research/disent/out/adversarial_data_approx/2021-09-06--05-42-06_INVERT-VSTRONG-cars3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06/data.h5' - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.76418207, 0.75554032, 0.75075393] - vis_std: [0.31892905, 0.32751031, 0.33319886] - -# TODO: this does not yet copy the data to /tmp/ and thus if run on a cluster of a network drive, this will hammer the network disk. Fix this! diff --git a/experiment/config/dataset/X--adv-dsprites--WARNING.yaml b/experiment/config/dataset/X--adv-dsprites--WARNING.yaml deleted file mode 100644 index 3965bf84..00000000 --- a/experiment/config/dataset/X--adv-dsprites--WARNING.yaml +++ /dev/null @@ -1,20 +0,0 @@ -defaults: - - _data_type_: gt - -name: adv_dsprites - -data: - _target_: disent.dataset.data.SelfContainedHdf5GroundTruthData - h5_path: '${oc.env:HOME}/workspace/research/disent/out/adversarial_data_approx/2021-09-06--03-17-28_INVERT-VSTRONG-dsprites_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06/data.h5' - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.20482841] - vis_std: [0.33634909] - -# TODO: this does not yet copy the data to /tmp/ and thus if run on a cluster of a network drive, this will hammer the network disk. Fix this! diff --git a/experiment/config/dataset/X--adv-shapes3d--WARNING.yaml b/experiment/config/dataset/X--adv-shapes3d--WARNING.yaml deleted file mode 100644 index 5983845a..00000000 --- a/experiment/config/dataset/X--adv-shapes3d--WARNING.yaml +++ /dev/null @@ -1,20 +0,0 @@ -defaults: - - _data_type_: gt - -name: adv_shapes3d - -data: - _target_: disent.dataset.data.SelfContainedHdf5GroundTruthData - h5_path: '${oc.env:HOME}/workspace/research/disent/out/adversarial_data_approx/2021-09-06--00-29-23_INVERT-VSTRONG-shapes3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06/data.h5' - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.47992192, 0.51311111, 0.54627272] - vis_std: [0.28653814, 0.29201543, 0.27395435] - -# TODO: this does not yet copy the data to /tmp/ and thus if run on a cluster of a network drive, this will hammer the network disk. Fix this! diff --git a/experiment/config/dataset/X--adv-smallnorb--WARNING.yaml b/experiment/config/dataset/X--adv-smallnorb--WARNING.yaml deleted file mode 100644 index fa483e82..00000000 --- a/experiment/config/dataset/X--adv-smallnorb--WARNING.yaml +++ /dev/null @@ -1,20 +0,0 @@ -defaults: - - _data_type_: gt - -name: adv_smallnorb - -data: - _target_: disent.dataset.data.SelfContainedHdf5GroundTruthData - h5_path: '${oc.env:HOME}/workspace/research/disent/out/adversarial_data_approx/2021-09-06--09-10-59_INVERT-VSTRONG-smallnorb_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06/data.h5' - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.69691603] - vis_std: [0.21310608] - -# TODO: this does not yet copy the data to /tmp/ and thus if run on a cluster of a network drive, this will hammer the network disk. Fix this! diff --git a/experiment/config/dataset/X--dsprites-imagenet-bg-100.yaml b/experiment/config/dataset/X--dsprites-imagenet-bg-100.yaml deleted file mode 100644 index 1ab49d2c..00000000 --- a/experiment/config/dataset/X--dsprites-imagenet-bg-100.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_bg_100 - -data: - _target_: disent.dataset.data.DSpritesImagenetData - visibility: 100 - mode: bg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${data.meta.vis_mean} - std: ${data.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.5020433619489952, 0.47206398913310593, 0.42380018909780404] - vis_std: [0.2505510666843685, 0.25007259803668697, 0.2562415603123114] diff --git a/experiment/config/dataset/X--dsprites-imagenet-bg-20.yaml b/experiment/config/dataset/X--dsprites-imagenet-bg-20.yaml deleted file mode 100644 index 00aa4955..00000000 --- a/experiment/config/dataset/X--dsprites-imagenet-bg-20.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_bg_20 - -data: - _target_: disent.dataset.data.DSpritesImagenetData - visibility: 20 - mode: bg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.13294969414492142, 0.12694375140936273, 0.11733572285575933] - vis_std: [0.18311250427586276, 0.1840916474752131, 0.18607373519458442] diff --git a/experiment/config/dataset/X--dsprites-imagenet-bg-40.yaml b/experiment/config/dataset/X--dsprites-imagenet-bg-40.yaml deleted file mode 100644 index ad4674ee..00000000 --- a/experiment/config/dataset/X--dsprites-imagenet-bg-40.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_bg_40 - -data: - _target_: disent.dataset.data.DSpritesImagenetData - visibility: 40 - mode: bg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.2248598986983768, 0.21285772298967615, 0.19359577132944206] - vis_std: [0.1841631708032332, 0.18554895825833284, 0.1893568926398198] diff --git a/experiment/config/dataset/X--dsprites-imagenet-bg-60.yaml b/experiment/config/dataset/X--dsprites-imagenet-bg-60.yaml deleted file mode 100644 index 5a0f6550..00000000 --- a/experiment/config/dataset/X--dsprites-imagenet-bg-60.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_bg_60 - -data: - _target_: disent.dataset.data.DSpritesImagenetData - visibility: 60 - mode: bg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.31676960943447674, 0.29877166834408025, 0.2698556821388113] - vis_std: [0.19745897110349003, 0.1986606891520453, 0.203808842880044] diff --git a/experiment/config/dataset/X--dsprites-imagenet-bg-80.yaml b/experiment/config/dataset/X--dsprites-imagenet-bg-80.yaml deleted file mode 100644 index f699681e..00000000 --- a/experiment/config/dataset/X--dsprites-imagenet-bg-80.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_bg_80 - -data: - _target_: disent.dataset.data.DSpritesImagenetData - visibility: 80 - mode: bg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.40867981393820857, 0.38468564002021527, 0.34611573047508204] - vis_std: [0.22048328737091344, 0.22102216869942384, 0.22692977053753477] diff --git a/experiment/config/dataset/X--dsprites-imagenet-fg-100.yaml b/experiment/config/dataset/X--dsprites-imagenet-fg-100.yaml deleted file mode 100644 index 82202433..00000000 --- a/experiment/config/dataset/X--dsprites-imagenet-fg-100.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_fg_100 - -data: - _target_: disent.dataset.data.DSpritesImagenetData - visibility: 100 - mode: fg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.02067051643494642, 0.018688392816012946, 0.01632900510079384] - vis_std: [0.10271307751834059, 0.09390213983525653, 0.08377594259970281] diff --git a/experiment/config/dataset/X--dsprites-imagenet-fg-20.yaml b/experiment/config/dataset/X--dsprites-imagenet-fg-20.yaml deleted file mode 100644 index df765265..00000000 --- a/experiment/config/dataset/X--dsprites-imagenet-fg-20.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_fg_20 - -data: - _target_: disent.dataset.data.DSpritesImagenetData - visibility: 20 - mode: fg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.038064750024334834, 0.03766780505193579, 0.03719798677641122] - vis_std: [0.17498878664096565, 0.17315570657628318, 0.1709923319496426] diff --git a/experiment/config/dataset/X--dsprites-imagenet-fg-40.yaml b/experiment/config/dataset/X--dsprites-imagenet-fg-40.yaml deleted file mode 100644 index 1d79f75d..00000000 --- a/experiment/config/dataset/X--dsprites-imagenet-fg-40.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_fg_40 - -data: - _target_: disent.dataset.data.DSpritesImagenetData - visibility: 40 - mode: fg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.03369999506331255, 0.03290657349801835, 0.03196482946320608] - vis_std: [0.155514074438101, 0.1518464537731621, 0.14750944591836743] diff --git a/experiment/config/dataset/X--dsprites-imagenet-fg-60.yaml b/experiment/config/dataset/X--dsprites-imagenet-fg-60.yaml deleted file mode 100644 index d65e3622..00000000 --- a/experiment/config/dataset/X--dsprites-imagenet-fg-60.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_fg_60 - -data: - _target_: disent.dataset.data.DSpritesImagenetData - visibility: 60 - mode: fg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.029335176871153983, 0.028145355435322966, 0.026731731769287146] - vis_std: [0.13663242436043319, 0.13114320478634894, 0.1246542727733097] diff --git a/experiment/config/dataset/X--dsprites-imagenet-fg-80.yaml b/experiment/config/dataset/X--dsprites-imagenet-fg-80.yaml deleted file mode 100644 index bb3c025c..00000000 --- a/experiment/config/dataset/X--dsprites-imagenet-fg-80.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_fg_80 - -data: - _target_: disent.dataset.data.DSpritesImagenetData - visibility: 80 - mode: fg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.024956427531012196, 0.02336780403840578, 0.021475119672280243] - vis_std: [0.11864125016313823, 0.11137998105649799, 0.10281424917834255] diff --git a/experiment/config/dataset/X--dsprites-imagenet.yaml b/experiment/config/dataset/X--dsprites-imagenet.yaml deleted file mode 100644 index 6329d035..00000000 --- a/experiment/config/dataset/X--dsprites-imagenet.yaml +++ /dev/null @@ -1,54 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_${dataset.mode}_${dataset.visibility} - -data: - _target_: disent.dataset.data.DSpritesImagenetData - visibility: 40 - mode: bg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: ${exit:EXITING... dsprites-imagenet has been disabled} # ${dataset.__STATS.${dataset.name}.vis_mean} - vis_std: ${exit:EXITING... dsprites-imagenet has been disabled} # ${dataset.__STATS.${dataset.name}.vis_std} - -__STATS: - dsprites_imagenet_fg_100: - vis_mean: [0.02067051643494642, 0.018688392816012946, 0.01632900510079384] - vis_std: [0.10271307751834059, 0.09390213983525653, 0.08377594259970281] - dsprites_imagenet_fg_80: - vis_mean: [0.024956427531012196, 0.02336780403840578, 0.021475119672280243] - vis_std: [0.11864125016313823, 0.11137998105649799, 0.10281424917834255] - dsprites_imagenet_fg_60: - vis_mean: [0.029335176871153983, 0.028145355435322966, 0.026731731769287146] - vis_std: [0.13663242436043319, 0.13114320478634894, 0.1246542727733097] - dsprites_imagenet_fg_40: - vis_mean: [0.03369999506331255, 0.03290657349801835, 0.03196482946320608] - vis_std: [0.155514074438101, 0.1518464537731621, 0.14750944591836743] - dsprites_imagenet_fg_20: - vis_mean: [0.038064750024334834, 0.03766780505193579, 0.03719798677641122] - vis_std: [0.17498878664096565, 0.17315570657628318, 0.1709923319496426] - dsprites_imagenet_bg_100: - vis_mean: [0.5020433619489952, 0.47206398913310593, 0.42380018909780404] - vis_std: [0.2505510666843685, 0.25007259803668697, 0.2562415603123114] - dsprites_imagenet_bg_80: - vis_mean: [0.40867981393820857, 0.38468564002021527, 0.34611573047508204] - vis_std: [0.22048328737091344, 0.22102216869942384, 0.22692977053753477] - dsprites_imagenet_bg_60: - vis_mean: [0.31676960943447674, 0.29877166834408025, 0.2698556821388113] - vis_std: [0.19745897110349003, 0.1986606891520453, 0.203808842880044] - dsprites_imagenet_bg_40: - vis_mean: [0.2248598986983768, 0.21285772298967615, 0.19359577132944206] - vis_std: [0.1841631708032332, 0.18554895825833284, 0.1893568926398198] - dsprites_imagenet_bg_20: - vis_mean: [0.13294969414492142, 0.12694375140936273, 0.11733572285575933] - vis_std: [0.18311250427586276, 0.1840916474752131, 0.18607373519458442] diff --git a/experiment/config/dataset/X--mask-adv-f-cars3d.yaml b/experiment/config/dataset/X--mask-adv-f-cars3d.yaml deleted file mode 100644 index 848a271e..00000000 --- a/experiment/config/dataset/X--mask-adv-f-cars3d.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_adv_f_cars3d - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.e06_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--21-23-27_EXP_cars3d_1000x256_all_std_mean/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--22-58-24_EXP_cars3d_1000x256_all_std_gmean/data.pkl.gz' - randomize: FALSE - data: - _target_: disent.dataset.data.Cars3dData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - size: 64 - mean: ${data.meta.vis_mean} - std: ${data.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.8976676149976628, 0.8891658020067508, 0.885147515814868] - vis_std: [0.22503195531503034, 0.2399461278981261, 0.24792106319684404] diff --git a/experiment/config/dataset/X--mask-adv-f-dsprites.yaml b/experiment/config/dataset/X--mask-adv-f-dsprites.yaml deleted file mode 100644 index 5517a992..00000000 --- a/experiment/config/dataset/X--mask-adv-f-dsprites.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_adv_f_dsprites - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.e06_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--21-45-46_EXP_dsprites_1000x256_all_std_mean/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--23-21-51_EXP_dsprites_1000x256_all_std_gmean/data.pkl.gz' - randomize: FALSE - data: - _target_: disent.dataset.data.DSpritesData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.042494423521889584] - vis_std: [0.19516645880626055] diff --git a/experiment/config/dataset/X--mask-adv-f-shapes3d.yaml b/experiment/config/dataset/X--mask-adv-f-shapes3d.yaml deleted file mode 100644 index 6871f130..00000000 --- a/experiment/config/dataset/X--mask-adv-f-shapes3d.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_adv_f_shapes3d - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.e06_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--21-33-57_EXP_shapes3d_1000x256_all_std_mean/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--23-09-05_EXP_shapes3d_1000x256_all_std_gmean/data.pkl.gz' - randomize: FALSE - data: - _target_: disent.dataset.data.Shapes3dData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.502584966788819, 0.5787597566089667, 0.6034499731859578] - vis_std: [0.2940814043555559, 0.3443979087517214, 0.3661685981524748] diff --git a/experiment/config/dataset/X--mask-adv-f-smallnorb.yaml b/experiment/config/dataset/X--mask-adv-f-smallnorb.yaml deleted file mode 100644 index 738a8abf..00000000 --- a/experiment/config/dataset/X--mask-adv-f-smallnorb.yaml +++ /dev/null @@ -1,29 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_adv_f_smallnorb - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.e06_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--21-28-42_EXP_smallnorb_1000x256_all_std_mean/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--23-03-51_EXP_smallnorb_1000x256_all_std_gmean/data.pkl.gz' - randomize: FALSE - data: - _target_: disent.dataset.data.SmallNorbData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - is_test: False - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - size: 64 - mean: ${dataset.meta.vis_mean} - std: ${data.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.7520918401088603] - vis_std: [0.09563879016827262] diff --git a/experiment/config/dataset/X--mask-adv-r-cars3d.yaml b/experiment/config/dataset/X--mask-adv-r-cars3d.yaml deleted file mode 100644 index d7c64191..00000000 --- a/experiment/config/dataset/X--mask-adv-r-cars3d.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_adv_r_cars3d - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.e06_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-10-19--14-49-26_DISTS-SCALED_cars3d_1000x384_random_256_True_std_False/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-10-19--18-41-14_DISTS-SCALED_cars3d_1000x384_random_256_True_range_False/data.pkl.gz' - randomize: FALSE - data: - _target_: disent.dataset.data.Cars3dData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - size: 64 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.8976676149976628, 0.8891658020067508, 0.885147515814868] - vis_std: [0.22503195531503034, 0.2399461278981261, 0.24792106319684404] diff --git a/experiment/config/dataset/X--mask-adv-r-dsprites.yaml b/experiment/config/dataset/X--mask-adv-r-dsprites.yaml deleted file mode 100644 index 26a16f75..00000000 --- a/experiment/config/dataset/X--mask-adv-r-dsprites.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_adv_r_dsprites - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.e06_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-10-19--16-31-56_DISTS-SCALED_dsprites_1000x384_random_256_True_std_False/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-10-19--19-58-39_DISTS-SCALED_dsprites_1000x384_random_256_True_range_False/data.pkl.gz' - randomize: FALSE - data: - _target_: disent.dataset.data.DSpritesData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.042494423521889584] - vis_std: [0.19516645880626055] diff --git a/experiment/config/dataset/X--mask-adv-r-shapes3d.yaml b/experiment/config/dataset/X--mask-adv-r-shapes3d.yaml deleted file mode 100644 index bc799876..00000000 --- a/experiment/config/dataset/X--mask-adv-r-shapes3d.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_adv_r_shapes3d - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.e06_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-10-19--15-20-48_DISTS-SCALED_shapes3d_1000x384_random_256_True_std_False/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-10-19--19-04-26_DISTS-SCALED_shapes3d_1000x384_random_256_True_range_False/data.pkl.gz' - randomize: FALSE - data: - _target_: disent.dataset.data.Shapes3dData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.502584966788819, 0.5787597566089667, 0.6034499731859578] - vis_std: [0.2940814043555559, 0.3443979087517214, 0.3661685981524748] diff --git a/experiment/config/dataset/X--mask-adv-r-smallnorb.yaml b/experiment/config/dataset/X--mask-adv-r-smallnorb.yaml deleted file mode 100644 index b36c799d..00000000 --- a/experiment/config/dataset/X--mask-adv-r-smallnorb.yaml +++ /dev/null @@ -1,29 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_adv_r_smallnorb - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.e06_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-10-19--15-10-07_DISTS-SCALED_smallnorb_1000x384_random_256_True_std_False/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-10-19--18-53-52_DISTS-SCALED_smallnorb_1000x384_random_256_True_range_False/data.pkl.gz' - randomize: FALSE - data: - _target_: disent.dataset.data.SmallNorbData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - is_test: False - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - size: 64 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.7520918401088603] - vis_std: [0.09563879016827262] diff --git a/experiment/config/dataset/X--mask-dthr-cars3d.yaml b/experiment/config/dataset/X--mask-dthr-cars3d.yaml deleted file mode 100644 index c643a64f..00000000 --- a/experiment/config/dataset/X--mask-dthr-cars3d.yaml +++ /dev/null @@ -1,24 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_dthr_cars3d - -data: - _target_: disent.dataset.wrapper.DitheredDataset - dither_n: 2 - keep_ratio: ${settings.framework_opt.usage_ratio} - gt_data: - _target_: disent.dataset.data.Cars3dData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - size: 64 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.8976676149976628, 0.8891658020067508, 0.885147515814868] - vis_std: [0.22503195531503034, 0.2399461278981261, 0.24792106319684404] diff --git a/experiment/config/dataset/X--mask-dthr-dsprites.yaml b/experiment/config/dataset/X--mask-dthr-dsprites.yaml deleted file mode 100644 index 03000f9b..00000000 --- a/experiment/config/dataset/X--mask-dthr-dsprites.yaml +++ /dev/null @@ -1,24 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_dthr_dsprites - -data: - _target_: disent.dataset.wrapper.DitheredDataset - dither_n: 2 - keep_ratio: ${settings.framework_opt.usage_ratio} - gt_data: - _target_: disent.dataset.data.DSpritesData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.042494423521889584] - vis_std: [0.19516645880626055] diff --git a/experiment/config/dataset/X--mask-dthr-shapes3d.yaml b/experiment/config/dataset/X--mask-dthr-shapes3d.yaml deleted file mode 100644 index 9aa229da..00000000 --- a/experiment/config/dataset/X--mask-dthr-shapes3d.yaml +++ /dev/null @@ -1,24 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_dthr_shapes3d - -data: - _target_: disent.dataset.wrapper.DitheredDataset - dither_n: 2 - keep_ratio: ${settings.framework_opt.usage_ratio} - gt_data: - _target_: disent.dataset.data.Shapes3dData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.502584966788819, 0.5787597566089667, 0.6034499731859578] - vis_std: [0.2940814043555559, 0.3443979087517214, 0.3661685981524748] diff --git a/experiment/config/dataset/X--mask-dthr-smallnorb.yaml b/experiment/config/dataset/X--mask-dthr-smallnorb.yaml deleted file mode 100644 index 28455e5f..00000000 --- a/experiment/config/dataset/X--mask-dthr-smallnorb.yaml +++ /dev/null @@ -1,25 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_dthr_smallnorb - -data: - _target_: disent.dataset.wrapper.DitheredDataset - dither_n: 2 - keep_ratio: ${settings.framework_opt.usage_ratio} - gt_data: - _target_: disent.dataset.data.SmallNorbData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - is_test: False - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - size: 64 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.7520918401088603] - vis_std: [0.09563879016827262] diff --git a/experiment/config/dataset/X--mask-ran-cars3d.yaml b/experiment/config/dataset/X--mask-ran-cars3d.yaml deleted file mode 100644 index 59afd87e..00000000 --- a/experiment/config/dataset/X--mask-ran-cars3d.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_ran_cars3d - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.e06_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--21-23-27_EXP_cars3d_1000x256_all_std_mean/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--22-58-24_EXP_cars3d_1000x256_all_std_gmean/data.pkl.gz' - randomize: TRUE - data: - _target_: disent.dataset.data.Cars3dData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - size: 64 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.8976676149976628, 0.8891658020067508, 0.885147515814868] - vis_std: [0.22503195531503034, 0.2399461278981261, 0.24792106319684404] diff --git a/experiment/config/dataset/X--mask-ran-dsprites.yaml b/experiment/config/dataset/X--mask-ran-dsprites.yaml deleted file mode 100644 index a9a1836a..00000000 --- a/experiment/config/dataset/X--mask-ran-dsprites.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_ran_dsprites - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.e06_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--21-45-46_EXP_dsprites_1000x256_all_std_mean/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--23-21-51_EXP_dsprites_1000x256_all_std_gmean/data.pkl.gz' - randomize: TRUE - data: - _target_: disent.dataset.data.DSpritesData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.042494423521889584] - vis_std: [0.19516645880626055] diff --git a/experiment/config/dataset/X--mask-ran-shapes3d.yaml b/experiment/config/dataset/X--mask-ran-shapes3d.yaml deleted file mode 100644 index c55396f5..00000000 --- a/experiment/config/dataset/X--mask-ran-shapes3d.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_ran_shapes3d - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.e06_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--21-33-57_EXP_shapes3d_1000x256_all_std_mean/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--23-09-05_EXP_shapes3d_1000x256_all_std_gmean/data.pkl.gz' - randomize: TRUE - data: - _target_: disent.dataset.data.Shapes3dData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.502584966788819, 0.5787597566089667, 0.6034499731859578] - vis_std: [0.2940814043555559, 0.3443979087517214, 0.3661685981524748] diff --git a/experiment/config/dataset/X--mask-ran-smallnorb.yaml b/experiment/config/dataset/X--mask-ran-smallnorb.yaml deleted file mode 100644 index f8d7267e..00000000 --- a/experiment/config/dataset/X--mask-ran-smallnorb.yaml +++ /dev/null @@ -1,29 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_ran_smallnorb - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.e06_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--21-28-42_EXP_smallnorb_1000x256_all_std_mean/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--23-03-51_EXP_smallnorb_1000x256_all_std_gmean/data.pkl.gz' - randomize: TRUE - data: - _target_: disent.dataset.data.SmallNorbData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - is_test: False - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - size: 64 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.7520918401088603] - vis_std: [0.09563879016827262] diff --git a/experiment/config/dataset/X--xyblocks.yaml b/experiment/config/dataset/X--xyblocks.yaml deleted file mode 100644 index 5eaf260d..00000000 --- a/experiment/config/dataset/X--xyblocks.yaml +++ /dev/null @@ -1,18 +0,0 @@ -defaults: - - _data_type_: gt - -name: xyblocks - -data: - _target_: disent.dataset.data.XYBlocksData - rgb: TRUE - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.10040509259259259, 0.10040509259259259, 0.10040509259259259] - vis_std: [0.21689087652106678, 0.21689087652106676, 0.21689087652106678] diff --git a/experiment/config/dataset/X--xyblocks_grey.yaml b/experiment/config/dataset/X--xyblocks_grey.yaml deleted file mode 100644 index 0faf884d..00000000 --- a/experiment/config/dataset/X--xyblocks_grey.yaml +++ /dev/null @@ -1,18 +0,0 @@ -defaults: - - _data_type_: gt - -name: xyblocks_grey - -data: - _target_: disent.dataset.data.XYBlocksData - rgb: FALSE - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: "${exit:EXITING... please compute the vis_mean and vis_std}" - vis_std: "${exit:EXITING... please compute the vis_mean and vis_std}" diff --git a/experiment/config/dataset/X--xysquares.yaml b/experiment/config/dataset/X--xysquares.yaml deleted file mode 100644 index e368ea3d..00000000 --- a/experiment/config/dataset/X--xysquares.yaml +++ /dev/null @@ -1,17 +0,0 @@ -defaults: - - _data_type_: gt - -name: xysquares_minimal - -data: - _target_: disent.dataset.data.XYSquaresMinimalData - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.015625, 0.015625, 0.015625] - vis_std: [0.12403473458920855, 0.12403473458920854, 0.12403473458920854] diff --git a/experiment/config/dataset/X--xysquares_grey.yaml b/experiment/config/dataset/X--xysquares_grey.yaml deleted file mode 100644 index 20088abd..00000000 --- a/experiment/config/dataset/X--xysquares_grey.yaml +++ /dev/null @@ -1,23 +0,0 @@ -defaults: - - _data_type_: gt - -name: xysquares_grey - -data: - _target_: disent.dataset.data.XYSquaresData - square_size: 8 # AFFECTS: mean and std - image_size: 64 # usually ok to adjust - grid_size: 8 # usually ok to adjust - grid_spacing: 8 # usually ok to adjust - num_squares: 3 # AFFECTS: mean and std - rgb: FALSE # AFFECTS: mean and std - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.046146392822265625] - vis_std: [0.2096506119375896] diff --git a/experiment/config/dataset/X--xysquares_rgb.yaml b/experiment/config/dataset/X--xysquares_rgb.yaml deleted file mode 100644 index 35a45110..00000000 --- a/experiment/config/dataset/X--xysquares_rgb.yaml +++ /dev/null @@ -1,23 +0,0 @@ -defaults: - - _data_type_: gt - -name: xysquares_rgb - -data: - _target_: disent.dataset.data.XYSquaresData - square_size: 8 # AFFECTS: mean and std - image_size: 64 # usually ok to adjust - grid_size: 8 # usually ok to adjust - grid_spacing: 8 # usually ok to adjust - num_squares: 3 # AFFECTS: mean and std - rgb: TRUE # AFFECTS: mean and std - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.015625, 0.015625, 0.015625] - vis_std: [0.12403473458920855, 0.12403473458920854, 0.12403473458920854] diff --git a/experiment/config/framework/X--adaae.yaml b/experiment/config/framework/X--adaae.yaml deleted file mode 100644 index d492ca75..00000000 --- a/experiment/config/framework/X--adaae.yaml +++ /dev/null @@ -1,19 +0,0 @@ -defaults: - - _input_mode_: pair - -name: adaae - -cfg: - _target_: disent.frameworks.ae.experimental.AdaAe.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # disable various components - disable_decoder: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # adavae - ada_thresh_ratio: 0.5 - -meta: - model_z_multiplier: 1 diff --git a/experiment/config/framework/X--adaae_os.yaml b/experiment/config/framework/X--adaae_os.yaml deleted file mode 100644 index 67a16b46..00000000 --- a/experiment/config/framework/X--adaae_os.yaml +++ /dev/null @@ -1,19 +0,0 @@ -defaults: - - _input_mode_: weak_pair - -name: adaae - -cfg: - _target_: disent.frameworks.ae.experimental.AdaAe.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # disable various components - disable_decoder: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # adavae - ada_thresh_ratio: 0.5 - -meta: - model_z_multiplier: 1 diff --git a/experiment/config/framework/X--adaavetvae.yaml b/experiment/config/framework/X--adaavetvae.yaml deleted file mode 100644 index 03ae727e..00000000 --- a/experiment/config/framework/X--adaavetvae.yaml +++ /dev/null @@ -1,45 +0,0 @@ -defaults: - - _input_mode_: triplet - -name: adaave_tvae - -cfg: - _target_: disent.frameworks.vae.experimental.AdaAveTripletVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - disable_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - disable_posterior_scale: NULL - # Beta-VAE - beta: ${settings.framework.beta} - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - # adavae - ada_average_mode: gvae - ada_thresh_mode: symmetric_kl # Only works for: adat_share_mask_mode == "posterior" --- kl, symmetric_kl, dist, sampled_dist - ada_thresh_ratio: 0.5 # >> USE WITH A SCHEDULE << - # ada_tvae - loss - adat_triplet_loss: triplet_soft_ave_all - adat_triplet_ratio: 1.0 # >> USE WITH A SCHEDULE << 0.5 is half of triplet and ada-triplet, 1.0 is all ada-triplet - adat_triplet_soft_scale: 1.0 # >> USE WITH A SCHEDULE << - adat_triplet_pull_weight: 0.1 # Only works for: adat_triplet_loss == "triplet_hard_neg_ave_pull" - adat_triplet_share_scale: 0.95 # >> USE WITH A SCHEDULE << only works for: adat_triplet_loss == "triplet_hard_neg_ave_scaled" - # ada_tvae - averaging - adat_share_mask_mode: posterior - adat_share_ave_mode: all # Only works for: adat_triplet_loss == "triplet_hard_ave_all" - # adaave_tvae - adaave_augment_orig: FALSE # triplet over original OR averaged embeddings - adaave_decode_orig: FALSE # decode & regularize original OR averaged embeddings - -meta: - model_z_multiplier: 2 diff --git a/experiment/config/framework/X--adanegtae.yaml b/experiment/config/framework/X--adanegtae.yaml deleted file mode 100644 index f5de4d33..00000000 --- a/experiment/config/framework/X--adanegtae.yaml +++ /dev/null @@ -1,27 +0,0 @@ -defaults: - - _input_mode_: triplet - -name: adanegtae - -cfg: - _target_: disent.frameworks.ae.experimental.AdaNegTripletAe.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # disable various components - disable_decoder: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - # adavae - ada_thresh_ratio: 0.5 # >> USE WITH A SCHEDULE << - # ada_tvae - loss - adat_triplet_share_scale: 0.95 # >> USE WITH A SCHEDULE << only works for: adat_triplet_loss == "triplet_hard_neg_ave_scaled" - -meta: - model_z_multiplier: 1 diff --git a/experiment/config/framework/X--adanegtvae.yaml b/experiment/config/framework/X--adanegtvae.yaml deleted file mode 100644 index a321400c..00000000 --- a/experiment/config/framework/X--adanegtvae.yaml +++ /dev/null @@ -1,37 +0,0 @@ -defaults: - - _input_mode_: triplet - -name: adanegtvae - -cfg: - _target_: disent.frameworks.vae.experimental.AdaNegTripletVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - disable_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - disable_posterior_scale: NULL - # Beta-VAE - beta: ${settings.framework.beta} - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - # adavae - ada_average_mode: gvae - ada_thresh_mode: symmetric_kl # Only works for: adat_share_mask_mode == "posterior" --- kl, symmetric_kl, dist, sampled_dist - ada_thresh_ratio: 0.5 # >> USE WITH A SCHEDULE << - # ada_tvae - loss - adat_triplet_share_scale: 0.95 # >> USE WITH A SCHEDULE << only works for: adat_triplet_loss == "triplet_hard_neg_ave_scaled" - # ada_tvae - averaging - adat_share_mask_mode: posterior - -meta: - model_z_multiplier: 2 diff --git a/experiment/config/framework/X--adatvae.yaml b/experiment/config/framework/X--adatvae.yaml deleted file mode 100644 index 0f822f24..00000000 --- a/experiment/config/framework/X--adatvae.yaml +++ /dev/null @@ -1,42 +0,0 @@ -defaults: - - _input_mode_: triplet - -name: adatvae - -cfg: - _target_: disent.frameworks.vae.experimental.AdaTripletVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - disable_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - disable_posterior_scale: NULL - # Beta-VAE - beta: ${settings.framework.beta} - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - # adavae - ada_average_mode: gvae - ada_thresh_mode: symmetric_kl # Only works for: adat_share_mask_mode == "posterior" --- kl, symmetric_kl, dist, sampled_dist - ada_thresh_ratio: 0.5 # >> USE WITH A SCHEDULE << - # ada_tvae - loss - adat_triplet_loss: triplet_soft_ave_all - adat_triplet_ratio: 1.0 # >> USE WITH A SCHEDULE << 0.5 is half of triplet and ada-triplet, 1.0 is all ada-triplet - adat_triplet_soft_scale: 1.0 # >> USE WITH A SCHEDULE << - adat_triplet_pull_weight: 0.1 # Only works for: adat_triplet_loss == "triplet_hard_neg_ave_pull" - adat_triplet_share_scale: 0.95 # >> USE WITH A SCHEDULE << only works for: adat_triplet_loss == "triplet_hard_neg_ave_scaled" - # ada_tvae - averaging - adat_share_mask_mode: posterior - adat_share_ave_mode: all # Only works for: adat_triplet_loss == "triplet_hard_ave_all" - -meta: - model_z_multiplier: 2 diff --git a/experiment/config/framework/X--augpos_tvae_os.yaml b/experiment/config/framework/X--augpos_tvae_os.yaml deleted file mode 100644 index d2f72dfd..00000000 --- a/experiment/config/framework/X--augpos_tvae_os.yaml +++ /dev/null @@ -1,46 +0,0 @@ -defaults: - - _input_mode_: weak_pair - -name: augpos_tvae_os - -cfg: - _target_: disent.frameworks.vae.experimental.AugPosTripletVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - disable_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - disable_posterior_scale: NULL - # Beta-VAE - beta: ${settings.framework.beta} - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - # overlap - overlap_augment: - _target_: disent.transform.FftBoxBlur - p: 1.0 - radius: [ 16, 16 ] - random_mode: "batch" - random_same_xy: TRUE - - # TODO: try original - # overlap_augment: - # size = a_x.shape[2:4] - # self._augment = torchvision.transforms.RandomOrder([ - # kornia.augmentation.ColorJitter(brightness=0.25, contrast=0.25, saturation=0, hue=0.15), - # kornia.augmentation.RandomCrop(size=size, padding=8), - # # kornia.augmentation.RandomPerspective(distortion_scale=0.05, p=1.0), - # # kornia.augmentation.RandomRotation(degrees=4), - # ]) - -meta: - model_z_multiplier: 2 diff --git a/experiment/config/framework/X--badavae.yaml b/experiment/config/framework/X--badavae.yaml deleted file mode 100644 index 000bd7f5..00000000 --- a/experiment/config/framework/X--badavae.yaml +++ /dev/null @@ -1,27 +0,0 @@ -defaults: - - _input_mode_: triplet - -name: badavae - -cfg: - _target_: disent.frameworks.vae.experimental.BoundedAdaVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - disable_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - disable_posterior_scale: NULL - # Beta-VAE - beta: ${settings.framework.beta} - # adavae - ada_average_mode: gvae # gvae or ml-vae - ada_thresh_mode: symmetric_kl - ada_thresh_ratio: 0.5 - -meta: - model_z_multiplier: 2 diff --git a/experiment/config/framework/X--dorvae.yaml b/experiment/config/framework/X--dorvae.yaml deleted file mode 100644 index 8a2cb997..00000000 --- a/experiment/config/framework/X--dorvae.yaml +++ /dev/null @@ -1,38 +0,0 @@ -defaults: - - _input_mode_: single - -name: dor_vae - -cfg: - _target_: disent.frameworks.vae.experimental.DataOverlapRankVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - disable_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - disable_posterior_scale: NULL - # Beta-VAE - beta: ${settings.framework.beta} - # compatibility - ada_thresh_mode: dist # kl, symmetric_kl, dist, sampled_dist - ada_thresh_ratio: 0.5 - adat_triplet_share_scale: 0.95 - # dorvae - overlap_loss: ${settings.framework_opt.overlap_loss} # any of the recon_loss values, or NULL to use the recon_loss value - overlap_num: 512 - # dorvae -- representation loss - overlap_repr: deterministic # deterministic, stochastic - overlap_rank_mode: spearman_rank # spearman_rank, mse_rank - overlap_inward_pressure_masked: FALSE - overlap_inward_pressure_scale: 0.01 - # dorvae -- augment - overlap_augment_mode: 'none' - overlap_augment: NULL - -meta: - model_z_multiplier: 2 diff --git a/experiment/config/framework/X--dorvae_aug.yaml b/experiment/config/framework/X--dorvae_aug.yaml deleted file mode 100644 index a2aacc27..00000000 --- a/experiment/config/framework/X--dorvae_aug.yaml +++ /dev/null @@ -1,43 +0,0 @@ -defaults: - - _input_mode_: single - -name: dor_vae_aug - -cfg: - _target_: disent.frameworks.vae.experimental.DataOverlapRankVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - disable_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - disable_posterior_scale: NULL - # Beta-VAE - beta: ${settings.framework.beta} - # compatibility - ada_thresh_mode: dist # kl, symmetric_kl, dist, sampled_dist - ada_thresh_ratio: 0.5 - adat_triplet_share_scale: 0.95 - # dorvae - overlap_loss: ${settings.framework_opt.overlap_loss} # any of the recon_loss values, or NULL to use the recon_loss value - overlap_num: 512 - # dorvae -- representation loss - overlap_repr: deterministic # deterministic, stochastic - overlap_rank_mode: spearman_rank # spearman_rank, mse_rank - overlap_inward_pressure_masked: FALSE - overlap_inward_pressure_scale: 0.01 - # dorvae -- augment - overlap_augment_mode: 'augment' - overlap_augment: - _target_: disent.transform.FftBoxBlur - p: 1.0 - radius: [16, 16] - random_mode: "batch" - random_same_xy: TRUE - -meta: - model_z_multiplier: 2 diff --git a/experiment/config/framework/X--dotae.yaml b/experiment/config/framework/X--dotae.yaml deleted file mode 100644 index b496247a..00000000 --- a/experiment/config/framework/X--dotae.yaml +++ /dev/null @@ -1,35 +0,0 @@ -defaults: - - _input_mode_: single - -name: dotae - -cfg: - _target_: disent.frameworks.ae.experimental.DataOverlapTripletAe.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # disable various components - disable_decoder: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - # adavae - ada_thresh_ratio: 0.5 # >> USE WITH A SCHEDULE << - # ada_tvae - loss - adat_triplet_share_scale: 0.95 # >> USE WITH A SCHEDULE << only works for: adat_triplet_loss == "triplet_hard_neg_ave_scaled" - # dotvae - overlap_loss: ${settings.framework_opt.overlap_loss} # any of the recon_loss values, or NULL to use the recon_loss value - overlap_num: 512 - overlap_mine_ratio: 0.1 - overlap_mine_triplet_mode: 'none' # none, hard_neg, semi_hard_neg, hard_pos, easy_pos, ran:hard_neg+hard_pos <- etc, dynamically evaluated, can chain multiple "+"s - # dotvae -- augment - overlap_augment_mode: 'none' - overlap_augment: NULL - -meta: - model_z_multiplier: 2 diff --git a/experiment/config/framework/X--dotvae.yaml b/experiment/config/framework/X--dotvae.yaml deleted file mode 100644 index c473f15d..00000000 --- a/experiment/config/framework/X--dotvae.yaml +++ /dev/null @@ -1,45 +0,0 @@ -defaults: - - _input_mode_: single - -name: do_tvae - -cfg: - _target_: disent.frameworks.vae.experimental.DataOverlapTripletVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - disable_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - disable_posterior_scale: NULL - # Beta-VAE - beta: ${settings.framework.beta} - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - # adavae - ada_average_mode: gvae - ada_thresh_mode: dist # Only works for: adat_share_mask_mode == "posterior" --- kl, symmetric_kl, dist, sampled_dist - ada_thresh_ratio: 0.5 # >> USE WITH A SCHEDULE << - # ada_tvae - loss - adat_triplet_share_scale: 0.95 # >> USE WITH A SCHEDULE << only works for: adat_triplet_loss == "triplet_hard_neg_ave_scaled" - # ada_tvae - averaging - adat_share_mask_mode: posterior - # dotvae - overlap_loss: ${settings.framework_opt.overlap_loss} # any of the recon_loss values, or NULL to use the recon_loss value - overlap_num: 512 - overlap_mine_ratio: 0.1 - overlap_mine_triplet_mode: 'none' # none, hard_neg, semi_hard_neg, hard_pos, easy_pos, ran:hard_neg+hard_pos <- etc, dynamically evaluated, can chain multiple "+"s - # dotvae -- augment - overlap_augment_mode: 'none' - overlap_augment: NULL - -meta: - model_z_multiplier: 2 diff --git a/experiment/config/framework/X--dotvae_aug.yaml b/experiment/config/framework/X--dotvae_aug.yaml deleted file mode 100644 index df6c527d..00000000 --- a/experiment/config/framework/X--dotvae_aug.yaml +++ /dev/null @@ -1,70 +0,0 @@ -defaults: - - _input_mode_: single - -name: do_tvae_aug - -cfg: - _target_: disent.frameworks.vae.experimental.DataOverlapTripletVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - disable_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - disable_posterior_scale: NULL - # Beta-VAE - beta: ${settings.framework.beta} - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - # adavae - ada_average_mode: gvae - ada_thresh_mode: dist # Only works for: adat_share_mask_mode == "posterior" --- kl, symmetric_kl, dist, sampled_dist - ada_thresh_ratio: 0.5 # >> USE WITH A SCHEDULE << - # ada_tvae - loss - adat_triplet_share_scale: 0.95 # >> USE WITH A SCHEDULE << only works for: adat_triplet_loss == "triplet_hard_neg_ave_scaled" - # ada_tvae - averaging - adat_share_mask_mode: posterior - # dotvae - overlap_loss: ${settings.framework_opt.overlap_loss} # any of the recon_loss values, or NULL to use the recon_loss value - overlap_num: 512 - overlap_mine_ratio: 0.1 - overlap_mine_triplet_mode: 'ran:hard_neg+easy_pos' # none, hard_neg, semi_hard_neg, hard_pos, easy_pos, ran:hard_neg+hard_pos <- etc, dynamically evaluated, can chain multiple "+"s - # dotvae -- augment - overlap_augment_mode: 'augment' - overlap_augment: - _target_: disent.transform.FftKernel - kernel: xy1_r47 - -# overlap_augment: -# _target_: disent.transform.FftBoxBlur -# p: 1.0 -# radius: [16, 16] -# random_mode: "batch" -# random_same_xy: TRUE -# - _target_: disent.transform.FftGaussianBlur -# p: 1.0 -# sigma: [0.1, 10.0] -# truncate: 3.0 -# random_mode: "batch" -# random_same_xy: FALSE -# - _target_: kornia.augmentation.RandomCrop -# p: 1.0 -# size: [64, 64] -# padding: 7 -# - _target_: kornia.augmentation.RandomPerspective -# p: 0.5 -# distortion_scale: 0.15 -# - _target_: kornia.augmentation.RandomRotation -# p: 0.5 -# degrees: 9 - -meta: - model_z_multiplier: 2 diff --git a/experiment/config/framework/X--gadavae.yaml b/experiment/config/framework/X--gadavae.yaml deleted file mode 100644 index 0a830662..00000000 --- a/experiment/config/framework/X--gadavae.yaml +++ /dev/null @@ -1,29 +0,0 @@ -defaults: - - _input_mode_: triplet - -name: gadavae - -cfg: - _target_: disent.frameworks.vae.experimental.GuidedAdaVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - disable_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - disable_posterior_scale: NULL - # Beta-VAE - beta: ${settings.framework.beta} - # adavae - ada_average_mode: gvae # gvae or ml-vae - ada_thresh_mode: symmetric_kl - ada_thresh_ratio: 0.5 - # guided adavae - gada_anchor_ave_mode: 'average' # [average, thresh] - -meta: - model_z_multiplier: 2 diff --git a/experiment/config/framework/X--st-adavae.yaml b/experiment/config/framework/X--st-adavae.yaml deleted file mode 100644 index ffcaf36f..00000000 --- a/experiment/config/framework/X--st-adavae.yaml +++ /dev/null @@ -1,29 +0,0 @@ -defaults: - - _input_mode_: pair - -name: st-adavae - -cfg: - _target_: disent.frameworks.vae.experimental.SwappedTargetAdaVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - disable_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - disable_posterior_scale: NULL - # Beta-VAE - beta: ${settings.framework.beta} - # adavae - ada_average_mode: gvae # gvae or ml-vae - ada_thresh_mode: symmetric_kl - ada_thresh_ratio: 0.5 - # swapped target - swap_chance: 0.1 - -meta: - model_z_multiplier: 2 diff --git a/experiment/config/framework/X--st-betavae.yaml b/experiment/config/framework/X--st-betavae.yaml deleted file mode 100644 index d2273212..00000000 --- a/experiment/config/framework/X--st-betavae.yaml +++ /dev/null @@ -1,25 +0,0 @@ -defaults: - - _input_mode_: pair - -name: st-betavae - -cfg: - _target_: disent.frameworks.vae.experimental.SwappedTargetBetaVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - disable_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - disable_posterior_scale: NULL - # Beta-VAE - beta: ${settings.framework.beta} - # swapped target - swap_chance: 0.1 - -meta: - model_z_multiplier: 2 diff --git a/experiment/config/framework/X--tbadavae.yaml b/experiment/config/framework/X--tbadavae.yaml deleted file mode 100644 index d6b4d3ad..00000000 --- a/experiment/config/framework/X--tbadavae.yaml +++ /dev/null @@ -1,33 +0,0 @@ -defaults: - - _input_mode_: triplet - -name: tbadavae - -cfg: - _target_: disent.frameworks.vae.experimental.TripletBoundedAdaVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - disable_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - disable_posterior_scale: NULL - # Beta-VAE - beta: ${settings.framework.beta} - # adavae - ada_average_mode: gvae # gvae or ml-vae - ada_thresh_mode: symmetric_kl - ada_thresh_ratio: 0.5 - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - -meta: - model_z_multiplier: 2 diff --git a/experiment/config/framework/X--tgadavae.yaml b/experiment/config/framework/X--tgadavae.yaml deleted file mode 100644 index 5d24f7e8..00000000 --- a/experiment/config/framework/X--tgadavae.yaml +++ /dev/null @@ -1,35 +0,0 @@ -defaults: - - _input_mode_: triplet - -name: tgadavae - -cfg: - _target_: disent.frameworks.vae.experimental.TripletGuidedAdaVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - disable_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - disable_posterior_scale: NULL - # Beta-VAE - beta: ${settings.framework.beta} - # adavae - ada_average_mode: gvae # gvae or ml-vae - ada_thresh_mode: symmetric_kl - ada_thresh_ratio: 0.5 - # guided adavae - gada_anchor_ave_mode: 'average' # [average, thresh] - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - -meta: - model_z_multiplier: 2 diff --git a/experiment/config/metrics/all.yaml b/experiment/config/metrics/all.yaml index fef33bb2..cc554da5 100644 --- a/experiment/config/metrics/all.yaml +++ b/experiment/config/metrics/all.yaml @@ -1,6 +1,4 @@ metric_list: - - flatness: {} # pragma: delete-on-release - - flatness_components: {} # pragma: delete-on-release - mig: {} - sap: {} - dci: diff --git a/experiment/config/metrics/fast.yaml b/experiment/config/metrics/fast.yaml index 1d776029..71853977 100644 --- a/experiment/config/metrics/fast.yaml +++ b/experiment/config/metrics/fast.yaml @@ -1,6 +1,4 @@ metric_list: - - flatness: {} # pragma: delete-on-release - - flatness_components: {} # pragma: delete-on-release - mig: {} - sap: {} - unsupervised: {} diff --git a/experiment/config/metrics/test.yaml b/experiment/config/metrics/test.yaml index d19f2326..698ed061 100644 --- a/experiment/config/metrics/test.yaml +++ b/experiment/config/metrics/test.yaml @@ -1,8 +1,4 @@ metric_list: - - flatness: # pragma: delete-on-release - every_n_steps: 110 # pragma: delete-on-release - - flatness_components: # pragma: delete-on-release - every_n_steps: 111 # pragma: delete-on-release - mig: every_n_steps: 112 - sap: diff --git a/experiment/config/run_location/griffin.yaml b/experiment/config/run_location/griffin.yaml deleted file mode 100644 index bd7634b0..00000000 --- a/experiment/config/run_location/griffin.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# @package _global_ - -dsettings: - trainer: - cuda: TRUE - storage: - logs_dir: 'logs' - data_root: '${oc.env:HOME}/workspace/research/disent/data/dataset' - dataset: - gpu_augment: FALSE - prepare: TRUE - try_in_memory: TRUE - -trainer: - prepare_data_per_node: TRUE - -dataloader: - num_workers: 32 # max 128, more than 16 doesn't really seem to help (tested on batch size 256*3)? - pin_memory: ${dsettings.trainer.cuda} - batch_size: ${settings.dataset.batch_size} - -hydra: - job: - name: 'disent' - run: - dir: '${dsettings.storage.logs_dir}/hydra_run/${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.name}' - sweep: - dir: '${dsettings.storage.logs_dir}/hydra_sweep/${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.name}' - subdir: '${hydra.job.id}' # hydra.job.id is not available for dir diff --git a/experiment/config/run_location/heartofgold.yaml b/experiment/config/run_location/heartofgold.yaml deleted file mode 100644 index 4e5bf8e3..00000000 --- a/experiment/config/run_location/heartofgold.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# @package _global_ - -dsettings: - trainer: - cuda: TRUE - storage: - logs_dir: 'logs' - data_root: '${oc.env:HOME}/workspace/research/disent/data/dataset' - dataset: - gpu_augment: FALSE - prepare: TRUE - try_in_memory: TRUE - -trainer: - prepare_data_per_node: TRUE - -dataloader: - num_workers: 12 - pin_memory: ${dsettings.trainer.cuda} - batch_size: ${settings.dataset.batch_size} - -hydra: - job: - name: 'disent' - run: - dir: '${dsettings.storage.logs_dir}/hydra_run/${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.name}' - sweep: - dir: '${dsettings.storage.logs_dir}/hydra_sweep/${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.name}' - subdir: '${hydra.job.id}' # hydra.job.id is not available for dir diff --git a/prepare_release.sh b/prepare_release.sh deleted file mode 100755 index da71f5c2..00000000 --- a/prepare_release.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -# prepare the project for a new release -# removing all the research components -# - yes this is terrible, but at the rate things are changing I -# don't want to rip things out into a separate repo... I will -# do that eventually, but not now. - -# ====== # -# HELPER # -# ====== # - -function remove_delete_commands() { - awk "!/pragma: delete-on-release/" "$1" > "$1.temp" && mv "$1.temp" "$1" -} - -function version_greater_equal() { - printf '%s\n%s\n' "$2" "$1" | sort --check=quiet --version-sort -} - -# check that we have the right version so -# that `shopt -s globstar` does not fail -if ! version_greater_equal "$BASH_VERSION" "4"; then - echo "bash version 4 is required, got: ${BASH_VERSION}" - exit 1 -fi - -# ============ # -# DELETE FILES # -# ============ # - -# RESEARCH: -rm requirements-research.txt -rm requirements-research-freeze.txt -rm -rf research/ - -# EXPERIMENT: -rm experiment/config/config_adversarial_dataset.yaml -rm experiment/config/config_adversarial_dataset_approx.yaml -rm experiment/config/config_adversarial_kernel.yaml -rm experiment/config/run_location/griffin.yaml -rm experiment/config/run_location/heartofgold.yaml -rm experiment/config/dataset/X--*.yaml -rm experiment/config/framework/X--*.yaml - -# DISENT: -# - metrics -rm disent/metrics/_flatness.py -rm disent/metrics/_flatness_components.py -# - frameworks -rm -rf disent/frameworks/ae/experimental -rm -rf disent/frameworks/vae/experimental -# - datasets -rm disent/dataset/data/_groundtruth__xcolumns.py -rm disent/dataset/data/_groundtruth__xysquares.py -rm disent/dataset/data/_groundtruth__xyblocks.py - -# DATA: -# - disent.framework.helper -rm -rf data/adversarial_kernel - -# ===================== # -# DELETE LINES OF FILES # -# ===================== # - -# enable recursive glob -shopt -s globstar - -# scan for all files that contain 'pragma: delete-on-release' -for file in **/*.{py,yaml}; do - if [ -n "$( grep -m 1 'pragma: delete-on-release' "$file" )" ]; then - echo "preparing: $file" - remove_delete_commands "$file" - fi -done - -# ===================== # -# CLEANUP THIS FILE # -# ===================== # - -rm prepare_release.sh -rm prepare_release_and_commit.sh diff --git a/prepare_release_and_commit.sh b/prepare_release_and_commit.sh deleted file mode 100755 index 1cd513bc..00000000 --- a/prepare_release_and_commit.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - - -# ====== # -# HELPER # -# ====== # - -function version_greater_equal() { - printf '%s\n%s\n' "$2" "$1" | sort --check=quiet --version-sort -} - -# check that we have the right version so -# that `shopt -s globstar` does not fail -if ! version_greater_equal "$BASH_VERSION" "4"; then - echo "bash version 4 is required, got: ${BASH_VERSION}" - exit 1 -fi - -# ====== # -# RUN # -# ====== # - -echo "(1/3) [GIT] Creating Prepare Branch" && \ - git checkout -b xdev-prepare && \ - ( git branch --unset-upstream 2>/dev/null || true ) && \ - \ - echo "(2/3) [PREPARE]" && \ - bash ./prepare_release.sh && \ - \ - echo "(3/3) [GIT] Committing Files" && \ - git add . && \ - git commit -m "run prepare_release.sh" - -# echo "(4/4) [GIT] Merging Changes" && \ -# git checkout dev && \ -# git merge xdev-prepare diff --git a/requirements-research-freeze.txt b/requirements-research-freeze.txt deleted file mode 100644 index 4e7f5b69..00000000 --- a/requirements-research-freeze.txt +++ /dev/null @@ -1,121 +0,0 @@ -# freeze from griffin on 2021-09-29 at 15:20 -# - Python 3.8.11 is used with miniconda-latest installed with pyenv -# - There are lots of unnecessary requirements in this list -# some have been generated with side experiments and other installs -# but experiments are confirmed to work locally with this list -# SLURM, still needs to be tested an might be broken with this. -# - install with: $ pip install --no-deps --ignore-installed -r requirements-research-freeze.txt -absl-py==0.13.0 -aiohttp==3.7.4.post0 -antlr4-python3-runtime==4.8 -argcomplete==1.12.3 -async-timeout==3.0.1 -attrs==21.2.0 -beautifulsoup4==4.10.0 -cachetools==4.2.2 -certifi==2021.5.30 -chardet==4.0.0 -charset-normalizer==2.0.4 -click==8.0.1 -cloudpickle==1.6.0 -colorlog==5.0.1 -configparser==5.0.2 -coverage==5.5 -cycler==0.10.0 -deap==1.3.1 -decorator==4.4.2 -deltas==0.7.0 -Deprecated==1.2.12 -diskcache==5.2.1 -docker-pycreds==0.4.0 -evaluations==0.0.5 -filelock==3.0.12 -fsspec==2021.7.0 -future==0.18.2 -generations==1.3.0 -gitdb==4.0.7 -GitPython==3.1.18 -google-auth==1.35.0 -google-auth-oauthlib==0.4.5 -grpcio==1.39.0 -h5py==3.3.0 -hydra-colorlog==1.0.1 -hydra-core==1.0.7 -hydra-submitit-launcher==1.1.1 -idna==3.2 -imageio==2.9.0 -imageio-ffmpeg==0.4.4 -importlib-resources==5.2.2 -iniconfig==1.1.1 -joblib==1.0.1 -kiwisolver==1.3.1 -llvmlite==0.37.0 -Logbook==1.5.3 -Markdown==3.3.4 -matplotlib==3.4.3 -member==0.0.1 -moviepy==1.0.3 -msgpack==1.0.2 -multidict==5.1.0 -numba==0.54.0 -numpy==1.20.3 -oauthlib==3.1.1 -offspring==0.1.1 -omegaconf==2.0.6 -packaging==21.0 -pathtools==0.1.2 -Pillow==8.3.1 -plotly==5.3.1 -pluggy==0.13.1 -population==0.0.1 -proglog==0.1.9 -promise==2.3 -protobuf==3.17.3 -psutil==5.8.0 -py==1.10.0 -pyasn1==0.4.8 -pyasn1-modules==0.2.8 -pyDeprecate==0.3.1 -pyparsing==2.4.7 -pytest==6.2.4 -pytest-cov==2.12.1 -python-dateutil==2.8.2 -pytorch-lightning==1.4.2 -PyYAML==5.4.1 -ray==1.6.0 -redis==3.5.3 -requests==2.26.0 -requests-oauthlib==1.3.0 -rsa==4.7.2 -ruck==0.2.2 -scikit-learn==0.24.2 -scipy==1.7.1 -sentry-sdk==1.3.1 -shortuuid==1.0.1 -six==1.16.0 -sklearn-genetic==0.4.1 -smmap==4.0.0 -soupsieve==2.2.1 -submitit==1.3.3 -subprocess32==3.5.4 -tenacity==8.0.1 -tensorboard==2.6.0 -tensorboard-data-server==0.6.1 -tensorboard-plugin-wit==1.8.0 -threadpoolctl==2.2.0 -tldr==2.0.0 -toml==0.10.2 -torch==1.9.1 -torchmetrics==0.5.0 -torchsort==0.1.6 -torchvision==0.10.1 -tqdm==4.62.1 -triton==1.0.0 -typing-extensions==3.10.0.0 -urllib3==1.26.6 -wandb==0.12.0 -Werkzeug==2.0.1 -wrapt==1.12.1 -yamlconf==0.2.4 -yarl==1.6.3 -zipp==3.5.0 diff --git a/requirements-research.txt b/requirements-research.txt deleted file mode 100644 index 93d0e876..00000000 --- a/requirements-research.txt +++ /dev/null @@ -1,19 +0,0 @@ - -# TODO: these requirements need to be cleaned up! - -# MISSING DEPS - these are imported in /research, but not included here, in requirements.txt OR in requirements-experiment.txt -# ============= - -# github -# matplotlib -# psutil - - -ray>=1.6.0 -ruck==0.2.4 - -seaborn>=0.11.0 -pandas>=1.3.0 -cachier>=1.5.0 - -statsmodels>=0.13.0 # required for seaborn, to estimate outliers in regression plots diff --git a/research/__init__.py b/research/__init__.py deleted file mode 100644 index 9a05a479..00000000 --- a/research/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ diff --git a/research/clog-batch.sh b/research/clog-batch.sh deleted file mode 100644 index 79602bf8..00000000 --- a/research/clog-batch.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export PROJECT="N/A" -export USERNAME="N/A" -export PARTITION="batch" -export PARALLELISM=24 - -# source the helper file -source "$(dirname "$(realpath -s "$0")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cuda_nodes "$PARTITION" 43200 "C-disent" # 12 hours diff --git a/research/clog-stampede.sh b/research/clog-stampede.sh deleted file mode 100644 index 5d53a26e..00000000 --- a/research/clog-stampede.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export PROJECT="N/A" -export USERNAME="N/A" -export PARTITION="stampede" -export PARALLELISM=24 - -# source the helper file -source "$(dirname "$(realpath -s "$0")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cuda_nodes "$PARTITION" 43200 "C-disent" # 12 hours diff --git a/research/e00_data_traversal/plots/.gitignore b/research/e00_data_traversal/plots/.gitignore deleted file mode 100644 index e33609d2..00000000 --- a/research/e00_data_traversal/plots/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.png diff --git a/research/e00_data_traversal/run_01_all_shared_data_prepare.sh b/research/e00_data_traversal/run_01_all_shared_data_prepare.sh deleted file mode 100644 index bdd2026e..00000000 --- a/research/e00_data_traversal/run_01_all_shared_data_prepare.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash - -# This script is intended to prepare all shared data on the wits cluster -# you can probably modify it for your own purposes -# - data is loaded and processed into ~/downloads/datasets which is a -# shared drive, instead of /tmp/, which is a local drive. - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="prepare-data" -export PARTITION="stampede" -export PARALLELISM=32 - -# source the helper file -source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -DATASETS=( - cars3d - dsprites - # monte_rollouts - # mpi3d_real - # mpi3d_realistic - # mpi3d_toy - shapes3d - smallnorb - #X--adv-cars3d--WARNING - #X--adv-dsprites--WARNING - #X--adv-shapes3d--WARNING - #X--adv-smallnorb--WARNING - #X--dsprites-imagenet - #X--dsprites-imagenet-bg-20 - #X--dsprites-imagenet-bg-40 - #X--dsprites-imagenet-bg-60 - #X--dsprites-imagenet-bg-80 - #X--dsprites-imagenet-bg-100 - #X--dsprites-imagenet-fg-20 - #X--dsprites-imagenet-fg-40 - #X--dsprites-imagenet-fg-60 - #X--dsprites-imagenet-fg-80 - #X--dsprites-imagenet-fg-100 - #X--mask-adv-f-cars3d - #X--mask-adv-f-dsprites - #X--mask-adv-f-shapes3d - #X--mask-adv-f-smallnorb - #X--mask-adv-r-cars3d - #X--mask-adv-r-dsprites - #X--mask-adv-r-shapes3d - #X--mask-adv-r-smallnorb - #X--mask-dthr-cars3d - #X--mask-dthr-dsprites - #X--mask-dthr-shapes3d - #X--mask-dthr-smallnorb - #X--mask-ran-cars3d - #X--mask-ran-dsprites - #X--mask-ran-shapes3d - #X--mask-ran-smallnorb - "X--xyblocks" - #X--xyblocks_grey - "X--xysquares" - #X--xysquares_grey - #X--xysquares_rgb - xyobject - #xyobject_grey - #xyobject_shaded - #xyobject_shaded_grey -) - -local_sweep \ - run_action=prepare_data \ - run_location=stampede_shr \ - run_launcher=local \ - dataset="$(IFS=, ; echo "${DATASETS[*]}")" diff --git a/research/e00_data_traversal/run_02_plot_data_overlap.py b/research/e00_data_traversal/run_02_plot_data_overlap.py deleted file mode 100644 index 6712defb..00000000 --- a/research/e00_data_traversal/run_02_plot_data_overlap.py +++ /dev/null @@ -1,187 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import os -from typing import Optional - -import numpy as np -from matplotlib import pyplot as plt - -import research.util as H -from disent.dataset import DisentDataset -from disent.dataset.data import Cars3dData -from disent.dataset.data import DSpritesData -from disent.dataset.data import DSpritesImagenetData -from disent.dataset.data import GroundTruthData -from disent.dataset.data import SelfContainedHdf5GroundTruthData -from disent.dataset.data import Shapes3dData -from disent.dataset.data import SmallNorbData -from disent.dataset.data import XYBlocksData -from disent.dataset.data import XYObjectData -from disent.dataset.data import XYObjectShadedData -from disent.dataset.data import XYSquaresData -from disent.util.function import wrapped_partial -from disent.util.seeds import TempNumpySeed - - -# ========================================================================= # -# core # -# ========================================================================= # - - -def ensure_rgb(img: np.ndarray) -> np.ndarray: - if img.shape[-1] == 1: - img = np.concatenate([img, img, img], axis=-1) - assert img.shape[-1] == 3, f'last channel of array is not of size 3 for RGB, got shape: {tuple(img.shape)}' - return img - - -def plot_dataset_overlap( - gt_data: GroundTruthData, - f_idxs=None, - obs_max: Optional[int] = None, - obs_spacing: int = 1, - rel_path=None, - plot_base: bool = False, - plot_combined: bool = True, - plot_sidebar: bool = False, - save=True, - seed=777, - plt_scale=4.5, - offset=0.75, -): - with TempNumpySeed(seed): - # choose an f_idx - f_idx = np.random.choice(gt_data.normalise_factor_idxs(f_idxs)) - f_name = gt_data.factor_names[f_idx] - num_cols = gt_data.factor_sizes[f_idx] - # get a traversal - factors, indices, obs = gt_data.sample_random_obs_traversal(f_idx=f_idx) - # get subset - if obs_max is not None: - max_obs_spacing, i = obs_spacing, 1 - while max_obs_spacing*obs_max > len(obs): - max_obs_spacing = obs_spacing-i - i += 1 - i = max((len(obs) - obs_max*max_obs_spacing) // 2, 0) - obs = obs[i:i+obs_max*obs_spacing:max_obs_spacing][:obs_max] - # convert - obs = np.array([ensure_rgb(x) for x in obs], dtype='float32') / 255 - # compute the distances - grid = np.zeros([len(obs), len(obs), *obs[0].shape]) - for i, i_obs in enumerate(obs): - for j, j_obs in enumerate(obs): - grid[i, j] = np.abs(i_obs - j_obs) - # normalize - grid /= grid.max() - - # make figure - factors, frames, _, _, c = grid.shape - assert c == 3 - - if plot_base: - # plot - fig, axs = H.plt_subplots_imshow(grid, label_size=18, title_size=24, title=f'{gt_data.name}: {f_name}', subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) - # save figure - if save and (rel_path is not None): - path = H.make_rel_path_add_ext(rel_path, ext='.png') - plt.savefig(path) - print(f'saved: {repr(path)}') - plt.show() - - if plot_combined: - # add obs - if True: - factors += 1 - frames += 1 - # scaled_obs = obs - scaled_obs = obs * 0.5 + 0.25 - # grid = 1 - grid - # grid = grid * 0.5 + 0.25 - grid = np.concatenate([scaled_obs[None, :], grid], axis=0) - add_row = np.concatenate([np.ones_like(obs[0:1]), scaled_obs], axis=0) - grid = np.concatenate([grid, add_row[:, None]], axis=1) - # plot - fig, axs = H.plt_subplots_imshow(grid, label_size=18, title_size=24, row_labels=["traversal"] + (["diff."] * len(obs)), col_labels=(["diff."] * len(obs)) + ["traversal"], title=f'{gt_data.name}: {f_name}', subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) - # save figure - if save and (rel_path is not None): - path = H.make_rel_path_add_ext(rel_path + '__combined', ext='.png') - plt.savefig(path) - print(f'saved: {repr(path)}') - plt.show() - - # plot - if plot_sidebar: - fig, axs = H.plt_subplots_imshow(obs[:, None], subplot_padding=None, figsize=(offset + (1/2.54)*1*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) - if save and (rel_path is not None): - path = H.make_rel_path_add_ext(rel_path + '__v', ext='.png') - plt.savefig(path) - print(f'saved: {repr(path)}') - plt.show() - fig, axs = H.plt_subplots_imshow(obs[None, :], subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(1+0.45)*plt_scale)) - if save and (rel_path is not None): - path = H.make_rel_path_add_ext(rel_path + '__h', ext='.png') - plt.savefig(path) - print(f'saved: {repr(path)}') - plt.show() - - -# ========================================================================= # -# entrypoint # -# ========================================================================= # - - -if __name__ == '__main__': - - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../gadfly.mplstyle')) - - # options - all_squares = True - add_random_traversal = True - num_cols = 7 - seed = 48 - - for gt_data_cls, name in [ - (wrapped_partial(XYSquaresData, grid_spacing=1, grid_size=8, no_warnings=True), f'xy-squares-spacing1'), - (wrapped_partial(XYSquaresData, grid_spacing=2, grid_size=8, no_warnings=True), f'xy-squares-spacing2'), - (wrapped_partial(XYSquaresData, grid_spacing=4, grid_size=8, no_warnings=True), f'xy-squares-spacing4'), - (wrapped_partial(XYSquaresData, grid_spacing=8, grid_size=8, no_warnings=True), f'xy-squares-spacing8'), - ]: - plot_dataset_overlap(gt_data_cls(), rel_path=f'plots/overlap__{name}', obs_max=3, obs_spacing=4, seed=seed-40) - - for gt_data_cls, name in [ - (DSpritesData, f'dsprites'), - (Shapes3dData, f'shapes3d'), - (Cars3dData, f'cars3d'), - (SmallNorbData, f'smallnorb'), - ]: - gt_data = gt_data_cls() - for f_idx, f_name in enumerate(gt_data.factor_names): - plot_dataset_overlap(gt_data, rel_path=f'plots/overlap__{name}__f{f_idx}-{f_name}', obs_max=3, obs_spacing=4, f_idxs=f_idx, seed=seed) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/e00_data_traversal/run_02_plot_traversals.py b/research/e00_data_traversal/run_02_plot_traversals.py deleted file mode 100644 index 15142441..00000000 --- a/research/e00_data_traversal/run_02_plot_traversals.py +++ /dev/null @@ -1,255 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import os -from typing import Optional -from typing import Sequence -from typing import Union - -import numpy as np -from matplotlib import pyplot as plt - -import research.util as H -from disent.dataset import DisentDataset -from disent.dataset.data import Cars3dData -from disent.dataset.data import DSpritesData -from disent.dataset.data import DSpritesImagenetData -from disent.dataset.data import GroundTruthData -from disent.dataset.data import SelfContainedHdf5GroundTruthData -from disent.dataset.data import Shapes3dData -from disent.dataset.data import SmallNorbData -from disent.dataset.data import XYBlocksData -from disent.dataset.data import XYObjectData -from disent.dataset.data import XYObjectShadedData -from disent.dataset.data import XYSquaresData -from disent.util.seeds import TempNumpySeed - - -# ========================================================================= # -# core # -# ========================================================================= # - - -def ensure_rgb(img: np.ndarray) -> np.ndarray: - if img.shape[-1] == 1: - img = np.concatenate([img, img, img], axis=-1) - assert img.shape[-1] == 3, f'last channel of array is not of size 3 for RGB, got shape: {tuple(img.shape)}' - return img - - -def plot_dataset_traversals( - gt_data: GroundTruthData, - f_idxs=None, - num_cols: Optional[int] = 8, - take_cols: Optional[int] = None, - base_factors=None, - add_random_traversal: bool = True, - pad: int = 8, - bg_color: int = 127, - border: bool = False, - rel_path: str = None, - save: bool = True, - seed: int = 777, - plt_scale: float = 4.5, - offset: float = 0.75, - transpose: bool = False, - title: Union[bool, str] = True, - label_size: int = 22, - title_size: int = 26, - labels_at_top: bool = False, -): - if take_cols is not None: - assert take_cols >= num_cols - # convert - dataset = DisentDataset(gt_data) - f_idxs = gt_data.normalise_factor_idxs(f_idxs) - num_cols = num_cols if (num_cols is not None) else min(max(gt_data.factor_sizes), 32) - # get traversal grid - row_labels = [gt_data.factor_names[i] for i in f_idxs] - grid, _, _ = H.visualize_dataset_traversal( - dataset=dataset, - data_mode='raw', - factor_names=f_idxs, - num_frames=num_cols if (take_cols is None) else take_cols, - seed=seed, - base_factors=base_factors, - traverse_mode='interval', - pad=pad, - bg_color=bg_color, - border=border, - ) - if take_cols is not None: - grid = grid[:, :num_cols, ...] - # add random traversal - if add_random_traversal: - with TempNumpySeed(seed): - row_labels = ['random'] + row_labels - row = dataset.dataset_sample_batch(num_samples=num_cols, mode='raw')[None, ...] # torch.Tensor - grid = np.concatenate([ensure_rgb(row), grid]) - # make figure - factors, frames, _, _, c = grid.shape - assert c == 3 - - # get title - if isinstance(title, bool): - title = gt_data.name if title else None - - if transpose: - col_titles = None - if labels_at_top: - col_titles, row_labels = row_labels, None - fig, axs = H.plt_subplots_imshow(np.swapaxes(grid, 0, 1), label_size=label_size, title_size=title_size, title=title, titles=col_titles, titles_size=label_size, col_labels=row_labels, subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)[::-1]) - else: - fig, axs = H.plt_subplots_imshow(grid, label_size=label_size, title_size=title_size, title=title, row_labels=row_labels, subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) - - # save figure - if save and (rel_path is not None): - path = H.make_rel_path_add_ext(rel_path, ext='.png') - plt.savefig(path) - print(f'saved: {repr(path)}') - plt.show() - # done! - return fig, axs - - -def plot_incr_overlap( - rel_path: Optional[str] = None, - spacings: Union[Sequence[int], bool] = False, - seed: int = 777, - fidx: int = 1, - traversal_size: int = 8, - traversal_lim: Optional[int] = None, - save: bool = True, - show: bool = True -): - if isinstance(spacings, bool): - spacings = ([1, 2, 3, 4, 5, 6, 7, 8] if spacings else [1, 4, 8]) - - if traversal_lim is None: - traversal_lim = traversal_size - assert traversal_size >= traversal_lim - - grid = [] - for s in spacings: - data = XYSquaresData(grid_spacing=s, grid_size=8, no_warnings=True) - with TempNumpySeed(seed): - factors, indices, obs = data.sample_random_obs_traversal(f_idx=data.normalise_factor_idx(fidx), num=traversal_size, mode='interval') - grid.append(obs[:traversal_lim]) - - w, h = traversal_lim * 2.54, len(spacings) * 2.54 - fig, axs = H.plt_subplots_imshow(grid, row_labels=[f'Space: {s}px' for s in spacings], figsize=(w, h), label_size=24) - fig.tight_layout() - - H.plt_rel_path_savefig(rel_path=rel_path, save=save, show=show) - - -# ========================================================================= # -# entrypoint # -# ========================================================================= # - - -if __name__ == '__main__': - - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../gadfly.mplstyle')) - - # options - all_squares = False - num_cols = 7 - mini_cols = 5 - transpose_cols = 3 - seed = 47 - - INCLUDE_RANDOM_TRAVERSAL = False - TITLE = False - TITLE_MINI = False - TITLE_TRANSPOSE = False - - # get name - prefix = 'traversal' if INCLUDE_RANDOM_TRAVERSAL else 'traversal-noran' - - # plot increasing levels of overlap - plot_incr_overlap(rel_path=f'plots/traversal-incr-overlap__xy-squares', save=True, show=True, traversal_lim=None) - - # mini versions - plot_dataset_traversals(XYSquaresData(), rel_path=f'plots/traversal-mini__xy-squares__spacing8', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(Shapes3dData(), rel_path=f'plots/traversal-mini__shapes3d', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(DSpritesData(), rel_path=f'plots/traversal-mini__dsprites', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(SmallNorbData(), rel_path=f'plots/traversal-mini__smallnorb', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(Cars3dData(), rel_path=f'plots/traversal-mini__cars3d', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols, take_cols=mini_cols+1) - - # transpose versions - plot_dataset_traversals(XYSquaresData(), rel_path=f'plots/traversal-transpose__xy-squares__spacing8', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(Shapes3dData(), rel_path=f'plots/traversal-transpose__shapes3d', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(DSpritesData(), rel_path=f'plots/traversal-transpose__dsprites', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(SmallNorbData(), rel_path=f'plots/traversal-transpose__smallnorb', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(Cars3dData(), rel_path=f'plots/traversal-transpose__cars3d', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols, take_cols=mini_cols+1) - - # save images - for i in ([1, 2, 3, 4, 5, 6, 7, 8] if all_squares else [1, 2, 4, 8]): - data = XYSquaresData(grid_spacing=i, grid_size=8, no_warnings=True) - plot_dataset_traversals(data, rel_path=f'plots/{prefix}__xy-squares__spacing{i}', title=TITLE, seed=seed-40, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(data, rel_path=f'plots/{prefix}__xy-squares__spacing{i}__some', title=TITLE, seed=seed-40, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols, f_idxs=[0, 3]) - - plot_dataset_traversals(Shapes3dData(), rel_path=f'plots/{prefix}__shapes3d', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(DSpritesData(), rel_path=f'plots/{prefix}__dsprites', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(SmallNorbData(), rel_path=f'plots/{prefix}__smallnorb', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(Cars3dData(), rel_path=f'plots/{prefix}__cars3d', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - - exit(1) - - plot_dataset_traversals(XYObjectData(), rel_path=f'plots/{prefix}__xy-object', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(XYObjectShadedData(), rel_path=f'plots/{prefix}__xy-object-shaded', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(XYBlocksData(), rel_path=f'plots/{prefix}__xy-blocks', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - - plot_dataset_traversals(DSpritesImagenetData(100, 'bg'), rel_path=f'plots/{prefix}__dsprites-imagenet-bg-100', title=TITLE, seed=seed-6, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(DSpritesImagenetData( 50, 'bg'), rel_path=f'plots/{prefix}__dsprites-imagenet-bg-50', title=TITLE, seed=seed-6, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(DSpritesImagenetData(100, 'fg'), rel_path=f'plots/{prefix}__dsprites-imagenet-fg-100', title=TITLE, seed=seed-6, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(DSpritesImagenetData( 50, 'fg'), rel_path=f'plots/{prefix}__dsprites-imagenet-fg-50', title=TITLE, seed=seed-6, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - - BASE = os.path.abspath(os.path.join(__file__, '../../../out/adversarial_data_approx')) - - for folder in [ - # 'const' datasets - ('2021-08-18--00-58-22_FINAL-dsprites_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('2021-08-18--01-33-47_FINAL-shapes3d_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('2021-08-18--02-20-13_FINAL-cars3d_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('2021-08-18--03-10-53_FINAL-smallnorb_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - # 'invert' datasets - ('2021-08-18--03-52-31_FINAL-dsprites_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('2021-08-18--04-29-25_FINAL-shapes3d_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('2021-08-18--05-13-15_FINAL-cars3d_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('2021-08-18--06-03-32_FINAL-smallnorb_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - # stronger 'invert' datasets - ('2021-09-06--00-29-23_INVERT-VSTRONG-shapes3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), - ('2021-09-06--03-17-28_INVERT-VSTRONG-dsprites_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), - ('2021-09-06--05-42-06_INVERT-VSTRONG-cars3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), - ('2021-09-06--09-10-59_INVERT-VSTRONG-smallnorb_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), - ]: - plot_dataset_traversals(SelfContainedHdf5GroundTruthData(f'{BASE}/{folder}/data.h5'), rel_path=f'plots/{prefix}__{folder}.png', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/e00_tuning/submit_param_tuning.sh b/research/e00_tuning/submit_param_tuning.sh deleted file mode 100644 index 4f554dfa..00000000 --- a/research/e00_tuning/submit_param_tuning.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash - - -# OVERVIEW: -# - this experiment is designed to find the optimal hyper-parameters for disentanglement, as well as investigate the -# effect of the adversarial XYSquares dataset against existing approaches. - - -# OUTCOMES: -# - Existing frameworks fail on the adversarial dataset -# - Much lower beta is required for adversarial dataset - - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="CVPR-00__basic-hparam-tuning" -export PARTITION="stampede" -export PARALLELISM=28 - -# source the helper file -source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 129600 "C-disent" # 36 hours - -# RUN SWEEP FOR GOOD BETA VALUES -# - beta: 0.01, 0.0316 seem good, 0.1 starts getting too strong, 0.00316 is a bit weak -# - z_size: higher means you can increase beta, eg. 25: beta=0.1 and 9: beta=0.01 -# - framework: adavae needs lower beta, eg. betavae: 0.1, adavae25: 0.0316, adavae9: 0.00316 -# - xy_squares really struggles to learn when non-overlapping, beta needs to be very low. -# might be worth using a warmup schedule -# betavae with zsize=25 and beta<=0.00316 -# betavae with zsize=09 and beta<=0.000316 -# adavae with zsize=25 does not work -# adavae with zsize=09 and beta<=0.001 (must get very lucky) - -# TODO: I should try lower the learning rate to 1e-4 from 1e-3, this might help with xysquares -# 1 * (2 * 8 * 2 * 5) = 160 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_beta' \ - hydra.job.name="vae_hparams" \ - \ - run_length=long \ - metrics=all \ - \ - settings.framework.beta=0.000316,0.001,0.00316,0.01,0.0316,0.1,0.316,1.0 \ - framework=betavae,adavae_os \ - schedule=none \ - settings.model.z_size=9,25 \ - \ - dataset=dsprites,shapes3d,cars3d,smallnorb,X--xysquares \ - sampling=default__bb - - -# TEST DISTANCES IN AEs VS VAEs -# -- supplementary material -# 3 * (1 * 5 = 2) = 15 -submit_sweep \ - +DUMMY.repeat=1,2,3 \ - +EXTRA.tags='sweep_ae' \ - hydra.job.name="ae_test" \ - \ - run_length=medium \ - metrics=all \ - \ - settings.framework.beta=0.0001 \ - framework=ae \ - schedule=none \ - settings.model.z_size=25 \ - \ - dataset=dsprites,shapes3d,cars3d,smallnorb,X--xysquares \ - sampling=default__bb - - -# RUN SWEEP FOR GOOD SCHEDULES -# -- unused -# 1 * (3 * 2 * 4 * 5) = 120 -#submit_sweep \ -# +DUMMY.repeat=1 \ -# +EXTRA.tags='sweep_schedule' \ -# \ -# run_length=long \ -# metrics=all \ -# \ -# settings.framework.beta=0.1,0.316,1.0 \ -# framework=betavae,adavae_os \ -# schedule=beta_cyclic,beta_cyclic_slow,beta_cyclic_fast,beta_decrease \ -# settings.model.z_size=25 \ -# \ -# dataset=dsprites,shapes3d,cars3d,smallnorb,X--xysquares \ -# sampling=default__bb diff --git a/research/e01_incr_overlap/run.py b/research/e01_incr_overlap/run.py deleted file mode 100644 index 9a96337a..00000000 --- a/research/e01_incr_overlap/run.py +++ /dev/null @@ -1,72 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - - -import numpy as np -from disent.dataset.data import XYSquaresData - - -class XYSquaresSampler(XYSquaresData): - - def sample_1d_boxes(self, size=None): - size = (2,) if (size is None) else ((size, 2) if isinstance(size, int) else (*size, 2)) - # sample x0, y0 - s0 = self._offset + self._spacing * np.random.randint(0, self._placements, size=size) - # sample x1, y1 - s1 = s0 + self._square_size - # return (x0, y0), (x1, y1) - return s0, s1 - - def sample_1d_overlap(self, size=None): - s0, s1 = self.sample_1d_boxes(size=size) - # compute overlap - return np.maximum(np.min(s1, axis=-1) - np.max(s0, axis=-1), 0) - - def sample_1d_delta(self, size=None): - s0, s1 = self.sample_1d_boxes(size=size) - # compute differences - l_delta = np.max(s0, axis=-1) - np.min(s0, axis=-1) - r_delta = np.max(s1, axis=-1) - np.min(s1, axis=-1) - # return delta - return np.minimum(l_delta + r_delta, self._square_size * 2) - - -if __name__ == '__main__': - - print('\nDecreasing Spacing & Increasing Size') - for ss, gs in [(8, 8), (9, 7), (17, 6), (25, 5), (33, 4), (41, 3), (49, 2), (57, 1)][::-1]: - d = XYSquaresSampler(square_size=ss, grid_spacing=gs, max_placements=8, no_warnings=True) - print('ss={:2d} gs={:1d} overlap={:7.4f} delta={:7.4f}'.format(ss, gs, d.sample_1d_overlap(size=1_000_000).mean(), d.sample_1d_delta(size=1_000_000).mean())) - - print('\nDecreasing Spacing') - for i in range(8): - ss, gs = 8, 8-i - d = XYSquaresSampler(square_size=ss, grid_spacing=gs, max_placements=8, no_warnings=True) - print('ss={:2d} gs={:1d} overlap={:7.4f} delta={:7.4f}'.format(ss, gs, d.sample_1d_overlap(size=1_000_000).mean(), d.sample_1d_delta(size=1_000_000).mean())) - - print('\nDecreasing Spacing & Keeping Dimension Size Constant') - for i in range(8): - ss, gs = 8, 8-i - d = XYSquaresSampler(square_size=ss, grid_spacing=gs, max_placements=None, no_warnings=True) - print('ss={:2d} gs={:1d} overlap={:7.4f} delta={:7.4f}'.format(ss, gs, d.sample_1d_overlap(size=1_000_000).mean(), d.sample_1d_delta(size=1_000_000).mean())) diff --git a/research/e01_incr_overlap/submit_incr_overlap.sh b/research/e01_incr_overlap/submit_incr_overlap.sh deleted file mode 100644 index 49c28043..00000000 --- a/research/e01_incr_overlap/submit_incr_overlap.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/bin/bash - - -# OVERVIEW: -# - this experiment is designed to check how increasing overlap (reducing -# the spacing between square positions on XYSquares) affects learning. - - -# OUTCOMES: -# - increasing overlap improves disentanglement & ability for the -# neural network to learn values. -# - decreasing overlap worsens disentanglement, but it also becomes -# very hard for the neural net to learn specific values needed. The -# average image does not correspond well to individual samples. -# Disentanglement performance is also a result of this fact, as -# the network can't always learn the dataset effectively. - - -# ========================================================================= # -# Settings # -# ========================================================================= # - - -export USERNAME="n_michlo" -export PROJECT="CVPR-01__incr_overlap" -export PARTITION="stampede" -export PARALLELISM=28 - -# source the helper file -source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" - - -# ========================================================================= # -# Experiment # -# ========================================================================= # - - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - - -# background launch various xysquares -# -- original experiment also had dfcvae -# -- beta is too high for adavae -# 5 * (2*2*8 = 32) = 160 -submit_sweep \ - +DUMMY.repeat=1,2,3,4,5 \ - +EXTRA.tags='sweep_xy_squares_overlap' \ - hydra.job.name="incr_ovlp" \ - \ - run_length=medium \ - metrics=all \ - \ - settings.framework.beta=0.001,0.00316 \ - framework=betavae,adavae_os \ - settings.model.z_size=9 \ - \ - sampling=default__bb \ - dataset=X--xysquares_rgb \ - dataset.data.grid_spacing=8,7,6,5,4,3,2,1 - - -# background launch various xysquares -# -- original experiment also had dfcvae -# -- beta is too high for adavae -# 5 * (2*8 = 16) = 80 -submit_sweep \ - +DUMMY.repeat=1,2,3,4,5 \ - +EXTRA.tags='sweep_xy_squares_overlap_small_beta' \ - hydra.job.name="sb_incr_ovlp" \ - \ - run_length=medium \ - metrics=all \ - \ - settings.framework.beta=0.0001,0.00001 \ - framework=adavae_os \ - settings.model.z_size=9 \ - \ - sampling=default__bb \ - dataset=X--xysquares_rgb \ - dataset.data.grid_spacing=8,7,6,5,4,3,2,1 - - -# background launch various xysquares -# - this time we try delay beta, so that it can learn properly... -# - NOTE: this doesn't actually work, the VAE loss often becomes -# NAN because the values are too small. -# 3 * (2*2*8 = 32) = 96 -# submit_sweep \ -# +DUMMY.repeat=1,2,3 \ -# +EXTRA.tags='sweep_xy_squares_overlap_delay' \ -# hydra.job.name="schd_incr_ovlp" \ -# \ -# schedule=beta_delay_long \ -# \ -# run_length=medium \ -# metrics=all \ -# \ -# settings.framework.beta=0.001 \ -# framework=betavae,adavae_os \ -# settings.model.z_size=9,25 \ -# \ -# sampling=default__bb \ -# dataset=X--xysquares_rgb \ -# dataset.data.grid_spacing=8,7,6,5,4,3,2,1 - - -# background launch traditional datasets -# -- original experiment also had dfcvae -# 5 * (2*2*4 = 16) = 80 -#submit_sweep \ -# +DUMMY.repeat=1,2,3,4,5 \ -# +EXTRA.tags='sweep_other' \ -# \ -# run_length=medium \ -# metrics=all \ -# \ -# settings.framework.beta=0.01,0.0316 \ -# framework=betavae,adavae_os \ -# settings.model.z_size=9 \ -# \ -# sampling=default__bb \ -# dataset=cars3d,shapes3d,dsprites,smallnorb - - -# ========================================================================= # -# DONE # -# ========================================================================= # diff --git a/research/e01_visual_overlap/plots/.gitignore b/research/e01_visual_overlap/plots/.gitignore deleted file mode 100644 index e33609d2..00000000 --- a/research/e01_visual_overlap/plots/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.png diff --git a/research/e01_visual_overlap/run_01_x_z_recon_dists.sh b/research/e01_visual_overlap/run_01_x_z_recon_dists.sh deleted file mode 100644 index 5abdc019..00000000 --- a/research/e01_visual_overlap/run_01_x_z_recon_dists.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-01__gt-vs-learnt-dists" -export PARTITION="stampede" -export PARALLELISM=28 - -# source the helper file -source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - - -# 1 * (3 * 6 * 4 * 2) = 144 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep' \ - \ - model=linear,vae_fc,vae_conv64 \ - \ - run_length=medium \ - metrics=all \ - \ - dataset=xyobject,xyobject_shaded,shapes3d,dsprites,cars3d,smallnorb \ - sampling=default__bb \ - framework=ae,X--adaae_os,betavae,adavae_os \ - \ - settings.framework.beta=0.0316 \ - settings.optimizer.lr=3e-4 \ - settings.model.z_size=9,25 diff --git a/research/e01_visual_overlap/run_plot_global_dists.py b/research/e01_visual_overlap/run_plot_global_dists.py deleted file mode 100644 index 08f1c239..00000000 --- a/research/e01_visual_overlap/run_plot_global_dists.py +++ /dev/null @@ -1,465 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - - -import os -from collections import defaultdict -from typing import Dict - -import seaborn as sns -import numpy as np -import pandas as pd -import torch -from matplotlib import pyplot as plt -from matplotlib.ticker import MultipleLocator -from tqdm import tqdm - -import research.util as H -from disent.dataset import DisentDataset -from disent.dataset.data import Cars3dData -from disent.dataset.data import DSpritesData -from disent.dataset.data import Shapes3dData -from disent.dataset.data import XYSquaresData -from disent.dataset.transform import ToImgTensorF32 -from disent.util import to_numpy -from disent.util.function import wrapped_partial - - -# ========================================================================= # -# plot # -# ========================================================================= # - - -def plot_overlap(a, b, mode='abs'): - a, b = np.transpose(to_numpy(a), (1, 2, 0)), np.transpose(to_numpy(b), (1, 2, 0)) - if mode == 'binary': - d = np.float32(a != b) - elif mode == 'abs': - d = np.abs(a - b) - elif mode == 'diff': - d = a - b - else: - raise KeyError - d = (d - d.min()) / (d.max() - d.min()) - a, b, d = np.uint8(a * 255), np.uint8(b * 255), np.uint8(d * 255) - fig, (ax_a, ax_b, ax_d) = plt.subplots(1, 3) - ax_a.imshow(a) - ax_b.imshow(b) - ax_d.imshow(d) - plt.show() - - -# ========================================================================= # -# CORE # -# ========================================================================= # - - -def generate_data(dataset: DisentDataset, data_name: str, batch_size=64, samples=100_000, plot_diffs=False, load_cache=True, save_cache=True, overlap_loss: str = 'mse'): - # cache - file_path = os.path.join(os.path.dirname(__file__), f'cache/{data_name}_{samples}.pkl') - if load_cache: - if os.path.exists(file_path): - print(f'loaded: {file_path}') - return pd.read_pickle(file_path, compression='gzip') - - # generate - with torch.no_grad(): - # dataframe - df = defaultdict(lambda: defaultdict(list)) - - # randomly overlapped data - name = 'random' - for i in tqdm(range((samples + (batch_size-1) - 1) // (batch_size-1)), desc=f'{data_name}: {name}'): - # get random batch of unique elements - idxs = H.sample_unique_batch_indices(num_obs=len(dataset), num_samples=batch_size) - batch = dataset.dataset_batch_from_indices(idxs, mode='input') - # plot - if plot_diffs and (i == 0): - plot_overlap(batch[0], batch[1]) - # store overlap results - o = to_numpy(H.pairwise_overlap(batch[:-1], batch[1:], mode=overlap_loss)) - df[True][name].extend(o) - df[False][name].extend(o) - - # traversal overlaps - for f_idx in range(dataset.gt_data.num_factors): - name = f'f_{dataset.gt_data.factor_names[f_idx]}' - for i in tqdm(range((samples + (dataset.gt_data.factor_sizes[f_idx] - 1) - 1) // (dataset.gt_data.factor_sizes[f_idx] - 1)), desc=f'{data_name}: {name}'): - # get random batch that is a factor traversal - factors = dataset.gt_data.sample_random_factor_traversal(f_idx) - batch = dataset.dataset_batch_from_factors(factors, mode='input') - # shuffle indices - idxs = np.arange(len(factors)) - np.random.shuffle(idxs) - # plot - if plot_diffs and (i == 0): plot_overlap(batch[0], batch[1]) - # store overlap results - df[True][name].extend(to_numpy(H.pairwise_overlap(batch[:-1], batch[1:], mode=overlap_loss))) - df[False][name].extend(to_numpy(H.pairwise_overlap(batch[idxs[:-1]], batch[idxs[1:]], mode=overlap_loss))) - - # make dataframe! - df = pd.DataFrame({ - 'overlap': [d for ordered, data in df.items() for name, dat in data.items() for d in dat], - 'samples': [name for ordered, data in df.items() for name, dat in data.items() for d in dat], - 'ordered': [ordered for ordered, data in df.items() for name, dat in data.items() for d in dat], - 'data': [data_name for ordered, data in df.items() for name, dat in data.items() for d in dat], - }) - - # save into cache - if save_cache: - os.makedirs(os.path.dirname(file_path), exist_ok=True) - df.to_pickle(file_path, compression='gzip') - print(f'cached: {file_path}') - - return df - - -# ========================================================================= # -# plotting # -# ========================================================================= # - - -def dual_plot_from_generated_data(df: pd.DataFrame, data_name: str = None, save_name: str = None, tick_size: float = None, fig_l_pad=1, fig_w=7, fig_h=13): - # make subplots - cm = 1 / 2.54 - fig, (ax0, ax1) = plt.subplots(1, 2, figsize=((fig_l_pad+2*fig_w)*cm, fig_h*cm)) - if data_name is not None: - fig.suptitle(data_name, fontsize=20) - ax0.set_ylim(-0.025, 1.025) - ax1.set_ylim(-0.025, 1.025) - # plot - ax0.set_title('Ordered Traversals') - sns.ecdfplot(ax=ax0, data=df[df['ordered']==True], x="distance", hue="samples") - ax1.set_title('Shuffled Traversals') - sns.ecdfplot(ax=ax1, data=df[df['ordered']==False], x="distance", hue="samples") - # edit plots - ax0.set_xlabel('Visual Distance') - ax1.set_xlabel('Visual Distance') - if tick_size is not None: - ax0.xaxis.set_major_locator(MultipleLocator(base=tick_size)) - ax1.xaxis.set_major_locator(MultipleLocator(base=tick_size)) - # ax0.xaxis.set_major_formatter(FormatStrFormatter('%.2f')) - # ax1.xaxis.set_major_formatter(FormatStrFormatter('%.2f')) - ax0.set_ylabel('Cumulative Proportion') - ax1.set_ylabel(None) - ax1.set_yticklabels([]) - ax1.get_legend().remove() - plt.tight_layout() - # save - if save_name is not None: - path = os.path.join(os.path.dirname(__file__), 'plots', save_name) - os.makedirs(os.path.dirname(path), exist_ok=True) - plt.savefig(path) - print(f'saved: {path}') - # show - return fig - - -def all_plot_from_all_generated_data(dfs: dict, ordered=True, save_name: str = None, tick_sizes: Dict[str, float] = None, hide_extra_legends=False, fig_l_pad=1, fig_w=7, fig_h=13): - if not dfs: - return None - # make subplots - cm = 1 / 2.54 - fig, axs = plt.subplots(1, len(dfs), figsize=((fig_l_pad+len(dfs)*fig_w)*cm, fig_h * cm)) - axs = np.array(axs, dtype=np.object).reshape((-1,)) - # plot all - for i, (ax, (data_name, df)) in enumerate(zip(axs, dfs.items())): - # plot - ax.set_title(data_name) - sns.ecdfplot(ax=ax, data=df[df['ordered']==ordered], x="distance", hue="samples") - # edit plots - ax.set_ylim(-0.025, 1.025) - ax.set_xlabel('Visual Distance') - if (tick_sizes is not None) and (data_name in tick_sizes): - ax.xaxis.set_major_locator(MultipleLocator(base=tick_sizes[data_name])) - if i == 0: - ax.set_ylabel('Cumulative Proportion') - else: - if hide_extra_legends: - ax.get_legend().remove() - ax.set_ylabel(None) - ax.set_yticklabels([]) - plt.tight_layout() - # save - if save_name is not None: - path = os.path.join(os.path.dirname(__file__), 'plots', save_name) - os.makedirs(os.path.dirname(path), exist_ok=True) - plt.savefig(path) - print(f'saved: {path}') - # show - return fig - - -def plot_all(exp_name: str, gt_data_classes, tick_sizes: dict, samples: int, load=True, save=True, show_plt=True, show_dual_plt=False, save_plt=True, hide_extra_legends=False, fig_l_pad=1, fig_w=7, fig_h=13): - # generate data and plot! - dfs = {} - for data_name, data_cls in gt_data_classes.items(): - df = generate_data( - DisentDataset(data_cls(), transform=ToImgTensorF32()), - data_name, - batch_size=64, - samples=samples, - plot_diffs=False, - load_cache=load, - save_cache=save, - ) - dfs[data_name] = df - # flip overlap - df['distance'] = - df['overlap'] - del df['overlap'] - # plot ordered + shuffled - fig = dual_plot_from_generated_data( - df, - data_name=data_name, - save_name=f'{exp_name}/{data_name}_{samples}.png' if save_plt else None, - tick_size=tick_sizes.get(data_name, None), - fig_l_pad=fig_l_pad, - fig_w=fig_w, - fig_h=fig_h, - ) - - if show_dual_plt: - plt.show() - else: - plt.close(fig) - - def _all_plot_generated(dfs, ordered: bool, suffix: str): - fig = all_plot_from_all_generated_data( - dfs, - ordered=ordered, - save_name=f'{exp_name}/{exp_name}-{"ordered" if ordered else "shuffled"}{suffix}.png' if save_plt else None, - tick_sizes=tick_sizes, - hide_extra_legends=hide_extra_legends, - fig_l_pad=fig_l_pad, - fig_w=fig_w, - fig_h=fig_h, - ) - if show_plt: - plt.show() - else: - plt.close(fig) - - # all ordered plots - _all_plot_generated(dfs, ordered=True, suffix='') - _all_plot_generated({k: v for k, v in dfs.items() if k.lower().startswith('xy')}, ordered=True, suffix='-xy') - _all_plot_generated({k: v for k, v in dfs.items() if not k.lower().startswith('xy')}, ordered=True, suffix='-normal') - # all shuffled plots - _all_plot_generated(dfs, ordered=False, suffix='') - _all_plot_generated({k: v for k, v in dfs.items() if k.lower().startswith('xy')}, ordered=False, suffix='-xy') - _all_plot_generated({k: v for k, v in dfs.items() if not k.lower().startswith('xy')}, ordered=False, suffix='-normal') - # done! - return dfs - - -def plot_dfs_stacked(dfs, title: str, save_name: str = None, show_plt=True, tick_size: float = None, fig_l_pad=1, fig_w=7, fig_h=13, **kwargs): - # make new dataframe - df = pd.concat((df[df['samples']=='random'] for df in dfs.values())) - # make plot - cm = 1 / 2.54 - fig, ax = plt.subplots(1, 1, figsize=((fig_l_pad+1*fig_w)*cm, fig_h*cm)) - ax.set_title(title) - # plot - # sns.kdeplot(ax=ax, data=df, x="overlap", hue="data", bw_adjust=2) - sns.ecdfplot(ax=ax, data=df, x="overlap", hue="data") - # edit settins - # ax.set_ylim(-0.025, 1.025) - ax.set_xlabel('Overlap') - if tick_size is not None: - ax.xaxis.set_major_locator(MultipleLocator(base=tick_size)) - ax.set_ylabel('Cumulative Proportion') - plt.tight_layout() - # save - if save_name is not None: - path = os.path.join(os.path.dirname(__file__), 'plots', save_name) - os.makedirs(os.path.dirname(path), exist_ok=True) - plt.savefig(path) - print(f'saved: {path}') - # show - if show_plt: - plt.show() - else: - plt.close(fig) - - -def plot_unique_count(dfs, save_name: str = None, show_plt: bool = True, fig_l_pad=1, fig_w=1.5*7, fig_h=13): - df_uniques = pd.DataFrame({ - 'Grid Spacing': ['/'.join(data_name.split('-')[1:]) for data_name, df in dfs.items()], - 'Unique Overlap Values': [len(np.unique(df['overlap'].values, return_counts=True)[1]) for data_name, df in dfs.items()] - }) - # make plot - cm = 1 / 2.54 - fig, ax = plt.subplots(1, 1, figsize=((fig_l_pad+fig_w)*cm, fig_h*cm)) - ax.set_title('Increasing Overlap') - sns.barplot(data=df_uniques, x='Grid Spacing', y='Unique Overlap Values') - plt.gca().invert_xaxis() - plt.tight_layout() - # save - if save_name is not None: - path = os.path.join(os.path.dirname(__file__), 'plots', save_name) - os.makedirs(os.path.dirname(path), exist_ok=True) - plt.savefig(path) - print(f'saved: {path}') - # show - if show_plt: - plt.show() - else: - plt.close(fig) - - -# ========================================================================= # -# entrypoint # -# ========================================================================= # - - -if __name__ == '__main__': - - # TODO: update to new classes - # TODO: update to use registry - - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../gadfly.mplstyle')) - - # common settings - SHARED_SETTINGS = dict( - samples=50_000, - load=True, - save=True, - show_plt=True, - save_plt=True, - show_dual_plt=False, - fig_l_pad=1, - fig_w=5.5, - fig_h=13, - tick_sizes={ - 'DSprites': 0.05, - 'Shapes3d': 0.2, - 'Cars3d': 0.05, - 'XYSquares': 0.01, - # increasing levels of overlap - 'XYSquares-1': 0.01, - 'XYSquares-2': 0.01, - 'XYSquares-3': 0.01, - 'XYSquares-4': 0.01, - 'XYSquares-5': 0.01, - 'XYSquares-6': 0.01, - 'XYSquares-7': 0.01, - 'XYSquares-8': 0.01, - # increasing levels of overlap 2 - 'XYSquares-1-8': 0.01, - 'XYSquares-2-8': 0.01, - 'XYSquares-3-8': 0.01, - 'XYSquares-4-8': 0.01, - 'XYSquares-5-8': 0.01, - 'XYSquares-6-8': 0.01, - 'XYSquares-7-8': 0.01, - 'XYSquares-8-8': 0.01, - }, - ) - - # EXPERIMENT 0 -- visual overlap on existing datasets - - dfs = plot_all( - exp_name='dataset-overlap', - gt_data_classes={ - # 'XYObject': wrapped_partial(XYObjectData), - # 'XYBlocks': wrapped_partial(XYBlocksData), - 'XYSquares': wrapped_partial(XYSquaresData), - 'DSprites': wrapped_partial(DSpritesData), - 'Shapes3d': wrapped_partial(Shapes3dData), - 'Cars3d': wrapped_partial(Cars3dData), - # 'SmallNorb': wrapped_partial(SmallNorbData), - # 'Mpi3d': wrapped_partial(Mpi3dData), - }, - hide_extra_legends=False, - **SHARED_SETTINGS - ) - - # EXPERIMENT 1 -- increasing visual overlap - - dfs = plot_all( - exp_name='increasing-overlap', - gt_data_classes={ - 'XYSquares-1': wrapped_partial(XYSquaresData, grid_spacing=1), - 'XYSquares-2': wrapped_partial(XYSquaresData, grid_spacing=2), - 'XYSquares-3': wrapped_partial(XYSquaresData, grid_spacing=3), - 'XYSquares-4': wrapped_partial(XYSquaresData, grid_spacing=4), - 'XYSquares-5': wrapped_partial(XYSquaresData, grid_spacing=5), - 'XYSquares-6': wrapped_partial(XYSquaresData, grid_spacing=6), - 'XYSquares-7': wrapped_partial(XYSquaresData, grid_spacing=7), - 'XYSquares-8': wrapped_partial(XYSquaresData, grid_spacing=8), - }, - hide_extra_legends=True, - **SHARED_SETTINGS - ) - - plot_unique_count( - dfs=dfs, - save_name='increasing-overlap/xysquares-increasing-overlap-counts.png', - ) - - plot_dfs_stacked( - dfs=dfs, - title='Increasing Overlap', - exp_name='increasing-overlap', - save_name='increasing-overlap/xysquares-increasing-overlap.png', - tick_size=0.01, - fig_w=13 - ) - - # EXPERIMENT 2 -- increasing visual overlap fixed dim size - - dfs = plot_all( - exp_name='increasing-overlap-fixed', - gt_data_classes={ - 'XYSquares-1-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=1, grid_size=8), - 'XYSquares-2-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=2, grid_size=8), - 'XYSquares-3-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=3, grid_size=8), - 'XYSquares-4-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=4, grid_size=8), - 'XYSquares-5-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=5, grid_size=8), - 'XYSquares-6-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=6, grid_size=8), - 'XYSquares-7-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=7, grid_size=8), - 'XYSquares-8-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=8, grid_size=8), - }, - hide_extra_legends=True, - **SHARED_SETTINGS - ) - - plot_unique_count( - dfs=dfs, - save_name='increasing-overlap-fixed/xysquares-increasing-overlap-fixed-counts.png', - ) - - plot_dfs_stacked( - dfs=dfs, - title='Increasing Overlap', - exp_name='increasing-overlap-fixed', - save_name='increasing-overlap-fixed/xysquares-increasing-overlap-fixed.png', - tick_size=0.01, - fig_w=13 - ) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/e01_visual_overlap/run_plot_traversal_dists.py b/research/e01_visual_overlap/run_plot_traversal_dists.py deleted file mode 100644 index 56283d35..00000000 --- a/research/e01_visual_overlap/run_plot_traversal_dists.py +++ /dev/null @@ -1,654 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import os -from collections import defaultdict -from typing import Any -from typing import Callable -from typing import Dict -from typing import List -from typing import Literal -from typing import Optional -from typing import Sequence -from typing import Tuple -from typing import Union - -import matplotlib.pyplot as plt -import numpy as np -import torch -import torch.nn.functional as F -from tqdm import tqdm - -import research.util as H -from disent.dataset.data import GroundTruthData -from disent.dataset.data import SelfContainedHdf5GroundTruthData -from disent.dataset.util.state_space import NonNormalisedFactors -from disent.dataset.transform import ToImgTensorF32 -from disent.dataset.util.stats import compute_data_mean_std -from disent.util.inout.paths import ensure_parent_dir_exists -from disent.util.profiling import Timer -from disent.util.seeds import TempNumpySeed - - -# ========================================================================= # -# Factor Traversal Stats # -# ========================================================================= # - - -SampleModeHint = Union[Literal['random'], Literal['near'], Literal['combinations']] - - -@torch.no_grad() -def sample_factor_traversal_info( - gt_data: GroundTruthData, - f_idx: Optional[int] = None, - circular_distance: bool = False, - sample_mode: SampleModeHint = 'random', -) -> dict: - # load traversal -- TODO: this is the bottleneck! not threaded - factors, indices, obs = gt_data.sample_random_obs_traversal(f_idx=f_idx, obs_collect_fn=torch.stack) - # get pairs - idxs_a, idxs_b = H.pair_indices(max_idx=len(indices), mode=sample_mode) - # compute deltas - deltas = F.mse_loss(obs[idxs_a], obs[idxs_b], reduction='none').mean(dim=[-3, -2, -1]).numpy() - fdists = H.np_factor_dists(factors[idxs_a], factors[idxs_b], factor_sizes=gt_data.factor_sizes, circular_if_factor_sizes=circular_distance, p=1) - # done! - return dict( - # traversals - factors=factors, # np.ndarray - indices=indices, # np.ndarray - obs=obs, # torch.Tensor - # pairs - idxs_a=idxs_a, # np.ndarray - idxs_b=idxs_b, # np.ndarray - deltas=deltas, # np.ndarray - fdists=fdists, # np.ndarray - ) - - -def sample_factor_traversal_info_and_distmat( - gt_data: GroundTruthData, - f_idx: Optional[int] = None, - circular_distance: bool = False, -) -> dict: - dat = sample_factor_traversal_info(gt_data=gt_data, f_idx=f_idx, sample_mode='combinations', circular_distance=circular_distance) - # extract - factors, idxs_a, idxs_b, deltas, fdists = dat['factors'], dat['idxs_a'], dat['idxs_b'], dat['deltas'], dat['fdists'] - # generate deltas matrix - deltas_matrix = np.zeros([factors.shape[0], factors.shape[0]]) - deltas_matrix[idxs_a, idxs_b] = deltas - deltas_matrix[idxs_b, idxs_a] = deltas - # generate distance matrix - fdists_matrix = np.zeros([factors.shape[0], factors.shape[0]]) - fdists_matrix[idxs_a, idxs_b] = fdists - fdists_matrix[idxs_b, idxs_a] = fdists - # done! - return dict(**dat, deltas_matrix=deltas_matrix, fdists_matrix=fdists_matrix) - - -# ========================================================================= # -# Factor Traversal Collector # -# ========================================================================= # - - -def _collect_stats_for_factors( - gt_data: GroundTruthData, - f_idxs: Sequence[int], - stats_fn: Callable[[GroundTruthData, int, int], Dict[str, Any]], - keep_keys: Sequence[str], - stats_callback: Optional[Callable[[Dict[str, List[Any]], int, int], None]] = None, - return_stats: bool = True, - num_traversal_sample: int = 100, -) -> List[Dict[str, List[Any]]]: - # prepare - f_idxs = gt_data.normalise_factor_idxs(f_idxs) - # generate data per factor - f_stats = [] - for i, f_idx in enumerate(f_idxs): - factor_name = gt_data.factor_names[f_idx] - factor_size = gt_data.factor_sizes[f_idx] - # repeatedly generate stats per factor - stats = defaultdict(list) - for _ in tqdm(range(num_traversal_sample), desc=f'{gt_data.name}: {factor_name}'): - data = stats_fn(gt_data, i, f_idx) - for key in keep_keys: - stats[key].append(data[key]) - # save factor stats - if return_stats: - f_stats.append(stats) - if stats_callback: - stats_callback(stats, i, f_idx) - # done! - if return_stats: - return f_stats - - -# ========================================================================= # -# Plot Traversal Stats # -# ========================================================================= # - - -_COLORS = { - 'blue': (None, 'Blues', 'Blues'), - 'red': (None, 'Reds', 'Reds'), - 'purple': (None, 'Purples', 'Purples'), - 'green': (None, 'Greens', 'Greens'), - 'orange': (None, 'Oranges', 'Oranges'), -} - - -def plot_traversal_stats( - dataset_or_name: Union[str, GroundTruthData], - num_repeats: int = 256, - f_idxs: Optional[NonNormalisedFactors] = None, - circular_distance: bool = False, - color='blue', - color_gt_dist='blue', - color_im_dist='purple', - suffix: Optional[str] = None, - save_path: Optional[str] = None, - plot_freq: bool = True, - plot_title: Union[bool, str] = False, - fig_block_size: float = 4.0, - col_titles: Union[bool, List[str]] = True, - hide_axis: bool = True, - hide_labels: bool = True, - y_size_offset: float = 0.0, - x_size_offset: float = 0.0, -): - # - - - - - - - - - - - - - - - - - # - - def stats_fn(gt_data, i, f_idx): - return sample_factor_traversal_info_and_distmat(gt_data=gt_data, f_idx=f_idx, circular_distance=circular_distance) - - def plot_ax(stats: dict, i: int, f_idx: int): - deltas = np.concatenate(stats['deltas']) - fdists = np.concatenate(stats['fdists']) - fdists_matrix = np.mean(stats['fdists_matrix'], axis=0) - deltas_matrix = np.mean(stats['deltas_matrix'], axis=0) - - # ensure that if we limit the number of points, that we get good values - with TempNumpySeed(777): np.random.shuffle(deltas) - with TempNumpySeed(777): np.random.shuffle(fdists) - - # subplot! - if plot_freq: - ax0, ax1, ax2, ax3 = axs[:, i] - else: - (ax0, ax1), (ax2, ax3) = (None, None), axs[:, i] - - # get title - curr_title = None - if isinstance(col_titles, bool): - if col_titles: - curr_title = gt_data.factor_names[f_idx] - else: - curr_title = col_titles[i] - - # set column titles - if curr_title is not None: - (ax0 if plot_freq else ax2).set_title(f'{curr_title}\n', fontsize=24) - - # plot the frequency stuffs - if plot_freq: - ax0.violinplot([deltas], vert=False) - ax0.set_xlabel('deltas') - ax0.set_ylabel('proportion') - - ax1.set_title('deltas vs. fdists') - ax1.scatter(x=deltas[:15_000], y=fdists[:15_000], s=20, alpha=0.1, c=c_points) - H.plt_2d_density( - x=deltas[:10_000], xmin=deltas.min(), xmax=deltas.max(), - y=fdists[:10_000], ymin=fdists.min() - 0.5, ymax=fdists.max() + 0.5, - n_bins=100, - ax=ax1, pcolormesh_kwargs=dict(cmap=cmap_density, alpha=0.5), - ) - ax1.set_xlabel('deltas') - ax1.set_ylabel('fdists') - - # ax2.set_title('fdists') - ax2.imshow(fdists_matrix, cmap=gt_cmap_img) - if not hide_labels: ax2.set_xlabel('f_idx') - if not hide_labels: ax2.set_ylabel('f_idx') - if hide_axis: H.plt_hide_axis(ax2) - - # ax3.set_title('divergence') - ax3.imshow(deltas_matrix, cmap=im_cmap_img) - if not hide_labels: ax3.set_xlabel('f_idx') - if not hide_labels: ax3.set_ylabel('f_idx') - if hide_axis: H.plt_hide_axis(ax3) - - - # - - - - - - - - - - - - - - - - - # - - # initialize - gt_data: GroundTruthData = H.make_data(dataset_or_name) if isinstance(dataset_or_name, str) else dataset_or_name - f_idxs = gt_data.normalise_factor_idxs(f_idxs) - - c_points, cmap_density, cmap_img = _COLORS[color] - im_c_points, im_cmap_density, im_cmap_img = _COLORS[color if (color_im_dist is None) else color_im_dist] - gt_c_points, gt_cmap_density, gt_cmap_img = _COLORS[color if (color_gt_dist is None) else color_gt_dist] - - n = 4 if plot_freq else 2 - - # get additional spacing - title_offset = 0 if (isinstance(col_titles, bool) and not col_titles) else 0.15 - - # settings - r, c = [n, len(f_idxs)] - h, w = [(n+title_offset)*fig_block_size + y_size_offset, len(f_idxs)*fig_block_size + x_size_offset] - - # initialize plot - fig, axs = plt.subplots(r, c, figsize=(w, h), squeeze=False) - - if isinstance(plot_title, str): - fig.suptitle(f'{plot_title}\n', fontsize=25) - elif plot_title: - fig.suptitle(f'{gt_data.name} [circular={circular_distance}]{f" {suffix}" if suffix else ""}\n', fontsize=25) - - # generate plot - _collect_stats_for_factors( - gt_data=gt_data, - f_idxs=f_idxs, - stats_fn=stats_fn, - keep_keys=['deltas', 'fdists', 'deltas_matrix', 'fdists_matrix'], - stats_callback=plot_ax, - num_traversal_sample=num_repeats, - ) - - # finalize plot - fig.tight_layout() # (pad=1.4 if hide_labels else 1.08) - - # save the path - if save_path is not None: - assert save_path.endswith('.png') - ensure_parent_dir_exists(save_path) - plt.savefig(save_path) - print(f'saved {gt_data.name} to: {save_path}') - - # show it! - plt.show() - - # - - - - - - - - - - - - - - - - - # - return fig - - -# TODO: fix -def plot_traversal_stats( - dataset_or_name: Union[str, GroundTruthData], - num_repeats: int = 256, - f_idxs: Optional[NonNormalisedFactors] = None, - circular_distance: bool = False, - color='blue', - color_gt_dist='blue', - color_im_dist='purple', - suffix: Optional[str] = None, - save_path: Optional[str] = None, - plot_freq: bool = True, - plot_title: Union[bool, str] = False, - plt_scale: float = 6, - col_titles: Union[bool, List[str]] = True, - hide_axis: bool = True, - hide_labels: bool = True, - y_size_offset: float = 0.45, - x_size_offset: float = 0.75, - disable_labels: bool = False, - bottom_labels: bool = False, - label_size: int = 23, -): - # - - - - - - - - - - - - - - - - - # - - def stats_fn(gt_data, i, f_idx): - return sample_factor_traversal_info_and_distmat( - gt_data=gt_data, f_idx=f_idx, circular_distance=circular_distance - ) - - grid_t = [] - grid_titles = [] - - def plot_ax(stats: dict, i: int, f_idx: int): - fdists_matrix = np.mean(stats['fdists_matrix'], axis=0) - deltas_matrix = np.mean(stats['deltas_matrix'], axis=0) - grid_t.append([fdists_matrix, deltas_matrix]) - # get the title - if isinstance(col_titles, bool): - if col_titles: - grid_titles.append(gt_data.factor_names[f_idx]) - else: - grid_titles.append(col_titles[i]) - - # initialize - gt_data: GroundTruthData = H.make_data(dataset_or_name) if isinstance(dataset_or_name, str) else dataset_or_name - f_idxs = gt_data.normalise_factor_idxs(f_idxs) - - # get title - if isinstance(plot_title, str): - suptitle = f'{plot_title}' - elif plot_title: - suptitle = f'{gt_data.name} [circular={circular_distance}]{f" {suffix}" if suffix else ""}' - else: - suptitle = None - - # generate plot - _collect_stats_for_factors( - gt_data=gt_data, - f_idxs=f_idxs, - stats_fn=stats_fn, - keep_keys=['deltas', 'fdists', 'deltas_matrix', 'fdists_matrix'], - stats_callback=plot_ax, - num_traversal_sample=num_repeats, - ) - - labels = None - if (not disable_labels) and grid_titles: - labels = grid_titles - - # settings - fig, axs = H.plt_subplots_imshow( - grid=list(zip(*grid_t)), - title=suptitle, - titles=None if bottom_labels else labels, - titles_size=label_size, - col_labels=labels if bottom_labels else None, - label_size=label_size, - subplot_padding=None, - figsize=((1/2.54) * len(f_idxs) * plt_scale + x_size_offset, (1/2.54) * (2) * plt_scale + y_size_offset) - ) - - # recolor axes - for (ax0, ax1) in axs.T: - ax0.images[0].set_cmap('Blues') - ax1.images[0].set_cmap('Purples') - - fig.tight_layout() - - # save the path - if save_path is not None: - assert save_path.endswith('.png') - ensure_parent_dir_exists(save_path) - plt.savefig(save_path) - print(f'saved {gt_data.name} to: {save_path}') - - # show it! - plt.show() - - # - - - - - - - - - - - - - - - - - # - return fig - - -# ========================================================================= # -# MAIN - DISTS # -# ========================================================================= # - - -@torch.no_grad() -def factor_stats(gt_data: GroundTruthData, f_idxs=None, min_samples: int = 100_000, min_repeats: int = 5000, recon_loss: str = 'mse', sample_mode: str = 'random') -> Tuple[Sequence[int], List[np.ndarray]]: - from disent.registry import RECON_LOSSES - from disent.frameworks.helper.reconstructions import ReconLossHandler - recon_loss: ReconLossHandler = RECON_LOSSES[recon_loss](reduction='mean') - - f_dists = [] - f_idxs = gt_data.normalise_factor_idxs(f_idxs) - # for each factor - for f_idx in f_idxs: - dists = [] - with tqdm(desc=gt_data.factor_names[f_idx], total=min_samples) as p: - # for multiple random factor traversals along the factor - while len(dists) < min_samples or p.n < min_repeats: - # based on: sample_factor_traversal_info(...) # TODO: should add recon loss to that function instead - factors, indices, obs = gt_data.sample_random_obs_traversal(f_idx=f_idx, obs_collect_fn=torch.stack) - # random pairs -- we use this because it does not include [i == i] - idxs_a, idxs_b = H.pair_indices(max_idx=len(indices), mode=sample_mode) - # get distances - d = recon_loss.compute_pairwise_loss(obs[idxs_a], obs[idxs_b]) - d = d.numpy().tolist() - # H.plt_subplots_imshow([[np.moveaxis(o.numpy(), 0, -1) for o in obs]]) - # plt.show() - dists.extend(d) - p.update(len(d)) - # aggregate the average distances - f_dists.append(np.array(dists)[:min_samples]) - - return f_idxs, f_dists - - -def get_random_dists(gt_data: GroundTruthData, num_samples: int = 100_000, recon_loss: str = 'mse'): - from disent.registry import RECON_LOSSES - from disent.frameworks.helper.reconstructions import ReconLossHandler - recon_loss: ReconLossHandler = RECON_LOSSES[recon_loss](reduction='mean') - - dists = [] - with tqdm(desc=gt_data.name, total=num_samples) as p: - # for multiple random factor traversals along the factor - while len(dists) < num_samples: - # random pair - i, j = np.random.randint(0, len(gt_data), size=2) - # get distance - d = recon_loss.compute_pairwise_loss(gt_data[i][None, ...], gt_data[j][None, ...]) - # plt.show() - dists.append(float(d.flatten())) - p.update() - # done! - return np.array(dists) - - -def print_ave_dists(gt_data: GroundTruthData, num_samples: int = 100_000, recon_loss: str = 'mse'): - dists = get_random_dists(gt_data=gt_data, num_samples=num_samples, recon_loss=recon_loss) - f_mean = np.mean(dists) - f_std = np.std(dists) - print(f'[{gt_data.name}] RANDOM ({len(gt_data)}, {len(dists)}) - mean: {f_mean:7.4f} std: {f_std:7.4f}') - - -def print_ave_factor_stats(gt_data: GroundTruthData, f_idxs=None, min_samples: int = 100_000, min_repeats: int = 5000, recon_loss: str = 'mse', sample_mode: str = 'random'): - # compute average distances - f_idxs, f_dists = factor_stats(gt_data=gt_data, f_idxs=f_idxs, min_repeats=min_repeats, min_samples=min_samples, recon_loss=recon_loss, sample_mode=sample_mode) - # compute dists - f_means = [np.mean(d) for d in f_dists] - f_stds = [np.std(d) for d in f_dists] - # sort in order of importance - order = np.argsort(f_means)[::-1] - # print information - for i in order: - f_idx, f_mean, f_std = f_idxs[i], f_means[i], f_stds[i] - print(f'[{gt_data.name}] {gt_data.factor_names[f_idx]} ({gt_data.factor_sizes[f_idx]}, {len(f_dists[f_idx])}) - mean: {f_mean:7.4f} std: {f_std:7.4f}') - - -def main_compute_dists(factor_samples: int = 50_000, min_repeats: int = 5000, random_samples: int = 50_000, recon_loss: str = 'mse', sample_mode: str = 'random', seed: int = 777): - # plot standard datasets - for name in ['dsprites', 'shapes3d', 'cars3d', 'smallnorb', 'xysquares_8x8_s8']: - gt_data = H.make_data(name) - if factor_samples is not None: - with TempNumpySeed(seed): - print_ave_factor_stats(gt_data, min_samples=factor_samples, min_repeats=min_repeats, recon_loss=recon_loss, sample_mode=sample_mode) - if random_samples is not None: - with TempNumpySeed(seed): - print_ave_dists(gt_data, num_samples=random_samples, recon_loss=recon_loss) - -# [dsprites] position_y (32, 50000) - mean: 0.0584 std: 0.0378 -# [dsprites] position_x (32, 50000) - mean: 0.0559 std: 0.0363 -# [dsprites] scale (6, 50000) - mean: 0.0250 std: 0.0148 -# [dsprites] shape (3, 50000) - mean: 0.0214 std: 0.0095 -# [dsprites] orientation (40, 50000) - mean: 0.0172 std: 0.0106 -# [dsprites] RANDOM (737280, 50000) - mean: 0.0754 std: 0.0289 - -# [3dshapes] wall_hue (10, 50000) - mean: 0.1122 std: 0.0661 -# [3dshapes] floor_hue (10, 50000) - mean: 0.1086 std: 0.0623 -# [3dshapes] object_hue (10, 50000) - mean: 0.0416 std: 0.0292 -# [3dshapes] shape (4, 50000) - mean: 0.0207 std: 0.0161 -# [3dshapes] scale (8, 50000) - mean: 0.0182 std: 0.0153 -# [3dshapes] orientation (15, 50000) - mean: 0.0116 std: 0.0079 -# [3dshapes] RANDOM (480000, 50000) - mean: 0.2432 std: 0.0918 - -# [cars3d] azimuth (24, 50000) - mean: 0.0355 std: 0.0185 -# [cars3d] object_type (183, 50000) - mean: 0.0349 std: 0.0176 -# [cars3d] elevation (4, 50000) - mean: 0.0174 std: 0.0100 -# [cars3d] RANDOM (17568, 50000) - mean: 0.0519 std: 0.0188 - -# [smallnorb] lighting (6, 50000) - mean: 0.0531 std: 0.0563 -# [smallnorb] category (5, 50000) - mean: 0.0113 std: 0.0066 -# [smallnorb] rotation (18, 50000) - mean: 0.0090 std: 0.0071 -# [smallnorb] instance (5, 50000) - mean: 0.0068 std: 0.0048 -# [smallnorb] elevation (9, 50000) - mean: 0.0034 std: 0.0030 -# [smallnorb] RANDOM (24300, 50000) - mean: 0.0535 std: 0.0529 - -# [xy_squares] y_B (8, 50000) - mean: 0.0104 std: 0.0000 -# [xy_squares] x_B (8, 50000) - mean: 0.0104 std: 0.0000 -# [xy_squares] y_G (8, 50000) - mean: 0.0104 std: 0.0000 -# [xy_squares] x_G (8, 50000) - mean: 0.0104 std: 0.0000 -# [xy_squares] y_R (8, 50000) - mean: 0.0104 std: 0.0000 -# [xy_squares] x_R (8, 50000) - mean: 0.0104 std: 0.0000 -# [xy_squares] RANDOM (262144, 50000) - mean: 0.0308 std: 0.0022 - -# ========================================================================= # -# MAIN - PLOTTING # -# ========================================================================= # - - -def _make_self_contained_dataset(h5_path): - return SelfContainedHdf5GroundTruthData(h5_path=h5_path, transform=ToImgTensorF32()) - - -def _print_data_mean_std(data_or_name, print_mean_std: bool = True): - if print_mean_std: - data = H.make_data(data_or_name) if isinstance(data_or_name, str) else data_or_name - name = data_or_name if isinstance(data_or_name, str) else data.name - mean, std = compute_data_mean_std(data) - print(f'{name}\n vis_mean: {mean.tolist()}\n vis_std: {std.tolist()}') - - -def main_plotting(plot_all=False, print_mean_std=False): - CIRCULAR = False - PLOT_FREQ = False - - def sp(name): - prefix = 'CIRCULAR_' if CIRCULAR else 'DIST_' - prefix = prefix + ('FREQ_' if PLOT_FREQ else 'NO-FREQ_') - return os.path.join(os.path.dirname(__file__), 'plots', f'{prefix}{name}.png') - - # plot xysquares with increasing overlap - for s in [1, 2, 3, 4, 5, 6, 7, 8]: - plot_traversal_stats(circular_distance=CIRCULAR, plt_scale=8, label_size=26, x_size_offset=0, y_size_offset=0.6, save_path=sp(f'xysquares_8x8_s{s}'), color='blue', dataset_or_name=f'xysquares_8x8_s{s}', f_idxs=[1], col_titles=[f'Space: {s}px'], plot_freq=PLOT_FREQ) - _print_data_mean_std(f'xysquares_8x8_s{s}', print_mean_std) - - # plot standard datasets - for name in ['dsprites', 'shapes3d', 'cars3d', 'smallnorb']: - plot_traversal_stats(circular_distance=CIRCULAR, x_size_offset=0, y_size_offset=0.6, num_repeats=256, disable_labels=False, save_path=sp(name), color='blue', dataset_or_name=name, plot_freq=PLOT_FREQ) - _print_data_mean_std(name, print_mean_std) - - if not plot_all: - return - - # plot adversarial dsprites datasets - for fg in [True, False]: - for vis in [100, 80, 60, 40, 20]: - name = f'dsprites_imagenet_{"fg" if fg else "bg"}_{vis}' - plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(name), color='orange', dataset_or_name=name, plot_freq=PLOT_FREQ, x_size_offset=0.4) - _print_data_mean_std(name, print_mean_std) - - BASE = os.path.abspath(os.path.join(__file__, '../../../out/adversarial_data_approx')) - - # plot adversarial datasets - for color, folder in [ - # 'const' datasets - ('purple', '2021-08-18--00-58-22_FINAL-dsprites_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('purple', '2021-08-18--01-33-47_FINAL-shapes3d_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('purple', '2021-08-18--02-20-13_FINAL-cars3d_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('purple', '2021-08-18--03-10-53_FINAL-smallnorb_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - # 'invert' datasets - ('orange', '2021-08-18--03-52-31_FINAL-dsprites_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('orange', '2021-08-18--04-29-25_FINAL-shapes3d_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('orange', '2021-08-18--05-13-15_FINAL-cars3d_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('orange', '2021-08-18--06-03-32_FINAL-smallnorb_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - # stronger 'invert' datasets - ('red', '2021-09-06--00-29-23_INVERT-VSTRONG-shapes3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), - ('red', '2021-09-06--03-17-28_INVERT-VSTRONG-dsprites_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), - ('red', '2021-09-06--05-42-06_INVERT-VSTRONG-cars3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), - ('red', '2021-09-06--09-10-59_INVERT-VSTRONG-smallnorb_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), - ]: - data = _make_self_contained_dataset(f'{BASE}/{folder}/data.h5') - plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(folder), color=color, dataset_or_name=data, plot_freq=PLOT_FREQ, x_size_offset=0.4) - _print_data_mean_std(data, print_mean_std) - - -# ========================================================================= # -# STATS # -# ========================================================================= # - - -if __name__ == '__main__': - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../gadfly.mplstyle')) - # run! - # main_plotting() - main_compute_dists() - - -# ========================================================================= # -# STATS # -# ========================================================================= # - - -# 2021-08-18--00-58-22_FINAL-dsprites_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.04375297] -# vis_std: [0.06837677] -# 2021-08-18--01-33-47_FINAL-shapes3d_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.48852729, 0.5872147 , 0.59863929] -# vis_std: [0.08931785, 0.18920148, 0.23331079] -# 2021-08-18--02-20-13_FINAL-cars3d_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.88888636, 0.88274618, 0.87782785] -# vis_std: [0.18967542, 0.20009377, 0.20805905] -# 2021-08-18--03-10-53_FINAL-smallnorb_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.74029344] -# vis_std: [0.06706581] -# -# 2021-08-18--03-52-31_FINAL-dsprites_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.0493243] -# vis_std: [0.09729655] -# 2021-08-18--04-29-25_FINAL-shapes3d_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.49514523, 0.58791172, 0.59616399] -# vis_std: [0.08637031, 0.1895267 , 0.23397072] -# 2021-08-18--05-13-15_FINAL-cars3d_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.88851889, 0.88029857, 0.87666017] -# vis_std: [0.200735 , 0.2151134, 0.2217553] -# 2021-08-18--06-03-32_FINAL-smallnorb_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.73232105] -# vis_std: [0.08755041] -# -# 2021-09-06--00-29-23_INVERT-VSTRONG-shapes3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.47992192, 0.51311111, 0.54627272] -# vis_std: [0.28653814, 0.29201543, 0.27395435] -# 2021-09-06--03-17-28_INVERT-VSTRONG-dsprites_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.20482841] -# vis_std: [0.33634909] -# 2021-09-06--05-42-06_INVERT-VSTRONG-cars3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.76418207, 0.75554032, 0.75075393] -# vis_std: [0.31892905, 0.32751031, 0.33319886] -# 2021-09-06--09-10-59_INVERT-VSTRONG-smallnorb_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.69691603] -# vis_std: [0.21310608] - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/e01_visual_overlap/util_compute_traversal_dist_pairs.py b/research/e01_visual_overlap/util_compute_traversal_dist_pairs.py deleted file mode 100644 index 3ecb9713..00000000 --- a/research/e01_visual_overlap/util_compute_traversal_dist_pairs.py +++ /dev/null @@ -1,274 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -from pathlib import Path -from typing import Optional - -import numpy as np -import psutil -import ray -import torch -from ray.util.queue import Queue -from tqdm import tqdm - -import research.util as H -from disent.dataset.data import GroundTruthData -from disent.util.inout.files import AtomicSaveFile -from disent.util.profiling import Timer -from disent.util.seeds import TempNumpySeed - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Dataset Distances # -# ========================================================================= # - - -@ray.remote -def _compute_given_dists(gt_data, idxs, obs_pair_idxs, progress_queue=None): - # checks - assert idxs.ndim == 1 - assert obs_pair_idxs.ndim == 2 - assert len(obs_pair_idxs) == len(idxs) - # storage - with torch.no_grad(), Timer() as timer: - obs_pair_dists = torch.zeros(*obs_pair_idxs.shape, dtype=torch.float32) - # progress - done = 0 - # for each observation - for i, (obs_idx, pair_idxs) in enumerate(zip(idxs, obs_pair_idxs)): - # load data - obs = gt_data[obs_idx].flatten() - batch = torch.stack([gt_data[i].flatten() for i in pair_idxs], dim=0) - # compute distances - obs_pair_dists[i, :] = torch.mean((batch - obs[None, :])**2, dim=-1, dtype=torch.float32) - # add progress - done += 1 - if progress_queue is not None: - if timer.elapsed > 0.2: - timer.restart() - progress_queue.put(done) - done = 0 - # final update - if progress_queue is not None: - if done > 0: - progress_queue.put(done) - # done! - return obs_pair_dists.numpy() - - -def compute_dists(gt_data: GroundTruthData, obs_pair_idxs: np.ndarray, jobs_per_cpu: int = 1): - """ - Compute all the distances for ground truth data. - - obs_pair_idxs is a 2D array (len(gt_dat), N) that is a list - of paired indices to each element in the dataset. - """ - # checks - assert obs_pair_idxs.ndim == 2 - assert obs_pair_idxs.shape[0] == len(gt_data) - assert jobs_per_cpu > 0 - # get workers - num_cpus = int(ray.available_resources().get('CPU', 1)) - num_workers = int(num_cpus * jobs_per_cpu) - # get chunks - pair_idxs_chunks = np.array_split(obs_pair_idxs, num_workers) - start_idxs = [0] + np.cumsum([len(c) for c in pair_idxs_chunks]).tolist() - # progress queue - progress_queue = Queue() - ref_gt_data = ray.put(gt_data) - # make workers - futures = [ - _compute_given_dists.remote(ref_gt_data, np.arange(i, i+len(chunk)), chunk, progress_queue) - for i, chunk in zip(start_idxs, pair_idxs_chunks) - ] - # check progress - with tqdm(desc=gt_data.name, total=len(gt_data)) as progress: - completed = 0 - while completed < len(gt_data): - done = progress_queue.get() - completed += done - progress.update(done) - # done - obs_pair_dists = np.concatenate(ray.get(futures), axis=0) - return obs_pair_dists - -# ========================================================================= # -# Distance Types # -# ========================================================================= # - - -def dataset_pair_idxs__random(gt_data: GroundTruthData, num_pairs: int = 25) -> np.ndarray: - # purely random pairs... - return np.random.randint(0, len(gt_data), size=[len(gt_data), num_pairs]) - - -def dataset_pair_idxs__nearby(gt_data: GroundTruthData, num_pairs: int = 10, radius: int = 5) -> np.ndarray: - radius = np.array(radius) - assert radius.ndim in (0, 1) - if radius.ndim == 1: - assert radius.shape == (gt_data.num_factors,) - # get all positions - pos = gt_data.idx_to_pos(np.arange(len(gt_data))) - # generate random offsets - offsets = np.random.randint(-radius, radius + 1, size=[len(gt_data), num_pairs, gt_data.num_factors]) - # broadcast random offsets & wrap around - nearby_pos = (pos[:, None, :] + offsets) % gt_data.factor_sizes - # convert back to indices - nearby_idxs = gt_data.pos_to_idx(nearby_pos) - # done! - return nearby_idxs - - -def dataset_pair_idxs__nearby_scaled(gt_data: GroundTruthData, num_pairs: int = 10, min_radius: int = 2, radius_ratio: float = 0.2) -> np.ndarray: - return dataset_pair_idxs__nearby( - gt_data=gt_data, - num_pairs=num_pairs, - radius=np.maximum((np.array(gt_data.factor_sizes) * radius_ratio).astype('int'), min_radius), - ) - - -_PAIR_IDXS_FNS = { - 'random': dataset_pair_idxs__random, - 'nearby': dataset_pair_idxs__nearby, - 'nearby_scaled': dataset_pair_idxs__nearby_scaled, -} - - -def dataset_pair_idxs(mode: str, gt_data: GroundTruthData, num_pairs: int = 10, **kwargs): - if mode not in _PAIR_IDXS_FNS: - raise KeyError(f'invalid mode: {repr(mode)}, must be one of: {sorted(_PAIR_IDXS_FNS.keys())}') - return _PAIR_IDXS_FNS[mode](gt_data, num_pairs=num_pairs, **kwargs) - - -# ========================================================================= # -# Cache Distances # -# ========================================================================= # - -def _get_default_seed( - pairs_per_obs: int, - pair_mode: str, - dataset_name: str, -): - import hashlib - seed_key = (pairs_per_obs, pair_mode, dataset_name) - seed_hash = hashlib.md5(str(seed_key).encode()) - seed = int(seed_hash.hexdigest()[:8], base=16) % (2**32) # [0, 2**32-1] - return seed - - -def cached_compute_dataset_pair_dists( - dataset_name: str = 'smallnorb', - pair_mode: str = 'nearby_scaled', # random, nearby, nearby_scaled - pairs_per_obs: int = 64, - seed: Optional[int] = None, - # cache settings - cache_dir: str = 'data/cache', - force: bool = False, - # normalize - scaled: bool = True, -): - # checks - assert (seed is None) or isinstance(seed, int), f'seed must be an int or None, got: {type(seed)}' - assert isinstance(pairs_per_obs, int), f'pairs_per_obs must be an int, got: {type(pairs_per_obs)}' - assert pair_mode in _PAIR_IDXS_FNS, f'pair_mode is invalid, got: {repr(pair_mode)}, must be one of: {sorted(_PAIR_IDXS_FNS.keys())}' - # get default seed - if seed is None: - seed = _get_default_seed(pairs_per_obs=pairs_per_obs, pair_mode=pair_mode, dataset_name=dataset_name) - # cache path - cache_path = Path(cache_dir, f'dist-pairs_{dataset_name}_{pairs_per_obs}_{pair_mode}_{seed}.npz') - # generate if it does not exist - if force or not cache_path.exists(): - log.info(f'generating cached distances for: {dataset_name} to: {cache_path}') - # load data - gt_data = H.make_data(dataset_name, transform_mode='float32') - # generate idxs - with TempNumpySeed(seed=seed): - obs_pair_idxs = dataset_pair_idxs(pair_mode, gt_data, num_pairs=pairs_per_obs) - obs_pair_dists = compute_dists(gt_data, obs_pair_idxs) - # generate & save - with AtomicSaveFile(file=cache_path, overwrite=force) as path: - np.savez(path, **{ - 'dataset_name': dataset_name, - 'seed': seed, - 'obs_pair_idxs': obs_pair_idxs, - 'obs_pair_dists': obs_pair_dists, - }) - # load cached data - else: - log.info(f'loading cached distances for: {dataset_name} from: {cache_path}') - data = np.load(cache_path) - obs_pair_idxs = data['obs_pair_idxs'] - obs_pair_dists = data['obs_pair_dists'] - # normalize the max distance to 1.0 - if scaled: - obs_pair_dists /= np.max(obs_pair_dists) - # done! - return obs_pair_idxs, obs_pair_dists - - -# ========================================================================= # -# TEST! # -# ========================================================================= # - - -def generate_common_cache(force=False, force_seed=None): - import itertools - # settings - sweep_pairs_per_obs = [128, 32, 256, 64, 16] - sweep_pair_modes = ['nearby_scaled', 'random', 'nearby'] - sweep_dataset_names = ['cars3d', 'smallnorb', 'shapes3d', 'dsprites', 'xysquares'] - # info - log.info(f'Computing distances for sweep of size: {len(sweep_pairs_per_obs)*len(sweep_pair_modes)*len(sweep_dataset_names)}') - # sweep - for i, (pairs_per_obs, pair_mode, dataset_name) in enumerate(itertools.product(sweep_pairs_per_obs, sweep_pair_modes, sweep_dataset_names)): - # deterministic seed based on settings - if force_seed is None: - seed = _get_default_seed(pairs_per_obs=pairs_per_obs, pair_mode=pair_mode, dataset_name=dataset_name) - else: - seed = force_seed - # info - log.info(f'[{i}] Computing distances for: {repr(dataset_name)} {repr(pair_mode)} {repr(pairs_per_obs)} {repr(seed)}') - # get the dataset and delete the transform - cached_compute_dataset_pair_dists( - dataset_name=dataset_name, - pair_mode=pair_mode, - pairs_per_obs=pairs_per_obs, - seed=seed, - force=force, - scaled=True - ) - - -if __name__ == '__main__': - logging.basicConfig(level=logging.INFO) - ray.init(num_cpus=psutil.cpu_count(logical=False)) - generate_common_cache() - - -# ========================================================================= # -# DONE # -# ========================================================================= # diff --git a/research/e01_visual_overlap/util_compute_traversal_dists.py b/research/e01_visual_overlap/util_compute_traversal_dists.py deleted file mode 100644 index 14f78d71..00000000 --- a/research/e01_visual_overlap/util_compute_traversal_dists.py +++ /dev/null @@ -1,303 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -import warnings -from typing import Sequence - -import psutil -import ray - -import logging -import os -from typing import Tuple - -import numpy as np -import torch -from matplotlib import pyplot as plt -from tqdm import tqdm - -import research.util as H -from disent.dataset.data import GroundTruthData -from disent.dataset.util.state_space import StateSpace -from disent.util.strings.fmt import bytes_to_human - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Dataset Stats # -# ========================================================================= # - - -def factor_dist_matrix_shapes(gt_data: GroundTruthData) -> np.ndarray: - # shape: (f_idx, num_factors + 1) - return np.array([factor_dist_matrix_shape(gt_data=gt_data, f_idx=f_idx) for f_idx in range(gt_data.num_factors)]) - - -def factor_dist_matrix_shape(gt_data: GroundTruthData, f_idx: int) -> Tuple[int, ...]: - # using triangular matrices complicates everything - # (np.prod(self._gt_data.factor_sizes) * self._gt_data.factor_sizes[i]) # symmetric, including diagonal in distance matrix - # (np.prod(self._gt_data.factor_sizes) * (self._gt_data.factor_sizes[i] - 1)) // 2 # upper triangular matrix excluding diagonal - # (np.prod(self._gt_data.factor_sizes) * (self._gt_data.factor_sizes[i] + 1)) // 2 # upper triangular matrix including diagonal - return (*np.delete(gt_data.factor_sizes, f_idx), gt_data.factor_sizes[f_idx], gt_data.factor_sizes[f_idx]) - - -def print_dist_matrix_stats(gt_data: GroundTruthData): - # assuming storage as f32 - num_pairs = factor_dist_matrix_shapes(gt_data).prod(axis=1).sum(axis=0) - pre_compute_bytes = num_pairs * (32 // 8) - pairwise_compute_bytes = num_pairs * (32 // 8) * np.prod(gt_data.x_shape) * 2 - traversal_compute_bytes = np.prod(gt_data.x_shape) * np.prod(gt_data.factor_sizes) * gt_data.num_factors - # string - print( - f'{f"{gt_data.name}:":12s} ' - f'{num_pairs:10d} (pairs) ' - f'{bytes_to_human(pre_compute_bytes):>22s} (pre-comp. f32) ' - f'{"x".join(str(i) for i in gt_data.img_shape):>11s} (obs. size)' - f'{bytes_to_human(pairwise_compute_bytes):>22s} (comp. f32) ' - f'{bytes_to_human(traversal_compute_bytes):>22s} (opt. f32)' - ) - - -# ========================================================================= # -# Dataset Compute # -# ========================================================================= # - - -def _iter_batch_ranges(total, batch_size): - assert total >= 0 - assert batch_size > 0 - for i in range(0, total, batch_size): - yield range(i, min(i + batch_size, total)) - - -def _check_gt_data(gt_data: GroundTruthData): - obs = gt_data[0] - # checks - assert isinstance(obs, torch.Tensor) - assert obs.dtype == torch.float32 - - -@ray.remote -def _compute_dists( - idxs: Sequence[int], - # thread data - f_states: StateSpace, - f_idx: int, - gt_data: GroundTruthData, - masked: bool, - a_idxs: np.ndarray, - b_idxs: np.ndarray, -): - results = [] - for idx in idxs: - # translate traversal position to dataset position - base_pos = f_states.idx_to_pos(int(idx)) - base_factors = np.insert(base_pos, f_idx, 0) - # load traversal: (f_size, H*W*C) - traversal = [gt_data[i].flatten().numpy() for i in gt_data.iter_traversal_indices(f_idx=f_idx, base_factors=base_factors)] - traversal = np.stack(traversal, axis=0) - # compute distances - if masked: - B, NUM = traversal.shape - # compute mask - mask = (traversal[0] != traversal[1]) - for item in traversal[2:]: - mask |= (traversal[0] != item) - traversal = traversal[:, mask] - # compute distances - dists = np.sum((traversal[a_idxs] - traversal[b_idxs]) ** 2, axis=1, dtype='float32') / NUM # might need to be float64 - else: - dists = np.mean((traversal[a_idxs] - traversal[b_idxs]) ** 2, axis=1, dtype='float32') - # return data - results.append((base_pos, dists)) - return results - - -def get_as_completed(obj_ids): - # https://github.com/ray-project/ray/issues/5554 - while obj_ids: - done, obj_ids = ray.wait(obj_ids) - yield ray.get(done[0]) - - -@torch.no_grad() -def compute_factor_dist_matrices( - gt_data: GroundTruthData, - f_idx: int, - masked: bool = True, - traversals_per_batch: int = 64, -): - if not ray.is_initialized(): - warnings.warn(f'Ray has not yet been initialized, consider calling `ray.init(...)` and specifying the CPU requirements.') - _check_gt_data(gt_data) - # load data - f_states = StateSpace(factor_sizes=np.delete(gt_data.factor_sizes, f_idx)) - a_idxs, b_idxs = H.pair_indices_combinations(gt_data.factor_sizes[f_idx]) - total = len(f_states) - # move to shared memory - ID_f_states = ray.put(f_states) - ID_gt_data = ray.put(gt_data) - ID_a_idxs = ray.put(a_idxs) - ID_b_idxs = ray.put(b_idxs) - # results - f_dist_matrices = np.zeros(factor_dist_matrix_shape(gt_data=gt_data, f_idx=f_idx), dtype='float32') - # generate futures - futures = [ - _compute_dists.remote( - idxs=sub_range, - f_idx=f_idx, - masked=masked, - f_states=ID_f_states, - gt_data=ID_gt_data, - a_idxs=ID_a_idxs, - b_idxs=ID_b_idxs, - ) - for sub_range in _iter_batch_ranges(total, batch_size=traversals_per_batch) - ] - # apply multithreading to compute traversal distances - with tqdm(total=total, desc=f'{gt_data.name}: {f_idx+1} of {gt_data.num_factors}') as p: - # compute distance matrices - for results in get_as_completed(futures): - for base_pos, dists in results: - f_dist_matrices[(*base_pos, a_idxs, b_idxs)] = dists - f_dist_matrices[(*base_pos, b_idxs, a_idxs)] = dists - p.update(len(results)) - # return distances - return f_dist_matrices - - -def compute_all_factor_dist_matrices( - gt_data: GroundTruthData, - masked: bool = True, - traversals_per_batch: int = 64, -): - """ - ALGORITHM: - for each factor: O(num_factors) - for each traversal: O(prod()) - for element in traversal: O(n) - -- compute overlapping mask - -- we use this mask to only transfer and compute over the needed data - -- we transfer the MASKED traversal to the GPU not the pairs - for each pair in the traversal: O(n*(n-1)/2) | O(n**2) - -- compute each unique pairs distance - -- return distances - """ - # for each factor, compute pairwise overlap - all_dist_matrices = [] - for f_idx in range(gt_data.num_factors): - f_dist_matrices = compute_factor_dist_matrices( - gt_data=gt_data, - f_idx=f_idx, - masked=masked, - traversals_per_batch=traversals_per_batch, - ) - all_dist_matrices.append(f_dist_matrices) - return all_dist_matrices - - -# TODO: replace this with cachier maybe? -def cached_compute_all_factor_dist_matrices( - dataset_name: str = 'smallnorb', - masked: bool = False, - traversals_per_batch: int = 64, - # cache settings - cache_dir: str = 'data/cache', - force: bool = False, - # normalize - normalize_mode: str = 'all', -): - import os - from disent.util.inout.files import AtomicSaveFile - # load data - gt_data = H.make_data(dataset_name, transform_mode='float32') - # check cache - name = f'dist-matrices_{dataset_name}_masked.npz' if masked else f'dist-matrices_{dataset_name}_full.npz' - cache_path = os.path.abspath(os.path.join(cache_dir, name)) - # generate if it does not exist - if force or not os.path.exists(cache_path): - log.info(f'generating cached distances for: {dataset_name} to: {cache_path}') - # generate & save - with AtomicSaveFile(file=cache_path, overwrite=force) as path: - all_dist_matrices = compute_all_factor_dist_matrices(gt_data, masked=masked, traversals_per_batch=traversals_per_batch) - np.savez(path, **{f_name: f_dists for f_name, f_dists in zip(gt_data.factor_names, all_dist_matrices)}) - # load data - log.info(f'loading cached distances for: {dataset_name} from: {cache_path}') - data = np.load(cache_path) - dist_mats = [data[f_name] for f_name in gt_data.factor_names] - # normalize the max distance to 1.0 - if (normalize_mode == 'none') or (normalize_mode is None): - pass - elif normalize_mode == 'all': - M = np.max([np.max(v) for v in dist_mats]) - dist_mats = [v / M for v in dist_mats] - log.info(f'normalized max over all distances: {M} to 1.0') - elif normalize_mode == 'each': - Ms = [v.max() for v in dist_mats] - dist_mats = [v / M for v, M in zip(dist_mats, Ms)] - log.info(f'normalized max over each factor distance: {Ms} to 1.0') - else: - raise KeyError(f'invalid normalize mode: {repr(normalize_mode)}') - - # done! - return dist_mats - - -# ========================================================================= # -# TEST! # -# ========================================================================= # - - -def generate_common_cache(): - for name in ['cars3d', 'smallnorb', 'shapes3d', 'dsprites', 'xysquares']: - # get the dataset and delete the transform - gt_data = H.make_data(name, transform_mode='float32') - print_dist_matrix_stats(gt_data) - f_dist_matrices = cached_compute_all_factor_dist_matrices( - dataset_name=name, - force=True, - masked=True, - traversals_per_batch=32, - ) - # plot distance matrices - H.plt_subplots_imshow( - grid=[[d.reshape([-1, *d.shape[-2:]]).mean(axis=0) for d in f_dist_matrices]], - subplot_padding=0.5, - figsize=(20, 10), - ) - plt.show() - - -def _test_masked_equals_unmasked(): - for name in ['cars3d', 'smallnorb', 'shapes3d', 'dsprites', 'xysquares']: - dists_a = compute_all_factor_dist_matrices(gt_data=H.make_data(name, transform_mode='float32'), masked=True, traversals_per_batch=32) - dists_b = compute_all_factor_dist_matrices(gt_data=H.make_data(name, transform_mode='float32'), masked=False, traversals_per_batch=32) - for a, b in zip(dists_a, dists_b): - assert np.allclose(a, b) - - -if __name__ == '__main__': - ray.init(num_cpus=min(os.cpu_count(), 32)) - generate_common_cache() diff --git a/research/e02_naive_triplet/submit_01_triplet_hparam_sweep.sh b/research/e02_naive_triplet/submit_01_triplet_hparam_sweep.sh deleted file mode 100644 index e6b2f644..00000000 --- a/research/e02_naive_triplet/submit_01_triplet_hparam_sweep.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-02__naive-triplet-hparams" -export PARTITION="batch" -export PARALLELISM=24 - -# source the helper file -source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -# general sweep of hyper parameters for triplet -# 1 * (3*3*3*2*3 = 162) = 162 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_tvae_params' \ - \ - run_length=long \ - metrics=all \ - \ - framework=tvae \ - settings.framework.beta=0.0316,0.01,0.1 \ - \ - framework.cfg.triplet_margin_max=0.1,1.0,10.0 \ - framework.cfg.triplet_scale=0.1,1.0,0.01 \ - framework.cfg.triplet_p=1,2 \ - \ - dataset=xysquares,cars3d,smallnorb \ - sampling=gt_dist__manhat - -# check sampling strategy -# 2 * (4 * 5 = 20) = 40 -echo PARAMS NOT SET FROM PREVIOUS SWEEP -exit 1 - -# TODO: set the parameters -submit_sweep \ - +DUMMY.repeat=1,2 \ - +EXTRA.tags='sweep_tvae_sampling' \ - \ - run_length=long \ - metrics=all \ - \ - framework=tvae \ - settings.framework.beta=??? \ - \ - framework.cfg.triplet_margin_max=??? \ - framework.cfg.triplet_scale=??? \ - framework.cfg.triplet_p=??? \ - \ - dataset=xysquares,cars3d,shapes3d,dsprites,smallnorb \ - sampling=gt_dist__manhat_scaled,gt_dist__manhat,gt__dist_combined,gt_dist__factors diff --git a/research/e02_naive_triplet/submit_02_check_vae_equivalence.sh b/research/e02_naive_triplet/submit_02_check_vae_equivalence.sh deleted file mode 100644 index a07c4783..00000000 --- a/research/e02_naive_triplet/submit_02_check_vae_equivalence.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-02__naive-triplet-equivalence" -export PARTITION="batch" -export PARALLELISM=24 - -# source the helper file -source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -# make sure the tvae is actually working -# like a vae when the triplet loss is disabled -# 1 * (4=4) = 4 -submit_sweep \ - +DUMMY.repeat=1,2 \ - +EXTRA.tags='check_equivalence' \ - \ - run_length=medium \ - metrics=all \ - \ - framework=tvae \ - framework.cfg.triplet_scale=0.0 \ - settings.framework.beta=0.0316 \ - \ - dataset=xysquares \ - sampling=gt_dist__manhat_scaled,gt_dist__manhat,gt__dist_combined,gt_dist__factors - -# check how sampling effects beta and adavae -# 2 * (2*3=6) = 12 -submit_sweep \ - +DUMMY.repeat=1,2 \ - +EXTRA.tags='check_vae_sampling' \ - \ - run_length=medium \ - metrics=all \ - \ - framework=betavae,adavae \ - settings.framework.beta=0.0316 \ - \ - dataset=xysquares \ - sampling=gt_dist__manhat_scaled,gt_dist__manhat,gt__dist_combined,gt_dist__factors diff --git a/research/e03_axis_triplet/submit_01.sh b/research/e03_axis_triplet/submit_01.sh deleted file mode 100644 index e86b94ea..00000000 --- a/research/e03_axis_triplet/submit_01.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-03__axis-triplet-3.0" -export PARTITION="batch" -export PARALLELISM=24 - -# source the helper file -source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 43200 "C-disent" # 12 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# SHORT RUNS: -# - test for best ada loss types -# 1 * (2*4*2*8=112) = 128 -submit_sweep \ - +DUMMY.repeat=1 \ - \ - framework=X--adatvae \ - dataset=xysquares \ - run_length=short \ - \ - framework.cfg.triplet_margin_max=1.0 \ - framework.cfg.triplet_scale=0.1 \ - framework.cfg.triplet_p=1 \ - sampling=gt_dist_manhat \ - \ - model.z_size=25,9 \ - \ - framework.cfg.thresh_ratio=0.5 \ - framework.cfg.ada_triplet_ratio=1.0 \ - schedule=adavae_thresh,adavae_all,adavae_ratio,none \ - framework.cfg.ada_triplet_sample=TRUE,FALSE \ - framework.cfg.ada_triplet_loss=framework.cfg.ada_triplet_loss=triplet,triplet_soft_ave,triplet_soft_neg_ave,triplet_all_soft_ave,triplet_hard_ave,triplet_hard_neg_ave,triplet_hard_neg_ave_pull,triplet_all_hard_ave - -# ADA TRIPLET LOSS MODES (short runs): -# - generally dont use sampling, except for: triplet_hard_neg_ave_pull -# - soft averages dont work if scheduling thresh or ratio separately, need to do both at the same time -# - hard averages perform well initially, but performance decays more toward the end of schedules -# ======================= -# [X] triplet -# -# [-] triplet_soft_ave [NOTE: OK, but just worse than, triplet_all_soft_ave] -# triplet_soft_neg_ave [NOTE: better disentanglement than triplet_all_soft_ave, but worse axis align] -# triplet_all_soft_ave -# -# triplet_hard_neg_ave -# triplet_hard_neg_ave_pull (weight = 0.1, triplet_hard_neg_ave_pull_soft) -# [X] triplet_hard_ave -# [X] triplet_hard_neg_ave_pull (weight = 1.0) -# [X] triplet_all_hard_ave diff --git a/research/e03_axis_triplet/submit_02.sh b/research/e03_axis_triplet/submit_02.sh deleted file mode 100644 index 76e4b1dd..00000000 --- a/research/e03_axis_triplet/submit_02.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-03__axis-triplet-3.0" -export PARTITION="batch" -export PARALLELISM=30 - -# source the helper file -source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# MED RUNS: -# - test for best hparams for all soft ave loss -# 2 * (2*3*3*3=54) = 104 -submit_sweep \ - +DUMMY.repeat=1,2 \ - +EXTRA.tags='med-run+soft-hparams' \ - \ - framework=X--adatvae \ - run_length=medium \ - model.z_size=25 \ - \ - framework.cfg.triplet_margin_max=1.0,5.0 \ - framework.cfg.triplet_scale=0.1,0.02,0.5 \ - framework.cfg.triplet_p=1 \ - sampling=gt_dist_manhat \ - \ - framework.cfg.thresh_ratio=0.5 \ - framework.cfg.ada_triplet_ratio=1.0 \ - framework.cfg.ada_triplet_soft_scale=0.25,1.0,4.0 \ - framework.cfg.ada_triplet_sample=FALSE \ - \ - schedule=adavae_all,adavae_thresh,adavae_ratio \ - framework.cfg.ada_triplet_loss=triplet_all_soft_ave \ - dataset=xysquares diff --git a/research/e03_axis_triplet/submit_03.sh b/research/e03_axis_triplet/submit_03.sh deleted file mode 100644 index 4317e923..00000000 --- a/research/e03_axis_triplet/submit_03.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-03__axis-triplet-3.0" -export PARTITION="stampede" -export PARALLELISM=32 - -# source the helper file -source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# LONG RUNS: -# - test best losses & best hparams from test1 on different datasets with long runs -# + [not tested] triplet_soft_neg_ave -# + triplet_all_soft_ave -# + triplet_hard_neg_ave -# + triplet_hard_neg_ave_pull - -# 1 * (2*3*4*4=96) = 96 -#submit_sweep \ -# +DUMMY.repeat=1 \ -# +EXTRA.tags='long-run' \ -# \ -# framework=X--adatvae \ -# run_length=long \ -# model.z_size=25 \ -# \ -# framework.cfg.triplet_margin_max=1.0 \ -# framework.cfg.triplet_scale=0.1 \ -# framework.cfg.triplet_p=1 \ -# sampling=gt_dist_manhat,gt_dist_manhat_scaled \ -# \ -# framework.cfg.thresh_ratio=0.5 \ -# framework.cfg.ada_triplet_ratio=1.0 \ -# framework.cfg.ada_triplet_soft_scale=1.0 \ -# framework.cfg.ada_triplet_sample=FALSE \ -# \ -# schedule=adavae_all,adavae_thresh,adavae_ratio \ -# framework.cfg.ada_triplet_loss=triplet,triplet_all_soft_ave,triplet_hard_neg_ave,triplet_hard_neg_ave_pull \ -# dataset=xysquares,shapes3d,cars3d,dsprites - -# 2*2*3*4*4 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='med-run+datasets+swap-chance+manhat-scaled' \ - \ - framework=X--adatvae \ - run_length=medium \ - model.z_size=25 \ - \ - sampling=gt_dist_manhat_scaled,gt_dist_manhat \ - schedule=adavae_all,adavae_thresh,adavae_ratio \ - sampling.triplet_swap_chance=0,0.1 \ - \ - framework.cfg.triplet_margin_max=1.0 \ - framework.cfg.triplet_scale=0.1 \ - framework.cfg.triplet_p=1 \ - \ - framework.cfg.thresh_ratio=0.5 \ - framework.cfg.ada_triplet_ratio=1.0 \ - framework.cfg.ada_triplet_soft_scale=1.0 \ - framework.cfg.ada_triplet_sample=FALSE \ - \ - framework.cfg.ada_triplet_loss=triplet,triplet_all_soft_ave,triplet_hard_neg_ave,triplet_hard_neg_ave_pull \ - dataset=xysquares,shapes3d,cars3d,dsprites diff --git a/research/e03_axis_triplet/submit_04.sh b/research/e03_axis_triplet/submit_04.sh deleted file mode 100644 index b44ae30f..00000000 --- a/research/e03_axis_triplet/submit_04.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-03__axis-triplet-4.0" -export PARTITION="stampede" -export PARALLELISM=24 - -# source the helper file -source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# RESULT: -# - BAD: ada_thresh_mode=symmetric_kl, rather use "dist" -# - BAD: framework.cfg.adaave_decode_orig=FALSE, rather use TRUE -# - adat_share_ave_mode depends on other settings, but usually doesnt matter -# - adaave_augment_orig depends on other settings, but usually doesnt matter -# - GOOD: adat_triplet_loss=triplet_hard_neg_ave -# - NOTE: schedule=adavae_up_ratio usually converges sooner -# - NOTE: schedule=adavae_up_all usually converges later (makes sense because its a doubling effect a ^ 2) -# - NOTE: schedule=adavae_up_thresh usually is worse at converging - - -# 3*2*4*2*2*2 == 192 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='short-run__ada-best-loss-combo' \ - \ - framework=X--adaavetvae \ - run_length=short \ - model.z_size=25 \ - \ - schedule=adavae_up_all,adavae_up_ratio,adavae_up_thresh \ - sampling=gt_dist_manhat \ - sampling.triplet_swap_chance=0 \ - dataset=xysquares \ - \ - framework.cfg.triplet_loss=triplet \ - framework.cfg.triplet_margin_min=0.001 \ - framework.cfg.triplet_margin_max=1 \ - framework.cfg.triplet_scale=0.1 \ - framework.cfg.triplet_p=1 \ - \ - framework.cfg.detach=FALSE \ - framework.cfg.detach_decoder=FALSE \ - framework.cfg.detach_no_kl=FALSE \ - framework.cfg.detach_std=NULL \ - \ - framework.module.ada_average_mode=gvae \ - framework.module.ada_thresh_mode=symmetric_kl,dist \ - framework.module.ada_thresh_ratio=0.5 \ - \ - framework.module.adat_triplet_loss=triplet,triplet_soft_ave_all,triplet_hard_neg_ave,triplet_hard_ave_all \ - framework.module.adat_triplet_ratio=1.0 \ - framework.module.adat_triplet_soft_scale=1.0 \ - framework.module.adat_triplet_pull_weight=0.1 \ - \ - framework.module.adat_share_mask_mode=posterior \ - framework.module.adat_share_ave_mode=all,neg \ - \ - framework.module.adaave_augment_orig=TRUE,FALSE \ - framework.module.adaave_decode_orig=TRUE,FALSE - -# TRY THESE TOO: -# framework.module.adat_share_ave_mode=all,neg,pos,pos_neg \ -# framework.module.adat_share_mask_mode=posterior,sample,sample_each \ -# framework.module.adat_triplet_loss=triplet,triplet_soft_ave_all,triplet_hard_neg_ave,triplet_hard_neg_ave_pull,triplet_hard_ave_all \ - -# # 3*2*8*2*3*2*2 -#submit_sweep \ -# +DUMMY.repeat=1 \ -# +EXTRA.tags='short-run__ada-best-loss-combo' \ -# \ -# framework=X--adaavetvae \ -# run_length=short \ -# model.z_size=25 \ -# \ -# schedule=adavae_all,adavae_thresh,adavae_ratio \ -# sampling=gt_dist_manhat \ -# sampling.triplet_swap_chance=0 \ -# dataset=xysquares \ -# \ -# triplet_loss=triplet \ -# triplet_margin_min=0.001 \ -# triplet_margin_max=1 \ -# triplet_scale=0.1 \ -# triplet_p=1 \ -# \ -# detach=FALSE \ -# disable_decoder=FALSE \ -# detach_no_kl=FALSE \ -# detach_std=NULL \ -# \ -# ada_average_mode=gvae \ -# ada_thresh_mode=symmetric_kl,dist \ -# ada_thresh_ratio=0.5 \ -# \ -# adat_triplet_loss=triplet,triplet_soft_ave_neg,triplet_soft_ave_p_n,triplet_soft_ave_all,triplet_hard_ave,triplet_hard_neg_ave,triplet_hard_neg_ave_pull,triplet_hard_ave_all \ -# adat_triplet_ratio=1.0 \ -# adat_triplet_soft_scale=1.0 \ -# adat_triplet_pull_weight=0.1 \ -# \ -# adat_share_mask_mode=posterior,dist \ -# adat_share_ave_mode=all,pos_neg,pos,neg \ -# \ -# adaave_augment_orig=TRUE,FALSE \ -# adaave_decode_orig=TRUE,FALSE diff --git a/research/e03_axis_triplet/submit_05.sh b/research/e03_axis_triplet/submit_05.sh deleted file mode 100644 index 5ea5025f..00000000 --- a/research/e03_axis_triplet/submit_05.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-03__axis-triplet-5.0" -export PARTITION="stampede" -export PARALLELISM=16 - -# source the helper file -source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# 1 * (3*6*5) == 90 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='ada-best-pull-weight' \ - \ - framework=X--adanegtvae \ - run_length=short,medium,long \ - model.z_size=25 \ - \ - schedule=adavae_down_all,adavae_up_all,adavae_down_ratio,adavae_up_ratio,adavae_down_thresh,adavae_up_thresh \ - sampling=gt_dist_manhat \ - sampling.triplet_swap_chance=0 \ - dataset=xysquares \ - \ - framework.cfg.triplet_loss=triplet \ - framework.cfg.triplet_margin_min=0.001 \ - framework.cfg.triplet_margin_max=1 \ - framework.cfg.triplet_scale=0.1 \ - framework.cfg.triplet_p=1 \ - \ - framework.cfg.detach=FALSE \ - framework.cfg.detach_decoder=FALSE \ - framework.cfg.detach_no_kl=FALSE \ - framework.cfg.detach_std=NULL \ - \ - framework.cfg.ada_average_mode=gvae \ - framework.cfg.ada_thresh_mode=dist \ - framework.cfg.ada_thresh_ratio=0.5 \ - \ - framework.cfg.adat_triplet_ratio=1.0 \ - framework.cfg.adat_triplet_pull_weight=-1.0,-0.1,0.0,0.1,1.0 \ - \ - framework.cfg.adat_share_mask_mode=posterior diff --git a/research/e04_data_overlap_triplet/submit_01.sh b/research/e04_data_overlap_triplet/submit_01.sh deleted file mode 100644 index 2c4f2630..00000000 --- a/research/e04_data_overlap_triplet/submit_01.sh +++ /dev/null @@ -1,62 +0,0 @@ -##!/bin/bash -# -## ========================================================================= # -## Settings # -## ========================================================================= # -# -#export USERNAME="n_michlo" -#export PROJECT="final-04__data-overlap-triplet" -#export PARTITION="stampede" -#export PARALLELISM=32 -# -## source the helper file -#source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" -# -## ========================================================================= # -## Experiment # -## ========================================================================= # -# -#clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours -# -## 1 * (3*2*2*5*2) == 120 -#submit_sweep \ -# +DUMMY.repeat=1 \ -# +EXTRA.tags='med-best' \ -# \ -# framework=X--dotvae_aug \ -# run_length=medium \ -# model.z_size=25 \ -# \ -# schedule=adavae_up_all,adavae_up_ratio,none \ -# sampling=gt_dist_manhat \ -# sampling.triplet_swap_chance=0 \ -# dataset=xysquares \ -# \ -# framework.cfg.triplet_loss=triplet \ -# framework.cfg.triplet_margin_min=0.001 \ -# framework.cfg.triplet_margin_max=1 \ -# framework.cfg.triplet_scale=0.1,0.01 \ -# framework.cfg.triplet_p=1 \ -# \ -# framework.cfg.detach=FALSE \ -# framework.cfg.disable_decoder=FALSE \ -# framework.cfg.detach_no_kl=FALSE \ -# framework.cfg.detach_std=NULL \ -# \ -# framework.cfg.ada_average_mode=gvae \ -# framework.cfg.ada_thresh_mode=dist \ -# framework.cfg.ada_thresh_ratio=0.5 \ -# \ -# framework.cfg.adat_triplet_share_scale=0.95 \ -# \ -# framework.cfg.adat_share_mask_mode=posterior \ -# \ -# framework.cfg.overlap_num=4096 \ -# framework.cfg.overlap_mine_ratio=0.05,0.1 \ -# framework.cfg.overlap_mine_triplet_mode=none,hard_neg,semi_hard_neg,hard_pos,easy_pos \ -# \ -# framework.cfg.overlap_augment_mode='augment' \ -# framework.cfg.overlap_augment.p=1.0 \ -# framework.cfg.overlap_augment.radius=[61,61],[0,61] \ -# framework.cfg.overlap_augment.random_mode='batch' \ -# framework.cfg.overlap_augment.random_same_xy=TRUE diff --git a/research/e04_data_overlap_triplet/submit_02.sh b/research/e04_data_overlap_triplet/submit_02.sh deleted file mode 100644 index 81865f40..00000000 --- a/research/e04_data_overlap_triplet/submit_02.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-04__data-overlap-triplet" -export PARTITION="batch" -export PARALLELISM=16 - -# source the helper file -source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# 1 * (2*8*4) == 64 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='best-augment-strength__alt' \ - \ - framework=X--dotvae_aug \ - run_length=short \ - model=conv64alt \ - model.z_size=25 \ - \ - schedule=adavae_up_ratio_full,adavae_up_all_full \ - sampling=gt_dist_manhat \ - sampling.triplet_swap_chance=0 \ - dataset=xysquares \ - \ - framework.cfg.triplet_loss=triplet \ - framework.cfg.triplet_margin_min=0.001 \ - framework.cfg.triplet_margin_max=1 \ - framework.cfg.triplet_scale=0.1 \ - framework.cfg.triplet_p=1 \ - \ - framework.cfg.detach=FALSE \ - framework.cfg.detach_decoder=FALSE \ - framework.cfg.detach_no_kl=FALSE \ - framework.cfg.detach_std=NULL \ - \ - framework.cfg.ada_average_mode=gvae \ - framework.cfg.ada_thresh_mode=dist \ - framework.cfg.ada_thresh_ratio=0.5 \ - \ - framework.cfg.adat_triplet_share_scale=1.0 \ - \ - framework.cfg.adat_share_mask_mode=posterior \ - \ - framework.cfg.overlap_augment_mode='augment' \ - framework.cfg.overlap_augment.kernel=xy1_r47,xy8_r47,box_r47,gau_r47 \ - \ - framework.cfg.overlap_num=4096 \ - framework.module.overlap_mine_ratio=0.1 \ - framework.module.overlap_mine_triplet_mode=none,hard_neg,semi_hard_neg,hard_pos,easy_pos,ran:hard_neg+hard_pos,ran:hard_neg+easy_pos,ran:hard_pos+easy_pos - - # framework.module.overlap_augment.kernel=xy1_r47,xy8_r47,box_r47,gau_r47,box_r15,box_r31,box_r63,gau_r15,gau_r31,gau_r63 diff --git a/research/e04_data_overlap_triplet/submit_03_test_softada_vs_ada.sh b/research/e04_data_overlap_triplet/submit_03_test_softada_vs_ada.sh deleted file mode 100644 index 494cc903..00000000 --- a/research/e04_data_overlap_triplet/submit_03_test_softada_vs_ada.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -# -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="test-hard-vs-soft-ada" -export PARTITION="stampede" -export PARALLELISM=16 - -# source the helper file -source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# 3 * (3 * 1) = 9 -submit_sweep \ - +DUMMY.repeat=1,2,3 \ - +EXTRA.tags='sweep_02' \ - \ - run_length=medium \ - metrics=all \ - \ - framework.beta=1 \ - framework=adavae_os,adagvae_minimal_os,X--softadagvae_minimal_os \ - model.z_size=25 \ - \ - dataset=shapes3d \ - \ - hydra.launcher.exclude='"mscluster93,mscluster94,mscluster97"' # we don't want to sweep over these diff --git a/research/e05_disentangle_kernel/run_01_sort_loss.py b/research/e05_disentangle_kernel/run_01_sort_loss.py deleted file mode 100644 index 710b2f3b..00000000 --- a/research/e05_disentangle_kernel/run_01_sort_loss.py +++ /dev/null @@ -1,80 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import torch -import torch.nn.functional as F -from torch.utils.data import DataLoader - -import research.util as H -from disent.nn.loss.softsort import multi_spearman_rank_loss -from disent.nn.loss.softsort import torch_soft_rank - - -# ========================================================================= # -# tests # -# ========================================================================= # - - -def run_differentiable_sorting_loss(dataset='dsprites', loss_mode='spearman', optimizer='adam', lr=1e-2): - """ - test that the differentiable sorting works over a batch of images. - """ - - dataset = H.make_dataset(dataset) - dataloader = DataLoader(dataset=dataset, batch_size=256, pin_memory=True, shuffle=True) - - y = H.get_single_batch(dataloader) - # y += torch.randn_like(y) * 0.001 # prevent nan errors - x = torch.randn_like(y, requires_grad=True) - - optimizer = H.make_optimizer(x, name=optimizer, lr=lr) - - for i in range(1001): - if loss_mode == 'spearman': - loss = multi_spearman_rank_loss(x, y, dims=(2, 3), nan_to_num=True) - elif loss_mode == 'mse_rank': - loss = 0. - loss += F.mse_loss(torch_soft_rank(x, dims=(-3, -1)), torch_soft_rank(y, dims=(-3, -1)), reduction='mean') - loss += F.mse_loss(torch_soft_rank(x, dims=(-3, -2)), torch_soft_rank(y, dims=(-3, -2)), reduction='mean') - elif loss_mode == 'mse': - loss += F.mse_loss(x, y, reduction='mean') - else: - raise KeyError(f'invalid loss mode: {repr(loss_mode)}') - - # update variables - H.step_optimizer(optimizer, loss) - if i % 250 == 0: - H.plt_imshow(H.to_img(x[0]), show=True) - - # compute loss - print(i, float(loss)) - - -# ========================================================================= # -# MAIN # -# ========================================================================= # - - -if __name__ == '__main__': - run_differentiable_sorting_loss() diff --git a/research/e05_disentangle_kernel/run_02_check_aug_gt_dists.py b/research/e05_disentangle_kernel/run_02_check_aug_gt_dists.py deleted file mode 100644 index 9dd69e0e..00000000 --- a/research/e05_disentangle_kernel/run_02_check_aug_gt_dists.py +++ /dev/null @@ -1,168 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - - -import numpy as np -import torch -import torch.nn.functional as F -from matplotlib import pyplot as plt -from tqdm import tqdm - -import research.util as H -from disent.nn.functional import torch_box_kernel_2d -from disent.nn.functional import torch_conv2d_channel_wise_fft -from disent.nn.functional import torch_gaussian_kernel_2d - - -# ========================================================================= # -# distance function # -# ========================================================================= # - - -def spearman_rank_dist( - pred: torch.Tensor, - targ: torch.Tensor, - reduction='mean', - nan_to_num=False, -): - # add missing dim - if pred.ndim == 1: - pred, targ = pred.reshape(1, -1), targ.reshape(1, -1) - assert pred.shape == targ.shape - assert pred.ndim == 2 - # sort the last dimension of the 2D tensors - pred = torch.argsort(pred).to(torch.float32) - targ = torch.argsort(targ).to(torch.float32) - # compute individual losses - # TODO: this can result in nan values, what to do then? - pred = pred - pred.mean(dim=-1, keepdim=True) - pred = pred / pred.norm(dim=-1, keepdim=True) - targ = targ - targ.mean(dim=-1, keepdim=True) - targ = targ / targ.norm(dim=-1, keepdim=True) - # replace nan values - if nan_to_num: - pred = torch.nan_to_num(pred, nan=0.0) - targ = torch.nan_to_num(targ, nan=0.0) - # compute the final loss - loss = (pred * targ).sum(dim=-1) - # reduce the loss - if reduction == 'mean': - return loss.mean() - elif reduction == 'none': - return loss - else: - raise KeyError(f'Invalid reduction mode: {repr(reduction)}') - - -def check_xy_squares_dists(kernel='box', repeats=100, samples=256, pairwise_samples=256, kernel_radius=32, show_prog=True): - if kernel == 'box': - kernel = torch_box_kernel_2d(radius=kernel_radius)[None, ...] - elif kernel == 'max_box': - crange = torch.abs(torch.arange(kernel_radius * 2 + 1) - kernel_radius) - y, x = torch.meshgrid(crange, crange) - d = torch.maximum(x, y) + 1 - d = d.max() - d - kernel = (d.to(torch.float32) / d.sum())[None, None, ...] - elif kernel == 'min_box': - crange = torch.abs(torch.arange(kernel_radius * 2 + 1) - kernel_radius) - y, x = torch.meshgrid(crange, crange) - d = torch.minimum(x, y) + 1 - d = d.max() - d - kernel = (d.to(torch.float32) / d.sum())[None, None, ...] - elif kernel == 'manhat_box': - crange = torch.abs(torch.arange(kernel_radius * 2 + 1) - kernel_radius) - y, x = torch.meshgrid(crange, crange) - d = (y + x) + 1 - d = d.max() - d - kernel = (d.to(torch.float32) / d.sum())[None, None, ...] - elif kernel == 'gaussian': - kernel = torch_gaussian_kernel_2d(sigma=kernel_radius / 4.0, truncate=4.0)[None, None, ...] - else: - raise KeyError(f'invalid kernel mode: {repr(kernel)}') - - # make dataset - dataset = H.make_dataset('xysquares') - - losses = [] - prog = tqdm(range(repeats), postfix={'loss': 0.0}) if show_prog else range(repeats) - - for i in prog: - # get random samples - factors = dataset.sample_factors(samples) - batch = dataset.dataset_batch_from_factors(factors, mode='target') - if torch.cuda.is_available(): - batch = batch.cuda() - kernel = kernel.cuda() - factors = torch.from_numpy(factors).to(dtype=torch.float32, device=batch.device) - - # random pairs - ia, ib = torch.randint(0, len(batch), size=(2, pairwise_samples), device=batch.device) - - # compute factor distances - f_dists = torch.abs(factors[ia] - factors[ib]).sum(dim=-1) - - # compute loss distances - aug_batch = torch_conv2d_channel_wise_fft(batch, kernel) - # TODO: aug - batch or aug - aug - # b_dists = torch.abs(aug_batch[ia] - aug_batch[ib]).sum(dim=(-3, -2, -1)) - b_dists = F.mse_loss(aug_batch[ia], aug_batch[ib], reduction='none').sum(dim=(-3, -2, -1)) - - # compute ranks - # losses.append(float(torch.clamp(torch_mse_rank_loss(b_dists, f_dists), 0, 100))) - # losses.append(float(torch.abs(torch.argsort(f_dists, descending=True) - torch.argsort(b_dists, descending=False)).to(torch.float32).mean())) - losses.append(float(spearman_rank_dist(b_dists, f_dists))) - - if show_prog: - prog.set_postfix({'loss': np.mean(losses)}) - - return np.mean(losses), aug_batch[0] - - -def run_check_all_xy_squares_dists(show=False): - for kernel in [ - 'box', - 'max_box', - 'min_box', - 'manhat_box', - 'gaussian', - ]: - rs = list(range(1, 33, 4)) - ys = [] - for r in rs: - ave_spearman, last_img = check_xy_squares_dists(kernel=kernel, repeats=32, samples=128, pairwise_samples=1024, kernel_radius=r, show_prog=False) - H.plt_imshow(H.to_img(last_img, scale=True), show=show) - ys.append(abs(ave_spearman)) - print(kernel, r, ':', r*2+1, abs(ave_spearman)) - plt.plot(rs, ys, label=kernel) - plt.legend() - plt.show() - - -# ========================================================================= # -# MAIN # -# ========================================================================= # - - -if __name__ == '__main__': - run_check_all_xy_squares_dists() diff --git a/research/e05_disentangle_kernel/run_03_train_disentangle_kernel.py b/research/e05_disentangle_kernel/run_03_train_disentangle_kernel.py deleted file mode 100644 index eb15d491..00000000 --- a/research/e05_disentangle_kernel/run_03_train_disentangle_kernel.py +++ /dev/null @@ -1,297 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -import os -from typing import List -from typing import Optional -from typing import Sequence - -import hydra -import numpy as np -import pytorch_lightning as pl -import torch -import wandb -from omegaconf import OmegaConf -from torch.nn import Parameter -from torch.utils.data import DataLoader - -import disent.util.seeds -import research.util as H -from disent.nn.functional import torch_conv2d_channel_wise_fft -from disent.nn.loss.softsort import spearman_rank_loss -from disent.nn.modules import DisentLightningModule -from disent.nn.modules import DisentModule -from disent.util.lightning.callbacks import BaseCallbackPeriodic -from disent.util.lightning.logger_util import wb_log_metrics -from disent.util.seeds import seed -from disent.util.strings.fmt import make_box_str -from experiment.run import hydra_append_progress_callback -from experiment.run import hydra_get_gpus -from experiment.run import hydra_make_logger -from experiment.util.hydra_utils import make_non_strict - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# EXP # -# ========================================================================= # - - -def disentangle_loss( - batch: torch.Tensor, - factors: torch.Tensor, - num_pairs: int, - f_idxs: Optional[List[int]] = None, - loss_fn: str = 'mse', - mean_dtype=None, -) -> torch.Tensor: - assert len(batch) == len(factors) - assert batch.ndim == 4 - assert factors.ndim == 2 - # random pairs - ia, ib = torch.randint(0, len(batch), size=(2, num_pairs), device=batch.device) - # get pairwise distances - b_dists = H.pairwise_loss(batch[ia], batch[ib], mode=loss_fn, mean_dtype=mean_dtype) # avoid precision errors - # compute factor distances - if f_idxs is not None: - f_dists = torch.abs(factors[ia][:, f_idxs] - factors[ib][:, f_idxs]).sum(dim=-1) - else: - f_dists = torch.abs(factors[ia] - factors[ib]).sum(dim=-1) - # optimise metric - loss = spearman_rank_loss(b_dists, -f_dists) # decreasing overlap should mean increasing factor dist - return loss - - -class DisentangleModule(DisentLightningModule): - - def __init__( - self, - model, - hparams, - disentangle_factor_idxs: Sequence[int] = None - ): - super().__init__() - self.model = model - self.hparams = hparams - self._disentangle_factors = None if (disentangle_factor_idxs is None) else np.array(disentangle_factor_idxs) - - def configure_optimizers(self): - return H.make_optimizer(self, name=self.hparams.optimizer.name, lr=self.hparams.optimizer.lr, weight_decay=self.hparams.optimizer.weight_decay) - - def training_step(self, batch, batch_idx): - (batch,), (factors,) = batch['x_targ'], batch['factors'] - # feed forward batch - aug_batch = self.model(batch) - # compute pairwise distances of factors and batch, and optimize to correspond - loss = disentangle_loss( - batch=aug_batch, - factors=factors, - num_pairs=int(len(batch) * self.hparams.train.pairs_ratio), - f_idxs=self._disentangle_factors, - loss_fn=self.hparams.train.loss, - mean_dtype=torch.float64, - ) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - if hasattr(self.model, 'augment_loss'): - loss_aug = self.model.augment_loss(self) - else: - loss_aug = 0 - loss += loss_aug - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - self.log('loss', loss) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - return loss - - def forward(self, batch): - return self.model(batch) - - -# ========================================================================= # -# MAIN # -# ========================================================================= # - - -class Kernel(DisentModule): - def __init__(self, radius: int = 33, channels: int = 1, offset: float = 0.0, scale: float = 0.001, train_symmetric_regularise: bool = True, train_norm_regularise: bool = True, train_nonneg_regularise: bool = True): - super().__init__() - assert channels in (1, 3) - kernel = torch.randn(1, channels, 2*radius+1, 2*radius+1, dtype=torch.float32) - kernel = offset + kernel * scale - # normalise - if train_nonneg_regularise: - kernel = torch.abs(kernel) - if train_norm_regularise: - kernel = kernel / kernel.sum(dim=[-1, -2], keepdim=True) - # store - self._kernel = Parameter(kernel) - # regularise options - self._train_symmetric_regularise = train_symmetric_regularise - self._train_norm_regularise = train_norm_regularise - self._train_nonneg_regularise = train_nonneg_regularise - - def forward(self, xs): - return torch_conv2d_channel_wise_fft(xs, self._kernel) - - def make_train_periodic_callback(self, cfg, dataset) -> BaseCallbackPeriodic: - class ImShowCallback(BaseCallbackPeriodic): - def do_step(self, trainer: pl.Trainer, pl_module: pl.LightningModule): - # get kernel image - kernel = H.to_img(pl_module.model._kernel[0], scale=True).numpy() - # augment function - def augment_fn(batch): - return H.to_imgs(pl_module.forward(batch.to(pl_module.device)), scale=True) - # get augmented traversals - with torch.no_grad(): - orig_wandb_image, orig_wandb_animation = H.visualize_dataset_traversal(dataset) - augm_wandb_image, augm_wandb_animation = H.visualize_dataset_traversal(dataset, augment_fn=augment_fn, data_mode='input') - # log images to WANDB - wb_log_metrics(trainer.logger, { - 'kernel': wandb.Image(kernel), - 'traversal_img_orig': orig_wandb_image, 'traversal_animation_orig': orig_wandb_animation, - 'traversal_img_augm': augm_wandb_image, 'traversal_animation_augm': augm_wandb_animation, - }) - return ImShowCallback(every_n_steps=cfg.exp.show_every_n_steps, begin_first_step=True) - - def augment_loss(self, framework: DisentLightningModule): - augment_loss = 0 - # symmetric loss - if self._train_symmetric_regularise: - k, kt = self._kernel[0], torch.transpose(self._kernel[0], -1, -2) - loss_symmetric = 0 - loss_symmetric += H.unreduced_loss(torch.flip(k, dims=[-1]), k, mode='mae').mean() - loss_symmetric += H.unreduced_loss(torch.flip(k, dims=[-2]), k, mode='mae').mean() - loss_symmetric += H.unreduced_loss(torch.flip(k, dims=[-1]), kt, mode='mae').mean() - loss_symmetric += H.unreduced_loss(torch.flip(k, dims=[-2]), kt, mode='mae').mean() - # log loss - framework.log('loss_symmetric', loss_symmetric) - # final loss - augment_loss += loss_symmetric - # sum of 1 loss, per channel - if self._train_norm_regularise: - k = self._kernel[0] - # sum over W & H resulting in: (C, W, H) -> (C,) - channel_sums = k.sum(dim=[-1, -2]) - channel_loss = H.unreduced_loss(channel_sums, torch.ones_like(channel_sums), mode='mae') - norm_loss = channel_loss.mean() - # log loss - framework.log('loss_norm', norm_loss) - # final loss - augment_loss += norm_loss - # no negatives regulariser - if self._train_nonneg_regularise: - k = self._kernel[0] - nonneg_loss = torch.abs(k[k < 0].sum()) - # log loss - framework.log('loss_non_negative', nonneg_loss) - # regularise negatives - augment_loss += nonneg_loss - # return! - return augment_loss - - -# ========================================================================= # -# Run Hydra # -# ========================================================================= # - - -ROOT_DIR = os.path.abspath(__file__ + '/../../../..') - - -@hydra.main(config_path=os.path.join(ROOT_DIR, 'experiment/config'), config_name="config_adversarial_kernel") -def run_disentangle_dataset_kernel(cfg): - cfg = make_non_strict(cfg) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # TODO: some of this code is duplicated between this and the main experiment run.py - # check CUDA setting - cfg.trainer.setdefault('cuda', 'try_cuda') - gpus = hydra_get_gpus(cfg) - # CREATE LOGGER - logger = hydra_make_logger(cfg) - # TRAINER CALLBACKS - callbacks = [] - hydra_append_progress_callback(callbacks, cfg) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - seed(disent.util.seeds.seed) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # initialise dataset and get factor names to disentangle - dataset = H.make_dataset(cfg.data.name, factors=True, data_root=cfg.default_settings.storage.data_root) - disentangle_factor_idxs = dataset.gt_data.normalise_factor_idxs(cfg.kernel.disentangle_factors) - cfg.kernel.disentangle_factors = tuple(dataset.gt_data.factor_names[i] for i in disentangle_factor_idxs) - log.info(f'Dataset has ground-truth factors: {dataset.gt_data.factor_names}') - log.info(f'Chosen ground-truth factors are: {tuple(cfg.kernel.disentangle_factors)}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # print everything - log.info('Final Config' + make_box_str(OmegaConf.to_yaml(cfg))) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - dataloader = DataLoader( - dataset, - batch_sampler=H.StochasticBatchSampler(dataset, batch_size=cfg.dataset.batch_size), - num_workers=cfg.dataset.num_workers, - pin_memory=cfg.dataset.pin_memory, - ) - model = Kernel(radius=cfg.kernel.radius, channels=cfg.kernel.channels, offset=0.002, scale=0.01, train_symmetric_regularise=cfg.kernel.regularize_symmetric, train_norm_regularise=cfg.kernel.regularize_norm, train_nonneg_regularise=cfg.kernel.regularize_nonneg) - callbacks.append(model.make_train_periodic_callback(cfg, dataset=dataset)) - framework = DisentangleModule(model, cfg, disentangle_factor_idxs=disentangle_factor_idxs) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - if framework.logger: - framework.logger.log_hyperparams(framework.hparams) - # train - trainer = pl.Trainer( - log_every_n_steps=cfg.log.setdefault('log_every_n_steps', 50), - flush_logs_every_n_steps=cfg.log.setdefault('flush_logs_every_n_steps', 100), - logger=logger, - callbacks=callbacks, - gpus=1 if gpus else 0, - max_epochs=cfg.trainer.setdefault('epochs', None), - max_steps=cfg.trainer.setdefault('steps', 10000), - progress_bar_refresh_rate=0, # ptl 0.9 - terminate_on_nan=True, # we do this here so we don't run the final metrics - checkpoint_callback=False, - ) - trainer.fit(framework, dataloader) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # save kernel - if cfg.exp.rel_save_dir is not None: - assert not os.path.isabs(cfg.exp.rel_save_dir), f'rel_save_dir must be relative: {repr(cfg.exp.rel_save_dir)}' - save_dir = os.path.join(ROOT_DIR, cfg.exp.rel_save_dir) - assert os.path.isabs(save_dir), f'save_dir must be absolute: {repr(save_dir)}' - # save kernel - H.torch_write(os.path.join(save_dir, cfg.exp.save_name), framework.model._kernel) - - -# ========================================================================= # -# Entry Point # -# ========================================================================= # - - -if __name__ == '__main__': - # HYDRA: - # run experiment (12min * 4*8*2) / 60 ~= 12 hours - # but speeds up as kernel size decreases, so might be shorter - # EXP ARGS: - # $ ... -m optimizer.weight_decay=1e-4,0.0 kernel.radius=63,55,47,39,31,23,15,7 dataset.spacing=8,4,2,1 - run_disentangle_dataset_kernel() diff --git a/research/e05_disentangle_kernel/submit_03.sh b/research/e05_disentangle_kernel/submit_03.sh deleted file mode 100644 index bd5e6e6a..00000000 --- a/research/e05_disentangle_kernel/submit_03.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-03__kernel-disentangle-xy" -export PARTITION="stampede" -export PARALLELISM=32 -export PY_RUN_FILE='experiment/exp/05_adversarial_data/run_03_train_disentangle_kernel.py' - -# source the helper file -source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# 1 * (2*8*4) == 64 -submit_sweep \ - optimizer.weight_decay=1e-4,0.0 \ - kernel.radius=63,55,47,39,31,23,15,7 \ - data.name=xysquares_8x8,xysquares_4x4,xysquares_2x2,xysquares_1x1 diff --git a/research/e06_adversarial_data/deprecated/run_01_gen_adversarial_disk.py b/research/e06_adversarial_data/deprecated/run_01_gen_adversarial_disk.py deleted file mode 100644 index 4d323ab5..00000000 --- a/research/e06_adversarial_data/deprecated/run_01_gen_adversarial_disk.py +++ /dev/null @@ -1,497 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - - -""" -Generate an adversarial dataset -- Stores the mutating dataset on disk -- Loads minibatches from disk that are optimized and the saved back to the disk -- No model is used, images are directly optimized against eachother, could decay in some cases? - -This is quite memory efficient, but it is quite old! -- Should probably be re-written using ray -""" - - -import logging -import multiprocessing.synchronize -import os -from concurrent.futures import Executor -from concurrent.futures import Future -from concurrent.futures import ProcessPoolExecutor -from typing import Optional -from typing import Sequence - -import h5py -import numpy as np -import psutil -import torch -from tqdm import tqdm - -import research.util as H -from disent.util.deprecate import deprecated -from disent.util.inout.paths import ensure_parent_dir_exists -from disent.util.profiling import Timer -from disent.util.seeds import seed - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# losses # -# ========================================================================= # - - -def stochastic_const_loss(pred: torch.Tensor, mask: torch.Tensor, num_pairs: int, num_samples: int, loss='mse', reg_out_of_bounds=True, top_k: int = None, constant_targ: float = None) -> torch.Tensor: - ia, ib = torch.randint(0, len(pred), size=(2, num_samples), device=pred.device) - # constant dist loss - x_ds = (H.unreduced_loss(pred[ia], pred[ib], mode=loss) * mask[None, ...]).mean(dim=(-3, -2, -1)) - # compute constant loss - if constant_targ is None: - iA, iB = torch.randint(0, len(x_ds), size=(2, num_pairs), device=pred.device) - lcst = H.unreduced_loss(x_ds[iA], x_ds[iB], mode=loss) - else: - lcst = H.unreduced_loss(x_ds, torch.full_like(x_ds, constant_targ), mode=loss) - # aggregate constant loss - if top_k is None: - lcst = lcst.mean() - else: - lcst = torch.topk(lcst, k=top_k, largest=True).values.mean() - # values over the required range - if reg_out_of_bounds: - m = torch.nan_to_num((0 - pred[pred < 0]) ** 2, nan=0).mean() - M = torch.nan_to_num((pred[pred > 1] - 1) ** 2, nan=0).mean() - mM = m + M - else: - mM = 0. - # done! - return mM + lcst - - -# ========================================================================= # -# h5py dataset helper # -# ========================================================================= # - - -NAME_DATA = 'data' -NAME_VISITS = 'visits' -NAME_OBS = 'x_targ' - -_SAVE_TYPE_LOOKUP = { - 'uint8': torch.uint8, - 'float16': torch.float16, - 'float32': torch.float32, -} - -SAVE_TYPE = 'float16' -assert SAVE_TYPE in _SAVE_TYPE_LOOKUP - - -def _make_hdf5_dataset(path, dataset, overwrite_mode: str = 'continue') -> str: - path = ensure_parent_dir_exists(path) - # get read/write mode - if overwrite_mode == 'overwrite': - rw_mode = 'w' # create new file, overwrite if exists - elif overwrite_mode == 'fail': - rw_mode = 'x' # create new file, fail if exists - elif overwrite_mode == 'continue': - rw_mode = 'a' # create if missing, append if exists - # clear file consistency flags - # if clear_consistency_flags: - # if os.path.isfile(path): - # cmd = ["h5clear", "-s", "'{path}'"] - # print(f'clearing file consistency flags: {" ".join(cmd)}') - # try: - # subprocess.check_output(cmd) - # except FileNotFoundError: - # raise FileNotFoundError('h5clear utility is not installed!') - else: - raise KeyError(f'invalid overwrite_mode={repr(overwrite_mode)}') - # open in read write mode - log.info(f'Opening hdf5 dataset: overwrite_mode={repr(overwrite_mode)} exists={repr(os.path.exists(path))} path={repr(path)}') - with h5py.File(path, rw_mode, libver='earliest') as f: - # get data - num_obs = len(dataset) - obs_shape = dataset[0][NAME_OBS][0].shape - # make dset - if NAME_DATA not in f: - f.create_dataset( - NAME_DATA, - shape=(num_obs, *obs_shape), - dtype=SAVE_TYPE, - chunks=(1, *obs_shape), - track_times=False, - ) - # make set_dset - if NAME_VISITS not in f: - f.create_dataset( - NAME_VISITS, - shape=(num_obs,), - dtype='int64', - chunks=(1,), - track_times=False, - ) - return path - - -# def _read_hdf5_batch(h5py_path: str, idxs, return_visits=False): -# batch, visits = [], [] -# with h5py.File(h5py_path, 'r', swmr=True) as f: -# for i in idxs: -# visits.append(f[NAME_VISITS][i]) -# obs = torch.as_tensor(f[NAME_DATA][i], dtype=torch.float32) -# if SAVE_TYPE == 'uint8': -# obs /= 255 -# batch.append(obs) -# # return values -# if return_visits: -# return torch.stack(batch, dim=0), np.array(visits, dtype=np.int64) -# else: -# return torch.stack(batch, dim=0) - - -def _load_hdf5_batch(dataset, h5py_path: str, idxs, initial_noise: Optional[float] = None, return_visits=True): - """ - Load a batch from the disk -- always return float32 - - Can be used by multiple threads at a time. - - returns an item from the original dataset if an - observation has not been saved into the hdf5 dataset yet. - """ - batch, visits = [], [] - with h5py.File(h5py_path, 'r', swmr=True) as f: - for i in idxs: - v = f[NAME_VISITS][i] - if v > 0: - obs = torch.as_tensor(f[NAME_DATA][i], dtype=torch.float32) - if SAVE_TYPE == 'uint8': - obs /= 255 - else: - (obs,) = dataset[i][NAME_OBS] - obs = obs.to(torch.float32) - if initial_noise is not None: - obs += (torch.randn_like(obs) * initial_noise) - batch.append(obs) - visits.append(v) - # stack and check values - batch = torch.stack(batch, dim=0) - assert batch.dtype == torch.float32 - # return values - if return_visits: - return batch, np.array(visits, dtype=np.int64) - else: - return batch - - -def _save_hdf5_batch(h5py_path: str, batch, idxs): - """ - Save a float32 batch to disk. - - Can only be used by one thread at a time! - """ - assert batch.dtype == torch.float32 - with h5py.File(h5py_path, 'r+', libver='earliest') as f: - for obs, idx in zip(batch, idxs): - if SAVE_TYPE == 'uint8': - f[NAME_DATA][idx] = torch.clamp(torch.round(obs * 255), 0, 255).to(torch.uint8) - else: - f[NAME_DATA][idx] = obs.to(_SAVE_TYPE_LOOKUP[SAVE_TYPE]) - f[NAME_VISITS][idx] += 1 - - -# ========================================================================= # -# multiproc h5py dataset helper # -# ========================================================================= # - - -class FutureList(object): - def __init__(self, futures: Sequence[Future]): - self._futures = futures - - def result(self): - return [future.result() for future in self._futures] - - -# ========================================================================= # -# multiproc h5py dataset helper # -# ========================================================================= # - - -# SUBMIT: - - -def _submit_load_batch_futures(executor: Executor, num_splits: int, dataset, h5py_path: str, idxs, initial_noise: Optional[float] = None) -> FutureList: - return FutureList([ - executor.submit(__inner__load_batch, dataset=dataset, h5py_path=h5py_path, idxs=idxs, initial_noise=initial_noise) - for idxs in np.array_split(idxs, num_splits) - ]) - - -def _submit_save_batch(executor: Executor, h5py_path: str, batch, idxs) -> Future: - return executor.submit(__inner__save_batch, h5py_path=h5py_path, batch=batch, idxs=idxs) - - -NUM_WORKERS = psutil.cpu_count() -_BARRIER = None - - -def __inner__load_batch(dataset, h5py_path: str, idxs, initial_noise: Optional[float] = None): - _BARRIER.wait() - result = _load_hdf5_batch(dataset=dataset, h5py_path=h5py_path, idxs=idxs, initial_noise=initial_noise) - _BARRIER.wait() - return result - - -def __inner__save_batch(h5py_path, batch, idxs): - _save_hdf5_batch(h5py_path=h5py_path, batch=batch, idxs=idxs) - - -# WAIT: - - -def _wait_for_load_future(future: FutureList): - with Timer() as t: - xs, visits = zip(*future.result()) - xs = torch.cat(xs, dim=0) - visits = np.concatenate(visits, axis=0).mean(dtype=np.float32) - return (xs, visits), t - - -def _wait_for_save_future(future: Future): - with Timer() as t: - future.result() - return t - - -# ========================================================================= # -# adversarial dataset generator # -# ========================================================================= # - - -def run_generate_and_save_adversarial_dataset_mp( - dataset_name: str = 'shapes3d', - dataset_load_into_memory: bool = False, - optimizer: str = 'adam', - lr: float = 1e-2, - obs_masked: bool = True, - obs_initial_noise: Optional[float] = None, - loss_fn: str = 'mse', - batch_size: int = 1024*12, # approx - batch_sample_mode: str = 'shuffle', # range, shuffle, random - loss_num_pairs: int = 1024*4, - loss_num_samples: int = 1024*4*2, # only applies if loss_const_targ=None - loss_top_k: Optional[int] = None, - loss_const_targ: Optional[float] = 0.1, # replace stochastic pairwise constant loss with deterministic loss target - loss_reg_out_of_bounds: bool = False, - train_epochs: int = 8, - train_optim_steps: int = 125, - # skipped params - save_folder: str = 'out/overlap', - save_prefix: str = '', - overwrite_mode: str = 'fail', # continue, overwrite, fail - seed_: Optional[int] = 777, -) -> str: - # checks - if obs_initial_noise is not None: - assert not obs_masked, '`obs_masked` cannot be `True`, if using initial noise, ie. `obs_initial_noise is not None`' - - # deterministic! - seed(seed_) - - # ↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓ # - # make dataset - dataset = H.make_dataset(dataset_name, load_into_memory=dataset_load_into_memory, load_memory_dtype=torch.float16) - # get save path - assert not ('/' in save_prefix or '\\' in save_prefix) - name = H.params_as_string(H.get_caller_params(exclude=["save_folder", "save_prefix", "overwrite_mode", "seed_"])) - path = _make_hdf5_dataset(os.path.join(save_folder, f'{save_prefix}{name}.hdf5'), dataset=dataset, overwrite_mode=overwrite_mode) - # ↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑ # - - train_batches = (len(dataset) + batch_size - 1) // batch_size - # loop vars & progress bar - save_time = Timer() - prog = tqdm(total=train_epochs * train_batches * train_optim_steps, postfix={'loss': 0.0, '💯': 0.0, '🔍': 'N/A', '💾': 'N/A'}, ncols=100) - # multiprocessing pool - global _BARRIER # TODO: this is a hack and should be unique to each run - _BARRIER = multiprocessing.Barrier(NUM_WORKERS) - executor = ProcessPoolExecutor(NUM_WORKERS) - - # EPOCHS: - for e in range(train_epochs): - # generate batches - batch_idxs = H.generate_epoch_batch_idxs(num_obs=len(dataset), num_batches=train_batches, mode=batch_sample_mode) - # first data load - load_future = _submit_load_batch_futures(executor, num_splits=NUM_WORKERS, dataset=dataset, h5py_path=path, idxs=batch_idxs[0], initial_noise=obs_initial_noise) - - # TODO: log to WANDB - # TODO: SAMPLING STRATEGY MIGHT NEED TO CHANGE! - # - currently random pairs are generated, but the pairs that matter are the nearby ones. - # - sample pairs that increase and decrease along an axis - # - sample pairs that are nearby according to the factor distance metric - - # BATCHES: - for n in range(len(batch_idxs)): - # ↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓ # - # get batch -- transfer to gpu is the bottleneck - (x, visits), load_time = _wait_for_load_future(load_future) - x = x.cuda().requires_grad_(True) - # ↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑ # - - # queue loading an extra batch - if (n+1) < len(batch_idxs): - load_future = _submit_load_batch_futures(executor, num_splits=NUM_WORKERS, dataset=dataset, h5py_path=path, idxs=batch_idxs[n + 1], initial_noise=obs_initial_noise) - - # ↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓ # - # make optimizers - mask = H.make_changed_mask(x, masked=obs_masked) - optim = H.make_optimizer(x, name=optimizer, lr=lr) - # ↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑ # - - # OPTIMIZE: - for _ in range(train_optim_steps): - # final loss & update - # ↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓ # - loss = stochastic_const_loss(x, mask, num_pairs=loss_num_pairs, num_samples=loss_num_samples, loss=loss_fn, reg_out_of_bounds=loss_reg_out_of_bounds, top_k=loss_top_k, constant_targ=loss_const_targ) - H.step_optimizer(optim, loss) - # ↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑ # - - # update progress bar - logs = {'loss': float(loss), '💯': visits, '🔍': load_time.pretty, '💾': save_time.pretty} - prog.update() - prog.set_postfix(logs) - - # save optimized minibatch - if n > 0: - save_time = _wait_for_save_future(save_future) - save_future = _submit_save_batch(executor, h5py_path=path, batch=x.detach().cpu(), idxs=batch_idxs[n]) - - # final save - save_time = _wait_for_save_future(save_future) - - # cleanup all - executor.shutdown() - # return the path to the dataset - return path - - -# ========================================================================= # -# test adversarial dataset generator # -# ========================================================================= # - - -@deprecated('Replaced with run_02_gen_adversarial_dataset_approx') -def run_generate_adversarial_data( - dataset: str ='shapes3d', - factor: str ='wall_hue', - factor_mode: str = 'sample_random', - optimizer: str ='radam', - lr: float = 1e-2, - obs_num: int = 1024 * 10, - obs_noise_weight: float = 0, - obs_masked: bool = True, - loss_fn: str = 'mse', - loss_num_pairs: int = 4096, - loss_num_samples: int = 4096*2, # only applies if loss_const_targ=None - loss_top_k: int = None, - loss_const_targ: float = None, # replace stochastic pairwise constant loss with deterministic loss target - loss_reg_out_of_bounds: bool = False, - train_steps: int = 2000, - display_period: int = 500, -): - seed(777) - # make dataset - dataset = H.make_dataset(dataset) - # make batches - factors = H.sample_factors(dataset, num_obs=obs_num, factor_mode=factor_mode, factor=factor) - x = dataset.dataset_batch_from_factors(factors, 'target') - # make tensors to optimize - if torch.cuda.is_available(): - x = x.cuda() - x = torch.tensor(x + torch.randn_like(x) * obs_noise_weight, requires_grad=True) - # generate mask - mask = H.make_changed_mask(x, masked=obs_masked) - H.plt_imshow(H.to_img(mask.to(torch.float32)), show=True) - # make optimizer - optimizer = H.make_optimizer(x, name=optimizer, lr=lr) - - # optimize differences according to loss - prog = tqdm(range(train_steps+1), postfix={'loss': 0.0}) - for i in prog: - # final loss - loss = stochastic_const_loss(x, mask, num_pairs=loss_num_pairs, num_samples=loss_num_samples, loss=loss_fn, reg_out_of_bounds=loss_reg_out_of_bounds, top_k=loss_top_k, constant_targ=loss_const_targ) - # update variables - H.step_optimizer(optimizer, loss) - if i % display_period == 0: - log.warning(f'visualisation of `x[:9]` was disabled') - prog.set_postfix({'loss': float(loss)}) - - -# ========================================================================= # -# entrypoint # -# ========================================================================= # - -# TODO: add WANDB support for visualisation of dataset -# TODO: add graphing of visual overlap like exp 01 - -def main(): - logging.basicConfig(level=logging.INFO, format='(%(asctime)s) %(name)s:%(lineno)d [%(levelname)s]: %(message)s') - - paths = [] - for i, kwargs in enumerate([ - # dict(save_prefix='e128__fixed_unmask_const_', obs_masked=False, loss_const_targ=0.1, obs_initial_noise=None, optimizer='adam', dataset_name='cars3d'), - # dict(save_prefix='e128__fixed_unmask_const_', obs_masked=False, loss_const_targ=0.1, obs_initial_noise=None, optimizer='adam', dataset_name='smallnorb'), - # dict(save_prefix='e128__fixed_unmask_randm_', obs_masked=False, loss_const_targ=None, obs_initial_noise=None, optimizer='adam', dataset_name='cars3d'), - # dict(save_prefix='e128__fixed_unmask_randm_', obs_masked=False, loss_const_targ=None, obs_initial_noise=None, optimizer='adam', dataset_name='smallnorb'), - ]): - # generate dataset - try: - path = run_generate_and_save_adversarial_dataset_mp( - train_epochs=128, - train_optim_steps=175, - seed_=777, - overwrite_mode='overwrite', - dataset_load_into_memory=True, - lr=5e-3, - # batch_sample_mode='range', - **kwargs - ) - paths.append(path) - except Exception as e: - log.error(f'[{i}] FAILED RUN: {e} -- {repr(kwargs)}', exc_info=True) - # load some samples and display them - try: - log.warning(f'visualisation of `_read_hdf5_batch(paths[-1], display_idxs)` was disabled') - except Exception as e: - log.warning(f'[{i}] FAILED SHOW: {e} -- {repr(kwargs)}') - - for path in paths: - print(path) - - -# ========================================================================= # -# main # -# ========================================================================= # - - -if __name__ == '__main__': - main() diff --git a/research/e06_adversarial_data/deprecated/run_02_adv_dataset.sh b/research/e06_adversarial_data/deprecated/run_02_adv_dataset.sh deleted file mode 100644 index e864de99..00000000 --- a/research/e06_adversarial_data/deprecated/run_02_adv_dataset.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# get the path to the script -PARENT_DIR="$(dirname "$(realpath -s "$0")")" -ROOT_DIR="$(dirname "$(dirname "$(dirname "$PARENT_DIR")")")" - -# TODO: fix this! -# TODO: this is out of date -PYTHONPATH="$ROOT_DIR" python3 "$PARENT_DIR/run_02_gen_adversarial_dataset.py" \ - -m \ - framework.sampler_name=same_k,close_far,same_factor,random_bb \ - framework.loss_mode=self,const,invert \ - framework.dataset_name=cars3d,smallnorb diff --git a/research/e06_adversarial_data/deprecated/run_02_gen_adversarial_dataset.py b/research/e06_adversarial_data/deprecated/run_02_gen_adversarial_dataset.py deleted file mode 100644 index 1260233b..00000000 --- a/research/e06_adversarial_data/deprecated/run_02_gen_adversarial_dataset.py +++ /dev/null @@ -1,436 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -""" -Generate an adversarial dataset -- images are directly optimized against each other, could decay in some cases? -- All data is stored in memory, with minibatches taken and optimized. -""" - -import logging -import os -import warnings -from datetime import datetime -from typing import Iterator -from typing import List -from typing import Optional -from typing import Sequence - -import hydra -import numpy as np -import pytorch_lightning as pl -import torch -import wandb -from omegaconf import OmegaConf -from torch.utils.data import DataLoader -from torch.utils.data import IterableDataset -from torch.utils.data.dataset import T_co - -import research.util as H -from disent.dataset import DisentDataset -from disent.dataset.sampling import BaseDisentSampler -from disent.dataset.util.hdf5 import H5Builder -from disent.util import to_numpy -from disent.util.deprecate import deprecated -from disent.util.inout.paths import ensure_parent_dir_exists -from disent.util.lightning.callbacks import BaseCallbackPeriodic -from disent.util.lightning.callbacks import LoggerProgressCallback -from disent.util.lightning.logger_util import wb_log_metrics -from disent.util.math.random import random_choice_prng -from disent.util.seeds import seed -from disent.util.seeds import TempNumpySeed -from disent.util.strings.fmt import bytes_to_human -from disent.util.strings.fmt import make_box_str -from disent.util.visualize.vis_util import make_image_grid -from experiment.run import hydra_get_callbacks -from experiment.run import hydra_get_gpus -from experiment.run import hydra_make_logger -from experiment.util.hydra_utils import make_non_strict -from experiment.util.run_utils import log_error_and_exit -from research.e06_adversarial_data.util_gen_adversarial_dataset import adversarial_loss -from research.e06_adversarial_data.util_gen_adversarial_dataset import make_adversarial_sampler - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# adversarial dataset generator # -# ========================================================================= # - - -class AdversarialModel(pl.LightningModule): - - def __init__( - self, - # optimizer options - optimizer_name: str = 'sgd', - optimizer_lr: float = 5e-2, - optimizer_kwargs: Optional[dict] = None, - # dataset config options - dataset_name: str = 'cars3d', - dataset_num_workers: int = min(os.cpu_count(), 16), - dataset_batch_size: int = 1024, # approx - data_root: str = 'data/dataset', - # data_load_into_memory: bool = False, - # adversarial loss options - adversarial_mode: str = 'self', - adversarial_swapped: bool = False, - adversarial_masking: bool = False, - adversarial_top_k: Optional[int] = None, - pixel_loss_mode: str = 'mse', - # loss extras - # loss_adversarial_weight: Optional[float] = 1.0, - # loss_same_stats_weight: Optional[float] = 0.0, - # loss_similarity_weight: Optional[float] = 0.0, - # loss_out_of_bounds_weight: Optional[float] = 0.0, - # sampling config - sampler_name: str = 'close_far', - # train options - train_batch_optimizer: bool = True, - train_dataset_fp16: bool = True, - train_is_gpu: bool = False, - # logging settings - # logging_scale_imgs: bool = False, - ): - super().__init__() - # check values - if train_dataset_fp16 and (not train_is_gpu): - warnings.warn('`train_dataset_fp16=True` is not supported on CPU, overriding setting to `False`') - train_dataset_fp16 = False - self._dtype_dst = torch.float32 - self._dtype_src = torch.float16 if train_dataset_fp16 else torch.float32 - # modify hparams - if optimizer_kwargs is None: - optimizer_kwargs = {} - # save hparams - self.save_hyperparameters() - # variables - self.dataset: DisentDataset = None - self.array: torch.Tensor = None - self.sampler: BaseDisentSampler = None - - # ================================== # - # setup # - # ================================== # - - def prepare_data(self) -> None: - # create dataset - self.dataset = H.make_dataset(self.hparams.dataset_name, load_into_memory=True, load_memory_dtype=self._dtype_src, data_root=self.hparams.data_root) - # load dataset into memory as fp16 - if self.hparams.train_batch_optimizer: - self.array = self.dataset.gt_data.array - else: - self.array = torch.nn.Parameter(self.dataset.gt_data.array, requires_grad=True) # move with model to correct device - # create sampler - self.sampler = make_adversarial_sampler(self.hparams.sampler_name) - self.sampler.init(self.dataset.gt_data) - - def _make_optimizer(self, params): - return H.make_optimizer( - params, - name=self.hparams.optimizer_name, - lr=self.hparams.optimizer_lr, - **self.hparams.optimizer_kwargs, - ) - - def configure_optimizers(self): - if self.hparams.train_batch_optimizer: - return None - else: - return self._make_optimizer(self.array) - - # ================================== # - # train step # - # ================================== # - - def training_step(self, batch, batch_idx): - # get indices - (a_idx, p_idx, n_idx) = batch['idx'] - # generate batches & transfer to correct device - if self.hparams.train_batch_optimizer: - (a_x, p_x, n_x), (params, param_idxs, optimizer) = self._load_batch(a_idx, p_idx, n_idx) - else: - a_x = self.array[a_idx] - p_x = self.array[p_idx] - n_x = self.array[n_idx] - # compute loss - loss = adversarial_loss( - ys=(a_x, p_x, n_x), - xs=None, - adversarial_mode=self.hparams.adversarial_mode, - adversarial_swapped=self.hparams.adversarial_swapped, - adversarial_masking=self.hparams.adversarial_masking, - adversarial_top_k=self.hparams.adversarial_top_k, - pixel_loss_mode=self.hparams.pixel_loss_mode, - ) - # log results - self.log_dict({ - 'loss': loss, - 'adv_loss': loss, - }, prog_bar=True) - # done! - if self.hparams.train_batch_optimizer: - self._update_with_batch(loss, params, param_idxs, optimizer) - return None - else: - return loss - - # ================================== # - # optimizer for each batch mode # - # ================================== # - - def _load_batch(self, a_idx, p_idx, n_idx): - with torch.no_grad(): - # get all indices - all_indices = np.stack([ - a_idx.detach().cpu().numpy(), - p_idx.detach().cpu().numpy(), - n_idx.detach().cpu().numpy(), - ], axis=0) - # find unique values - param_idxs, inverse_indices = np.unique(all_indices.flatten(), return_inverse=True) - inverse_indices = inverse_indices.reshape(all_indices.shape) - # load data with values & move to gpu - # - for batch size (256*3, 3, 64, 64) with num_workers=0, this is 5% faster - # than .to(device=self.device, dtype=DST_DTYPE) in one call, as we reduce - # the memory overhead in the transfer. This does slightly increase the - # memory usage on the target device. - # - for batch size (1024*3, 3, 64, 64) with num_workers=12, this is 15% faster - # but consumes slightly more memory: 2492MiB vs. 2510MiB - params = self.array[param_idxs].to(device=self.device).to(dtype=self._dtype_dst) - # make params and optimizer - params = torch.nn.Parameter(params, requires_grad=True) - optimizer = self._make_optimizer(params) - # get batches -- it is ok to index by a numpy array without conversion - a_x = params[inverse_indices[0, :]] - p_x = params[inverse_indices[1, :]] - n_x = params[inverse_indices[2, :]] - # return values - return (a_x, p_x, n_x), (params, param_idxs, optimizer) - - def _update_with_batch(self, loss, params, param_idxs, optimizer): - with TempNumpySeed(777): - std, mean = torch.std_mean(self.array[np.random.randint(0, len(self.array), size=128)]) - std, mean = std.cpu().numpy().tolist(), mean.cpu().numpy().tolist() - self.log_dict({'approx_mean': mean, 'approx_std': std}, prog_bar=True) - # backprop - H.step_optimizer(optimizer, loss) - # save values to dataset - with torch.no_grad(): - self.array[param_idxs] = params.detach().cpu().to(self._dtype_src) - - # ================================== # - # dataset # - # ================================== # - - def train_dataloader(self): - # sampling in dataloader - sampler = self.sampler - data_len = len(self.dataset.gt_data) - # generate the indices in a multi-threaded environment -- this is not deterministic if num_workers > 0 - class SamplerIndicesDataset(IterableDataset): - def __getitem__(self, index) -> T_co: - raise RuntimeError('this should never be called on an iterable dataset') - def __iter__(self) -> Iterator[T_co]: - while True: - yield {'idx': sampler(np.random.randint(0, data_len))} - # create data loader! - return DataLoader( - SamplerIndicesDataset(), - batch_size=self.hparams.dataset_batch_size, - num_workers=self.hparams.dataset_num_workers, - shuffle=False, - ) - - def make_train_periodic_callbacks(self, cfg) -> Sequence[pl.Callback]: - class ImShowCallback(BaseCallbackPeriodic): - def do_step(this, trainer: pl.Trainer, pl_module: pl.LightningModule): - if self.dataset is None: - log.warning('dataset not initialized, skipping visualisation') - # get dataset images - with TempNumpySeed(777): - # get scaling values - samples = self.dataset.dataset_sample_batch(num_samples=128, mode='raw').to(torch.float32) - m, M = float(torch.min(samples)), float(torch.max(samples)) - # add transform to dataset - self.dataset._transform = lambda x: H.to_img((x.to(torch.float32) - m) / (M - m)) # this is hacky, scale values to [0, 1] then to [0, 255] - # get images - image = make_image_grid(self.dataset.dataset_sample_batch(num_samples=16, mode='input')) - # get augmented traversals - with torch.no_grad(): - wandb_image, wandb_animation = H.visualize_dataset_traversal(self.dataset, data_mode='input', output_wandb=True) - # log images to WANDB - wb_log_metrics(trainer.logger, { - 'random_images': wandb.Image(image), - 'traversal_image': wandb_image, 'traversal_animation': wandb_animation, - }) - return [ImShowCallback(every_n_steps=cfg.exp.show_every_n_steps, begin_first_step=True)] - - -# ========================================================================= # -# Run Hydra # -# ========================================================================= # - - -ROOT_DIR = os.path.abspath(__file__ + '/../../../..') - - -@deprecated('Replaced with run_02_gen_adversarial_dataset_approx') -def run_gen_adversarial_dataset(cfg): - time_string = datetime.today().strftime('%Y-%m-%d--%H-%M-%S') - log.info(f'Starting run at time: {time_string}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # cleanup from old runs: - try: - wandb.finish() - except: - pass - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - cfg = make_non_strict(cfg) - # - - - - - - - - - - - - - - - # - # check CUDA setting - gpus = hydra_get_gpus(cfg) - # create logger - logger = hydra_make_logger(cfg) - # create callbacks - callbacks: List[pl.Callback] = [c for c in hydra_get_callbacks(cfg) if isinstance(c, LoggerProgressCallback)] - # - - - - - - - - - - - - - - - # - # check save dirs - assert not os.path.isabs(cfg.settings.exp.rel_save_dir), f'rel_save_dir must be relative: {repr(cfg.settings.exp.rel_save_dir)}' - save_dir = os.path.join(ROOT_DIR, cfg.settings.exp.rel_save_dir) - assert os.path.isabs(save_dir), f'save_dir must be absolute: {repr(save_dir)}' - # - - - - - - - - - - - - - - - # - # get the logger and initialize - if logger is not None: - logger.log_hyperparams(cfg) - # print the final config! - log.info('Final Config' + make_box_str(OmegaConf.to_yaml(cfg))) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # | | | | | | | | | | | | | | | # - seed(cfg.settings.job.seed) - # | | | | | | | | | | | | | | | # - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # make framework - framework = AdversarialModel(train_is_gpu=cfg.trainer.cuda, **cfg.adv_system) - callbacks.extend(framework.make_train_periodic_callbacks(cfg)) - # train - trainer = pl.Trainer( - logger=logger, - callbacks=callbacks, - # cfg.dsettings.trainer - gpus=gpus, - # cfg.trainer - max_epochs=cfg.trainer.max_epochs, - max_steps=cfg.trainer.max_steps, - log_every_n_steps=cfg.trainer.log_every_n_steps, - flush_logs_every_n_steps=cfg.trainer.flush_logs_every_n_steps, - progress_bar_refresh_rate=cfg.trainer.progress_bar_refresh_rate, - prepare_data_per_node=cfg.trainer.prepare_data_per_node, - # we do this here so we don't run the final metrics - terminate_on_nan=True, - checkpoint_callback=False, - ) - trainer.fit(framework) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # get save paths - save_prefix = f'{cfg.settings.exp.save_prefix}_' if cfg.settings.exp.save_prefix else '' - save_path_data = os.path.join(save_dir, f'{save_prefix}{time_string}_{cfg.settings.job.name}', f'data.h5') - # create directories - if cfg.settings.exp.save_data: ensure_parent_dir_exists(save_path_data) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # compute standard deviation when saving and scale so - # that we have mean=0 and std=1 of the saved data! - with TempNumpySeed(777): - std, mean = torch.std_mean(framework.array[random_choice_prng(len(framework.array), size=2048, replace=False)]) - std, mean = float(std), float(mean) - log.info(f'normalizing saved dataset of shape: {tuple(framework.array.shape)} and dtype: {framework.array.dtype} with mean: {repr(mean)} and std: {repr(std)}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # save adversarial dataset - if cfg.settings.exp.save_data: - log.info(f'saving data to path: {repr(save_path_data)}') - # transfer to GPU - if torch.cuda.is_available(): - framework = framework.cuda() - # create new h5py file -- TODO: use this in other places! - with H5Builder(path=save_path_data, mode='atomic_w') as builder: - # this dataset is self-contained and can be loaded by SelfContainedHdf5GroundTruthData - # we normalize the values to have approx mean of 0 and std of 1 - builder.add_dataset_from_gt_data( - data=framework.dataset, # produces tensors - mutator=lambda x: np.moveaxis((to_numpy(x).astype('float32') - mean) / std, -3, -1).astype('float16'), # consumes tensors -> np.ndarrays - img_shape=(64, 64, None), - compression_lvl=9, - batch_size=32, - dtype='float16', - attrs=dict( - norm_mean=mean, - norm_std=std, - ) - ) - log.info(f'saved data size: {bytes_to_human(os.path.getsize(save_path_data))}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - -# ========================================================================= # -# Entry Point # -# ========================================================================= # - - -if __name__ == '__main__': - - # BENCHMARK (batch_size=256, optimizer=sgd, lr=1e-2, dataset_num_workers=0): - # - batch_optimizer=False, gpu=True, fp16=True : [3168MiB/5932MiB, 3.32/11.7G, 5.52it/s] - # - batch_optimizer=False, gpu=True, fp16=False : [5248MiB/5932MiB, 3.72/11.7G, 4.84it/s] - # - batch_optimizer=False, gpu=False, fp16=True : [same as fp16=False] - # - batch_optimizer=False, gpu=False, fp16=False : [0003MiB/5932MiB, 4.60/11.7G, 1.05it/s] - # --------- - # - batch_optimizer=True, gpu=True, fp16=True : [1284MiB/5932MiB, 3.45/11.7G, 4.31it/s] - # - batch_optimizer=True, gpu=True, fp16=False : [1284MiB/5932MiB, 3.72/11.7G, 4.31it/s] - # - batch_optimizer=True, gpu=False, fp16=True : [same as fp16=False] - # - batch_optimizer=True, gpu=False, fp16=False : [0003MiB/5932MiB, 1.80/11.7G, 4.18it/s] - - # BENCHMARK (batch_size=1024, optimizer=sgd, lr=1e-2, dataset_num_workers=12): - # - batch_optimizer=True, gpu=True, fp16=True : [2510MiB/5932MiB, 4.10/11.7G, 4.75it/s, 20% gpu util] (to(device).to(dtype)) - # - batch_optimizer=True, gpu=True, fp16=True : [2492MiB/5932MiB, 4.10/11.7G, 4.12it/s, 19% gpu util] (to(device, dtype)) - - @hydra.main(config_path=os.path.join(ROOT_DIR, 'experiment/config'), config_name="config_adversarial_dataset") - def main(cfg): - try: - run_gen_adversarial_dataset(cfg) - except Exception as e: - # truncate error - err_msg = str(e) - err_msg = err_msg[:244] + ' ' if len(err_msg) > 244 else err_msg - # log something at least - log.error(f'exiting: experiment error | {err_msg}', exc_info=True) - - # EXP ARGS: - # $ ... -m dataset=smallnorb,shapes3d - try: - main() - except KeyboardInterrupt as e: - log_error_and_exit(err_type='interrupted', err_msg=str(e), exc_info=False) - except Exception as e: - log_error_and_exit(err_type='hydra error', err_msg=str(e)) diff --git a/research/e06_adversarial_data/deprecated/run_03_check.py b/research/e06_adversarial_data/deprecated/run_03_check.py deleted file mode 100644 index d15c8eeb..00000000 --- a/research/e06_adversarial_data/deprecated/run_03_check.py +++ /dev/null @@ -1,86 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - - -""" -Check the adversarial data generated in previous exerpiments -- This is old and outdated... -- Should use `e01_visual_overlap/run_plot_traversal_dists.py` instead! -""" - - -import numpy as np -import torch -import torch.nn.functional as F -import torchvision -import matplotlib.pyplot as plt - -from disent.dataset.data import Shapes3dData -from research.util._data import AdversarialOptimizedData - - -if __name__ == '__main__': - - def ave_pairwise_dist(data, n_samples=1000): - """ - Get the average distance between observations in the dataset - """ - # get stats - diff = [] - for i in range(n_samples): - a, b = np.random.randint(0, len(data), size=2) - a, b = data[a], data[b] - diff.append(F.mse_loss(a, b, reduction='mean').item()) - return np.mean(diff) - - def plot_samples(data, name=None): - """ - Display random observations from the dataset - """ - # get image - img = torchvision.utils.make_grid([data[i*1000] for i in range(9)], nrow=3) - img = torch.moveaxis(img, 0, -1).numpy() - # plot - if name is not None: - plt.title(name) - plt.imshow(img) - plt.show() - - - def main(): - base_data = Shapes3dData(in_memory=False, prepare=True, transform=torchvision.transforms.ToTensor()) - plot_samples(base_data) - print(ave_pairwise_dist(base_data)) - - for path in [ - 'out/overlap/fixed_masked_const_shapes3d_adam_0.01_True_None_mse_12288_shuffle_5120_10240_None_0.1_False_8_125.hdf5', - 'out/overlap/fixed_masked_randm_shapes3d_adam_0.01_True_None_mse_12288_shuffle_5120_10240_None_None_False_8_125.hdf5', - 'out/overlap/noise_unmask_randm_shapes3d_adam_0.01_False_0.001_mse_12288_shuffle_5120_10240_None_None_False_8_125.hdf5', - 'out/overlap/noise_unmask_randm_shapes3d_adam_0.01_False_0.1_mse_12288_shuffle_5120_10240_None_None_False_8_125.hdf5', - ]: - data = AdversarialOptimizedData(path, base_data, transform=torchvision.transforms.ToTensor()) - plot_samples(data) - print(ave_pairwise_dist(data)) - - main() diff --git a/research/e06_adversarial_data/deprecated/run_04_gen_adversarial_ruck.py b/research/e06_adversarial_data/deprecated/run_04_gen_adversarial_ruck.py deleted file mode 100644 index 9aee9e08..00000000 --- a/research/e06_adversarial_data/deprecated/run_04_gen_adversarial_ruck.py +++ /dev/null @@ -1,585 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -""" -This file generates pareto-optimal solutions to the multi-objective -optimisation problem of masking a dataset as to minimize some metric -for overlap, while maximizing the amount of data kept. - -- We solve this problem using the NSGA2 algorithm and save all the results - to disk to be loaded with `get_closest_mask` from `util_load_adversarial_mask.py` -""" - -import gzip -import logging -import os -import pickle -import random -import warnings -from datetime import datetime -from typing import Any -from typing import Dict -from typing import List -from typing import Optional -from typing import Tuple - -import numpy as np -import ray -import ruck -from matplotlib import pyplot as plt -from ruck import R -from ruck.external.ray import ray_map -from ruck.external.ray import ray_remote_put -from ruck.external.ray import ray_remote_puts -from ruck.external.deap import select_nsga2 - -import research.util as H -from disent.dataset.wrapper import MaskedDataset -from disent.util.function import wrapped_partial -from disent.util.inout.paths import ensure_parent_dir_exists -from disent.util.profiling import Timer -from disent.util.seeds import seed -from disent.util.visualize.vis_util import get_idx_traversal -from research.e01_visual_overlap.util_compute_traversal_dists import cached_compute_all_factor_dist_matrices -from research.e06_adversarial_data.util_eval_adversarial import eval_individual - - -log = logging.getLogger(__name__) - - -''' -NOTES ON MULTI-OBJECTIVE OPTIMIZATION: - https://en.wikipedia.org/wiki/Pareto_efficiency - https://en.wikipedia.org/wiki/Multi-objective_optimization - https://www.youtube.com/watch?v=SL-u_7hIqjA - - IDEAL MULTI-OBJECTIVE OPTIMIZATION - 1. generate set of pareto-optimal solutions (solutions lying along optimal boundary) -- (A posteriori methods) - - converge to pareto optimal front - - maintain as diverse a population as possible along the front (nsga-ii?) - 2. choose one from set using higher level information - - NOTE: - most multi-objective problems are just - converted into single objective functions. - - WEIGHTED SUMS - -- need to know weights - -- non-uniform in pareto-optimal solutions - -- cannot find some pareto-optimal solutions in non-convex regions - `return w0 * score0 + w1 * score1 + ...` - - ε-CONSTRAINT: constrain all but one objective - -- need to know ε vectors - -- non-uniform in pareto-optimal solutions - -- any pareto-optimal solution can be found - * EMO is a generalisation? -''' - - -# ========================================================================= # -# Ruck Helper # -# ========================================================================= # - - -def mutate_oneof(*mutate_fns): - # TODO: move this into ruck - def mutate_fn(value): - fn = random.choice(mutate_fns) - return fn(value) - return mutate_fn - - -def plt_pareto_solutions( - population, - label_fitness_0: str, - label_fitness_1: str, - title: str = None, - plot: bool = True, - chosen_idxs_f0=None, - chosen_idxs_f1=None, - random_points=None, - **fig_kw, -): - # fitness values must be of type Tuple[float, float] for this function to work! - fig, axs = H.plt_subplots(1, 1, title=title if title else 'Pareto-Optimal Solutions', **fig_kw) - # plot fitness values - xs, ys = zip(*(m.fitness for m in population)) - axs[0, 0].set_xlabel(label_fitness_0) - axs[0, 0].set_ylabel(label_fitness_1) - # plot random - if random_points is not None: - axs[0, 0].scatter(*np.array(random_points).T, c='orange') - # plot normal - axs[0, 0].scatter(xs, ys) - # plot chosen - if chosen_idxs_f0 is not None: - axs[0, 0].scatter(*np.array([population[i].fitness for i in chosen_idxs_f0]).T, c='purple') - if chosen_idxs_f1 is not None: - axs[0, 0].scatter(*np.array([population[i].fitness for i in chosen_idxs_f1]).T, c='green') - # label axes - # layout - fig.tight_layout() - # plot - if plot: - plt.show() - # done! - return fig, axs - - -def individual_ave(dataset, individual, print_=False): - if isinstance(dataset, str): - dataset = H.make_data(dataset, transform_mode='none') - # masked - sub_data = MaskedDataset(data=dataset, mask=individual.flatten()) - if print_: - print(', '.join(f'{individual.reshape(sub_data._data.factor_sizes).sum(axis=f_idx).mean():2f}' for f_idx in range(sub_data._data.num_factors))) - # make obs - ave_obs = np.zeros_like(sub_data[0], dtype='float64') - for obs in sub_data: - ave_obs += obs - return ave_obs / ave_obs.max() - - -def plot_averages(dataset_name: str, values: list, subtitle: str, title_prefix: str = None, titles=None, show: bool = False): - data = H.make_data(dataset_name, transform_mode='none') - # average individuals - ave_imgs = [individual_ave(data, v) for v in values] - col_lbls = [f'{np.sum(v)} / {np.prod(v.shape)}' for v in values] - # make plots - fig_ave_imgs, _ = H.plt_subplots_imshow( - [ave_imgs], - col_labels=col_lbls, - titles=titles, - show=show, - vmin=0.0, - vmax=1.0, - figsize=(10, 3), - title=f'{f"{title_prefix} " if title_prefix else ""}Average Datasets\n{subtitle}', - ) - return fig_ave_imgs - - -def get_spaced(array, num: int): - return [array[i] for i in get_idx_traversal(len(array), num)] - - -# ========================================================================= # -# Evaluation # -# ========================================================================= # - - -@ray.remote -def evaluate_member( - value: np.ndarray, - gt_dist_matrices: np.ndarray, - factor_sizes: Tuple[int, ...], - fitness_overlap_mode: str, - fitness_overlap_aggregate: str, - fitness_overlap_include_singles: bool, -) -> Tuple[float, float]: - overlap_score, usage_ratio = eval_individual( - individual=value, - gt_dist_matrices=gt_dist_matrices, - factor_sizes=factor_sizes, - fitness_overlap_mode=fitness_overlap_mode, - fitness_overlap_aggregate=fitness_overlap_aggregate, - exclude_diag=True, - increment_single=fitness_overlap_include_singles, - backend='numba', - ) - - # weight components - # assert fitness_overlap_weight >= 0 - # assert fitness_usage_weight >= 0 - # w_ovrlp = fitness_overlap_weight * overlap_score - # w_usage = fitness_usage_weight * usage_ratio - - # GOALS: minimize overlap, maximize usage - # [min, max] objective -> target - # [ 0, 1] factor_score -> 0 - # [ 0, 1] kept_ratio -> 1 - - # linear scalarization - # loss = w_ovrlp - w_usage - - # No-preference method - # -- norm(f(x) - z_ideal) - # -- preferably scale variables - # z_ovrlp = fitness_overlap_weight * (overlap_score - 0.0) - # z_usage = fitness_usage_weight * (usage_ratio - 1.0) - # loss = np.linalg.norm([z_ovrlp, z_usage], ord=2) - - # convert minimization problem into maximization - # return - loss - - if overlap_score < 0: - log.warning(f'member has invalid overlap_score: {repr(overlap_score)}') - overlap_score = 1000 # minimizing target to 0 in range [0, 1] so this is bad - if usage_ratio < 0: - log.warning(f'member has invalid usage_ratio: {repr(usage_ratio)}') - usage_ratio = -1000 # maximizing target to 1 in range [0, 1] so this is bad - - return (-overlap_score, usage_ratio) - - -# ========================================================================= # -# Type Hints # -# ========================================================================= # - - -Values = List[ray.ObjectRef] -Population = List[ruck.Member[ray.ObjectRef]] - - -# ========================================================================= # -# Evolutionary System # -# ========================================================================= # - - -class DatasetMaskModule(ruck.EaModule): - - # STATISTICS - - def get_stats_groups(self): - remote_sum = ray.remote(np.mean).remote - return { - **super().get_stats_groups(), - 'mask': ruck.StatsGroup(lambda pop: ray.get([remote_sum(m.value) for m in pop]), min=np.min, max=np.max, mean=np.mean), - } - - def get_progress_stats(self): - return ('evals', 'fit:mean', 'mask:mean') - - # POPULATION - - def gen_starting_values(self) -> Values: - return [ - ray.put(np.random.random(np.prod(self.hparams.factor_sizes)) < (0.1 + np.random.random() * 0.8)) - for _ in range(self.hparams.population_size) - ] - - def select_population(self, population: Population, offspring: Population) -> Population: - return select_nsga2(population + offspring, len(population), weights=(1.0, 1.0)) - - def evaluate_values(self, values: Values) -> List[float]: - return ray.get([self._evaluate_value_fn(v) for v in values]) - - # INITIALISE - - def __init__( - self, - dataset_name: str = 'cars3d', - dist_normalize_mode: str = 'all', - population_size: int = 128, - # fitness settings - fitness_overlap_aggregate: str = 'mean', - fitness_overlap_mode: str = 'std', - fitness_overlap_include_singles: bool = True, - # ea settings - p_mate: float = 0.5, - p_mutate: float = 0.5, - p_mutate_flip: float = 0.05, - ): - # load the dataset - gt_data = H.make_data(dataset_name) - factor_sizes = gt_data.factor_sizes - # save hyper parameters to .hparams - self.save_hyperparameters(include=['factor_sizes']) - # compute all distances - gt_dist_matrices = cached_compute_all_factor_dist_matrices(dataset_name, normalize_mode=dist_normalize_mode) - gt_dist_matrices = ray.put(gt_dist_matrices) - # get offspring function - self.generate_offspring = wrapped_partial( - R.apply_mate_and_mutate, - mate_fn=ray_remote_puts(R.mate_crossover_nd).remote, - mutate_fn=ray_remote_put(mutate_oneof( - wrapped_partial(R.mutate_flip_bits, p=p_mutate_flip), - wrapped_partial(R.mutate_flip_bit_groups, p=p_mutate_flip), - )).remote, - p_mate=p_mate, - p_mutate=p_mutate, - map_fn=ray_map # parallelize - ) - # get evaluation function - self._evaluate_value_fn = wrapped_partial( - evaluate_member.remote, - gt_dist_matrices=gt_dist_matrices, - factor_sizes=factor_sizes, - fitness_overlap_mode=fitness_overlap_mode, - fitness_overlap_aggregate=fitness_overlap_aggregate, - fitness_overlap_include_singles=fitness_overlap_include_singles, - ) - - -# ========================================================================= # -# RUNNER # -# ========================================================================= # - - -def run( - dataset_name: str = 'shapes3d', # xysquares_8x8_toy_s4, xcolumns_8x_toy_s1 - dist_normalize_mode: str = 'all', # all, each, none - # population - generations: int = 250, - population_size: int = 128, - # fitness settings - fitness_overlap_mode: str = 'std', - fitness_overlap_aggregate: str = 'mean', - fitness_overlap_include_singles: bool = True, - # save settings - save: bool = False, - save_prefix: str = '', - seed_: Optional[int] = None, - # plot settings - plot: bool = False, - # wandb_settings - wandb_enabled: bool = True, - wandb_init: bool = True, - wandb_project: str = 'exp-adversarial-mask', - wandb_user: str = 'n_michlo', - wandb_job_name: str = None, - wandb_tags: Optional[List[str]] = None, - wandb_finish: bool = True, -) -> Dict[str, Any]: - # save the starting time for the save path - time_string = datetime.today().strftime('%Y-%m-%d--%H-%M-%S') - log.info(f'Starting run at time: {time_string}') - - # get hparams - hparams = dict(dataset_name=dataset_name, dist_normalize_mode=dist_normalize_mode, generations=generations, population_size=population_size, fitness_overlap_mode=fitness_overlap_mode, fitness_overlap_aggregate=fitness_overlap_aggregate, fitness_overlap_include_singles=fitness_overlap_include_singles, save=save, save_prefix=save_prefix, seed_=seed_, plot=plot, wandb_enabled=wandb_enabled, wandb_init=wandb_init, wandb_project=wandb_project, wandb_user=wandb_user, wandb_job_name=wandb_job_name) - # name - name = f'{(save_prefix + "_" if save_prefix else "")}{dataset_name}_{generations}x{population_size}_{dist_normalize_mode}_{fitness_overlap_mode}_{fitness_overlap_aggregate}_{fitness_overlap_include_singles}' - log.info(f'- Run name is: {name}') - - # enable wandb - wandb = None - if wandb_enabled: - import wandb - # cleanup from old runs: - if wandb_init: - if wandb_finish: - try: - wandb.finish() - except: - pass - # initialize - wandb.init( - entity=wandb_user, - project=wandb_project, - name=wandb_job_name if (wandb_job_name is not None) else name, - group=None, - tags=wandb_tags, - ) - # track hparams - wandb.config.update({f'adv/{k}': v for k, v in hparams.items()}) - - # This is not completely deterministic with ray - # although the starting population will always be the same! - seed_ = seed_ if (seed_ is not None) else int(np.random.randint(1, 2**31-1)) - seed(seed_) - - # run! - with Timer('ruck:onemax'): - problem = DatasetMaskModule( - dataset_name=dataset_name, - dist_normalize_mode=dist_normalize_mode, - population_size=population_size, - fitness_overlap_mode=fitness_overlap_mode, - fitness_overlap_aggregate=fitness_overlap_aggregate, - fitness_overlap_include_singles=fitness_overlap_include_singles, - ) - # train - population, logbook, halloffame = ruck.Trainer(generations=generations, progress=True).fit(problem) - # retrieve stats - log.info(f'start population: {logbook[0]}') - log.info(f'end population: {logbook[-1]}') - values = [ray.get(m.value) for m in halloffame] - - # log to wandb as steps - if wandb_enabled: - for i, stats in enumerate(logbook): - stats = {f'stats/{k}': v for k, v in stats.items()} - stats['current_step'] = i - wandb.log(stats, step=i) - - # generate average images - if plot or wandb_enabled: - # plot average - fig_ave_imgs_hof = plot_averages(dataset_name, values, title_prefix='HOF', subtitle=name, show=plot) - # get individuals -- this is not ideal because not evenly spaced - idxs_chosen_f0 = get_spaced(np.argsort([m.fitness[0] for m in population])[::-1], 5) # overlap - idxs_chosen_f1 = get_spaced(np.argsort([m.fitness[1] for m in population]), 5) # usage - chosen_values_f0 = [ray.get(population[i].value) for i in idxs_chosen_f0] - chosen_values_f1 = [ray.get(population[i].value) for i in idxs_chosen_f1] - random_fitnesses = problem.evaluate_values([ray.put(np.random.random(values[0].shape) < p) for p in np.linspace(0.025, 1, num=population_size+2)[1:-1]]) - # plot averages - fig_ave_imgs_f0 = plot_averages(dataset_name, chosen_values_f0, subtitle=name, titles=[f'{population[i].fitness[0]:2f}' for i in idxs_chosen_f0], title_prefix='Overlap -', show=plot) - fig_ave_imgs_f1 = plot_averages(dataset_name, chosen_values_f1, subtitle=name, titles=[f'{population[i].fitness[1]:2f}' for i in idxs_chosen_f1], title_prefix='Usage -', show=plot) - # plot parento optimal solutions - fig_pareto_sol, axs = plt_pareto_solutions( - population, - label_fitness_0='Overlap Score', - label_fitness_1='Usage Score', - title=f'Pareto-Optimal Solutions\n{name}', - plot=plot, - chosen_idxs_f0=idxs_chosen_f0, - chosen_idxs_f1=idxs_chosen_f1, - random_points=random_fitnesses, - figsize=(7, 7), - ) - # log average - if wandb_enabled: - wandb.log({ - 'ave_images_hof': wandb.Image(fig_ave_imgs_hof), - 'ave_images_overlap': wandb.Image(fig_ave_imgs_f0), - 'ave_images_usage': wandb.Image(fig_ave_imgs_f1), - 'pareto_solutions': wandb.Image(fig_pareto_sol), - }) - - # get summary - use_elems = np.sum(values[0]) - num_elems = np.prod(values[0].shape) - use_ratio = (use_elems / num_elems) - - # log summary - if wandb_enabled: - wandb.summary['num_elements'] = num_elems - wandb.summary['used_elements'] = use_elems - wandb.summary['used_elements_ratio'] = use_ratio - for k, v in logbook[0].items(): wandb.summary[f'log:start:{k}'] = v - for k, v in logbook[-1].items(): wandb.summary[f'log:end:{k}'] = v - - # generate paths - job_name = f'{time_string}_{name}' - - # collect results - results = { - 'hparams': hparams, - 'job_name': job_name, - 'save_path': None, - 'time_string': time_string, - 'values': [ray.get(m.value) for m in population], - 'scores': [m.fitness for m in population], - # score components - 'scores_overlap': [m.fitness[0] for m in population], - 'scores_usage': [m.fitness[1] for m in population], - # history data - 'logbook_history': logbook.history, - # we don't want these because they store object refs, and - # it means we need ray to unpickle them. - # 'population': population, - # 'halloffame_members': halloffame.members, - } - - if save: - # get save path, make parent dir & save! - results['save_path'] = ensure_parent_dir_exists(ROOT_DIR, 'out/adversarial_mask', job_name, 'data.pkl.gz') - # NONE : 122943493 ~= 118M (100.%) : 103.420ms - # lvl=1 : 23566691 ~= 23M (19.1%) : 1.223s - # lvl=2 : 21913595 ~= 21M (17.8%) : 1.463s - # lvl=3 : 20688319 ~= 20M (16.8%) : 2.504s - # lvl=4 : 18325859 ~= 18M (14.9%) : 1.856s # good - # lvl=5 : 17467772 ~= 17M (14.2%) : 3.332s # good - # lvl=6 : 16594660 ~= 16M (13.5%) : 7.163s # starting to slow - # lvl=7 : 16242279 ~= 16M (13.2%) : 12.407s - # lvl=8 : 15586416 ~= 15M (12.7%) : 1m:4s # far too slow - # lvl=9 : 15023324 ~= 15M (12.2%) : 3m:11s # far too slow - log.info(f'saving data to: {results["save_path"]}') - with gzip.open(results["save_path"], 'wb', compresslevel=5) as fp: - pickle.dump(results, fp) - log.info(f'saved data to: {results["save_path"]}') - - # cleanup wandb - if wandb_enabled: - if wandb_finish: - try: - wandb.finish() - except: - pass - - # done - return results - - -# ========================================================================= # -# ENTRYPOINT # -# ========================================================================= # - - -ROOT_DIR = os.path.abspath(__file__ + '/../../..') - - -def main(): - from itertools import product - - # (3 * 2 * 2 * 5) - for (fitness_overlap_include_singles, dist_normalize_mode, fitness_overlap_aggregate, fitness_overlap_mode, dataset_name) in product( - [True, False], - ['all', 'each', 'none'], - ['gmean', 'mean'], - ['std', 'range'], - ['xysquares_8x8_toy_s2', 'cars3d', 'smallnorb', 'shapes3d', 'dsprites'], - ): - print('='*100) - print(f'[STARTING]: dataset_name={repr(dataset_name)} dist_normalize_mode={repr(dist_normalize_mode)} fitness_overlap_mode={repr(fitness_overlap_mode)} fitness_overlap_aggregate={repr(fitness_overlap_aggregate)} fitness_overlap_include_singles={repr(fitness_overlap_include_singles)}') - try: - run( - dataset_name=dataset_name, - dist_normalize_mode=dist_normalize_mode, - # fitness - fitness_overlap_aggregate=fitness_overlap_aggregate, - fitness_overlap_mode=fitness_overlap_mode, - fitness_overlap_include_singles=fitness_overlap_include_singles, - # population - generations=1000, - population_size=256, - seed_=42, - save=True, - save_prefix='EXP', - plot=True, - wandb_enabled=True, - wandb_project='exp-adversarial-mask', - wandb_tags=['exp_factor_dists'] - ) - except KeyboardInterrupt: - warnings.warn('Exiting early') - exit(1) - # except: - # warnings.warn(f'[FAILED]: dataset_name={repr(dataset_name)} dist_normalize_mode={repr(dist_normalize_mode)} fitness_overlap_mode={repr(fitness_overlap_mode)} fitness_overlap_aggregate={repr(fitness_overlap_aggregate)}') - print('='*100) - - -if __name__ == '__main__': - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../gadfly.mplstyle')) - - # run - logging.basicConfig(level=logging.INFO) - ray.init(num_cpus=64) - main() - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/e06_adversarial_data/deprecated/run_04_gen_adversarial_ruck_dist_pairs.py b/research/e06_adversarial_data/deprecated/run_04_gen_adversarial_ruck_dist_pairs.py deleted file mode 100644 index ebfb7555..00000000 --- a/research/e06_adversarial_data/deprecated/run_04_gen_adversarial_ruck_dist_pairs.py +++ /dev/null @@ -1,601 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -""" -This file generates pareto-optimal solutions to the multi-objective -optimisation problem of masking a dataset as to minimize some metric -for overlap, while maximizing the amount of data kept. - -- We solve this problem using the NSGA2 algorithm and save all the results - to disk to be loaded with `get_closest_mask` from `util_load_adversarial_mask.py` -""" - -import gzip -import logging -import os -import pickle -import random -import warnings -from datetime import datetime -from typing import Any -from typing import Dict -from typing import List -from typing import Optional -from typing import Tuple - -import numpy as np -import psutil -import ray -import ruck -from matplotlib import pyplot as plt -from ruck import R -from ruck.external.ray import ray_map -from ruck.external.ray import ray_remote_put -from ruck.external.ray import ray_remote_puts - -from ruck.external.deap import select_nsga2 as select_nsga2_deap -# from ruck.functional import select_nsga2 as select_nsga2_ruck # should rather use this! - -import research.util as H -from disent.dataset.wrapper import MaskedDataset -from disent.util.function import wrapped_partial -from disent.util.inout.paths import ensure_parent_dir_exists -from disent.util.profiling import Timer -from disent.util.seeds import seed -from disent.util.visualize.vis_util import get_idx_traversal -from research.e01_visual_overlap.util_compute_traversal_dist_pairs import cached_compute_dataset_pair_dists -from research.e06_adversarial_data.util_eval_adversarial_dist_pairs import eval_masked_dist_pairs - - -log = logging.getLogger(__name__) - - -''' -NOTES ON MULTI-OBJECTIVE OPTIMIZATION: - https://en.wikipedia.org/wiki/Pareto_efficiency - https://en.wikipedia.org/wiki/Multi-objective_optimization - https://www.youtube.com/watch?v=SL-u_7hIqjA - - IDEAL MULTI-OBJECTIVE OPTIMIZATION - 1. generate set of pareto-optimal solutions (solutions lying along optimal boundary) -- (A posteriori methods) - - converge to pareto optimal front - - maintain as diverse a population as possible along the front (nsga-ii?) - 2. choose one from set using higher level information - - NOTE: - most multi-objective problems are just - converted into single objective functions. - - WEIGHTED SUMS - -- need to know weights - -- non-uniform in pareto-optimal solutions - -- cannot find some pareto-optimal solutions in non-convex regions - `return w0 * score0 + w1 * score1 + ...` - - ε-CONSTRAINT: constrain all but one objective - -- need to know ε vectors - -- non-uniform in pareto-optimal solutions - -- any pareto-optimal solution can be found - * EMO is a generalisation? -''' - - -# ========================================================================= # -# Ruck Helper # -# ========================================================================= # - - -def mutate_oneof(*mutate_fns): - # TODO: move this into ruck - def mutate_fn(value): - fn = random.choice(mutate_fns) - return fn(value) - return mutate_fn - - -def plt_pareto_solutions( - population, - label_fitness_0: str, - label_fitness_1: str, - title: str = None, - plot: bool = True, - chosen_idxs_f0=None, - chosen_idxs_f1=None, - random_points=None, - **fig_kw, -): - # fitness values must be of type Tuple[float, float] for this function to work! - fig, axs = H.plt_subplots(1, 1, title=title if title else 'Pareto-Optimal Solutions', **fig_kw) - # plot fitness values - xs, ys = zip(*(m.fitness for m in population)) - axs[0, 0].set_xlabel(label_fitness_0) - axs[0, 0].set_ylabel(label_fitness_1) - # plot random - if random_points is not None: - axs[0, 0].scatter(*np.array(random_points).T, c='orange') - # plot normal - axs[0, 0].scatter(xs, ys) - # plot chosen - if chosen_idxs_f0 is not None: - axs[0, 0].scatter(*np.array([population[i].fitness for i in chosen_idxs_f0]).T, c='purple') - if chosen_idxs_f1 is not None: - axs[0, 0].scatter(*np.array([population[i].fitness for i in chosen_idxs_f1]).T, c='green') - # label axes - # layout - fig.tight_layout() - # plot - if plot: - plt.show() - # done! - return fig, axs - - -def individual_ave(dataset, individual, print_=False): - if isinstance(dataset, str): - dataset = H.make_data(dataset, transform_mode='none') - # masked - sub_data = MaskedDataset(data=dataset, mask=individual.flatten()) - if print_: - print(', '.join(f'{individual.reshape(sub_data._data.factor_sizes).sum(axis=f_idx).mean():2f}' for f_idx in range(sub_data._data.num_factors))) - # make obs - ave_obs = np.zeros_like(sub_data[0], dtype='float64') - for obs in sub_data: - ave_obs += obs - return ave_obs / ave_obs.max() - - -def plot_averages(dataset_name: str, values: list, subtitle: str, title_prefix: str = None, titles=None, show: bool = False): - data = H.make_data(dataset_name, transform_mode='none') - # average individuals - ave_imgs = [individual_ave(data, v) for v in values] - col_lbls = [f'{np.sum(v)} / {np.prod(v.shape)}' for v in values] - # make plots - fig_ave_imgs, _ = H.plt_subplots_imshow( - [ave_imgs], - col_labels=col_lbls, - titles=titles, - show=show, - vmin=0.0, - vmax=1.0, - figsize=(10, 3), - title=f'{f"{title_prefix} " if title_prefix else ""}Average Datasets\n{subtitle}', - ) - return fig_ave_imgs - - -def get_spaced(array, num: int): - return [array[i] for i in get_idx_traversal(len(array), num)] - - -# ========================================================================= # -# Evaluation # -# ========================================================================= # - - -@ray.remote -def evaluate_member( - value: np.ndarray, - pair_obs_dists: np.ndarray, - pair_obs_idxs: np.ndarray, - fitness_overlap_mode: str, - fitness_overlap_include_singles: bool = True, -) -> Tuple[float, float]: - overlap_score, usage_ratio = eval_masked_dist_pairs( - mask=value, - pair_obs_dists=pair_obs_dists, - pair_obs_idxs=pair_obs_idxs, - fitness_mode=fitness_overlap_mode, - increment_single=fitness_overlap_include_singles, - backend='numba', - ) - - # weight components - # assert fitness_overlap_weight >= 0 - # assert fitness_usage_weight >= 0 - # w_ovrlp = fitness_overlap_weight * overlap_score - # w_usage = fitness_usage_weight * usage_ratio - - # GOALS: minimize overlap, maximize usage - # [min, max] objective -> target - # [ 0, 1] factor_score -> 0 - # [ 0, 1] kept_ratio -> 1 - - # linear scalarization - # loss = w_ovrlp - w_usage - - # No-preference method - # -- norm(f(x) - z_ideal) - # -- preferably scale variables - # z_ovrlp = fitness_overlap_weight * (overlap_score - 0.0) - # z_usage = fitness_usage_weight * (usage_ratio - 1.0) - # loss = np.linalg.norm([z_ovrlp, z_usage], ord=2) - - # convert minimization problem into maximization - # return - loss - - if overlap_score < 0: - log.warning(f'member has invalid overlap_score: {repr(overlap_score)}') - overlap_score = 1000 # minimizing target to 0 in range [0, 1] so this is bad - if usage_ratio < 0: - log.warning(f'member has invalid usage_ratio: {repr(usage_ratio)}') - usage_ratio = -1000 # maximizing target to 1 in range [0, 1] so this is bad - - return (-overlap_score, usage_ratio) - - -# ========================================================================= # -# Type Hints # -# ========================================================================= # - - -Values = List[ray.ObjectRef] -Population = List[ruck.Member[ray.ObjectRef]] - - -# ========================================================================= # -# Evolutionary System # -# ========================================================================= # - - -class DatasetDistPairMaskModule(ruck.EaModule): - - # STATISTICS - - def get_stats_groups(self): - remote_sum = ray.remote(np.mean).remote - return { - **super().get_stats_groups(), - 'mask': ruck.StatsGroup(lambda pop: ray.get([remote_sum(m.value) for m in pop]), min=np.min, max=np.max, mean=np.mean), - } - - def get_progress_stats(self): - return ('evals', 'fit:mean', 'mask:mean') - - # POPULATION - - def gen_starting_values(self) -> Values: - return [ - ray.put(np.random.random(np.prod(self.hparams.factor_sizes)) < (0.1 + np.random.random() * 0.8)) - for _ in range(self.hparams.population_size) - ] - - def select_population(self, population: Population, offspring: Population) -> Population: - return select_nsga2_deap(population + offspring, len(population)) - - def evaluate_values(self, values: Values) -> List[float]: - return ray.get([self._evaluate_value_fn(v) for v in values]) - - # INITIALISE - - def __init__( - self, - dataset_name: str = 'smallnorb', - pair_mode: str = 'nearby_scaled', # random, nearby, nearby_scaled - pairs_per_obs: int = 100, - pairs_seed: Optional[int] = None, - dists_scaled: bool = True, - # population - population_size: int = 128, - # fitness settings - fitness_overlap_mode: str = 'std', - fitness_overlap_include_singles: bool = True, - # ea settings - p_mate: float = 0.5, - p_mutate: float = 0.5, - p_mutate_flip: float = 0.05, - ): - # load the dataset - gt_data = H.make_data(dataset_name) - factor_sizes = gt_data.factor_sizes - # save hyper parameters to .hparams - self.save_hyperparameters(include=['factor_sizes']) - # compute all distances - obs_pair_idxs, obs_pair_dists = cached_compute_dataset_pair_dists(dataset_name, pair_mode=pair_mode, pairs_per_obs=pairs_per_obs, seed=pairs_seed, scaled=dists_scaled) - obs_pair_idxs = ray.put(obs_pair_idxs) - obs_pair_dists = ray.put(obs_pair_dists) - # get offspring function - self.generate_offspring = wrapped_partial( - R.apply_mate_and_mutate, - mate_fn=ray_remote_puts(R.mate_crossover_nd).remote, - mutate_fn=ray_remote_put(mutate_oneof( - wrapped_partial(R.mutate_flip_bits, p=p_mutate_flip), - wrapped_partial(R.mutate_flip_bit_groups, p=p_mutate_flip), - )).remote, - p_mate=p_mate, - p_mutate=p_mutate, - map_fn=ray_map # parallelize - ) - # get evaluation function - self._evaluate_value_fn = wrapped_partial( - evaluate_member.remote, - pair_obs_dists=obs_pair_dists, - pair_obs_idxs=obs_pair_idxs, - fitness_overlap_mode=fitness_overlap_mode, - fitness_overlap_include_singles=fitness_overlap_include_singles, - ) - - -# ========================================================================= # -# RUNNER # -# ========================================================================= # - - -def run( - dataset_name: str = 'shapes3d', # xysquares_8x8_toy_s4, xcolumns_8x_toy_s1 - pair_mode: str = 'nearby_scaled', - pairs_per_obs: int = 64, - dists_scaled: bool = True, - # population - generations: int = 250, - population_size: int = 128, - # fitness settings - fitness_overlap_mode: str = 'std', - fitness_overlap_include_singles: bool = True, - # save settings - save: bool = False, - save_prefix: str = '', - seed_: Optional[int] = None, - # plot settings - plot: bool = False, - # wandb_settings - wandb_enabled: bool = True, - wandb_init: bool = True, - wandb_project: str = 'exp-adversarial-mask', - wandb_user: str = 'n_michlo', - wandb_job_name: str = None, - wandb_tags: Optional[List[str]] = None, - wandb_finish: bool = True, -) -> Dict[str, Any]: - # save the starting time for the save path - time_string = datetime.today().strftime('%Y-%m-%d--%H-%M-%S') - log.info(f'Starting run at time: {time_string}') - - # get hparams - hparams = dict(dataset_name=dataset_name, pair_mode=pair_mode, pairs_per_obs=pairs_per_obs, dists_scaled=dists_scaled, generations=generations, population_size=population_size, fitness_overlap_mode=fitness_overlap_mode, fitness_overlap_include_singles=fitness_overlap_include_singles, save=save, save_prefix=save_prefix, seed_=seed_, plot=plot, wandb_enabled=wandb_enabled, wandb_init=wandb_init, wandb_project=wandb_project, wandb_user=wandb_user, wandb_job_name=wandb_job_name, wandb_tags=wandb_tags, wandb_finish=wandb_finish) - # name - name = f'{(save_prefix + "_" if save_prefix else "")}{dataset_name}_{generations}x{population_size}_{pair_mode}_{pairs_per_obs}_{dists_scaled}_{fitness_overlap_mode}_{fitness_overlap_include_singles}' - log.info(f'- Run name is: {name}') - - # enable wandb - wandb = None - if wandb_enabled: - import wandb - # cleanup from old runs: - if wandb_init: - if wandb_finish: - try: - wandb.finish() - except: - pass - # initialize - wandb.init( - entity=wandb_user, - project=wandb_project, - name=wandb_job_name if (wandb_job_name is not None) else name, - group=None, - tags=wandb_tags, - ) - # track hparams - wandb.config.update({f'adv/{k}': v for k, v in hparams.items()}) - - # This is not completely deterministic with ray - # although the starting population will always be the same! - seed_ = seed_ if (seed_ is not None) else int(np.random.randint(1, 2**31-1)) - seed(seed_) - - # run! - with Timer('ruck:onemax'): - problem = DatasetDistPairMaskModule( - dataset_name=dataset_name, - pair_mode=pair_mode, - pairs_per_obs=pairs_per_obs, - # pairs_seed=pairs_seed, - dists_scaled=dists_scaled, - # population - population_size=population_size, - # fitness settings - fitness_overlap_mode=fitness_overlap_mode, - fitness_overlap_include_singles=fitness_overlap_include_singles, - # ea settings - # p_mate=p_mate, - # p_mutate=p_mutate, - # p_mutate_flip=p_mutate_flip, - ) - # train - population, logbook, halloffame = ruck.Trainer(generations=generations, progress=True).fit(problem) - # retrieve stats - log.info(f'start population: {logbook[0]}') - log.info(f'end population: {logbook[-1]}') - values = [ray.get(m.value) for m in halloffame] - - # log to wandb as steps - if wandb_enabled: - for i, stats in enumerate(logbook): - stats = {f'stats/{k}': v for k, v in stats.items()} - stats['current_step'] = i - wandb.log(stats, step=i) - - # generate average images - if plot or wandb_enabled: - # plot average - fig_ave_imgs_hof = plot_averages(dataset_name, values, title_prefix='HOF', subtitle=name, show=plot) - # get individuals -- this is not ideal because not evenly spaced - idxs_chosen_f0 = get_spaced(np.argsort([m.fitness[0] for m in population])[::-1], 5) # overlap - idxs_chosen_f1 = get_spaced(np.argsort([m.fitness[1] for m in population]), 5) # usage - chosen_values_f0 = [ray.get(population[i].value) for i in idxs_chosen_f0] - chosen_values_f1 = [ray.get(population[i].value) for i in idxs_chosen_f1] - random_fitnesses = problem.evaluate_values([ray.put(np.random.random(values[0].shape) < p) for p in np.linspace(0.025, 1, num=population_size+2)[1:-1]]) - # plot averages - fig_ave_imgs_f0 = plot_averages(dataset_name, chosen_values_f0, subtitle=name, titles=[f'{population[i].fitness[0]:2f}' for i in idxs_chosen_f0], title_prefix='Overlap -', show=plot) - fig_ave_imgs_f1 = plot_averages(dataset_name, chosen_values_f1, subtitle=name, titles=[f'{population[i].fitness[1]:2f}' for i in idxs_chosen_f1], title_prefix='Usage -', show=plot) - # plot parento optimal solutions - fig_pareto_sol, axs = plt_pareto_solutions( - population, - label_fitness_0='Overlap Score', - label_fitness_1='Usage Score', - title=f'Pareto-Optimal Solutions\n{name}', - plot=plot, - chosen_idxs_f0=idxs_chosen_f0, - chosen_idxs_f1=idxs_chosen_f1, - random_points=random_fitnesses, - figsize=(7, 7), - ) - # plot factor usage ratios - # TODO: PLOT 2D matrix of all permutations of factors aggregated - # log average - if wandb_enabled: - wandb.log({ - 'ave_images_hof': wandb.Image(fig_ave_imgs_hof), - 'ave_images_overlap': wandb.Image(fig_ave_imgs_f0), - 'ave_images_usage': wandb.Image(fig_ave_imgs_f1), - 'pareto_solutions': wandb.Image(fig_pareto_sol), - }) - - # get summary - use_elems = np.sum(values[0]) - num_elems = np.prod(values[0].shape) - use_ratio = (use_elems / num_elems) - - # log summary - if wandb_enabled: - wandb.summary['num_elements'] = num_elems - wandb.summary['used_elements'] = use_elems - wandb.summary['used_elements_ratio'] = use_ratio - for k, v in logbook[0].items(): wandb.summary[f'log:start:{k}'] = v - for k, v in logbook[-1].items(): wandb.summary[f'log:end:{k}'] = v - - # generate paths - job_name = f'{time_string}_{name}' - - # collect results - results = { - 'hparams': hparams, - 'job_name': job_name, - 'save_path': None, - 'time_string': time_string, - 'values': [ray.get(m.value) for m in population], - 'scores': [m.fitness for m in population], - # score components - 'scores_overlap': [m.fitness[0] for m in population], - 'scores_usage': [m.fitness[1] for m in population], - # history data - 'logbook_history': logbook.history, - # we don't want these because they store object refs, and - # it means we need ray to unpickle them. - # 'population': population, - # 'halloffame_members': halloffame.members, - } - - if save: - # get save path, make parent dir & save! - results['save_path'] = ensure_parent_dir_exists(ROOT_DIR, 'out/adversarial_mask', job_name, 'data.pkl.gz') - # NONE : 122943493 ~= 118M (100.%) : 103.420ms - # lvl=1 : 23566691 ~= 23M (19.1%) : 1.223s - # lvl=2 : 21913595 ~= 21M (17.8%) : 1.463s - # lvl=3 : 20688319 ~= 20M (16.8%) : 2.504s - # lvl=4 : 18325859 ~= 18M (14.9%) : 1.856s # good - # lvl=5 : 17467772 ~= 17M (14.2%) : 3.332s # good - # lvl=6 : 16594660 ~= 16M (13.5%) : 7.163s # starting to slow - # lvl=7 : 16242279 ~= 16M (13.2%) : 12.407s - # lvl=8 : 15586416 ~= 15M (12.7%) : 1m:4s # far too slow - # lvl=9 : 15023324 ~= 15M (12.2%) : 3m:11s # far too slow - log.info(f'saving data to: {results["save_path"]}') - with gzip.open(results["save_path"], 'wb', compresslevel=5) as fp: - pickle.dump(results, fp) - log.info(f'saved data to: {results["save_path"]}') - - # cleanup wandb - if wandb_enabled: - if wandb_finish: - try: - wandb.finish() - except: - pass - - # done - return results - - -# ========================================================================= # -# ENTRYPOINT # -# ========================================================================= # - - -ROOT_DIR = os.path.abspath(__file__ + '/../../..') - - -def main(): - from itertools import product - - # (2*1 * 3*1*2 * 5) = 60 - for i, (fitness_overlap_include_singles, dists_scaled, pair_mode, pairs_per_obs, fitness_overlap_mode, dataset_name) in enumerate(product( - [True, False], - [True], # [True, False] - ['nearby_scaled', 'nearby', 'random'], - [256], # [64, 16, 256] - ['std', 'range'], - ['xysquares_8x8_toy_s2', 'cars3d', 'smallnorb', 'shapes3d', 'dsprites'], # ['xysquares_8x8_toy_s2'] - )): - print('='*100) - print(f'[STARTING]: i={i} dataset_name={repr(dataset_name)} pair_mode={repr(pair_mode)} pairs_per_obs={repr(pairs_per_obs)} dists_scaled={repr(dists_scaled)} fitness_overlap_mode={repr(fitness_overlap_mode)} fitness_overlap_include_singles={repr(fitness_overlap_include_singles)}') - try: - run( - dataset_name=dataset_name, - pair_mode=pair_mode, - pairs_per_obs=pairs_per_obs, - dists_scaled=dists_scaled, - fitness_overlap_mode=fitness_overlap_mode, - fitness_overlap_include_singles=fitness_overlap_include_singles, - # population - generations=1000, # 1000 - population_size=384, - seed_=42, - save=True, - save_prefix='DISTS-SCALED', - plot=True, - wandb_enabled=True, - wandb_project='exp-adversarial-mask', - wandb_tags=['exp_pair_dists'] - ) - except KeyboardInterrupt: - warnings.warn('Exiting early') - exit(1) - except: - warnings.warn(f'[FAILED] i={i}') - print('='*100) - - -if __name__ == '__main__': - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../gadfly.mplstyle')) - - # run - logging.basicConfig(level=logging.INFO) - ray.init(num_cpus=psutil.cpu_count(logical=False)) - main() - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/e06_adversarial_data/deprecated/submit_02_train_adversarial_data.sh b/research/e06_adversarial_data/deprecated/submit_02_train_adversarial_data.sh deleted file mode 100644 index ef6cf701..00000000 --- a/research/e06_adversarial_data/deprecated/submit_02_train_adversarial_data.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-06__adversarial-modified-data" -export PARTITION="stampede" -export PARALLELISM=28 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# 1 * (4 * 2 * 2) = 16 -local_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_griffin' \ - run_location='griffin' \ - \ - run_length=short \ - metrics=fast \ - \ - framework.beta=0.001,0.00316,0.01,0.000316 \ - framework=betavae,adavae_os \ - model.z_size=25 \ - \ - dataset=X--adv-dsprites--WARNING,X--adv-shapes3d--WARNING \ - sampling=default__bb # \ - # \ - # hydra.launcher.exclude='"mscluster93,mscluster94,mscluster97"' # we don't want to sweep over these - -# 2 * (8 * 2 * 4) = 128 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_beta' \ - \ - run_length=short \ - metrics=fast \ - \ - framework.beta=0.000316,0.001,0.00316,0.01,0.0316,0.1,0.316,1.0 \ - framework=betavae,adavae_os \ - model.z_size=25 \ - \ - dataset=dsprites,shapes3d,cars3d,smallnorb \ - sampling=default__bb \ - \ - hydra.launcher.exclude='"mscluster93,mscluster94,mscluster97,mscluster99"' # we don't want to sweep over these diff --git a/research/e06_adversarial_data/deprecated/submit_04_train_dsprites_imagenet.sh b/research/e06_adversarial_data/deprecated/submit_04_train_dsprites_imagenet.sh deleted file mode 100644 index 5737db33..00000000 --- a/research/e06_adversarial_data/deprecated/submit_04_train_dsprites_imagenet.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-06__dsprites-imagenet" -export PARTITION="stampede" -export PARALLELISM=36 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# (3*2*2*11) = 132 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_dsprites_imagenet' \ - \ - run_callbacks=vis \ - run_length=medium \ - metrics=fast \ - \ - model.z_size=9,16 \ - framework.beta=0.0316,0.01,0.1 \ - framework=adavae_os,betavae \ - \ - dataset=dsprites,X--dsprites-imagenet-bg-20,X--dsprites-imagenet-bg-40,X--dsprites-imagenet-bg-60,X--dsprites-imagenet-bg-80,X--dsprites-imagenet-bg-100,X--dsprites-imagenet-fg-20,X--dsprites-imagenet-fg-40,X--dsprites-imagenet-fg-60,X--dsprites-imagenet-fg-80,X--dsprites-imagenet-fg-100 \ - sampling=default__bb \ - \ - hydra.launcher.exclude='"mscluster93,mscluster94,mscluster97,mscluster99"' # we don't want to sweep over these diff --git a/research/e06_adversarial_data/deprecated/submit_04_train_masked_data.sh b/research/e06_adversarial_data/deprecated/submit_04_train_masked_data.sh deleted file mode 100644 index 4a5f57dc..00000000 --- a/research/e06_adversarial_data/deprecated/submit_04_train_masked_data.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-06__masked-datasets" -export PARTITION="stampede" -export PARALLELISM=28 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# 3 * (12 * 2 * 2) = 144 -#submit_sweep \ -# +DUMMY.repeat=1,2,3 \ -# +EXTRA.tags='sweep_01' \ -# \ -# run_length=medium \ -# \ -# framework.beta=0.001 \ -# framework=betavae,adavae_os \ -# model.z_size=9 \ -# \ -# dataset=X--mask-adv-f-dsprites,X--mask-ran-dsprites,dsprites,X--mask-adv-f-shapes3d,X--mask-ran-shapes3d,shapes3d,X--mask-adv-f-smallnorb,X--mask-ran-smallnorb,smallnorb,X--mask-adv-f-cars3d,X--mask-ran-cars3d,cars3d \ -# sampling=random \ -# \ -# hydra.launcher.exclude='"mscluster93,mscluster94,mscluster97"' # we don't want to sweep over these - -# TODO: beta needs to be tuned! -# 3 * (12*3*2 = 72) = 216 -submit_sweep \ - +DUMMY.repeat=1,2,3 \ - +EXTRA.tags='sweep_usage_ratio' \ - \ - run_callbacks=vis \ - run_length=short \ - metrics=all \ - \ - framework.beta=0.001 \ - framework=betavae,adavae_os \ - model.z_size=25 \ - framework.optional.usage_ratio=0.5,0.2,0.05 \ - \ - dataset=X--mask-adv-f-dsprites,X--mask-ran-dsprites,dsprites,X--mask-adv-f-shapes3d,X--mask-ran-shapes3d,shapes3d,X--mask-adv-f-smallnorb,X--mask-ran-smallnorb,smallnorb,X--mask-adv-f-cars3d,X--mask-ran-cars3d,cars3d \ - sampling=random \ - \ - hydra.launcher.exclude='"mscluster93,mscluster94,mscluster97,mscluster99"' # we don't want to sweep over these diff --git a/research/e06_adversarial_data/deprecated/submit_04_train_masked_data_dist_pairs.sh b/research/e06_adversarial_data/deprecated/submit_04_train_masked_data_dist_pairs.sh deleted file mode 100644 index 59d9ad11..00000000 --- a/research/e06_adversarial_data/deprecated/submit_04_train_masked_data_dist_pairs.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-06__masked-datasets-dist-pairs" -export PARTITION="stampede" -export PARALLELISM=36 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# (3*2*3*12 = 72) = 216 -# TODO: z_size needs tuning -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_dist_pairs_usage_ratio' \ - \ - run_callbacks=vis \ - run_length=short \ - metrics=all \ - \ - framework.beta=0.0316,0.01,0.1 \ - framework=betavae,adavae_os \ - model.z_size=16 \ - framework.optional.usage_ratio=0.5,0.2,0.05 \ - \ - dataset=X--mask-adv-r-dsprites,X--mask-ran-dsprites,dsprites,X--mask-adv-r-shapes3d,X--mask-ran-shapes3d,shapes3d,X--mask-adv-r-smallnorb,X--mask-ran-smallnorb,smallnorb,X--mask-adv-r-cars3d,X--mask-ran-cars3d,cars3d \ - sampling=random \ - \ - hydra.launcher.exclude='"mscluster93,mscluster94,mscluster97,mscluster99"' # we don't want to sweep over these diff --git a/research/e06_adversarial_data/run_02_adv_dataset_approx.sh b/research/e06_adversarial_data/run_02_adv_dataset_approx.sh deleted file mode 100644 index 75dd8b44..00000000 --- a/research/e06_adversarial_data/run_02_adv_dataset_approx.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# get the path to the script -PARENT_DIR="$(dirname "$(realpath -s "$0")")" -ROOT_DIR="$(dirname "$(dirname "$PARENT_DIR")")" - -# maybe lower lr or increase batch size? -#PYTHONPATH="$ROOT_DIR" python3 "$PARENT_DIR/run_02_gen_adversarial_dataset_approx.py" \ -# -m \ -# adv_system.sampler_name=close_p_random_n,same_k1_close \ -# adv_system.adversarial_mode=self,invert_margin_0.005 \ -# adv_system.dataset_name=dsprites,shapes3d,cars3d,smallnorb - -#PYTHONPATH="$ROOT_DIR" python3 "$PARENT_DIR/run_02_gen_adversarial_dataset_approx.py" \ -# -m \ -# settings.dataset.batch_size=32,256 \ -# adv_system.loss_out_of_bounds_weight=0.0,1.0 \ -# \ -# adv_system.sampler_name=close_p_random_n \ -# adv_system.adversarial_mode=invert_margin_0.05,invert_margin_0.005,invert_margin_0.0005 \ -# adv_system.dataset_name=smallnorb - -PYTHONPATH="$ROOT_DIR" python3 "$PARENT_DIR/run_02_gen_adversarial_dataset_approx.py" \ - -m "$@" \ - \ - +meta.tag='unbounded_manhat' \ - settings.job.name_prefix=MANHAT \ - \ - adv_system.sampler_name=same_k_close,random_swap_manhattan,close_p_random_n \ - adv_system.samples_sort_mode=swap,sort_reverse,none,sort_inorder \ - \ - adv_system.adversarial_mode=triplet_unbounded \ - adv_system.dataset_name=smallnorb \ - \ - trainer.max_steps=7500 \ - trainer.max_epochs=7500 \ - \ - adv_system.optimizer_lr=5e-3 \ - settings.exp.show_every_n_steps=500 \ - \ - settings.dataset.batch_size=128 diff --git a/research/e06_adversarial_data/run_02_gen_adversarial_dataset_approx.py b/research/e06_adversarial_data/run_02_gen_adversarial_dataset_approx.py deleted file mode 100644 index 0caf72ec..00000000 --- a/research/e06_adversarial_data/run_02_gen_adversarial_dataset_approx.py +++ /dev/null @@ -1,620 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -""" -Generate an adversarial dataset by approximating the difference between -the dataset and the target adversarial images using a model. - adv = obs + diff(obs) -""" - -import logging -import os -from datetime import datetime -from typing import List -from typing import Optional -from typing import Sequence -from typing import Tuple - -import hydra -import numpy as np -import pytorch_lightning as pl -import torch -import torch.nn.functional as F -import wandb -from omegaconf import OmegaConf -from torch.utils.data import DataLoader - -import research.util as H -from disent import registry -from disent.dataset import DisentDataset -from disent.dataset.sampling import BaseDisentSampler -from disent.dataset.util.hdf5 import H5Builder -from disent.model import AutoEncoder -from disent.nn.activations import Swish -from disent.nn.modules import DisentModule -from disent.nn.weights import init_model_weights -from disent.util import to_numpy -from disent.util.function import wrapped_partial -from disent.util.inout.paths import ensure_parent_dir_exists -from disent.util.lightning.callbacks import BaseCallbackPeriodic -from disent.util.lightning.callbacks import LoggerProgressCallback -from disent.util.lightning.logger_util import wb_has_logger -from disent.util.lightning.logger_util import wb_log_metrics -from disent.util.seeds import seed -from disent.util.seeds import TempNumpySeed -from disent.util.strings.fmt import bytes_to_human -from disent.util.strings.fmt import make_box_str -from disent.util.visualize.vis_util import make_image_grid -from experiment.run import hydra_get_gpus -from experiment.run import hydra_get_callbacks -from experiment.run import hydra_make_logger -from experiment.util.hydra_utils import make_non_strict -from experiment.util.run_utils import log_error_and_exit -from research.e06_adversarial_data.util_gen_adversarial_dataset import adversarial_loss -from research.e06_adversarial_data.util_gen_adversarial_dataset import make_adversarial_sampler -from research.e06_adversarial_data.util_gen_adversarial_dataset import sort_samples - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Dataset Mask # -# ========================================================================= # - -@torch.no_grad() -def _sample_stacked_batch(dataset: DisentDataset) -> torch.Tensor: - batch = next(iter(DataLoader(dataset, batch_size=1024, num_workers=0, shuffle=True))) - batch = torch.cat(batch['x_targ'], dim=0) - return batch - -@torch.no_grad() -def gen_approx_dataset_mask(dataset: DisentDataset, model_mask_mode: Optional[str]) -> Optional[torch.Tensor]: - if model_mask_mode in ('none', None): - mask = None - elif model_mask_mode == 'diff': - batch = _sample_stacked_batch(dataset) - mask = ~torch.all(batch[1:] == batch[0:1], dim=0) - elif model_mask_mode == 'std': - batch = _sample_stacked_batch(dataset) - mask = torch.std(batch, dim=0) - m, M = torch.min(mask), torch.max(mask) - mask = (mask - m) / (M - m) - else: - raise KeyError(f'invalid `model_mask_mode`: {repr(model_mask_mode)}') - # done - return mask - - -# ========================================================================= # -# adversarial dataset generator # -# ========================================================================= # - - -class AeModel(AutoEncoder): - def forward(self, x): - return self.decode(self.encode(x)) - - -def make_delta_model(model_type: str, x_shape: Tuple[int, ...]): - C, H, W = x_shape - # get model - if model_type.startswith('ae_'): - return AeModel( - encoder=registry.MODELS[f'encoder_{model_type[len("ae_"):]}'](x_shape=x_shape, z_size=64, z_multiplier=1), - decoder=registry.MODELS[f'decoder_{model_type[len("ae_"):]}'](x_shape=x_shape, z_size=64, z_multiplier=1), - ) - elif model_type == 'fcn_small': - return torch.nn.Sequential( - torch.nn.ReflectionPad2d(1), torch.nn.Conv2d(in_channels=C, out_channels=5, kernel_size=3), Swish(), - torch.nn.ReflectionPad2d(1), torch.nn.Conv2d(in_channels=5, out_channels=7, kernel_size=3), Swish(), - torch.nn.ReflectionPad2d(1), torch.nn.Conv2d(in_channels=7, out_channels=9, kernel_size=3), Swish(), - torch.nn.ReflectionPad2d(1), torch.nn.Conv2d(in_channels=9, out_channels=7, kernel_size=3), Swish(), - torch.nn.ReflectionPad2d(1), torch.nn.Conv2d(in_channels=7, out_channels=5, kernel_size=3), Swish(), - torch.nn.ReflectionPad2d(1), torch.nn.Conv2d(in_channels=5, out_channels=C, kernel_size=3), - ) - else: - raise KeyError(f'invalid model type: {repr(model_type)}') - - -class AdversarialAugmentModel(DisentModule): - - def __init__(self, model_type: str, x_shape=(3, 64, 64), mask=None, meta: dict = None): - super().__init__() - # make layers - self.delta_model = make_delta_model(model_type=model_type, x_shape=x_shape) - self.meta = meta if meta else {} - # mask - if mask is not None: - self.register_buffer('mask', mask[None, ...]) - assert self.mask.ndim == 4 # (1, C, H, W) - - def forward(self, x): - assert x.ndim == 4 - # compute - if hasattr(self, 'mask'): - return x + self.delta_model(x) * self.mask - else: - return x + self.delta_model(x) - - -# ========================================================================= # -# adversarial dataset generator # -# ========================================================================= # - - -class AdversarialModel(pl.LightningModule): - - def __init__( - self, - # optimizer options - optimizer_name: str = 'sgd', - optimizer_lr: float = 5e-2, - optimizer_kwargs: Optional[dict] = None, - # dataset config options - dataset_name: str = 'cars3d', - dataset_num_workers: int = min(os.cpu_count(), 16), - dataset_batch_size: int = 1024, # approx - data_root: str = 'data/dataset', - data_load_into_memory: bool = False, - # adversarial loss options - adversarial_mode: str = 'self', - adversarial_swapped: bool = False, - adversarial_masking: bool = False, - adversarial_top_k: Optional[int] = None, - pixel_loss_mode: str = 'mse', - # loss extras - loss_adversarial_weight: Optional[float] = 1.0, - loss_same_stats_weight: Optional[float] = 0.0, - loss_similarity_weight: Optional[float] = 0.0, - loss_out_of_bounds_weight: Optional[float] = 0.0, - # sampling config - sampler_name: str = 'close_far', - samples_sort_mode: str = 'none', - # model settings - model_type: str = 'ae_linear', - model_mask_mode: Optional[str] = 'none', - model_weight_init: str = 'xavier_normal', - # logging settings - logging_scale_imgs: bool = False, - # log_wb_stats_table: bool = True, - ): - super().__init__() - # modify hparams - if optimizer_kwargs is None: - optimizer_kwargs = {} # value used by save_hyperparameters - # save hparams - self.save_hyperparameters() - # variables - self.dataset: DisentDataset = None - self.sampler: BaseDisentSampler = None - self.model: DisentModule = None - - # ================================== # - # setup # - # ================================== # - - def prepare_data(self) -> None: - # create dataset - self.dataset = H.make_dataset( - self.hparams.dataset_name, - load_into_memory=self.hparams.data_load_into_memory, - load_memory_dtype=torch.float32, - data_root=self.hparams.data_root, - sampler=make_adversarial_sampler(self.hparams.sampler_name), - ) - # make the model - self.model = AdversarialAugmentModel( - model_type=self.hparams.model_type, - x_shape=(self.dataset.gt_data.img_channels, 64, 64), - mask=gen_approx_dataset_mask(dataset=self.dataset, model_mask_mode=self.hparams.model_mask_mode), - # if we save the model we can restore things! - meta=dict( - dataset_name=self.hparams.dataset_name, - dataset_factor_sizes=self.dataset.gt_data.factor_sizes, - dataset_factor_names=self.dataset.gt_data.factor_names, - sampler_name=self.hparams.sampler_name, - hparams=dict(self.hparams) - ), - ) - # initialize model - self.model = init_model_weights(self.model, mode=self.hparams.model_weight_init) - - def train_dataloader(self): - return DataLoader( - self.dataset, - batch_size=self.hparams.dataset_batch_size, - num_workers=self.hparams.dataset_num_workers, - shuffle=True, - ) - - def configure_optimizers(self): - return H.make_optimizer( - self.model, - name=self.hparams.optimizer_name, - lr=self.hparams.optimizer_lr, - **self.hparams.optimizer_kwargs, - ) - - # ================================== # - # train step # - # ================================== # - - def forward(self, x): - return self.model(x) - - def training_step(self, batch, batch_idx): - (a_x, p_x, n_x) = batch['x_targ'] - # sort inputs - a_x, p_x, n_x = sort_samples(a_x, p_x, n_x, sort_mode=self.hparams.samples_sort_mode, pixel_loss_mode=self.hparams.pixel_loss_mode) - # feed forward - a_y = self.model(a_x) - p_y = self.model(p_x) - n_y = self.model(n_x) - # compute loss - loss_adv = 0 - if (self.hparams.loss_adversarial_weight is not None) and (self.hparams.loss_adversarial_weight > 0): - loss_adv, loss_adv_stats = adversarial_loss( - ys=(a_y, p_y, n_y), - xs=(a_x, p_x, n_x), - adversarial_mode=self.hparams.adversarial_mode, - adversarial_swapped=self.hparams.adversarial_swapped, - adversarial_masking=self.hparams.adversarial_masking, - adversarial_top_k=self.hparams.adversarial_top_k, - pixel_loss_mode=self.hparams.pixel_loss_mode, - return_stats=True, - ) - loss_adv *= self.hparams.loss_adversarial_weight - self.log_dict(loss_adv_stats) - # additional loss components - # - keep stats the same - loss_stats = 0 - if (self.hparams.loss_same_stats_weight is not None) and (self.hparams.loss_same_stats_weight > 0): - loss_stats += (self.hparams.loss_same_stats_weight/3) * (( - F.mse_loss(a_y.mean(dim=[-3, -2, -1]), a_x.mean(dim=[-3, -2, -1]), reduction='mean') + - F.mse_loss(p_y.mean(dim=[-3, -2, -1]), p_x.mean(dim=[-3, -2, -1]), reduction='mean') + - F.mse_loss(n_y.mean(dim=[-3, -2, -1]), n_x.mean(dim=[-3, -2, -1]), reduction='mean') - ) + ( - F.mse_loss(a_y.std(dim=[-3, -2, -1]), a_x.std(dim=[-3, -2, -1]), reduction='mean') + - F.mse_loss(p_y.std(dim=[-3, -2, -1]), p_x.std(dim=[-3, -2, -1]), reduction='mean') + - F.mse_loss(n_y.std(dim=[-3, -2, -1]), n_x.std(dim=[-3, -2, -1]), reduction='mean') - )) - # - try keep similar to inputs - loss_sim = 0 - if (self.hparams.loss_similarity_weight is not None) and (self.hparams.loss_similarity_weight > 0): - loss_sim = (self.hparams.loss_similarity_weight / 3) * ( - F.mse_loss(a_y, a_x, reduction='mean') + - F.mse_loss(p_y, p_x, reduction='mean') + - F.mse_loss(n_y, n_x, reduction='mean') - ) - # - regularize if out of bounds - loss_out = 0 - if (self.hparams.loss_out_of_bounds_weight is not None) and (self.hparams.loss_out_of_bounds_weight > 0): - zeros = torch.zeros_like(a_y) - loss_out = (self.hparams.loss_out_of_bounds_weight / 6) * ( - torch.where(a_y < 0, -a_y, zeros).mean() + torch.where(a_y > 1, a_y-1, zeros).mean() + - torch.where(p_y < 0, -p_y, zeros).mean() + torch.where(p_y > 1, p_y-1, zeros).mean() + - torch.where(n_y < 0, -n_y, zeros).mean() + torch.where(n_y > 1, n_y-1, zeros).mean() - ) - # final loss - loss = loss_adv + loss_sim + loss_out - # log everything - self.log_dict({ - 'loss': loss, - 'loss_stats': loss_stats, - 'loss_adv': loss_adv, - 'loss_out': loss_out, - 'loss_sim': loss_sim, - }, prog_bar=True) - # done! - return loss - - # ================================== # - # dataset # - # ================================== # - - @torch.no_grad() - def batch_to_adversarial_imgs(self, batch: torch.Tensor, m=0, M=1, mode='uint8') -> np.ndarray: - batch = batch.to(device=self.device, dtype=torch.float32) - batch = self.model(batch) - batch = (batch - m) / (M - m) - if mode == 'uint8': return H.to_imgs(batch).numpy() - elif mode == 'float32': return torch.moveaxis(batch, -3, -1).to(torch.float32).cpu().numpy() - elif mode == 'float16': return torch.moveaxis(batch, -3, -1).to(torch.float16).cpu().numpy() - else: raise KeyError(f'invalid output mode: {repr(mode)}') - - def make_train_periodic_callbacks(self, cfg) -> Sequence[BaseCallbackPeriodic]: - - # dataset transform helper - @TempNumpySeed(42) - @torch.no_grad() - def make_scale_uint8_transform(): - # get scaling values - if self.hparams.logging_scale_imgs: - samples = self.dataset.dataset_sample_batch(num_samples=128, mode='raw').to(torch.float32) - samples = self.model(samples.to(self.device)).cpu() - m, M = float(torch.min(samples)), float(torch.max(samples)) - else: - m, M = 0, 1 - return lambda x: self.batch_to_adversarial_imgs(x[None, ...], m=m, M=M)[0] - - # show image callback - class _BaseDatasetCallback(BaseCallbackPeriodic): - @TempNumpySeed(777) - @torch.no_grad() - def do_step(this, trainer: pl.Trainer, system: AdversarialModel): - if not wb_has_logger(trainer.logger): - log.warning(f'no wandb logger found, skipping visualisation: {system.__class__.__name__}') - return - if system.dataset is None: - log.warning(f'dataset not initialized, skipping visualisation: {system.__class__.__name__}') - return - log.info(f'visualising: {this.__class__.__name__}') - try: - this._do_step(trainer, system) - except: - log.error('Failed to do visualise callback step!', exc_info=True) - - # override this - def _do_step(this, trainer: pl.Trainer, system: AdversarialModel): - raise NotImplementedError - - # show image callback - class ImShowCallback(_BaseDatasetCallback): - def _do_step(this, trainer: pl.Trainer, system: AdversarialModel): - # make dataset with required transform - # -- this is inefficient for multiple subclasses of this class, we need to recompute the transform each time - dataset = system.dataset.shallow_copy(transform=make_scale_uint8_transform()) - # get images & traversal - image = make_image_grid(dataset.dataset_sample_batch(num_samples=16, mode='input')) - wandb_image, wandb_animation = H.visualize_dataset_traversal(dataset, data_mode='input', output_wandb=True) - # log images to WANDB - wb_log_metrics(trainer.logger, { - 'random_images': wandb.Image(image), - 'traversal_image': wandb_image, - 'traversal_animation': wandb_animation, - }) - - # factor distances callback - class DistsPlotCallback(_BaseDatasetCallback): - def _do_step(this, trainer: pl.Trainer, system: AdversarialModel): - from disent.util.lightning.callbacks._callbacks_vae import compute_factor_distances, plt_factor_distances - - # make distances function - def dists_fn(xs_a, xs_b): - dists = H.pairwise_loss(xs_a, xs_b, mode=system.hparams.pixel_loss_mode, mean_dtype=torch.float32, mask=None) - return [dists] - - def transform_batch(batch): - return system.model(batch.to(device=system.device)) - - # compute various distances matrices for each factor - dists_names, f_grid = compute_factor_distances( - dataset=system.dataset, - dists_fn=dists_fn, - dists_names=['dists'], - traversal_repeats=100, - batch_size=system.hparams.dataset_batch_size, - include_gt_factor_dists=True, - transform_batch=transform_batch, - seed=777, - data_mode='input', - ) - # plot these results - fig, axs = plt_factor_distances( - gt_data=system.dataset.gt_data, - f_grid=f_grid, - dists_names=dists_names, - title=f'{system.hparams.model_type.capitalize()}: {system.hparams.dataset_name.capitalize()} Distances', - plt_block_size=1.25, - plt_transpose=True, - plt_cmap='Blues', - ) - # recolour dists axis - for ax in axs[-1, :]: - ax.images[0].set_cmap('Reds') - # generate image & close matplotlib instace - from matplotlib import pyplot as plt - img = wandb.Image(fig) - plt.close() - # log the plot to wandb - if True: - wb_log_metrics(trainer.logger, { - 'factor_distances': img - }) - - # show stats callback - class StatsShowCallback(_BaseDatasetCallback): - def _do_step(this, trainer: pl.Trainer, system: AdversarialModel): - # make dataset with required transform - # -- this is inefficient for multiple subclasses of this class, we need to recompute the transform each time - dataset = system.dataset.shallow_copy(transform=make_scale_uint8_transform()) - # get batches - batch, factors = dataset.dataset_sample_batch_with_factors(num_samples=512, mode='input') - batch = batch.to(torch.float32) - a_idx = torch.randint(0, len(batch), size=[4*len(batch)]) - b_idx = torch.randint(0, len(batch), size=[4*len(batch)]) - mask = (a_idx != b_idx) - # TODO: check that this is deterministic - # compute distances - deltas = to_numpy(H.pairwise_overlap(batch[a_idx[mask]], batch[b_idx[mask]], mode='mse')) - fdists = to_numpy(torch.abs(factors[a_idx[mask]] - factors[b_idx[mask]]).sum(dim=-1)) - sdists = to_numpy((torch.abs(factors[a_idx[mask]] - factors[b_idx[mask]]) / to_numpy(dataset.gt_data.factor_sizes)[None, :]).sum(dim=-1)) - # log to wandb - from matplotlib import pyplot as plt - plt.scatter(fdists, deltas); img_fdists = wandb.Image(plt); plt.close() - plt.scatter(sdists, deltas); img_sdists = wandb.Image(plt); plt.close() - wb_log_metrics(trainer.logger, { - 'fdists_vs_overlap': img_fdists, - 'sdists_vs_overlap': img_sdists, - }) - - # done! - return [ - ImShowCallback(every_n_steps=cfg.settings.exp.show_every_n_steps, begin_first_step=True), - DistsPlotCallback(every_n_steps=cfg.settings.exp.show_every_n_steps, begin_first_step=True), - StatsShowCallback(every_n_steps=cfg.settings.exp.show_every_n_steps, begin_first_step=True), - ] - - -# ========================================================================= # -# Run Hydra # -# ========================================================================= # - - -ROOT_DIR = os.path.abspath(__file__ + '/../../..') - - -def run_gen_adversarial_dataset(cfg): - time_string = datetime.today().strftime('%Y-%m-%d--%H-%M-%S') - log.info(f'Starting run at time: {time_string}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # cleanup from old runs: - try: - wandb.finish() - except: - pass - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - cfg = make_non_strict(cfg) - # - - - - - - - - - - - - - - - # - # check CUDA setting - gpus = hydra_get_gpus(cfg) - # create logger - logger = hydra_make_logger(cfg) - # create callbacks - callbacks: List[pl.Callback] = [c for c in hydra_get_callbacks(cfg) if isinstance(c, LoggerProgressCallback)] - # - - - - - - - - - - - - - - - # - # check save dirs - assert not os.path.isabs(cfg.settings.exp.rel_save_dir), f'rel_save_dir must be relative: {repr(cfg.settings.exp.rel_save_dir)}' - save_dir = os.path.join(ROOT_DIR, cfg.settings.exp.rel_save_dir) - assert os.path.isabs(save_dir), f'save_dir must be absolute: {repr(save_dir)}' - # - - - - - - - - - - - - - - - # - # get the logger and initialize - if logger is not None: - logger.log_hyperparams(cfg) - # print the final config! - log.info('Final Config' + make_box_str(OmegaConf.to_yaml(cfg))) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # | | | | | | | | | | | | | | | # - seed(cfg.settings.job.seed) - # | | | | | | | | | | | | | | | # - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # make framework - framework = AdversarialModel(**cfg.adv_system) - callbacks.extend(framework.make_train_periodic_callbacks(cfg)) - # train - trainer = pl.Trainer( - logger=logger, - callbacks=callbacks, - # cfg.dsettings.trainer - gpus=gpus, - # cfg.trainer - max_epochs=cfg.trainer.max_epochs, - max_steps=cfg.trainer.max_steps, - log_every_n_steps=cfg.trainer.log_every_n_steps, - flush_logs_every_n_steps=cfg.trainer.flush_logs_every_n_steps, - progress_bar_refresh_rate=cfg.trainer.progress_bar_refresh_rate, - prepare_data_per_node=cfg.trainer.prepare_data_per_node, - # we do this here so we don't run the final metrics - terminate_on_nan=True, - checkpoint_callback=False, - ) - trainer.fit(framework) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # get save paths - save_prefix = f'{cfg.settings.exp.save_prefix}_' if cfg.settings.exp.save_prefix else '' - save_path_model = os.path.join(save_dir, f'{save_prefix}{time_string}_{cfg.settings.job.name}', f'model.pt') - save_path_data = os.path.join(save_dir, f'{save_prefix}{time_string}_{cfg.settings.job.name}', f'data.h5') - # create directories - if cfg.settings.exp.save_model: ensure_parent_dir_exists(save_path_model) - if cfg.settings.exp.save_data: ensure_parent_dir_exists(save_path_data) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # save adversarial model - if cfg.settings.exp.save_model: - log.info(f'saving model to path: {repr(save_path_model)}') - torch.save(framework.model, save_path_model) - log.info(f'saved model size: {bytes_to_human(os.path.getsize(save_path_model))}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # save adversarial dataset - if cfg.settings.exp.save_data: - log.info(f'saving data to path: {repr(save_path_data)}') - # transfer to GPU - if torch.cuda.is_available(): - framework = framework.cuda() - # create new h5py file -- TODO: use this in other places! - with H5Builder(path=save_path_data, mode='atomic_w') as builder: - # this dataset is self-contained and can be loaded by SelfContainedHdf5GroundTruthData - builder.add_dataset_from_gt_data( - data=framework.dataset, # produces tensors - mutator=wrapped_partial(framework.batch_to_adversarial_imgs, mode=cfg.settings.exp.save_dtype), # consumes tensors -> np.ndarrays - img_shape=(64, 64, None), - compression_lvl=4, - dtype=cfg.settings.exp.save_dtype, - batch_size=32, - ) - log.info(f'saved data size: {bytes_to_human(os.path.getsize(save_path_data))}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - -# ========================================================================= # -# Entry Point # -# ========================================================================= # - - -if __name__ == '__main__': - - # BENCHMARK (batch_size=256, optimizer=sgd, lr=1e-2, dataset_num_workers=0): - # - batch_optimizer=False, gpu=True, fp16=True : [3168MiB/5932MiB, 3.32/11.7G, 5.52it/s] - # - batch_optimizer=False, gpu=True, fp16=False : [5248MiB/5932MiB, 3.72/11.7G, 4.84it/s] - # - batch_optimizer=False, gpu=False, fp16=True : [same as fp16=False] - # - batch_optimizer=False, gpu=False, fp16=False : [0003MiB/5932MiB, 4.60/11.7G, 1.05it/s] - # --------- - # - batch_optimizer=True, gpu=True, fp16=True : [1284MiB/5932MiB, 3.45/11.7G, 4.31it/s] - # - batch_optimizer=True, gpu=True, fp16=False : [1284MiB/5932MiB, 3.72/11.7G, 4.31it/s] - # - batch_optimizer=True, gpu=False, fp16=True : [same as fp16=False] - # - batch_optimizer=True, gpu=False, fp16=False : [0003MiB/5932MiB, 1.80/11.7G, 4.18it/s] - - # BENCHMARK (batch_size=1024, optimizer=sgd, lr=1e-2, dataset_num_workers=12): - # - batch_optimizer=True, gpu=True, fp16=True : [2510MiB/5932MiB, 4.10/11.7G, 4.75it/s, 20% gpu util] (to(device).to(dtype)) - # - batch_optimizer=True, gpu=True, fp16=True : [2492MiB/5932MiB, 4.10/11.7G, 4.12it/s, 19% gpu util] (to(device, dtype)) - - @hydra.main(config_path=os.path.join(ROOT_DIR, 'experiment/config'), config_name="config_adversarial_dataset_approx") - def main(cfg): - try: - run_gen_adversarial_dataset(cfg) - except Exception as e: - # truncate error - err_msg = str(e) - err_msg = err_msg[:244] + ' ' if len(err_msg) > 244 else err_msg - # log something at least - log.error(f'exiting: experiment error | {err_msg}', exc_info=True) - - # EXP ARGS: - # $ ... -m dataset=smallnorb,shapes3d - try: - main() - except KeyboardInterrupt as e: - log_error_and_exit(err_type='interrupted', err_msg=str(e), exc_info=False) - except Exception as e: - log_error_and_exit(err_type='hydra error', err_msg=str(e)) diff --git a/research/e06_adversarial_data/util_eval_adversarial.py b/research/e06_adversarial_data/util_eval_adversarial.py deleted file mode 100644 index 8b4c9b5c..00000000 --- a/research/e06_adversarial_data/util_eval_adversarial.py +++ /dev/null @@ -1,348 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from typing import Tuple - -import numpy as np -from numba import njit -from scipy.stats import gmean - - -# ========================================================================= # -# Aggregate # -# ========================================================================= # - - -_NP_AGGREGATE_FNS = { - 'sum': np.sum, - 'mean': np.mean, - 'gmean': gmean, # no negatives - 'max': lambda a, axis, dtype: np.amax(a, axis=axis), # propagate NaNs - 'min': lambda a, axis, dtype: np.amin(a, axis=axis), # propagate NaNs - 'std': np.std, -} - - -def np_aggregate(array, mode: str, axis=0, dtype=None): - try: - fn = _NP_AGGREGATE_FNS[mode] - except KeyError: - raise KeyError(f'invalid aggregate mode: {repr(mode)}, must be one of: {sorted(_NP_AGGREGATE_FNS.keys())}') - result = fn(array, axis=axis, dtype=dtype) - if dtype is not None: - result = result.astype(dtype) - return result - - -# ========================================================================= # -# Factor Evaluation - SLOW # -# ========================================================================= # - - -def eval_factor_fitness_numpy( - individual: np.ndarray, - f_idx: int, - f_dist_matrices: np.ndarray, - factor_sizes: Tuple[int, ...], - fitness_mode: str, - exclude_diag: bool, - increment_single: bool = True, -) -> float: - assert increment_single, f'`increment_single=False` is not supported for numpy fitness evaluation' - # generate missing mask axis - mask = individual.reshape(factor_sizes) - mask = np.moveaxis(mask, f_idx, -1) - f_mask = mask[..., :, None] & mask[..., None, :] - # the diagonal can change statistics - if exclude_diag: - diag = np.arange(f_mask.shape[-1]) - f_mask[..., diag, diag] = False - # mask the distance array | we negate the mask so that TRUE means the item is disabled - f_dists = np.ma.masked_where(~f_mask, f_dist_matrices) - - # get distances - if fitness_mode == 'range': agg_vals = np.ma.max(f_dists, axis=-1) - np.ma.min(f_dists, axis=-1) - elif fitness_mode == 'max': agg_vals = np.ma.max(f_dists, axis=-1) - elif fitness_mode == 'std': agg_vals = np.ma.std(f_dists, axis=-1) - else: raise KeyError(f'invalid fitness_mode: {repr(fitness_mode)}') - - # mean -- there is still a slight difference between this version - # and the numba version, but this helps improve things... - # It might just be a precision error? - fitness_sparse = np.ma.masked_where(~mask, agg_vals).mean() - - # combined scores - return fitness_sparse - - -# ========================================================================= # -# Factor Evaluation - FAST # -# ========================================================================= # - - -@njit -def eval_factor_fitness_numba__std_nodiag( - mask: np.ndarray, - f_dists: np.ndarray, - increment_single: bool = True -): - """ - This is about 10x faster than the built in numpy version - """ - assert f_dists.shape == (*mask.shape, mask.shape[-1]) - # totals - total = 0.0 - count = 0 - # iterate over values -- np.ndindex is usually quite fast - for I in np.ndindex(mask.shape[:-1]): - # mask is broadcast to the distance matrix - m_row = mask[I] - d_mat = f_dists[I] - # handle each distance matrix -- enumerate is usually faster than range - for i, m in enumerate(m_row): - if not m: - continue - # get vars - dists = d_mat[i] - # init vars - n = 0 - s = 0.0 - s2 = 0.0 - # handle each row -- enumerate is usually faster than range - for j, d in enumerate(dists): - if i == j: - continue - if not m_row[j]: - continue - n += 1 - s += d - s2 += d*d - # ^^^ END j - # update total - if n > 1: - mean2 = (s * s) / (n * n) - m2 = (s2 / n) - # is this just needed because of precision errors? - if m2 > mean2: - total += np.sqrt(m2 - mean2) - count += 1 - elif increment_single and (n == 1): - total += 0. - count += 1 - # ^^^ END i - if count == 0: - return -1 - else: - return total / count - - -@njit -def eval_factor_fitness_numba__range_nodiag( - mask: np.ndarray, - f_dists: np.ndarray, - increment_single: bool = True, -): - """ - This is about 10x faster than the built in numpy version - """ - assert f_dists.shape == (*mask.shape, mask.shape[-1]) - # totals - total = 0.0 - count = 0 - # iterate over values -- np.ndindex is usually quite fast - for I in np.ndindex(mask.shape[:-1]): - # mask is broadcast to the distance matrix - m_row = mask[I] - d_mat = f_dists[I] - # handle each distance matrix -- enumerate is usually faster than range - for i, m in enumerate(m_row): - if not m: - continue - # get vars - dists = d_mat[i] - # init vars - num_checked = False - m = 0.0 - M = 0.0 - # handle each row -- enumerate is usually faster than range - for j, d in enumerate(dists): - if i == j: - continue - if not m_row[j]: - continue - # update range - if num_checked > 0: - if d < m: - m = d - if d > M: - M = d - else: - m = d - M = d - # update num checked - num_checked += 1 - # ^^^ END j - # update total - if (num_checked > 1) or (increment_single and num_checked == 1): - total += (M - m) - count += 1 - # ^^^ END i - if count == 0: - return -1 - else: - return total / count - - -def eval_factor_fitness_numba( - individual: np.ndarray, - f_idx: int, - f_dist_matrices: np.ndarray, - factor_sizes: Tuple[int, ...], - fitness_mode: str, - exclude_diag: bool, - increment_single: bool = True, -): - """ - We only keep this function as a compatibility layer between: - - eval_factor_fitness_numpy - - eval_factor_fitness_numba__range_nodiag - """ - assert exclude_diag, 'fast version of eval only supports `exclude_diag=True`' - # usually a view - mask = np.moveaxis(individual.reshape(factor_sizes), f_idx, -1) - # call - if fitness_mode == 'range': - return eval_factor_fitness_numba__range_nodiag(mask=mask, f_dists=f_dist_matrices, increment_single=increment_single) - elif fitness_mode == 'std': - return eval_factor_fitness_numba__std_nodiag(mask=mask, f_dists=f_dist_matrices, increment_single=increment_single) - else: - raise KeyError(f'fast version of eval only supports `fitness_mode in ("range", "std")`, got: {repr(fitness_mode)}') - - -# ========================================================================= # -# Individual Evaluation # -# ========================================================================= # - - -_EVAL_BACKENDS = { - 'numpy': eval_factor_fitness_numpy, - 'numba': eval_factor_fitness_numba, -} - - -def eval_individual( - individual: np.ndarray, - gt_dist_matrices: np.ndarray, - factor_sizes: Tuple[int, ...], - fitness_overlap_mode: str, - fitness_overlap_aggregate: str, - exclude_diag: bool, - increment_single: bool = True, - backend: str = 'numba', -) -> Tuple[float, float]: - # get function - if backend not in _EVAL_BACKENDS: - raise KeyError(f'invalid backend: {repr(backend)}, must be one of: {sorted(_EVAL_BACKENDS.keys())}') - eval_fn = _EVAL_BACKENDS[backend] - # evaluate all factors - factor_scores = np.array([ - [eval_fn(individual, f_idx, f_dist_matrices, factor_sizes=factor_sizes, fitness_mode=fitness_overlap_mode, exclude_diag=exclude_diag, increment_single=increment_single)] - for f_idx, f_dist_matrices in enumerate(gt_dist_matrices) - ]) - # aggregate - factor_score = np_aggregate(factor_scores[:, 0], mode=fitness_overlap_aggregate, dtype='float64') - kept_ratio = individual.mean() - # check values just in case something goes wrong! - factor_score = np.nan_to_num(factor_score, nan=float('-inf')) - kept_ratio = np.nan_to_num(kept_ratio, nan=float('-inf')) - # return values! - return float(factor_score), float(kept_ratio) - - -# ========================================================================= # -# Equality Checks # -# ========================================================================= # - - -def _check_equal( - dataset_name: str = 'dsprites', - fitness_mode: str = 'std', # range, std - n: int = 5, -): - from research.e01_visual_overlap.util_compute_traversal_dists import cached_compute_all_factor_dist_matrices - from timeit import timeit - import research.util as H - - # load data - gt_data = H.make_data(dataset_name) - print(f'{dataset_name} {gt_data.factor_sizes} : {fitness_mode}') - - # get distances & individual - all_dist_matrices = cached_compute_all_factor_dist_matrices(dataset_name) # SHAPE FOR: s=factor_sizes, i=f_idx | (*s[:i], *s[i+1:], s[i], s[i]) - mask = np.random.random(len(gt_data)) < 0.5 # SHAPE: (-1,) - - def eval_factor(backend: str, f_idx: int, increment_single=True): - return _EVAL_BACKENDS[backend]( - individual=mask, - f_idx=f_idx, - f_dist_matrices=all_dist_matrices[f_idx], - factor_sizes=gt_data.factor_sizes, - fitness_mode=fitness_mode, - exclude_diag=True, - increment_single=increment_single, - ) - - def eval_all(backend: str, increment_single=True): - return np.around([eval_factor(backend, i, increment_single=increment_single) for i in range(gt_data.num_factors)], decimals=15) - - new_vals = eval_all('numba', increment_single=False) - new_time = timeit(lambda: eval_all('numba', increment_single=False), number=n) / n - print(f'- NEW {new_time:.5f}s {new_vals} (increment_single=False)') - - new_vals = eval_all('numba') - new_time = timeit(lambda: eval_all('numba'), number=n) / n - print(f'- NEW {new_time:.5f}s {new_vals}') - - old_vals = eval_all('numpy') - old_time = timeit(lambda: eval_all('numpy'), number=n) / n - print(f'- OLD {old_time:.5f}s {old_vals}') - print(f'* speedup: {np.around(old_time/new_time, decimals=2)}x') - - if not np.allclose(new_vals, old_vals): - print('[WARNING]: values are not close!') - - -if __name__ == '__main__': - - for dataset_name in ['smallnorb', 'shapes3d', 'dsprites']: - print('='*100) - _check_equal(dataset_name, fitness_mode='std') - print() - _check_equal(dataset_name, fitness_mode='range') - print('='*100) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/e06_adversarial_data/util_eval_adversarial_dist_pairs.py b/research/e06_adversarial_data/util_eval_adversarial_dist_pairs.py deleted file mode 100644 index 0146b254..00000000 --- a/research/e06_adversarial_data/util_eval_adversarial_dist_pairs.py +++ /dev/null @@ -1,291 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -from typing import Tuple - -import numpy as np -from numba import njit - - -# ========================================================================= # -# Factor Evaluation - SLOW # -# ========================================================================= # -from disent.util.profiling import Timer - - -def eval_dist_pairs_numpy( - mask: np.ndarray, - pair_obs_dists: np.ndarray, - pair_obs_idxs: np.ndarray, - fitness_mode: str, - increment_single: bool = True -) -> float: - assert increment_single, f'`increment_single=False` is not supported for numpy fitness evaluation' - # mask the distance array | we negate the mask so that TRUE means the item is disabled - dists = np.ma.masked_where(~mask[pair_obs_idxs], pair_obs_dists) - # get distances - if fitness_mode == 'range': agg_vals = np.ma.max(dists, axis=-1) - np.ma.min(dists, axis=-1) - elif fitness_mode == 'std': agg_vals = np.ma.std(dists, axis=-1) - else: raise KeyError(f'invalid fitness_mode: {repr(fitness_mode)}') - # mean -- there is still a slight difference between this version - # and the numba version, but this helps improve things... - # It might just be a precision error? - fitness_sparse = np.ma.masked_where(~mask, agg_vals).mean() - # combined scores - return fitness_sparse - - -# ========================================================================= # -# Factor Evaluation - FAST # -# ========================================================================= # - - -@njit -def eval_dist_pairs_numba__std( - mask: np.ndarray, - pair_obs_dists: np.ndarray, - pair_obs_idxs: np.ndarray, - increment_single: bool = True -): - """ - This is about 10x faster than the built in numpy version - -- something is wrong compared to the numpy version, maybe the - numpy version is wrong because of the mean taken after masking? - """ - assert len(mask) == len(pair_obs_dists) - assert len(mask) == len(pair_obs_idxs) - assert pair_obs_dists.shape == pair_obs_idxs.shape - # totals - total = 0.0 - count = 0 - # iterate over values -- np.ndindex is usually quite fast - for i, m in enumerate(mask): - # skip if invalid - if not m: - continue - # get pair info - dists = pair_obs_dists[i] - idxs = pair_obs_idxs[i] - # init vars - n = 0 - s = 0.0 - s2 = 0.0 - # handle each distance matrix -- enumerate is usually faster than range - for j, d in zip(idxs, dists): - # skip if invalid - if not mask[j]: - continue - # compute std - n += 1 - s += d - s2 += d*d - # update total -- TODO: numpy includes this, but we might not want to? - if n > 1: - mean2 = (s * s) / (n * n) - m2 = (s2 / n) - # is this just needed because of precision errors? - if m2 > mean2: - total += np.sqrt(m2 - mean2) - count += 1 - elif increment_single and (n == 1): - total += 0. - count += 1 - # ^^^ END i - if count == 0: - return -1 - else: - return total / count - - -@njit -def eval_dist_pairs_numba__range( - mask: np.ndarray, - pair_obs_dists: np.ndarray, - pair_obs_idxs: np.ndarray, - increment_single: bool = True -): - """ - This is about 10x faster than the built in numpy version - """ - assert len(mask) == len(pair_obs_dists) - assert len(mask) == len(pair_obs_idxs) - assert pair_obs_dists.shape == pair_obs_idxs.shape - # totals - total = 0.0 - count = 0 - # iterate over values -- np.ndindex is usually quite fast - for i, m in enumerate(mask): - # skip if invalid - if not m: - continue - # get pair info - dists = pair_obs_dists[i] - idxs = pair_obs_idxs[i] - # init vars - num_checked = 0 - m = 0.0 - M = 0.0 - # handle each distance matrix -- enumerate is usually faster than range - for j, d in zip(idxs, dists): - # skip if invalid - if not mask[j]: - continue - # update range - if num_checked > 0: - if d < m: m = d - if d > M: M = d - else: - m = d - M = d - # update num checked - num_checked += 1 - # update total - if (num_checked > 1) or (increment_single and num_checked == 1): - total += (M - m) - count += 1 - # ^^^ END i - if count == 0: - return -1 - else: - return total / count - - -def eval_dist_pairs_numba( - mask: np.ndarray, - pair_obs_dists: np.ndarray, - pair_obs_idxs: np.ndarray, - fitness_mode: str, - increment_single: bool = True -): - """ - We only keep this function as a compatibility layer between: - - eval_numpy - - eval_numba__range_nodiag - """ - # call - if fitness_mode == 'range': - return eval_dist_pairs_numba__range(mask=mask, pair_obs_dists=pair_obs_dists, pair_obs_idxs=pair_obs_idxs, increment_single=increment_single) - elif fitness_mode == 'std': - return eval_dist_pairs_numba__std(mask=mask, pair_obs_dists=pair_obs_dists, pair_obs_idxs=pair_obs_idxs, increment_single=increment_single) - else: - raise KeyError(f'fast version of eval only supports `fitness_mode in ("range", "std")`, got: {repr(fitness_mode)}') - - -# ========================================================================= # -# Individual Evaluation # -# ========================================================================= # - - -_EVAL_BACKENDS = { - 'numpy': eval_dist_pairs_numpy, - 'numba': eval_dist_pairs_numba, -} - - -def eval_masked_dist_pairs( - mask: np.ndarray, - pair_obs_dists: np.ndarray, - pair_obs_idxs: np.ndarray, - fitness_mode: str, - increment_single: bool = True, - backend: str = 'numba', -) -> Tuple[float, float]: - # get function - if backend not in _EVAL_BACKENDS: - raise KeyError(f'invalid backend: {repr(backend)}, must be one of: {sorted(_EVAL_BACKENDS.keys())}') - eval_fn = _EVAL_BACKENDS[backend] - # evaluate - factor_score = eval_fn( - mask=mask, - pair_obs_dists=pair_obs_dists, - pair_obs_idxs=pair_obs_idxs, - fitness_mode=fitness_mode, - increment_single=increment_single, - ) - # aggregate - kept_ratio = mask.mean() - # check values just in case something goes wrong! - factor_score = np.nan_to_num(factor_score, nan=float('-inf')) - kept_ratio = np.nan_to_num(kept_ratio, nan=float('-inf')) - # return values! - return float(factor_score), float(kept_ratio) - - -# ========================================================================= # -# Equality Checks # -# ========================================================================= # - - -def _check_equal( - dataset_name: str = 'dsprites', - pair_mode: str = 'nearby_scaled', - pairs_per_obs: int = 8, - fitness_mode: str = 'std', # range, std - n: int = 5, -): - from research.e01_visual_overlap.util_compute_traversal_dist_pairs import cached_compute_dataset_pair_dists - from timeit import timeit - - # get distances & individual # (len(gt_data), pairs_per_obs) & (len(gt_data),) - obs_pair_idxs, obs_pair_dists = cached_compute_dataset_pair_dists(dataset_name=dataset_name, pair_mode=pair_mode, pairs_per_obs=pairs_per_obs, scaled=True) - mask = np.random.random(len(obs_pair_idxs)) < 0.5 - - def eval_all(backend: str, increment_single=True): - return _EVAL_BACKENDS[backend]( - mask=mask, - pair_obs_dists=obs_pair_dists, - pair_obs_idxs=obs_pair_idxs, - fitness_mode=fitness_mode, - increment_single=increment_single, - ) - - new_vals = eval_all('numba', increment_single=False) - new_time = timeit(lambda: eval_all('numba', increment_single=False), number=n) / n - print(f'- NEW {new_time:.5f}s {new_vals} (increment_single=False)') - - new_vals = eval_all('numba') - new_time = timeit(lambda: eval_all('numba'), number=n) / n - print(f'- NEW {new_time:.5f}s {new_vals}') - - old_vals = eval_all('numpy') - old_time = timeit(lambda: eval_all('numpy'), number=n) / n - print(f'- OLD {old_time:.5f}s {old_vals}') - print(f'* speedup: {np.around(old_time/new_time, decimals=2)}x') - - if not np.allclose(new_vals, old_vals): - print('[WARNING]: values are not close!') - - -if __name__ == '__main__': - - for dataset_name in ['smallnorb', 'shapes3d', 'dsprites']: - print('='*100) - _check_equal(dataset_name, fitness_mode='std') - print() - _check_equal(dataset_name, fitness_mode='range') - print('='*100) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/e06_adversarial_data/util_gen_adversarial_dataset.py b/research/e06_adversarial_data/util_gen_adversarial_dataset.py deleted file mode 100644 index 4db566f3..00000000 --- a/research/e06_adversarial_data/util_gen_adversarial_dataset.py +++ /dev/null @@ -1,446 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -""" -General helper utilities for generating -adversarial datasets using triplet sampling. -""" - -import logging -from functools import lru_cache -from typing import Literal -from typing import Optional -from typing import Tuple -from typing import Union - -import numpy as np -import torch - -import research.util as H -from disent.dataset.data import GroundTruthData -from disent.dataset.sampling import BaseDisentSampler -from disent.dataset.sampling import GroundTruthPairSampler -from disent.dataset.sampling import GroundTruthTripleSampler -from disent.dataset.sampling import RandomSampler -from disent.util.strings import colors as c - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Samplers # -# ========================================================================= # - - -class AdversarialSampler_SwappedRandom(BaseDisentSampler): - - def uninit_copy(self) -> 'AdversarialSampler_SwappedRandom': - return AdversarialSampler_SwappedRandom(swap_metric=self._swap_metric) - - def __init__(self, swap_metric='manhattan'): - super().__init__(3) - assert swap_metric in {'k', 'manhattan', 'manhattan_norm', 'euclidean', 'euclidean_norm'} - self._swap_metric = swap_metric - self._sampler = GroundTruthTripleSampler(swap_metric=swap_metric) - self._gt_data: GroundTruthData = None - - def _init(self, gt_data: GroundTruthData): - self._sampler.init(gt_data) - self._gt_data = gt_data - - def _sample_idx(self, idx: int) -> Tuple[int, ...]: - anchor, pos, neg = self._gt_data.idx_to_pos([ - idx, - *np.random.randint(0, len(self._gt_data), size=2) - ]) - # swap values - pos, neg = self._sampler._swap_factors(anchor_factors=anchor, positive_factors=pos, negative_factors=neg) - # return triple - return tuple(self._gt_data.pos_to_idx([anchor, pos, neg])) - - -class AdversarialSampler_CloseFar(BaseDisentSampler): - - def uninit_copy(self) -> 'AdversarialSampler_CloseFar': - return AdversarialSampler_CloseFar( - p_k_range=self._p_k_range, - p_radius_range=self._p_radius_range, - n_k_range=self._n_k_range, - n_radius_range=self._n_radius_range, - ) - - def __init__( - self, - p_k_range=(1, 1), - p_radius_range=(1, 1), - n_k_range=(1, -1), - n_radius_range=(1, -1), - ): - super().__init__(3) - self._p_k_range = p_k_range - self._p_radius_range = p_radius_range - self._n_k_range = n_k_range - self._n_radius_range = n_radius_range - self.sampler_close = GroundTruthPairSampler(p_k_range=p_k_range, p_radius_range=p_radius_range) - self.sampler_far = GroundTruthPairSampler(p_k_range=n_k_range, p_radius_range=n_radius_range) - - def _init(self, gt_data: GroundTruthData): - self.sampler_close.init(gt_data) - self.sampler_far.init(gt_data) - - def _sample_idx(self, idx: int) -> Tuple[int, ...]: - # sample indices - anchor, pos = self.sampler_close(idx) - _anchor, neg = self.sampler_far(idx) - assert anchor == _anchor - # return triple - return anchor, pos, neg - - -class AdversarialSampler_SameK(BaseDisentSampler): - - def uninit_copy(self) -> 'AdversarialSampler_SameK': - return AdversarialSampler_SameK( - k=self._k, - sample_p_close=self._sample_p_close, - ) - - def __init__(self, k: Union[Literal['random'], int] = 'random', sample_p_close: bool = False): - super().__init__(3) - self._gt_data: GroundTruthData = None - self._sample_p_close = sample_p_close - self._k = k - assert (isinstance(k, int) and k > 0) or (k == 'random') - - def _init(self, gt_data: GroundTruthData): - self._gt_data = gt_data - - def _sample_idx(self, idx: int) -> Tuple[int, ...]: - a_factors = self._gt_data.idx_to_pos(idx) - # SAMPLE FACTOR INDICES - k = self._k - if k == 'random': - k = np.random.randint(1, self._gt_data.num_factors+1) # end exclusive, ie. [1, num_factors+1) - # get shared mask - shared_indices = np.random.choice(self._gt_data.num_factors, size=self._gt_data.num_factors-k, replace=False) - shared_mask = np.zeros(a_factors.shape, dtype='bool') - shared_mask[shared_indices] = True - # generate values - p_factors = self._sample_shared(a_factors, shared_mask, sample_close=self._sample_p_close) - n_factors = self._sample_shared(a_factors, shared_mask, sample_close=False) - # swap values if wrong - # TODO: this might give errors! - # - one factor might be less than another - if np.sum(np.abs(a_factors - p_factors)) > np.sum(np.abs(a_factors - n_factors)): - p_factors, n_factors = n_factors, p_factors - # check values - assert np.sum(a_factors != p_factors) == k, 'this should never happen!' - assert np.sum(a_factors != n_factors) == k, 'this should never happen!' - # return values - return tuple(self._gt_data.pos_to_idx([ - a_factors, - p_factors, - n_factors, - ])) - - def _sample_shared(self, base_factors, shared_mask, tries=100, sample_close: bool = False): - sampled_factors = base_factors.copy() - generate_mask = ~shared_mask - # generate values - for i in range(tries): - if sample_close: - sampled_values = (base_factors + np.random.randint(-1, 1+1, size=self._gt_data.num_factors)) - sampled_values = np.clip(sampled_values, 0, np.array(self._gt_data.factor_sizes) - 1)[generate_mask] - else: - sampled_values = np.random.randint(0, np.array(self._gt_data.factor_sizes)[generate_mask]) - # overwrite values that are not different - sampled_factors[generate_mask] = sampled_values - # update mask - sampled_shared_mask = (sampled_factors == base_factors) - generate_mask &= sampled_shared_mask - # check everything - if np.sum(sampled_shared_mask) == np.sum(shared_mask): - assert np.sum(generate_mask) == 0 - return sampled_factors - # we need to try again! - raise RuntimeError('could not generate factors: {}') - - -def sampler_print_test(sampler: Union[str, BaseDisentSampler], gt_data: GroundTruthData = None, steps=100): - # make data - if gt_data is None: - gt_data = H.make_dataset('xysquares_8x8_mini').gt_data - # make sampler - if isinstance(sampler, str): - prefix = sampler - sampler = make_adversarial_sampler(sampler) - else: - prefix = sampler.__class__.__name__ - if not sampler.is_init: - sampler.init(gt_data) - # print everything - count_pn_k0, count_pn_d0 = 0, 0 - for i in range(min(steps, len(gt_data))): - a, p, n = gt_data.idx_to_pos(sampler(i)) - ap_k = np.sum(a != p); ap_d = np.sum(np.abs(a - p)) - an_k = np.sum(a != n); an_d = np.sum(np.abs(a - n)) - pn_k = np.sum(p != n); pn_d = np.sum(np.abs(p - n)) - print(f'{prefix}: [{c.lGRN}ap{c.RST}:{ap_k:2d}:{ap_d:2d}] [{c.lRED}an{c.RST}:{an_k:2d}:{an_d:2d}] [{c.lYLW}pn{c.RST}:{pn_k:2d}:{pn_d:2d}] {a} {p} {n}') - count_pn_k0 += (pn_k == 0) - count_pn_d0 += (pn_d == 0) - print(f'count pn:(k=0) = {count_pn_k0} pn:(d=0) = {count_pn_d0}') - - -def make_adversarial_sampler(mode: str = 'close_far'): - if mode in ['random_swap_k', 'random_swap_manhattan', 'random_swap_manhattan_norm', 'random_swap_euclidean', 'random_swap_euclidean_norm']: - # NOTE # -- random_swap_manhattan -- probability is too low of encountering nearby obs, don't use this! - metric = mode[len('random_swap_'):] - return AdversarialSampler_SwappedRandom(swap_metric=metric) - elif mode in ['close_far', 'close_p_random_n']: - # *NB* # - return AdversarialSampler_CloseFar( - p_k_range=(1, 1), n_k_range=(1, -1), - p_radius_range=(1, 1), n_radius_range=(1, -1), - ) - elif mode in ['close_far_random', 'close_p_random_n_bb']: - # *NB* # - return GroundTruthTripleSampler( - p_k_range=(1, 1), n_k_range=(1, -1), n_k_sample_mode='bounded_below', n_k_is_shared=True, - p_radius_range=(1, 1), n_radius_range=(1, -1), n_radius_sample_mode='bounded_below', - ) - elif mode in ['same_k']: - # *NB* # - return AdversarialSampler_SameK(k='random', sample_p_close=False) - elif mode in ['same_k_close']: - # *NB* # - return AdversarialSampler_SameK(k='random', sample_p_close=True) - elif mode in ['same_k1_close']: - # *NB* # - return AdversarialSampler_SameK(k=1, sample_p_close=True) - elif mode == 'close_factor_far_random': - return GroundTruthTripleSampler( - p_k_range=(1, 1), n_k_range=(1, -1), n_k_sample_mode='bounded_below', n_k_is_shared=True, - p_radius_range=(1, -1), n_radius_range=(0, -1), n_radius_sample_mode='bounded_below', - ) - elif mode == 'close_far_same_factor': - # TODO: problematic for dsprites - return GroundTruthTripleSampler( - p_k_range=(1, 1), n_k_range=(1, 1), n_k_sample_mode='bounded_below', n_k_is_shared=True, - p_radius_range=(1, 1), n_radius_range=(2, -1), n_radius_sample_mode='bounded_below', - ) - elif mode == 'same_factor': - return GroundTruthTripleSampler( - p_k_range=(1, 1), n_k_range=(1, 1), n_k_sample_mode='bounded_below', n_k_is_shared=True, - p_radius_range=(1, -2), n_radius_range=(2, -1), n_radius_sample_mode='bounded_below', # bounded below does not always work, still relies on random chance :/ - ) - elif mode == 'random_bb': - return GroundTruthTripleSampler( - p_k_range=(0, -1), n_k_range=(0, -1), n_k_sample_mode='bounded_below', n_k_is_shared=True, - p_radius_range=(0, -1), n_radius_range=(0, -1), n_radius_sample_mode='bounded_below', - ) - elif mode == 'random_swap_manhat': - return GroundTruthTripleSampler( - p_k_range=(0, -1), n_k_range=(0, -1), n_k_sample_mode='random', n_k_is_shared=False, - p_radius_range=(0, -1), n_radius_range=(0, -1), n_radius_sample_mode='random', - swap_metric='manhattan' - ) - elif mode == 'random_swap_manhat_norm': - return GroundTruthTripleSampler( - p_k_range=(0, -1), n_k_range=(0, -1), n_k_sample_mode='random', n_k_is_shared=False, - p_radius_range=(0, -1), n_radius_range=(0, -1), n_radius_sample_mode='random', - swap_metric='manhattan_norm' - ) - elif mode == 'random': - return RandomSampler(num_samples=3) - else: - raise KeyError(f'invalid adversarial sampler: mode={repr(mode)}') - - -# ========================================================================= # -# Adversarial Sort # -# ========================================================================= # - - -@torch.no_grad() -def sort_samples(a_x: torch.Tensor, p_x: torch.Tensor, n_x: torch.Tensor, sort_mode: str = 'none', pixel_loss_mode: str = 'mse'): - # NOTE: this function may mutate its inputs, however - # the returned values should be used. - # do not sort! - if sort_mode == 'none': - return (a_x, p_x, n_x) - elif sort_mode == 'swap': - return (a_x, n_x, p_x) - # compute deltas - p_deltas = H.pairwise_loss(a_x, p_x, mode=pixel_loss_mode, mean_dtype=torch.float32, mask=None) - n_deltas = H.pairwise_loss(a_x, n_x, mode=pixel_loss_mode, mean_dtype=torch.float32, mask=None) - # get swap mask - if sort_mode == 'sort_inorder': swap_mask = p_deltas > n_deltas - elif sort_mode == 'sort_reverse': swap_mask = p_deltas < n_deltas - else: raise KeyError(f'invalid sort_mode: {repr(sort_mode)}, must be one of: ["none", "swap", "sort_inorder", "sort_reverse"]') - # handle mutate or copy - idx_swap = torch.where(swap_mask) - # swap memory values -- TODO: `p_x[idx_swap], n_x[idx_swap] = n_x[idx_swap], p_x[idx_swap]` is this fine? - temp = torch.clone(n_x[idx_swap]) - n_x[idx_swap] = p_x[idx_swap] - p_x[idx_swap] = temp - # done! - return (a_x, p_x, n_x) - - -# ========================================================================= # -# Adversarial Loss # -# ========================================================================= # - -# anchor, positive, negative -TensorTriple = Tuple[torch.Tensor, torch.Tensor, torch.Tensor] - - -def _get_triple(x: TensorTriple, adversarial_swapped: bool): - if not adversarial_swapped: - a, p, n = x - else: - a, n, p = x - return a, p, n - - -_MARGIN_MODES = { - 'invert_margin', - 'triplet_margin', -} - - -@lru_cache() -def _parse_margin_mode(adversarial_mode: str): - # parse the MARGIN_MODES -- linear search - for margin_mode in _MARGIN_MODES: - if adversarial_mode == margin_mode: - raise KeyError(f'`{margin_mode}` is not valid, specify the margin in the name, eg. `{margin_mode}_0.01`') - elif adversarial_mode.startswith(f'{margin_mode}_'): - margin = float(adversarial_mode[len(f'{margin_mode}_'):]) - return margin_mode, margin - # done! - return adversarial_mode, None - - -def adversarial_loss( - ys: TensorTriple, - xs: Optional[TensorTriple] = None, # only used if mask_deltas==True - # adversarial loss settings - adversarial_mode: str = 'invert_shift', - adversarial_swapped: bool = False, - adversarial_masking: bool = False, # requires `xs` to be set - adversarial_top_k: Optional[int] = None, - # pixel loss to get deltas settings - pixel_loss_mode: str = 'mse', - # statistics - return_stats: bool = False, -): - a_y, p_y, n_y = _get_triple(ys, adversarial_swapped=adversarial_swapped) - - # get mask - if adversarial_masking: - a_x, p_x, n_x = _get_triple(xs, adversarial_swapped=adversarial_swapped) - ap_mask, an_mask = (a_x != p_x), (a_x != n_x) - else: - ap_mask, an_mask = None, None - - # compute deltas - p_deltas = H.pairwise_loss(a_y, p_y, mode=pixel_loss_mode, mean_dtype=torch.float32, mask=ap_mask) - n_deltas = H.pairwise_loss(a_y, n_y, mode=pixel_loss_mode, mean_dtype=torch.float32, mask=an_mask) - deltas = (n_deltas - p_deltas) - - # parse mode - adversarial_mode, margin = _parse_margin_mode(adversarial_mode) - - # compute loss deltas - # AUTO-CONSTANT - if adversarial_mode == 'self': loss_deltas = torch.abs(deltas) - elif adversarial_mode == 'self_random': - # the above should be equivalent with the right sampling strategy? - all_deltas = torch.cat([p_deltas, n_deltas], dim=0) - indices = np.arange(len(all_deltas)) - np.random.shuffle(indices) - deltas = all_deltas[indices[len(deltas):]] - all_deltas[indices[:len(deltas)]] - loss_deltas = torch.abs(deltas) - # INVERT - elif adversarial_mode == 'invert': loss_deltas = torch.maximum(deltas, torch.zeros_like(deltas)) - elif adversarial_mode == 'invert_margin': loss_deltas = torch.maximum(margin + deltas, torch.zeros_like(deltas)) # invert_loss = torch.clamp_min(n_dist - p_dist + margin_max, 0) - elif adversarial_mode == 'invert_unbounded': loss_deltas = deltas - # TRIPLET - elif adversarial_mode == 'triplet': loss_deltas = torch.maximum(-deltas, torch.zeros_like(deltas)) - elif adversarial_mode == 'triplet_margin': loss_deltas = torch.maximum(margin - deltas, torch.zeros_like(deltas)) # triplet_loss = torch.clamp_min(p_dist - n_dist + margin_max, 0) - elif adversarial_mode == 'triplet_unbounded': loss_deltas = -deltas - # OTHER - else: - raise KeyError(f'invalid `adversarial_mode`: {repr(adversarial_mode)}') - - # checks - assert deltas.shape == loss_deltas.shape, 'this is a bug' - - # top k deltas - if adversarial_top_k is not None: - loss_deltas = torch.topk(loss_deltas, k=adversarial_top_k, largest=True).values - - # get average loss - loss = loss_deltas.mean() - - # return early - if not return_stats: - return loss - - # compute stats! - with torch.no_grad(): - loss_stats = { - 'stat/p_delta:mean': float(p_deltas.mean().cpu()), 'stat/p_delta:std': float(p_deltas.std().cpu()), - 'stat/n_delta:mean': float(n_deltas.mean().cpu()), 'stat/n_delta:std': float(n_deltas.std().cpu()), - 'stat/deltas:mean': float(loss_deltas.mean().cpu()), 'stat/deltas:std': float(loss_deltas.std().cpu()), - } - - return loss, loss_stats - - -# ========================================================================= # -# END # -# ========================================================================= # - - -# if __name__ == '__main__': -# -# def _main(): -# from disent.dataset.data import XYObjectData -# -# # NB: -# # close_p_random_n -# # close_p_random_n_bb -# # same_k -# # same_k_close -# # same_k1_close -# -# sampler_print_test( -# sampler='close_p_random_n', -# gt_data=XYObjectData() -# ) -# -# _main() diff --git a/research/e06_adversarial_data/util_load_adversarial_mask.py b/research/e06_adversarial_data/util_load_adversarial_mask.py deleted file mode 100644 index 481a1329..00000000 --- a/research/e06_adversarial_data/util_load_adversarial_mask.py +++ /dev/null @@ -1,78 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import gzip -import pickle -import numpy as np -import logging - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# HELPER # -# ========================================================================= # - - -def get_closest_mask(usage_ratio: float, pickle_file: str, print_n_best: int = 3) -> np.ndarray: - """ - This function is intended to be used with the data - generated by `run_04_gen_adversarial_ruck.py` - - The function finds the closest member in the population with - the matching statistic. The reason this function works is that - the population should consist only of near-pareto-optimal solutions. - - These solutions are found using NSGA2 - - Usage With Hydra Config: - _target_: research.e06_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: 0.5 - pickle_file: data.pkl.gz - """ - # load pickled data - with gzip.open(pickle_file, mode='rb') as fp: - data = pickle.load(fp) - values = np.array(data['values'], dtype='bool') - scores = np.array(data['scores'], dtype='float64') - del data - # check shapes - assert values.ndim == 2 - assert scores.ndim == 2 - assert scores.shape == (len(values), 2) - # get closest - best_indices = np.argsort(np.abs(scores[:, 1] - usage_ratio)) - # print stats - if print_n_best > 0: - log.info(f'The {print_n_best} closest members to target usage={usage_ratio:7f}') - for i, idx in enumerate(best_indices[:print_n_best]): - assert np.isclose(np.mean(values[idx]), scores[idx, 1]), 'member fitness_usage is not close to the actual mask usage. The data is invalid.' - log.info(f' [{i+1}] idx={idx:04d} overlap={scores[idx, 0]:7f} usage={scores[idx, 1]:7f}') - # return the best! - return values[best_indices[0]] - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/e07_metric/make_graphs.py b/research/e07_metric/make_graphs.py deleted file mode 100644 index 2ba8e720..00000000 --- a/research/e07_metric/make_graphs.py +++ /dev/null @@ -1,436 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import itertools -import os -from typing import Optional -from typing import Sequence -from typing import Tuple - -import numpy as np -import torch -from matplotlib import cm -from matplotlib import pyplot as plt -from tqdm import tqdm - -import research.util as H -from disent.metrics._flatness_components import compute_axis_score -from disent.metrics._flatness_components import compute_linear_score -from disent.util.seeds import seed - - -# ========================================================================= # -# distance function # -# ========================================================================= # - - -def _rotation_matrix(d, i, j, deg): - assert 0 <= i < j <= d - mat = torch.eye(d, dtype=torch.float32) - r = np.deg2rad(deg) - s, c = np.sin(r), np.cos(r) - mat[i, i] = c - mat[j, j] = c - mat[j, i] = -s - mat[i, j] = s - return mat - - -def rotation_matrix_2d(deg): - return _rotation_matrix(d=2, i=0, j=1, deg=deg) - - -def _random_rotation_matrix(d): - mat = torch.eye(d, dtype=torch.float32) - for i in range(d): - for j in range(i+1, d): - mat @= _rotation_matrix(d, i, j, np.random.randint(0, 360)) - return mat - - -def make_2d_line_points(n: int = 100, deg: float = 30, std_x: float = 1.0, std_y: float = 0.005): - points = torch.randn(n, 2, dtype=torch.float32) * torch.as_tensor([[std_x, std_y]], dtype=torch.float32) - points = points @ rotation_matrix_2d(deg) - return points - - -def make_nd_line_points(n: int = 100, dims: int = 4, std_x: float = 1.0, std_y: float = 0.005): - if not isinstance(dims, int): - m, M = dims - dims = np.randint(m, M) - # generate numbers - xs = torch.randn(n, dims, dtype=torch.float32) - # axis standard deviations - if isinstance(std_y, (float, int)): - std_y = torch.full((dims-1,), fill_value=std_y, dtype=torch.float32) - else: - m, M = std_y - std_y = torch.rand(dims-1, dtype=torch.float32) * (M - m) + m - # scale axes - std = torch.cat([torch.as_tensor([std_x]), std_y]) - xs = xs * std[None, :] - # rotate - return xs @ _random_rotation_matrix(dims) - - -def make_line_points(n: int = 100, deg: float = None, dims: int = 2, std_x: float = 1.0, std_y: float = 0.1): - if deg is None: - return make_nd_line_points(n=n, dims=dims, std_x=std_x, std_y=std_y) - else: - assert dims == 2, f'if "deg" is not None, then "dims" must equal 2, currently set to: {repr(dims)}' - return make_2d_line_points(n=n, deg=deg, std_x=std_x, std_y=std_y) - - -# def random_line(std, n=100): -# std = torch.as_tensor(std, dtype=torch.float32) -# (d,) = std.shape -# # generate numbers -# xs = torch.randn(n, d, dtype=torch.float32) -# # scale axes -# xs = xs * std[None, :] -# # rotate -# return xs @ _random_rotation_matrix(d) - - -# ========================================================================= # -# GAUSSIAN # -# ========================================================================= # - - -def gaussian_1d(x, s): return 1 / (np.sqrt(2 * np.pi) * s) * torch.exp(-(x**2)/(2*s**2)) -def gaussian_1d_dx(x, s): return gaussian_1d(x, s) * (-x/s**2) -def gaussian_1d_dx2(x, s): return gaussian_1d(x, s) * ((x**2 - s**2)/s**4) - - -def gaussian_2d(x, y, sx, sy): return gaussian_1d(x, sx) * gaussian_1d(y, sy) -def gaussian_2d_dy(x, y, sx, sy): return gaussian_1d(x, sx) * gaussian_1d_dx(y, sy) -def gaussian_2d_dy2(x, y, sx, sy): return gaussian_1d(x, sx) * gaussian_1d_dx2(y, sy) - - -def rotated_radius_meshgrid(radius: float, num_points: int, deg: float = 0, device=None, return_orig=False) -> Tuple[torch.Tensor, torch.Tensor]: - # x & y values centered around zero - # p = torch.arange(size, device=device) - (size-1)/2 - p = torch.linspace(-radius, radius, num_points, device=device) - x, y = torch.meshgrid(p, p) - # matrix multiplication along first axis | https://pytorch.org/docs/stable/generated/torch.einsum.html - rx, ry = torch.einsum('dxy,kd->kxy', torch.stack([x, y]), rotation_matrix_2d(deg)) - # result - if return_orig: - return (rx, ry), (x, y) - return rx, ry - - -def rotated_guassian2d(std_x: float, std_y: float, deg: float, trunc_sigma: Optional[float] = None, num_points: int = 511): - radius = (2.25*max(std_x, std_y)) if (trunc_sigma is None) else trunc_sigma - (xs_r, ys_r), (xs, ys) = rotated_radius_meshgrid(radius=radius, num_points=num_points, deg=deg, return_orig=True) - zs = gaussian_2d(xs_r, ys_r, sx=std_x, sy=std_y) - zs /= zs.sum() - return xs, ys, zs - - -def plot_gaussian( - deg: float = 0.0, - std_x: float = 1.0, - std_y: float = 0.1, - # contour - contour_resolution: int = 255, - contour_trunc_sigma: Optional[float] = None, - contour_kwargs: Optional[dict] = None, - # dots - dots_num: Optional[int] = None, - dots_kwargs: Optional[dict] = None, - # axis - ax=None, -): - if ax is None: - fig = plt.figure() - ax = fig.gca() - # set limits - trunc_sigma = (2.05 * max(std_x, std_y)) if (contour_trunc_sigma is None) else contour_trunc_sigma - ax.set_xlim([-trunc_sigma, trunc_sigma]) - ax.set_ylim([-trunc_sigma, trunc_sigma]) - # plot contour - xs, ys, zs = rotated_guassian2d(std_x=std_x, std_y=std_y, deg=deg, trunc_sigma=trunc_sigma, num_points=contour_resolution) - ax.contourf(xs, ys, zs, **({} if contour_kwargs is None else contour_kwargs)) - # plot dots - if dots_num is not None: - points = make_line_points(n=dots_num, dims=2, deg=deg, std_x=std_x, std_y=std_y) - ax.scatter(*points.T, **({} if dots_kwargs is None else dots_kwargs)) - # done - return ax - - -# ========================================================================= # -# Generate Average Plots # -# ========================================================================= # - - -def score_grid( - deg_rotations: Sequence[Optional[float]], - y_std_ratios: Sequence[float], - x_std: float = 1.0, - num_points: int = 1000, - num_dims: int = 2, - use_std: bool = True, - use_max: bool = False, - norm: bool = True, - return_points: bool = False, -): - h, w = len(y_std_ratios), len(deg_rotations) - # grids - axis_scores = torch.zeros([h, w], dtype=torch.float64) - linear_scores = torch.zeros([h, w], dtype=torch.float64) - if return_points: - all_points = torch.zeros([h, w, num_points, num_dims], dtype=torch.float64) - # compute scores - for i, y_std_ratio in enumerate(y_std_ratios): - for j, deg in enumerate(deg_rotations): - points = make_line_points(n=num_points, dims=num_dims, deg=deg, std_x=x_std, std_y=x_std * y_std_ratio) - axis_scores[i, j] = compute_axis_score(points, use_std=use_std, use_max=use_max, norm=norm) - linear_scores[i, j] = compute_linear_score(points, use_std=use_std, use_max=use_max, norm=norm) - if return_points: - all_points[i, j] = points - # results - if return_points: - return axis_scores, linear_scores, all_points - return axis_scores, linear_scores - - -def ave_score_grid( - deg_rotations: Sequence[Optional[float]], - y_std_ratios: Sequence[float], - x_std: float = 1.0, - num_points: int = 1000, - num_dims: int = 2, - use_std: bool = True, - use_max: bool = False, - norm: bool = True, - repeats: int = 10, -): - results = [] - # repeat - for i in tqdm(range(repeats)): - results.append(score_grid(deg_rotations=deg_rotations, y_std_ratios=y_std_ratios, x_std=x_std, num_points=num_points, num_dims=num_dims, use_std=use_std, use_max=use_max, norm=norm)) - # average results - all_axis_scores, all_linear_scores = zip(*results) - axis_scores = torch.mean(torch.stack(all_axis_scores, dim=0), dim=0) - linear_scores = torch.mean(torch.stack(all_linear_scores, dim=0), dim=0) - # results - return axis_scores, linear_scores - - -def make_ave_scores_plot( - std_num: int = 21, - deg_num: int = 21, - ndim: Optional[int] = None, - # extra - num_points: int = 1000, - repeats: int = 25, - x_std: float = 1.0, - use_std: bool = True, - use_max: bool = False, - norm: bool = True, - # cmap - cmap_axis: str = 'GnBu_r', # 'RdPu_r', 'GnBu_r', 'Blues_r', 'viridis', 'plasma', 'magma' - cmap_linear: str = 'RdPu_r', # 'RdPu_r', 'GnBu_r', 'Blues_r', 'viridis', 'plasma', 'magma' - vertical: bool = True, - # subplot settings - subplot_size: float = 4., - subplot_padding: float = 1.5, -): - # make sure to handle the random case - deg_num = std_num if (ndim is None) else deg_num - axis_scores, linear_scores = ave_score_grid( - deg_rotations=np.linspace(0., 180., num=deg_num) if (ndim is None) else [None], - y_std_ratios=np.linspace(0., 1., num=std_num), - x_std=x_std, - num_points=num_points, - num_dims=2 if (ndim is None) else ndim, - use_std=use_std, - use_max=use_max, - norm=norm, - repeats=repeats, - ) - # make plot - fig, axs = H.plt_subplots( - nrows=1+int(vertical), - ncols=1+int(not vertical), - titles=['Linear', 'Axis'], - row_labels=f'$σ_y$ - Standard Deviation', - col_labels=f'θ - Rotation Degrees', - figsize=(subplot_size + 0.5, subplot_size * 2 * (deg_num / std_num) + 0.75)[::1 if vertical else -1] - ) - (ax0, ax1) = axs.flatten() - # subplots - ax0.imshow(linear_scores, cmap=cmap_linear, extent=[0., 180., 1., 0.]) - ax1.imshow(axis_scores, cmap=cmap_axis, extent=[0., 180., 1., 0.]) - for ax in axs.flatten(): - ax.set_aspect(180 * (std_num / deg_num)) - if len(ax.get_xticks()): - ax.set_xticks(np.linspace(0., 180., 5)) - # layout - fig.tight_layout(pad=subplot_padding) - # done - return fig, axs - - -# ========================================================================= # -# HELPER # -# ========================================================================= # - - -def plot_scores(ax, axis_score, linear_score): - from matplotlib.lines import Line2D - assert 0 <= linear_score <= 1 - assert 0 <= axis_score <= 1 - linear_rgb = cm.get_cmap('RdPu_r')(np.clip(linear_score, 0., 1.)) - axis_rgb = cm.get_cmap('GnBu_r')(np.clip(axis_score, 0., 1.)) - ax.legend(handles=[ - Line2D([0], [0], label=f'Linear: {float(linear_score):.2f}', color=linear_rgb, marker='o', markersize=10, linestyle='None'), - Line2D([0], [0], label=f'Axis: {float(axis_score):.2f}', color=axis_rgb, marker='o', markersize=10, linestyle='None'), - ]) - return ax - - -# ========================================================================= # -# Generate Grid Plots # -# ========================================================================= # - - -def make_grid_gaussian_score_plot( - # grid - y_stds: Sequence[float] = (0.8, 0.2, 0.05)[::-1], # (0.8, 0.4, 0.2, 0.1, 0.05), - deg_rotations: Sequence[float] = (0, 22.5, 45, 67.5, 90, 112.5, 135, 157.5), # (0, 15, 30, 45, 60, 75, 90, 105, 120, 135, 150, 165), - # plot dot options - dots_num: Optional[int] = None, - # score options - num_points: int = 10000, - repeats: int = 100, - use_std: bool = True, - use_max: bool = False, - norm: bool = True, - # grid options - subplot_size: float = 2.125, - subplot_padding: float = 0.5, - subplot_contour_kwargs: Optional[dict] = None, - subplot_dots_kwargs: Optional[dict] = None, -): - # defaults - if subplot_contour_kwargs is None: subplot_contour_kwargs = dict(cmap='Blues') - if subplot_dots_kwargs is None: subplot_dots_kwargs = dict(cmap='Purples') - - # make figure - nrows, ncols = len(y_stds), len(deg_rotations) - fig, axs = H.plt_subplots( - nrows=nrows, ncols=ncols, - row_labels=[f'$σ_y$ = {std_y}' for std_y in y_stds], - col_labels=[f'θ = {deg}°' for deg in deg_rotations], - hide_axis='all', - figsize=(ncols*subplot_size, nrows*subplot_size), - ) - - # progress - p = tqdm(total=axs.size, desc='generating_plot') - # generate plot - for (y, std_y), (x, deg) in itertools.product(enumerate(y_stds), enumerate(deg_rotations)): - # compute scores - axis_score, linear_score = [], [] - for k in range(repeats): - points = make_2d_line_points(n=num_points, deg=deg, std_x=1.0, std_y=std_y) - axis_score.append(compute_axis_score(points, use_std=use_std, use_max=use_max, norm=norm)) - linear_score.append(compute_linear_score(points, use_std=use_std, use_max=use_max, norm=norm)) - axis_score, linear_score = np.mean(axis_score), np.mean(linear_score) - # generate subplots - plot_gaussian(ax=axs[y, x], deg=deg, std_x=1.0, std_y=std_y, dots_num=dots_num, contour_trunc_sigma=2.05, contour_kwargs=subplot_contour_kwargs, dots_kwargs=subplot_dots_kwargs) - plot_scores(ax=axs[y, x], axis_score=axis_score, linear_score=linear_score) - # update progress - p.update() - plt.tight_layout(pad=subplot_padding) - - return fig, axs - - -# ========================================================================= # -# MAIN # -# ========================================================================= # - - -if __name__ == '__main__': - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../gadfly.mplstyle')) - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # - - # plot everything - seed(777) - make_grid_gaussian_score_plot( - repeats=250, - num_points=25000, - ) - plt.savefig(H.make_rel_path_add_ext('plots/metric_grid', ext='.png')) - plt.show() - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # - - # plot everything -- minimal - seed(777) - make_grid_gaussian_score_plot( - y_stds=(0.8, 0.4, 0.2, 0.1, 0.05)[::-1], # (0.8, 0.4, 0.2, 0.1, 0.05), - deg_rotations=(0, 22.5, 45, 67.5, 90), - repeats=250, - num_points=25000, - ) - plt.savefig(H.make_rel_path_add_ext('plots/metric_grid_minimal_5x5', ext='.png')) - plt.show() - - # plot everything -- minimal - seed(777) - make_grid_gaussian_score_plot( - y_stds=(0.8, 0.4, 0.2, 0.05)[::-1], # (0.8, 0.4, 0.2, 0.1, 0.05), - deg_rotations=(0, 22.5, 45, 67.5, 90), - repeats=250, - num_points=25000, - ) - plt.savefig(H.make_rel_path_add_ext('plots/metric_grid_minimal_4x5', ext='.png')) - plt.show() - - # plot everything -- minimal - seed(777) - fig, axs = make_grid_gaussian_score_plot( - y_stds=(0.8, 0.2, 0.05)[::-1], # (0.8, 0.4, 0.2, 0.1, 0.05), - deg_rotations=(0, 22.5, 45, 67.5, 90), - repeats=250, - num_points=25000, - ) - plt.savefig(H.make_rel_path_add_ext('plots/metric_grid_minimal_3x5', ext='.png')) - plt.show() - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # - - seed(777) - make_ave_scores_plot(repeats=250, num_points=10000, use_max=False) - plt.savefig(H.make_rel_path_add_ext('plots/metric_scores', ext='.png')) - plt.show() - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # diff --git a/research/e08_autoencoders/submit_01.sh b/research/e08_autoencoders/submit_01.sh deleted file mode 100644 index 8e5086a5..00000000 --- a/research/e08_autoencoders/submit_01.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-08__autoencoder-versions" -export PARTITION="stampede" -export PARALLELISM=32 - -# source the helper file -source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# 1 * (2*2*3*3*8) == 288 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='various-auto-encoders' \ - \ - run_length=short,long \ - schedule=adavae_up_ratio_full,adavae_up_all_full,none \ - \ - dataset=xysquares,cars3d,shapes3d \ - framework=ae,tae,X--adaae,X--adanegtae,vae,tvae,adavae,X--adanegtvae \ - model=conv64alt \ - model.z_size=25 \ - \ - sampling=gt_dist_manhat,gt_dist_manhat_scaled diff --git a/research/e09_vae_overlap_loss/submit_overlap_loss.sh b/research/e09_vae_overlap_loss/submit_overlap_loss.sh deleted file mode 100644 index d51572b2..00000000 --- a/research/e09_vae_overlap_loss/submit_overlap_loss.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash - -# OVERVIEW: -# - this experiment is designed to test how changing the reconstruction loss to match the -# ground-truth distances allows datasets to be disentangled. - - -# OUTCOMES: -# - When the reconstruction loss is used as a distance function between observations, and those -# distances match the ground truth, it enables disentanglement. -# - Loss must still be able to reconstruct the inputs correctly. -# - AEs have no incentive to learn the same distances as VAEs - - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="CVPR-09__vae_overlap_loss" -export PARTITION="stampede" -export PARALLELISM=28 - -# source the helper file -source "$(dirname "$(dirname "$(realpath -s "$0")")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TEST MSE vs BoxBlur MSE (with different beta values over different datasets) -# - mse boxblur weight is too strong, need to lower significantly -# 1 * (5 * 2*4*2) = 80 -#submit_sweep \ -# +DUMMY.repeat=1 \ -# +EXTRA.tags='sweep_overlap_boxblur' \ -# hydra.job.name="ovlp_loss" \ -# \ -# +VAR.recon_loss_weight=1.0 \ -# +VAR.kernel_loss_weight=3969.0 \ -# +VAR.kernel_radius=31 \ -# \ -# run_length=medium \ -# metrics=all \ -# \ -# dataset=X--xysquares,dsprites,shapes3d,smallnorb,cars3d \ -# \ -# framework=betavae,adavae_os \ -# settings.framework.beta=0.0316,0.316,0.1,0.01 \ -# settings.model.z_size=25,9 \ -# settings.framework.recon_loss='mse_box_r${VAR.kernel_radius}_l${VAR.recon_loss_weight}_k${VAR.kernel_loss_weight}' \ -# \ -# sampling=default__bb - - -# TEST MSE vs BoxBlur MSE -# - changing the reconstruction loss enables disentanglement -# 5 * (2*2*2 = 8) = 40 -submit_sweep \ - +DUMMY.repeat=1,2,3,4,5 \ - +EXTRA.tags='sweep_overlap_boxblur_specific' \ - hydra.job.name="s_ovlp_loss" \ - \ - +VAR.recon_loss_weight=1.0 \ - +VAR.kernel_loss_weight=3969.0 \ - +VAR.kernel_radius=31 \ - \ - run_length=medium \ - metrics=all \ - \ - dataset=X--xysquares \ - \ - framework=betavae,adavae_os \ - settings.framework.beta=0.0316,0.0001 \ - settings.model.z_size=25 \ - settings.framework.recon_loss=mse,'mse_box_r${VAR.kernel_radius}_l${VAR.recon_loss_weight}_k${VAR.kernel_loss_weight}' \ - \ - sampling=default__bb - - -# TEST DISTANCES IN AEs VS VAEs -# -- supplementary material -# 3 * (1 * 2 = 2) = 6 -submit_sweep \ - +DUMMY.repeat=1,2,3 \ - +EXTRA.tags='sweep_overlap_boxblur_autoencoders' \ - hydra.job.name="e_ovlp_loss" \ - \ - +VAR.recon_loss_weight=1.0 \ - +VAR.kernel_loss_weight=3969.0 \ - +VAR.kernel_radius=31 \ - \ - run_length=medium \ - metrics=all \ - \ - dataset=X--xysquares \ - \ - framework=ae \ - settings.framework.beta=0.0001 \ - settings.model.z_size=25 \ - settings.framework.recon_loss=mse,'mse_box_r${VAR.kernel_radius}_l${VAR.recon_loss_weight}_k${VAR.kernel_loss_weight}' \ - \ - sampling=default__bb - - -# HPARAM SWEEP -- TODO: update -# -- old, unused -# 1 * (2 * 8 * 2 * 2) = 160 -#submit_sweep \ -# +DUMMY.repeat=1 \ -# +EXTRA.tags='sweep_beta' \ -# hydra.job.name="vae_hparams" \ -# \ -# run_length=long \ -# metrics=all \ -# \ -# settings.framework.beta=0.000316,0.001,0.00316,0.01,0.0316,0.1,0.316,1.0 \ -# framework=betavae,adavae_os \ -# schedule=none \ -# settings.model.z_size=9,25 \ -# \ -# dataset=X--xysquares \ -# sampling=default__bb diff --git a/research/gadfly.mplstyle b/research/gadfly.mplstyle deleted file mode 100644 index 00b215ac..00000000 --- a/research/gadfly.mplstyle +++ /dev/null @@ -1,627 +0,0 @@ -#### MATPLOTLIBRC FORMAT - -# FROM: https://towardsdatascience.com/a-new-plot-theme-for-matplotlib-gadfly-2cffc745ff84 - -## This is a sample matplotlib configuration file - you can find a copy -## of it on your system in -## site-packages/matplotlib/mpl-data/matplotlibrc. If you edit it -## there, please note that it will be overwritten in your next install. -## If you want to keep a permanent local copy that will not be -## overwritten, place it in the following location: -## unix/linux: -## $HOME/.config/matplotlib/matplotlibrc or -## $XDG_CONFIG_HOME/matplotlib/matplotlibrc (if $XDG_CONFIG_HOME is set) -## other platforms: -## $HOME/.matplotlib/matplotlibrc -## -## See http://matplotlib.org/users/customizing.html#the-matplotlibrc-file for -## more details on the paths which are checked for the configuration file. -## -## This file is best viewed in a editor which supports python mode -## syntax highlighting. Blank lines, or lines starting with a comment -## symbol, are ignored, as are trailing comments. Other lines must -## have the format -## key : val ## optional comment -## -## Colors: for the color values below, you can either use - a -## matplotlib color string, such as r, k, or b - an rgb tuple, such as -## (1.0, 0.5, 0.0) - a hex string, such as ff00ff - a scalar -## grayscale intensity such as 0.75 - a legal html color name, e.g., red, -## blue, darkslategray - -##### CONFIGURATION BEGINS HERE - -## The default backend; one of GTK3Agg GTK3Cairo MacOSX Qt4Agg Qt5Agg TkAgg -## WX WXAgg Agg Cairo PS PDF SVG Template. -## You can also deploy your own backend outside of matplotlib by -## referring to the module name (which must be in the PYTHONPATH) as -## 'module://my_backend'. -## -## If you omit this parameter, the backend will be determined by fallback. -#backend : Agg - -## Note that this can be overridden by the environment variable -## QT_API used by Enthought Tool Suite (ETS); valid values are -## "pyqt" and "pyside". The "pyqt" setting has the side effect of -## forcing the use of Version 2 API for QString and QVariant. - -## The port to use for the web server in the WebAgg backend. -#webagg.port : 8988 - -## The address on which the WebAgg web server should be reachable -#webagg.address : 127.0.0.1 - -## If webagg.port is unavailable, a number of other random ports will -## be tried until one that is available is found. -#webagg.port_retries : 50 - -## When True, open the webbrowser to the plot that is shown -#webagg.open_in_browser : True - -## if you are running pyplot inside a GUI and your backend choice -## conflicts, we will automatically try to find a compatible one for -## you if backend_fallback is True -#backend_fallback: True - -#interactive : False -#toolbar : toolbar2 ## None | toolbar2 ("classic" is deprecated) -#timezone : UTC ## a pytz timezone string, e.g., US/Central or Europe/Paris - -## Where your matplotlib data lives if you installed to a non-default -## location. This is where the matplotlib fonts, bitmaps, etc reside -#datapath : /home/jdhunter/mpldata - - -#### LINES -## See http://matplotlib.org/api/artist_api.html#module-matplotlib.lines for more -## information on line properties. -lines.linewidth : 2 ## line width in points -#lines.linestyle : - ## solid line -#lines.color : C0 ## has no affect on plot(); see axes.prop_cycle -#lines.marker : None ## the default marker -# lines.markerfacecolor : auto ## the default markerfacecolor -lines.markeredgecolor : white ## the default markeredgecolor -lines.markeredgewidth : 1 ## the line width around the marker symbol -lines.markersize : 7 ## markersize, in points -#lines.dash_joinstyle : round ## miter|round|bevel -#lines.dash_capstyle : butt ## butt|round|projecting -#lines.solid_joinstyle : round ## miter|round|bevel -#lines.solid_capstyle : projecting ## butt|round|projecting -#lines.antialiased : True ## render lines in antialiased (no jaggies) - -## The three standard dash patterns. These are scaled by the linewidth. -#lines.dashed_pattern : 3.7, 1.6 -#lines.dashdot_pattern : 6.4, 1.6, 1, 1.6 -#lines.dotted_pattern : 1, 1.65 -#lines.scale_dashes : True - -#markers.fillstyle: full ## full|left|right|bottom|top|none - -#### PATCHES -## Patches are graphical objects that fill 2D space, like polygons or -## circles. See -## http://matplotlib.org/api/artist_api.html#module-matplotlib.patches -## information on patch properties -patch.linewidth : 1 ## edge width in points. -patch.facecolor : C0 -patch.edgecolor : black ## if forced, or patch is not filled -#patch.force_edgecolor : False ## True to always use edgecolor -#patch.antialiased : True ## render patches in antialiased (no jaggies) - -#### HATCHES -#hatch.color : black -#hatch.linewidth : 1.0 - -#### Boxplot -#boxplot.notch : False -#boxplot.vertical : True -#boxplot.whiskers : 1.5 -# boxplot.bootstrap : None -boxplot.patchartist : True -#boxplot.showmeans : False -#boxplot.showcaps : True -#boxplot.showbox : True -#boxplot.showfliers : True -#boxplot.meanline : False - -boxplot.flierprops.color : C0 -boxplot.flierprops.marker : o -boxplot.flierprops.markerfacecolor : auto -boxplot.flierprops.markeredgecolor : white -boxplot.flierprops.markersize : 7 -boxplot.flierprops.linestyle : none -boxplot.flierprops.linewidth : 1.0 - -boxplot.boxprops.color : 9ae1f9 -boxplot.boxprops.linewidth : 0 -boxplot.boxprops.linestyle : - - -boxplot.whiskerprops.color : C0 -boxplot.whiskerprops.linewidth : 1.0 -boxplot.whiskerprops.linestyle : - - -boxplot.capprops.color : C0 -boxplot.capprops.linewidth : 1.0 -boxplot.capprops.linestyle : - - -boxplot.medianprops.color : 9ae1f9 -boxplot.medianprops.linewidth : 1 -boxplot.medianprops.linestyle : - - -boxplot.meanprops.color : C1 -boxplot.meanprops.marker : ^ -boxplot.meanprops.markerfacecolor : C1 -boxplot.meanprops.markeredgecolor : C1 -boxplot.meanprops.markersize : 7 -boxplot.meanprops.linestyle : -- -boxplot.meanprops.linewidth : 1.0 - - -#### FONT - -## font properties used by text.Text. See -## http://matplotlib.org/api/font_manager_api.html for more -## information on font properties. The 6 font properties used for font -## matching are given below with their default values. -## -## The font.family property has five values: 'serif' (e.g., Times), -## 'sans-serif' (e.g., Helvetica), 'cursive' (e.g., Zapf-Chancery), -## 'fantasy' (e.g., Western), and 'monospace' (e.g., Courier). Each of -## these font families has a default list of font names in decreasing -## order of priority associated with them. When text.usetex is False, -## font.family may also be one or more concrete font names. -## -## The font.style property has three values: normal (or roman), italic -## or oblique. The oblique style will be used for italic, if it is not -## present. -## -## The font.variant property has two values: normal or small-caps. For -## TrueType fonts, which are scalable fonts, small-caps is equivalent -## to using a font size of 'smaller', or about 83%% of the current font -## size. -## -## The font.weight property has effectively 13 values: normal, bold, -## bolder, lighter, 100, 200, 300, ..., 900. Normal is the same as -## 400, and bold is 700. bolder and lighter are relative values with -## respect to the current weight. -## -## The font.stretch property has 11 values: ultra-condensed, -## extra-condensed, condensed, semi-condensed, normal, semi-expanded, -## expanded, extra-expanded, ultra-expanded, wider, and narrower. This -## property is not currently implemented. -## -## The font.size property is the default font size for text, given in pts. -## 10 pt is the standard value. - -#font.family : sans-serif -#font.style : normal -#font.variant : normal -#font.weight : normal -#font.stretch : normal -## note that font.size controls default text sizes. To configure -## special text sizes tick labels, axes, labels, title, etc, see the rc -## settings for axes and ticks. Special text sizes can be defined -## relative to font.size, using the following values: xx-small, x-small, -## small, medium, large, x-large, xx-large, larger, or smaller -#font.size : 10.0 -#font.serif : DejaVu Serif, Bitstream Vera Serif, Computer Modern Roman, New Century Schoolbook, Century Schoolbook L, Utopia, ITC Bookman, Bookman, Nimbus Roman No9 L, Times New Roman, Times, Palatino, Charter, serif -#font.sans-serif : DejaVu Sans, Bitstream Vera Sans, Computer Modern Sans Serif, Lucida Grande, Verdana, Geneva, Lucid, Arial, Helvetica, Avant Garde, sans-serif -#font.cursive : Apple Chancery, Textile, Zapf Chancery, Sand, Script MT, Felipa, cursive -#font.fantasy : Comic Sans MS, Chicago, Charcoal, ImpactWestern, Humor Sans, xkcd, fantasy -#font.monospace : DejaVu Sans Mono, Bitstream Vera Sans Mono, Computer Modern Typewriter, Andale Mono, Nimbus Mono L, Courier New, Courier, Fixed, Terminal, monospace - -#### TEXT -## text properties used by text.Text. See -## http://matplotlib.org/api/artist_api.html#module-matplotlib.text for more -## information on text properties -text.color : 707074 - -#### LaTeX customizations. See http://wiki.scipy.org/Cookbook/Matplotlib/UsingTex -#text.usetex : False ## use latex for all text handling. The following fonts - ## are supported through the usual rc parameter settings: - ## new century schoolbook, bookman, times, palatino, - ## zapf chancery, charter, serif, sans-serif, helvetica, - ## avant garde, courier, monospace, computer modern roman, - ## computer modern sans serif, computer modern typewriter - ## If another font is desired which can loaded using the - ## LaTeX \usepackage command, please inquire at the - ## matplotlib mailing list -#text.latex.preamble : ## IMPROPER USE OF THIS FEATURE WILL LEAD TO LATEX FAILURES - ## AND IS THEREFORE UNSUPPORTED. PLEASE DO NOT ASK FOR HELP - ## IF THIS FEATURE DOES NOT DO WHAT YOU EXPECT IT TO. - ## preamble is a comma separated list of LaTeX statements - ## that are included in the LaTeX document preamble. - ## An example: - ## text.latex.preamble : \usepackage{bm},\usepackage{euler} - ## The following packages are always loaded with usetex, so - ## beware of package collisions: color, geometry, graphicx, - ## type1cm, textcomp. Adobe Postscript (PSSNFS) font packages - ## may also be loaded, depending on your font settings -#text.latex.preview : False - -#text.hinting : auto ## May be one of the following: - ## none: Perform no hinting - ## auto: Use FreeType's autohinter - ## native: Use the hinting information in the - # font file, if available, and if your - # FreeType library supports it - ## either: Use the native hinting information, - # or the autohinter if none is available. - ## For backward compatibility, this value may also be - ## True === 'auto' or False === 'none'. -#text.hinting_factor : 8 ## Specifies the amount of softness for hinting in the - ## horizontal direction. A value of 1 will hint to full - ## pixels. A value of 2 will hint to half pixels etc. -#text.antialiased : True ## If True (default), the text will be antialiased. - ## This only affects the Agg backend. - -## The following settings allow you to select the fonts in math mode. -## They map from a TeX font name to a fontconfig font pattern. -## These settings are only used if mathtext.fontset is 'custom'. -## Note that this "custom" mode is unsupported and may go away in the -## future. -#mathtext.cal : cursive -#mathtext.rm : sans -#mathtext.tt : monospace -#mathtext.it : sans:italic -#mathtext.bf : sans:bold -#mathtext.sf : sans -#mathtext.fontset : dejavusans ## Should be 'dejavusans' (default), - ## 'dejavuserif', 'cm' (Computer Modern), 'stix', - ## 'stixsans' or 'custom' -#mathtext.fallback_to_cm : True ## When True, use symbols from the Computer Modern - ## fonts when a symbol can not be found in one of - ## the custom math fonts. -#mathtext.default : it ## The default font to use for math. - ## Can be any of the LaTeX font names, including - ## the special name "regular" for the same font - ## used in regular text. - -#### AXES -## default face and edge color, default tick sizes, -## default fontsizes for ticklabels, and so on. See -## http://matplotlib.org/api/axes_api.html#module-matplotlib.axes -#axes.facecolor : white ## axes background color -axes.edgecolor : D0D0E0 ## axes edge color -#axes.linewidth : 0.8 ## edge linewidth -axes.grid : True ## display grid or not -axes.grid.axis : both ## which axis the grid should apply to -#axes.grid.which : major ## gridlines at major, minor or both ticks -axes.titlesize : 18 ## fontsize of the axes title -#axes.titleweight : normal ## font weight of title -#axes.titlepad : 6.0 ## pad between axes and title in points -axes.labelsize : 14 ## fontsize of the x any y labels -#axes.labelpad : 4.0 ## space between label and axis -#axes.labelweight : normal ## weight of the x and y labels -axes.labelcolor : 707074 -#axes.axisbelow : line ## draw axis gridlines and ticks below - ## patches (True); above patches but below - ## lines ('line'); or above all (False) -#axes.formatter.limits : -7, 7 ## use scientific notation if log10 - ## of the axis range is smaller than the - ## first or larger than the second -#axes.formatter.use_locale : False ## When True, format tick labels - ## according to the user's locale. - ## For example, use ',' as a decimal - ## separator in the fr_FR locale. -#axes.formatter.use_mathtext : False ## When True, use mathtext for scientific - ## notation. -#axes.formatter.min_exponent: 0 ## minimum exponent to format in scientific notation -#axes.formatter.useoffset : True ## If True, the tick label formatter - ## will default to labeling ticks relative - ## to an offset when the data range is - ## small compared to the minimum absolute - ## value of the data. -#axes.formatter.offset_threshold : 4 ## When useoffset is True, the offset - ## will be used when it can remove - ## at least this number of significant - ## digits from tick labels. -axes.spines.left : False ## display axis spines -axes.spines.bottom : False -axes.spines.top : False -axes.spines.right : False -#axes.unicode_minus : True ## use unicode for the minus symbol - ## rather than hyphen. See - ## http://en.wikipedia.org/wiki/Plus_and_minus_signs#Character_codes -## ========================================================================================== ## -## ========================================================================================== ## -## ========================================================================================== ## -## COLOR PALETTE -# v1 https://coolors.co/2364aa-3da5d9-4ebc93-b4da1b-fbcc23-ec8232-e40066-df26cf-ae5ce6-9b899f -# v2 https://coolors.co/3482d5-66b8e1-5cc19c-b9d548-fbc737-f2822c-ff338f-d54ee4-a072e9-9b899f -axes.prop_cycle : cycler('color', ['3482d5', '66b8e1', '5cc19c', 'b9d548', 'fbc737', 'f2822c', 'ff338f', 'd54ee4', 'a072e9', '9b899f']) ## CUSTOM -## axes.prop_cycle : cycler('color', ['00BEFF', 'D4CA3A', 'FF6DAE', '67E1B5', 'EBACFA', '9E9E9E', 'F1988E', '5DB15A', 'E28544', '52B8AA']) ## ORIG - ## color cycle for plot lines as list of string - ## colorspecs: single letter, long name, or web-style hex - ## Note the use of string escapes here ('1f77b4', instead of 1f77b4) - ## as opposed to the rest of this file. -## ========================================================================================== ## -## ========================================================================================== ## -## ========================================================================================== ## -#axes.autolimit_mode : data ## How to scale axes limits to the data. - ## Use "data" to use data limits, plus some margin - ## Use "round_number" move to the nearest "round" number -#axes.xmargin : .05 ## x margin. See `axes.Axes.margins` -#axes.ymargin : .05 ## y margin See `axes.Axes.margins` -#polaraxes.grid : True ## display grid on polar axes -#axes3d.grid : True ## display grid on 3d axes - -#### DATES -## These control the default format strings used in AutoDateFormatter. -## Any valid format datetime format string can be used (see the python -## `datetime` for details). For example using '%%x' will use the locale date representation -## '%%X' will use the locale time representation and '%%c' will use the full locale datetime -## representation. -## These values map to the scales: -## {'year': 365, 'month': 30, 'day': 1, 'hour': 1/24, 'minute': 1 / (24 * 60)} - -#date.autoformatter.year : %Y -#date.autoformatter.month : %Y-%m -#date.autoformatter.day : %Y-%m-%d -#date.autoformatter.hour : %m-%d %H -#date.autoformatter.minute : %d %H:%M -#date.autoformatter.second : %H:%M:%S -#date.autoformatter.microsecond : %M:%S.%f - -#### TICKS -## see http://matplotlib.org/api/axis_api.html#matplotlib.axis.Tick -#xtick.top : False ## draw ticks on the top side -#xtick.bottom : True ## draw ticks on the bottom side -#xtick.labeltop : False ## draw label on the top -#xtick.labelbottom : True ## draw label on the bottom -#xtick.major.size : 3.5 ## major tick size in points -#xtick.minor.size : 2 ## minor tick size in points -#xtick.major.width : 0.8 ## major tick width in points -#xtick.minor.width : 0.6 ## minor tick width in points -#xtick.major.pad : 3.5 ## distance to major tick label in points -#xtick.minor.pad : 3.4 ## distance to the minor tick label in points -xtick.color : 707074 ## color of the tick labels -xtick.labelsize : 12 ## fontsize of the tick labels -#xtick.direction : out ## direction: in, out, or inout -#xtick.minor.visible : False ## visibility of minor ticks on x-axis -#xtick.major.top : True ## draw x axis top major ticks -#xtick.major.bottom : True ## draw x axis bottom major ticks -#xtick.minor.top : True ## draw x axis top minor ticks -#xtick.minor.bottom : True ## draw x axis bottom minor ticks -#xtick.alignment : center ## alignment of xticks - -#ytick.left : True ## draw ticks on the left side -#ytick.right : False ## draw ticks on the right side -#ytick.labelleft : True ## draw tick labels on the left side -#ytick.labelright : False ## draw tick labels on the right side -#ytick.major.size : 3.5 ## major tick size in points -#ytick.minor.size : 2 ## minor tick size in points -#ytick.major.width : 0.8 ## major tick width in points -#ytick.minor.width : 0.6 ## minor tick width in points -#ytick.major.pad : 3.5 ## distance to major tick label in points -#ytick.minor.pad : 3.4 ## distance to the minor tick label in points -ytick.color : 707074 ## color of the tick labels -ytick.labelsize : 12 ## fontsize of the tick labels -#ytick.direction : out ## direction: in, out, or inout -#ytick.minor.visible : False ## visibility of minor ticks on y-axis -#ytick.major.left : True ## draw y axis left major ticks -#ytick.major.right : True ## draw y axis right major ticks -#ytick.minor.left : True ## draw y axis left minor ticks -#ytick.minor.right : True ## draw y axis right minor ticks -#ytick.alignment : center_baseline ## alignment of yticks - -#### GRIDS -grid.color : 93939c ## grid color -grid.linestyle : -- ## solid -#grid.linewidth : 0.8 ## in points -grid.alpha : 0.2 ## transparency, between 0.0 and 1.0 - -#### Legend -#legend.loc : best -#legend.frameon : True ## if True, draw the legend on a background patch -#legend.framealpha : 0.8 ## legend patch transparency -#legend.facecolor : inherit ## inherit from axes.facecolor; or color spec -#legend.edgecolor : 0.8 ## background patch boundary color -#legend.fancybox : True ## if True, use a rounded box for the - ## legend background, else a rectangle -#legend.shadow : False ## if True, give background a shadow effect -#legend.numpoints : 1 ## the number of marker points in the legend line -#legend.scatterpoints : 1 ## number of scatter points -#legend.markerscale : 1.0 ## the relative size of legend markers vs. original -#legend.fontsize : medium -#legend.title_fontsize : None ## None sets to the same as the default axes. -## Dimensions as fraction of fontsize: -#legend.borderpad : 0.4 ## border whitespace -#legend.labelspacing : 0.5 ## the vertical space between the legend entries -#legend.handlelength : 2.0 ## the length of the legend lines -#legend.handleheight : 0.7 ## the height of the legend handle -#legend.handletextpad : 0.8 ## the space between the legend line and legend text -#legend.borderaxespad : 0.5 ## the border between the axes and legend edge -#legend.columnspacing : 2.0 ## column separation - -#### FIGURE -## See http://matplotlib.org/api/figure_api.html#matplotlib.figure.Figure -#figure.titlesize : large ## size of the figure title (Figure.suptitle()) -#figure.titleweight : normal ## weight of the figure title -#figure.figsize : 6.4, 4.8 ## figure size in inches -#figure.dpi : 100 ## figure dots per inch -#figure.facecolor : white ## figure facecolor -#figure.edgecolor : white ## figure edgecolor -#figure.frameon : True ## enable figure frame -#figure.max_open_warning : 20 ## The maximum number of figures to open through - ## the pyplot interface before emitting a warning. - ## If less than one this feature is disabled. -## The figure subplot parameters. All dimensions are a fraction of the -#figure.subplot.left : 0.125 ## the left side of the subplots of the figure -#figure.subplot.right : 0.9 ## the right side of the subplots of the figure -#figure.subplot.bottom : 0.11 ## the bottom of the subplots of the figure -#figure.subplot.top : 0.88 ## the top of the subplots of the figure -#figure.subplot.wspace : 0.2 ## the amount of width reserved for space between subplots, - ## expressed as a fraction of the average axis width -#figure.subplot.hspace : 0.2 ## the amount of height reserved for space between subplots, - ## expressed as a fraction of the average axis height - -## Figure layout -#figure.autolayout : False ## When True, automatically adjust subplot - ## parameters to make the plot fit the figure - ## using `tight_layout` -#figure.constrained_layout.use: False ## When True, automatically make plot - ## elements fit on the figure. (Not compatible - ## with `autolayout`, above). -#figure.constrained_layout.h_pad : 0.04167 ## Padding around axes objects. Float representing -#figure.constrained_layout.w_pad : 0.04167 ## inches. Default is 3./72. inches (3 pts) -#figure.constrained_layout.hspace : 0.02 ## Space between subplot groups. Float representing -#figure.constrained_layout.wspace : 0.02 ## a fraction of the subplot widths being separated. - -#### IMAGES -#image.aspect : equal ## equal | auto | a number -#image.interpolation : nearest ## see help(imshow) for options -#image.cmap : viridis ## A colormap name, gray etc... -#image.lut : 256 ## the size of the colormap lookup table -#image.origin : upper ## lower | upper -#image.resample : True -#image.composite_image : True ## When True, all the images on a set of axes are - ## combined into a single composite image before - ## saving a figure as a vector graphics file, - ## such as a PDF. - -#### CONTOUR PLOTS -#contour.negative_linestyle : dashed ## string or on-off ink sequence -#contour.corner_mask : True ## True | False | legacy - -#### ERRORBAR PLOTS -#errorbar.capsize : 0 ## length of end cap on error bars in pixels - -#### HISTOGRAM PLOTS -#hist.bins : 10 ## The default number of histogram bins. - ## If Numpy 1.11 or later is - ## installed, may also be `auto` - -#### SCATTER PLOTS -#scatter.marker : o ## The default marker type for scatter plots. - -#### Agg rendering -#### Warning: experimental, 2008/10/10 -#agg.path.chunksize : 0 ## 0 to disable; values in the range - ## 10000 to 100000 can improve speed slightly - ## and prevent an Agg rendering failure - ## when plotting very large data sets, - ## especially if they are very gappy. - ## It may cause minor artifacts, though. - ## A value of 20000 is probably a good - ## starting point. -#### PATHS -#path.simplify : True ## When True, simplify paths by removing "invisible" - ## points to reduce file size and increase rendering - ## speed -#path.simplify_threshold : 0.111111111111 ## The threshold of similarity below which - ## vertices will be removed in the - ## simplification process -#path.snap : True ## When True, rectilinear axis-aligned paths will be snapped to - ## the nearest pixel when certain criteria are met. When False, - ## paths will never be snapped. -#path.sketch : None ## May be none, or a 3-tuple of the form (scale, length, - ## randomness). - ## *scale* is the amplitude of the wiggle - ## perpendicular to the line (in pixels). *length* - ## is the length of the wiggle along the line (in - ## pixels). *randomness* is the factor by which - ## the length is randomly scaled. -#path.effects : [] ## - -#### SAVING FIGURES -## the default savefig params can be different from the display params -## e.g., you may want a higher resolution, or to make the figure -## background white -#savefig.dpi : figure ## figure dots per inch or 'figure' -#savefig.facecolor : white ## figure facecolor when saving -#savefig.edgecolor : white ## figure edgecolor when saving -#savefig.format : png ## png, ps, pdf, svg -#savefig.bbox : standard ## 'tight' or 'standard'. - ## 'tight' is incompatible with pipe-based animation - ## backends but will workd with temporary file based ones: - ## e.g. setting animation.writer to ffmpeg will not work, - ## use ffmpeg_file instead -#savefig.pad_inches : 0.1 ## Padding to be used when bbox is set to 'tight' -#savefig.jpeg_quality: 95 ## when a jpeg is saved, the default quality parameter. -#savefig.directory : ~ ## default directory in savefig dialog box, - ## leave empty to always use current working directory -#savefig.transparent : False ## setting that controls whether figures are saved with a - ## transparent background by default -#savefig.frameon : True ## enable frame of figure when saving -#savefig.orientation : portrait ## Orientation of saved figure - -### tk backend params -#tk.window_focus : False ## Maintain shell focus for TkAgg - -### ps backend params -#ps.papersize : letter ## auto, letter, legal, ledger, A0-A10, B0-B10 -#ps.useafm : False ## use of afm fonts, results in small files -#ps.usedistiller : False ## can be: None, ghostscript or xpdf - ## Experimental: may produce smaller files. - ## xpdf intended for production of publication quality files, - ## but requires ghostscript, xpdf and ps2eps -#ps.distiller.res : 6000 ## dpi -#ps.fonttype : 3 ## Output Type 3 (Type3) or Type 42 (TrueType) - -### pdf backend params -#pdf.compression : 6 ## integer from 0 to 9 - ## 0 disables compression (good for debugging) -#pdf.fonttype : 3 ## Output Type 3 (Type3) or Type 42 (TrueType) -#pdf.use14corefonts : False -#pdf.inheritcolor : False - -### svg backend params -#svg.image_inline : True ## write raster image data directly into the svg file -#svg.fonttype : path ## How to handle SVG fonts: - ## none: Assume fonts are installed on the machine where the SVG will be viewed. - ## path: Embed characters as paths -- supported by most SVG renderers - ## svgfont: Embed characters as SVG fonts -- supported only by Chrome, - ## Opera and Safari -#svg.hashsalt : None ## if not None, use this string as hash salt - ## instead of uuid4 -### pgf parameter -#pgf.rcfonts : True -#pgf.preamble : -#pgf.texsystem : xelatex - -### docstring params -##docstring.hardcopy = False ## set this when you want to generate hardcopy docstring - -## Event keys to interact with figures/plots via keyboard. -## Customize these settings according to your needs. -## Leave the field(s) empty if you don't need a key-map. (i.e., fullscreen : '') -#keymap.fullscreen : f, ctrl+f ## toggling -#keymap.home : h, r, home ## home or reset mnemonic -#keymap.back : left, c, backspace ## forward / backward keys to enable -#keymap.forward : right, v ## left handed quick navigation -#keymap.pan : p ## pan mnemonic -#keymap.zoom : o ## zoom mnemonic -#keymap.save : s, ctrl+s ## saving current figure -#keymap.help : f1 ## display help about active tools -#keymap.quit : ctrl+w, cmd+w, q ## close the current figure -#keymap.quit_all : W, cmd+W, Q ## close all figures -#keymap.grid : g ## switching on/off major grids in current axes -#keymap.grid_minor : G ## switching on/off minor grids in current axes -#keymap.yscale : l ## toggle scaling of y-axes ('log'/'linear') -#keymap.xscale : k, L ## toggle scaling of x-axes ('log'/'linear') -#keymap.all_axes : a ## enable all axes -#keymap.copy : ctrl+c, cmd+c ## Copy figure to clipboard - -###ANIMATION settings -#animation.html : none ## How to display the animation as HTML in - ## the IPython notebook. 'html5' uses - ## HTML5 video tag; 'jshtml' creates a - ## Javascript animation -#animation.writer : ffmpeg ## MovieWriter 'backend' to use -#animation.codec : h264 ## Codec to use for writing movie -#animation.bitrate: -1 ## Controls size/quality tradeoff for movie. - ## -1 implies let utility auto-determine -#animation.frame_format: png ## Controls frame format used by temp files -#animation.html_args: ## Additional arguments to pass to html writer -#animation.ffmpeg_path: ffmpeg ## Path to ffmpeg binary. Without full path - ## $PATH is searched -#animation.ffmpeg_args: ## Additional arguments to pass to ffmpeg -#animation.avconv_path: avconv ## Path to avconv binary. Without full path - ## $PATH is searched -#animation.avconv_args: ## Additional arguments to pass to avconv -#animation.convert_path: convert ## Path to ImageMagick's convert binary. - ## On Windows use the full path since convert - ## is also the name of a system tool. -#animation.convert_args: ## Additional arguments to pass to convert -#animation.embed_limit : 20.0 \ No newline at end of file diff --git a/research/helper.sh b/research/helper.sh deleted file mode 100644 index c0635998..00000000 --- a/research/helper.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Description # -# ========================================================================= # - -# before sourcing this script, it requires the following variables to be exported: -# - PROJECT: str -# - PARTITION: str -# - PARALLELISM: int - -# source this script from the script you use to run the experiment -# 1. gets and exports the path to the root -# 2. changes the working directory to the root -# 3. exports a helper function that runs the script in the background, with -# the correct python path and settings - -if [ -z "$PROJECT" ]; then echo "PROJECT is not set"; exit 1; fi -if [ -z "$PARTITION" ]; then echo "PARTITION is not set"; exit 1; fi -if [ -z "$PARALLELISM" ]; then echo "PARALLELISM is not set"; exit 1; fi -if [ -z "$USERNAME" ]; then echo "USERNAME is not set"; exit 1; fi -if [ -z "$PY_RUN_FILE" ]; then PY_RUN_FILE='experiment/run.py'; fi - -export PY_RUN_FILE - -# ========================================================================= # -# Helper # -# ========================================================================= # - -# get the root directory -SCRIPT_DIR=$(dirname "$(realpath -s "$0")") -ROOT_DIR="$(realpath -s "$SCRIPT_DIR/../..")" - -# cd into the root, exit on failure -cd "$ROOT_DIR" || exit 1 -echo "working directory is: $(pwd)" - -function submit_sweep() { - echo "SUBMITTING SWEEP:" "$@" - PYTHONPATH="$ROOT_DIR" python3 "$PY_RUN_FILE" -m \ - run_launcher=slurm \ - dsettings.launcher.partition="$PARTITION" \ - settings.job.project="$PROJECT" \ - settings.job.user="$USERNAME" \ - hydra.launcher.array_parallelism="$PARALLELISM" \ - "$@" \ - & # run in background -} - -function local_run() { - echo "RUNNING:" "$@" - PYTHONPATH="$ROOT_DIR" python3 "$PY_RUN_FILE" \ - run_launcher=local \ - settings.job.project="$PROJECT" \ - settings.job.user="$USERNAME" \ - "$@" -} - -function local_sweep() { - echo "RUNNING SWEEP:" "$@" - PYTHONPATH="$ROOT_DIR" python3 "$PY_RUN_FILE" -m \ - run_launcher=local \ - settings.job.project="$PROJECT" \ - settings.job.user="$USERNAME" \ - "$@" -} - -# export -export ROOT_DIR -export submit_sweep -export local_run - -# debug hydra -HYDRA_FULL_ERROR=1 -export HYDRA_FULL_ERROR - -# ========================================================================= # -# Slurm Helper # -# ========================================================================= # - - -function num_idle_nodes() { - if [ -z "$1" ]; then echo "partition (first arg) is not set"; exit 1; fi - # number of idle nodes - num=$(sinfo --partition="$1" --noheader -O Nodes,Available,StateCompact | awk '{if($2 == "up" && $3 == "idle"){print $1}}') - if [ -z "$num" ]; then num=0; fi - echo $num -} - -function clog_cudaless_nodes() { - if [ -z "$1" ]; then echo "partition is not set"; exit 1; fi - if [ -z "$2" ]; then echo wait=120; else wait="$2"; fi - if [ -z "$3" ]; then echo name="NO-CUDA"; else name="$3"; fi - # clog idle nodes - n=$(num_idle_nodes "$1") - if [ "$n" -lt "1" ]; then - echo -e "\e[93mclogging skipped! no idle nodes found on partition '$1'\e[0m"; - else - echo -e "\e[92mclogging $n nodes on partition '$1' for ${wait}s if cuda is not available!\e[0m"; - sbatch --array=1-"$n" --partition="$1" --job-name="$name" --output=/dev/null --error=/dev/null \ - --wrap='python -c "import torch; import time; cuda=torch.cuda.is_available(); print(\"CUDA:\", cuda, flush=True); print(flush=True); time.sleep(5 if cuda else '"$wait"');"' - fi -} - -function clog_cuda_nodes() { - if [ -z "$1" ]; then echo "partition is not set"; exit 1; fi - if [ -z "$2" ]; then echo wait=120; else wait="$2"; fi - if [ -z "$3" ]; then echo name="HAS-CUDA"; else name="$3"; fi - # clog idle nodes - n=$(num_idle_nodes "$1") - if [ "$n" -lt "1" ]; then - echo -e "\e[93mclogging skipped! no idle nodes found on partition '$1'\e[0m"; - else - echo -e "\e[92mclogging $n nodes on partition '$1' for ${wait}s if cuda is available!\e[0m"; - sbatch --array=1-"$n" --partition="$1" --job-name="$name" --output=/dev/null --error=/dev/null \ - --wrap='python -c "import torch; import time; cuda=torch.cuda.is_available(); print(\"CUDA:\", cuda, flush=True); print(flush=True); time.sleep(5 if not cuda else '"$wait"');"' - fi -} - -export num_idle_nodes -export clog_cudaless_nodes -export clog_cuda_nodes - -# ========================================================================= # -# End # -# ========================================================================= # diff --git a/research/plot_wandb_experiments/plot_experiments.py b/research/plot_wandb_experiments/plot_experiments.py deleted file mode 100644 index 6233eb38..00000000 --- a/research/plot_wandb_experiments/plot_experiments.py +++ /dev/null @@ -1,373 +0,0 @@ -import os -from typing import List -from typing import Optional - -import pandas as pd -import seaborn as sns -import wandb -from cachier import cachier as _cachier -from matplotlib import pyplot as plt -from tqdm import tqdm - -import research.util as H -from disent.util.function import wrapped_partial - - -# ========================================================================= # -# Helper # -# ========================================================================= # - - -cachier = wrapped_partial(_cachier, cache_dir='./cache') -DF = pd.DataFrame - - -def clear_cache(): - load_runs.clear_cache() - - -# ========================================================================= # -# Load WANDB Data # -# ========================================================================= # - - -@cachier() -def load_runs(project: str) -> pd.DataFrame: - api = wandb.Api() - - runs = api.runs(project) - - info_list, summary_list, config_list, name_list = [], [], [], [] - for run in tqdm(runs, desc=f'loading: {project}'): - info_list.append({ - 'id': run.id, - 'name': run.name, - 'state': run.state, - 'storage_id': run.storage_id, - 'url': run.url, - }) - summary_list.append(run.summary._json_dict) - config_list.append({k: v for k, v in run.config.items() if not k.startswith('_')}) - name_list.append(run.name) - - return pd.DataFrame({ - "info": info_list, - "summary": summary_list, - "config": config_list, - "name": name_list - }) - - -def load_expanded_runs(project: str) -> pd.DataFrame: - # load the data - df_runs: DF = load_runs(project) - # expand the dictionaries - df_info: DF = df_runs['info'].apply(pd.Series) - df_summary: DF = df_runs['summary'].apply(pd.Series) - df_config: DF = df_runs['config'].apply(pd.Series) - # merge the data - df: DF = df_config.join(df_summary).join(df_info) - assert len(df.columns) == len(df_info.columns) + len(df_summary.columns) + len(df_config.columns) - # done! - return df - - -def drop_unhashable(df: pd.DataFrame, inplace: bool = False) -> (pd.DataFrame, List[str]): - dropped = [] - for col in df.columns: - try: - df[col].unique() - except: - dropped.append(col) - df = df.drop(col, inplace=inplace, axis=1) - return df, dropped - - -def drop_non_diverse_cols(df: pd.DataFrame, inplace: bool = False) -> (pd.DataFrame, List[str]): - dropped = [] - for col in df.columns: - if len(df[col].unique()) == 1: - dropped.append(col) - df = df.drop(col, inplace=inplace, axis=1) - return df, dropped - - -# ========================================================================= # -# Prepare Data # -# ========================================================================= # - - -# common keys -K_GROUP = 'Run Group' -K_DATASET = 'Dataset' -K_FRAMEWORK = 'Framework' -K_SPACING = 'Grid Spacing' -K_BETA = 'Beta' -K_LOSS = 'Recon. Loss' -K_Z_SIZE = 'Latent Dims.' -K_REPEAT = 'Repeat' -K_STATE = 'State' -K_MIG = 'MIG Score' -K_DCI = 'DCI Score' - - -def load_general_data(project: str): - # load data - df = load_expanded_runs(project) - # filter out unneeded columns - df, dropped_hash = drop_unhashable(df) - df, dropped_diverse = drop_non_diverse_cols(df) - # rename columns - return df.rename(columns={ - 'EXTRA/tags': K_GROUP, - 'dataset/name': K_DATASET, - 'framework/name': K_FRAMEWORK, - 'dataset/data/grid_spacing': K_SPACING, - 'settings/framework/beta': K_BETA, - 'settings/framework/recon_loss': K_LOSS, - 'settings/model/z_size': K_Z_SIZE, - 'DUMMY/repeat': K_REPEAT, - 'state': K_STATE, - 'final_metric/mig.discrete_score.max': K_MIG, - 'final_metric/dci.disentanglement.max': K_DCI, - }) - - -# ========================================================================= # -# Plot Experiments # -# ========================================================================= # - -PINK = '#FE375F' -PURPLE = '#5E5BE5' -BLUE = '#0A83FE' -LBLUE = '#63D2FE' -ORANGE = '#FE9F0A' -GREEN = '#2FD157' - - -def plot_incr_overlap_exp( - rel_path: Optional[str] = None, - save: bool = True, - show: bool = True, - reg_order: int = 4, - color_betavae: str = PINK, - color_adavae: str = ORANGE, - titles: bool = False, -): - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - df = load_general_data(f'{os.environ["WANDB_USER"]}/CVPR-01__incr_overlap') - # select run groups - df = df[df[K_GROUP].isin(['sweep_xy_squares_overlap', 'sweep_xy_squares_overlap_small_beta'])] - # print common key values - print('K_GROUP: ', list(df[K_GROUP].unique())) - print('K_FRAMEWORK:', list(df[K_FRAMEWORK].unique())) - print('K_SPACING: ', list(df[K_SPACING].unique())) - print('K_BETA: ', list(df[K_BETA].unique())) - print('K_REPEAT: ', list(df[K_REPEAT].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - BETA = 0.00316 # if grid_spacing < 6 - BETA = 0.001 # if grid_spacing >= 6 - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - orig = df - # select runs - # df = df[df[K_STATE] == 'finished'] - # df = df[df[K_REPEAT].isin([1, 2, 3])] - # select adavae - adavae_selector = (df[K_FRAMEWORK] == 'adavae_os') & (df[K_BETA] == 0.001) # 0.001, 0.0001 - data_adavae = df[adavae_selector] - # select - betavae_selector_a = (df[K_FRAMEWORK] == 'betavae') & (df[K_BETA] == 0.001) & (df[K_SPACING] >= 3) - betavae_selector_b = (df[K_FRAMEWORK] == 'betavae') & (df[K_BETA] == 0.00316) & (df[K_SPACING] < 3) - data_betavae = df[betavae_selector_a | betavae_selector_b] - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - print('ADAGVAE', len(orig), '->', len(data_adavae)) - print('BETAVAE', len(orig), '->', len(data_betavae)) - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - fig, axs = plt.subplots(1, 2, figsize=(10, 5)) - (ax0, ax1) = axs - # PLOT: MIG - sns.regplot(ax=ax0, x=K_SPACING, y=K_MIG, data=data_adavae, seed=777, order=reg_order, robust=False, color=color_adavae, marker='o') - sns.regplot(ax=ax0, x=K_SPACING, y=K_MIG, data=data_betavae, seed=777, order=reg_order, robust=False, color=color_betavae, marker='x', line_kws=dict(linestyle='dashed')) - ax0.legend(labels=["Ada-GVAE", "Beta-VAE"], fontsize=14) - ax0.set_ylim([-0.1, 1.1]) - ax0.set_xlim([0.8, 8.2]) - if titles: ax0.set_title('Framework Mig Scores') - # PLOT: DCI - sns.regplot(ax=ax1, x=K_SPACING, y=K_DCI, data=data_adavae, seed=777, order=reg_order, robust=False, color=color_adavae, marker='o') - sns.regplot(ax=ax1, x=K_SPACING, y=K_DCI, data=data_betavae, seed=777, order=reg_order, robust=False, color=color_betavae, marker='x', line_kws=dict(linestyle='dashed')) - ax1.legend(labels=["Ada-GVAE", "Beta-VAE"], fontsize=14) - ax1.set_ylim([-0.1, 1.1]) - ax1.set_xlim([0.8, 8.2]) - if titles: ax1.set_title('Framework DCI Scores') - # PLOT: - fig.tight_layout() - H.plt_rel_path_savefig(rel_path, save=save, show=show) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - return fig, axs - - - - - -def plot_hparams_exp( - rel_path: Optional[str] = None, - save: bool = True, - show: bool = True, - color_betavae: str = PINK, - color_adavae: str = ORANGE, -): - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - df = load_general_data(f'{os.environ["WANDB_USER"]}/CVPR-00__basic-hparam-tuning') - # select run groups - df = df[df[K_GROUP].isin(['sweep_beta'])] - # print common key values - print('K_GROUP: ', list(df[K_GROUP].unique())) - print('K_DATASET: ', list(df[K_DATASET].unique())) - print('K_FRAMEWORK:', list(df[K_FRAMEWORK].unique())) - print('K_BETA: ', list(df[K_BETA].unique())) - print('K_Z_SIZE: ', list(df[K_Z_SIZE].unique())) - print('K_REPEAT: ', list(df[K_REPEAT].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - orig = df - # select runs - df = df[df[K_STATE] == 'finished'] - # [1.0, 0.316, 0.1, 0.0316, 0.01, 0.00316, 0.001, 0.000316] - # df = df[(0.000316 < df[K_BETA]) & (df[K_BETA] < 1.0)] - print('NUM', len(orig), '->', len(df)) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - df = df[[K_DATASET, K_FRAMEWORK, K_MIG, K_DCI]] - df[K_DATASET].replace('xysquares_minimal', 'XYSquares', inplace=True) - df[K_DATASET].replace('smallnorb', 'NORB', inplace=True) - df[K_DATASET].replace('cars3d', 'Cars3D', inplace=True) - df[K_DATASET].replace('3dshapes', 'Shapes3D', inplace=True) - df[K_DATASET].replace('dsprites', 'dSprites', inplace=True) - df[K_FRAMEWORK].replace('adavae_os', 'Ada-GVAE', inplace=True) - df[K_FRAMEWORK].replace('betavae', 'Beta-VAE', inplace=True) - PALLETTE = {'Ada-GVAE': color_adavae, 'Beta-VAE': color_betavae} - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - fig, axs = plt.subplots(1, 2, figsize=(10, 4)) - (ax0, ax1) = axs - # PLOT: MIG - sns.violinplot(x=K_DATASET, y=K_MIG, hue=K_FRAMEWORK, palette=PALLETTE, split=True, cut=0, width=0.75, data=df, ax=ax0, scale='width', inner='quartile') - ax0.set_ylim([-0.1, 1.1]) - ax0.legend(bbox_to_anchor=(0.425, 0.9), fontsize=13) - sns.violinplot(x=K_DATASET, y=K_DCI, hue=K_FRAMEWORK, palette=PALLETTE, split=True, cut=0, width=0.75, data=df, ax=ax1, scale='width', inner='quartile') - ax1.set_ylim([-0.1, 1.1]) - ax1.get_legend().remove() - # PLOT: - fig.tight_layout() - H.plt_rel_path_savefig(rel_path, save=save, show=show, dpi=300) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - return fig, axs - - - -def plot_overlap_loss_exp( - rel_path: Optional[str] = None, - save: bool = True, - show: bool = True, - color_betavae: str = PINK, - color_adavae: str = ORANGE, - color_mse: str = '#9FD911', - color_mse_overlap: str = '#36CFC8', -): - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - df = load_general_data(f'{os.environ["WANDB_USER"]}/CVPR-09__vae_overlap_loss') - # select run groups - df = df[df[K_GROUP].isin(['sweep_overlap_boxblur_specific', 'sweep_overlap_boxblur'])] - # print common key values - print('K_GROUP: ', list(df[K_GROUP].unique())) - print() - print('K_DATASET: ', list(df[K_DATASET].unique())) - print('K_FRAMEWORK:', list(df[K_FRAMEWORK].unique())) - print('K_Z_SIZE: ', list(df[K_Z_SIZE].unique())) - print('K_LOSS: ', list(df[K_LOSS].unique())) - print('K_BETA: ', list(df[K_BETA].unique())) - print() - print('K_REPEAT: ', list(df[K_REPEAT].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - orig = df - # select runs - df = df[df[K_STATE] == 'finished'] # TODO: update - df = df[df[K_DATASET] == 'xysquares_minimal'] - df = df[df[K_BETA].isin([0.0001, 0.0316])] - df = df[df[K_Z_SIZE] == 25] - # df = df[df[K_FRAMEWORK] == 'betavae'] # 18 - # df = df[df[K_FRAMEWORK] == 'adavae_os'] # 21 - # df = df[df[K_LOSS] == 'mse'] # 20 - # df = df[df[K_LOSS] != 'mse'] # 19 - # # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # TEMP - # df[K_MIG] = df['final_metric/mig.discrete_score.max'] - # df[K_DCI] = df['final_metric/dci.disentanglement.max'] - - print('NUM', len(orig), '->', len(df)) - - df = df[[K_DATASET, K_FRAMEWORK, K_LOSS, K_BETA, K_MIG, K_DCI]] - df[K_DATASET].replace('xysquares_minimal', 'XYSquares', inplace=True) - df[K_FRAMEWORK].replace('adavae_os', 'Ada-GVAE', inplace=True) - df[K_FRAMEWORK].replace('betavae', 'Beta-VAE', inplace=True) - df[K_LOSS].replace('mse_box_r31_l1.0_k3969.0', 'MSE-boxblur', inplace=True) - df[K_LOSS].replace('mse', 'MSE', inplace=True) - PALLETTE = {'Ada-GVAE': color_adavae, 'Beta-VAE': color_betavae, 'MSE': color_mse, 'MSE-boxblur': color_mse_overlap} - - print(df) - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - fig, axs = plt.subplots(1, 2, figsize=(10, 4)) - (ax0, ax1) = axs - # PLOT: MIG - sns.violinplot(x=K_FRAMEWORK, y=K_MIG, hue=K_LOSS, palette=PALLETTE, split=True, cut=0, width=0.5, data=df, ax=ax0, scale='width', inner='quartile') - ax0.set_ylim([-0.1, 1.1]) - ax0.legend(fontsize=13) - # ax0.legend(bbox_to_anchor=(0.425, 0.9), fontsize=13) - sns.violinplot(x=K_FRAMEWORK, y=K_DCI, hue=K_LOSS, palette=PALLETTE, split=True, cut=0, width=0.5, data=df, ax=ax1, scale='width', inner='quartile') - ax1.set_ylim([-0.1, 1.1]) - ax1.get_legend().remove() - # PLOT: - fig.tight_layout() - H.plt_rel_path_savefig(rel_path, save=save, show=show, dpi=300) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - -# ========================================================================= # -# Entrypoint # -# ========================================================================= # - - -if __name__ == '__main__': - - assert 'WANDB_USER' in os.environ, 'specify "WANDB_USER" environment variable' - - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../gadfly.mplstyle')) - - # clear_cache() - - def main(): - # plot_hparams_exp(rel_path='plots/exp_hparams-exp', show=True) - plot_overlap_loss_exp(rel_path='plots/exp_overlap-loss', show=True) - # plot_incr_overlap_exp(rel_path='plots/exp_incr-overlap', show=True) - - main() - - -# ========================================================================= # -# DONE # -# ========================================================================= # diff --git a/research/plot_wandb_experiments/plots/.gitignore b/research/plot_wandb_experiments/plots/.gitignore deleted file mode 100644 index e33609d2..00000000 --- a/research/plot_wandb_experiments/plots/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.png diff --git a/research/util/__init__.py b/research/util/__init__.py deleted file mode 100644 index e02c2205..00000000 --- a/research/util/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ - -from ._fn_util import * -from ._dataset import * -from ._io_util import * -from ._loss import * - -# disent exports to make life easy -from disent.util.visualize.plot import to_img -from disent.util.visualize.plot import to_imgs -from disent.util.visualize.plot import plt_imshow -from disent.util.visualize.plot import plt_subplots -from disent.util.visualize.plot import plt_subplots_imshow -from disent.util.visualize.plot import plt_hide_axis -from disent.util.visualize.plot import visualize_dataset_traversal -from disent.util.visualize.plot import plt_2d_density diff --git a/research/util/_data.py b/research/util/_data.py deleted file mode 100644 index cce196e9..00000000 --- a/research/util/_data.py +++ /dev/null @@ -1,82 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from typing import Tuple - -import numpy as np - -from disent.dataset.data import GroundTruthData -from disent.dataset.data._raw import Hdf5Dataset - - -# TODO: these classes are old... -# TODO: these classes are old... -# TODO: these classes are old... - - -class TransformDataset(GroundTruthData): - - # TODO: all data should be datasets - # TODO: file preparation should be separate from datasets - # TODO: disent/data should be datasets, and disent/datasets should be samplers that wrap disent/data - - def __init__(self, base_data: GroundTruthData, transform=None): - self.base_data = base_data - super().__init__(transform=transform) - - @property - def factor_names(self) -> Tuple[str, ...]: - return self.base_data.factor_names - - @property - def factor_sizes(self) -> Tuple[int, ...]: - return self.base_data.factor_sizes - - @property - def img_shape(self) -> Tuple[int, ...]: - return self.base_data.img_shape - - def _get_observation(self, idx): - return self.base_data[idx] - - -class AdversarialOptimizedData(TransformDataset): - - def __init__(self, h5_path: str, base_data: GroundTruthData, transform=None): - # normalize hd5f data - def _normalize_hdf5(x): - c, h, w = x.shape - if c in (1, 3): - return np.moveaxis(x, 0, -1) - return x - # get the data - self.hdf5_data = Hdf5Dataset(h5_path, transform=_normalize_hdf5) - # checks - assert isinstance(base_data, GroundTruthData), f'base_data must be an instance of {repr(GroundTruthData.__name__)}, got: {repr(base_data)}' - assert len(base_data) == len(self.hdf5_data), f'length of base_data: {len(base_data)} does not match length of hd5f data: {len(self.hdf5_data)}' - # initialize - super().__init__(base_data=base_data, transform=transform) - - def _get_observation(self, idx): - return self.hdf5_data[idx] diff --git a/research/util/_dataset.py b/research/util/_dataset.py deleted file mode 100644 index 8a30e174..00000000 --- a/research/util/_dataset.py +++ /dev/null @@ -1,457 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import os -import warnings -from typing import List -from typing import Literal -from typing import Optional -from typing import Sequence -from typing import Sized -from typing import Tuple -from typing import Union - -import numpy as np -import torch -import torch.utils.data - -from disent.dataset import DisentDataset -from disent.dataset.data import Cars3dData -from disent.dataset.data import DSpritesData -from disent.dataset.data import DSpritesImagenetData -from disent.dataset.data import GroundTruthData -from disent.dataset.data import Shapes3dData -from disent.dataset.data import SmallNorbData -from disent.dataset.data import XColumnsData -from disent.dataset.data import XYBlocksData -from disent.dataset.data import XYObjectData -from disent.dataset.data import XYSquaresData -from disent.dataset.sampling import BaseDisentSampler -from disent.dataset.sampling import GroundTruthSingleSampler -from disent.dataset.transform import Noop -from disent.dataset.transform import ToImgTensorF32 -from disent.dataset.transform import ToImgTensorU8 - - -# ========================================================================= # -# dataset io # -# ========================================================================= # - - -# TODO: this is much faster! -# -# import psutil -# import multiprocessing as mp -# -# def copy_batch_into(src: GroundTruthData, dst: torch.Tensor, i: int, j: int): -# for k in range(i, min(j, len(dst))): -# dst[k, ...] = src[k] -# return (i, j) -# -# def load_dataset_into_memory( -# gt_data: GroundTruthData, -# workers: int = min(psutil.cpu_count(logical=False), 16), -# ) -> ArrayGroundTruthData: -# # make data and tensors -# tensor = torch.zeros(len(gt_data), *gt_data.obs_shape, dtype=gt_data[0].dtype).share_memory_() -# # compute batch size -# n = len(gt_data) -# batch_size = (n + workers - 1) // workers -# # load in batches -# with mp.Pool(processes=workers) as POOL: -# POOL.starmap( -# copy_batch_into, [ -# (gt_data, tensor, i, i + batch_size) -# for i in range(0, n, batch_size) -# ] -# ) -# # return array -# return ArrayGroundTruthData.new_like(tensor, gt_data, array_chn_is_last=False) - - -def load_dataset_into_memory(gt_data: GroundTruthData, x_shape: Optional[Tuple[int, ...]] = None, batch_size=64, num_workers=min(os.cpu_count(), 16), dtype=torch.float32, raw_array=False): - assert dtype in {torch.float16, torch.float32} - # TODO: this should be part of disent? - from torch.utils.data import DataLoader - from tqdm import tqdm - from disent.dataset.data import ArrayGroundTruthData - # get observation shape - # - manually specify this if the gt_data has a transform applied that resizes the observations for example! - if x_shape is None: - x_shape = gt_data.x_shape - # load dataset into memory manually! - data = torch.zeros(len(gt_data), *x_shape, dtype=dtype) - # load all batches - dataloader = DataLoader(gt_data, batch_size=batch_size, shuffle=False, num_workers=num_workers, drop_last=False) - idx = 0 - for batch in tqdm(dataloader, desc='loading dataset into memory'): - data[idx:idx+len(batch)] = batch.to(dtype) - idx += len(batch) - # done! - if raw_array: - return data - else: - # channels get swapped by the below ToImgTensorF32(), maybe allow `array_chn_is_last` as param - return ArrayGroundTruthData.new_like(array=data, gt_data=gt_data, array_chn_is_last=False) - - -# ========================================================================= # -# dataset # -# ========================================================================= # - - -TransformTypeHint = Union[Literal['uint8'], Literal['float'], Literal['float32'], Literal['none']] - - -def make_data( - name: str = 'xysquares', - factors: bool = False, - data_root: str = 'data/dataset', - try_in_memory: bool = False, - load_into_memory: bool = False, - load_memory_dtype: torch.dtype = torch.float16, - transform_mode: TransformTypeHint = 'float32' -) -> GroundTruthData: - # override values - if load_into_memory and try_in_memory: - warnings.warn('`load_into_memory==True` is incompatible with `try_in_memory==True`, setting `try_in_memory=False`!') - try_in_memory = False - # transform object - TransformCls = { - 'uint8': ToImgTensorU8, - 'float32': ToImgTensorF32, - 'none': Noop, - }[transform_mode] - # make data - if name == 'xysquares': data = XYSquaresData(transform=TransformCls()) # equivalent: [xysquares, xysquares_8x8, xysquares_8x8_s8] - elif name == 'xysquares_1x1': data = XYSquaresData(square_size=1, transform=TransformCls()) - elif name == 'xysquares_2x2': data = XYSquaresData(square_size=2, transform=TransformCls()) - elif name == 'xysquares_4x4': data = XYSquaresData(square_size=4, transform=TransformCls()) - elif name == 'xysquares_8x8': data = XYSquaresData(square_size=8, transform=TransformCls()) # 8x8x8x8x8x8 = 262144 # equivalent: [xysquares, xysquares_8x8, xysquares_8x8_s8] - elif name == 'xysquares_8x8_mini': data = XYSquaresData(square_size=8, grid_spacing=14, transform=TransformCls()) # 5x5x5x5x5x5 = 15625 - # TOY DATASETS - elif name == 'xysquares_8x8_toy': data = XYSquaresData(square_size=8, grid_spacing=8, rgb=False, num_squares=1, transform=TransformCls()) # 8x8 = ? - elif name == 'xysquares_8x8_toy_s1': data = XYSquaresData(square_size=8, grid_spacing=1, rgb=False, num_squares=1, transform=TransformCls()) # ?x? = ? - elif name == 'xysquares_8x8_toy_s2': data = XYSquaresData(square_size=8, grid_spacing=2, rgb=False, num_squares=1, transform=TransformCls()) # ?x? = ? - elif name == 'xysquares_8x8_toy_s4': data = XYSquaresData(square_size=8, grid_spacing=4, rgb=False, num_squares=1, transform=TransformCls()) # ?x? = ? - elif name == 'xysquares_8x8_toy_s8': data = XYSquaresData(square_size=8, grid_spacing=8, rgb=False, num_squares=1, transform=TransformCls()) # 8x8 = ? - # TOY DATASETS ALT - elif name == 'xcolumns_8x_toy': data = XColumnsData(square_size=8, grid_spacing=8, rgb=False, num_squares=1, transform=TransformCls()) # 8 = ? - elif name == 'xcolumns_8x_toy_s1': data = XColumnsData(square_size=8, grid_spacing=1, rgb=False, num_squares=1, transform=TransformCls()) # ? = ? - elif name == 'xcolumns_8x_toy_s2': data = XColumnsData(square_size=8, grid_spacing=2, rgb=False, num_squares=1, transform=TransformCls()) # ? = ? - elif name == 'xcolumns_8x_toy_s4': data = XColumnsData(square_size=8, grid_spacing=4, rgb=False, num_squares=1, transform=TransformCls()) # ? = ? - elif name == 'xcolumns_8x_toy_s8': data = XColumnsData(square_size=8, grid_spacing=8, rgb=False, num_squares=1, transform=TransformCls()) # 8 = ? - # OVERLAPPING DATASETS - elif name == 'xysquares_8x8_s1': data = XYSquaresData(square_size=8, grid_size=8, grid_spacing=1, transform=TransformCls()) # ?x?x?x?x?x? = ? - elif name == 'xysquares_8x8_s2': data = XYSquaresData(square_size=8, grid_size=8, grid_spacing=2, transform=TransformCls()) # ?x?x?x?x?x? = ? - elif name == 'xysquares_8x8_s3': data = XYSquaresData(square_size=8, grid_size=8, grid_spacing=3, transform=TransformCls()) # ?x?x?x?x?x? = ? - elif name == 'xysquares_8x8_s4': data = XYSquaresData(square_size=8, grid_size=8, grid_spacing=4, transform=TransformCls()) # ?x?x?x?x?x? = ? - elif name == 'xysquares_8x8_s5': data = XYSquaresData(square_size=8, grid_size=8, grid_spacing=5, transform=TransformCls()) # ?x?x?x?x?x? = ? - elif name == 'xysquares_8x8_s6': data = XYSquaresData(square_size=8, grid_size=8, grid_spacing=6, transform=TransformCls()) # ?x?x?x?x?x? = ? - elif name == 'xysquares_8x8_s7': data = XYSquaresData(square_size=8, grid_size=8, grid_spacing=7, transform=TransformCls()) # ?x?x?x?x?x? = ? - elif name == 'xysquares_8x8_s8': data = XYSquaresData(square_size=8, grid_size=8, grid_spacing=8, transform=TransformCls()) # 8x8x8x8x8x8 = 262144 # equivalent: [xysquares, xysquares_8x8, xysquares_8x8_s8] - # OTHER SYNTHETIC DATASETS - elif name == 'xyobject': data = XYObjectData(transform=TransformCls()) - elif name == 'xyblocks': data = XYBlocksData(transform=TransformCls()) - # NORMAL DATASETS - elif name == 'cars3d': data = Cars3dData(data_root=data_root, prepare=True, transform=TransformCls(size=64)) - elif name == 'smallnorb': data = SmallNorbData(data_root=data_root, prepare=True, transform=TransformCls(size=64)) - elif name == 'shapes3d': data = Shapes3dData(data_root=data_root, prepare=True, transform=TransformCls(), in_memory=try_in_memory) - elif name == 'dsprites': data = DSpritesData(data_root=data_root, prepare=True, transform=TransformCls(), in_memory=try_in_memory) - # CUSTOM DATASETS - elif name == 'dsprites_imagenet_bg_100': data = DSpritesImagenetData(visibility=100, mode='bg', data_root=data_root, prepare=True, transform=TransformCls(), in_memory=try_in_memory) - elif name == 'dsprites_imagenet_bg_80': data = DSpritesImagenetData(visibility=80, mode='bg', data_root=data_root, prepare=True, transform=TransformCls(), in_memory=try_in_memory) - elif name == 'dsprites_imagenet_bg_60': data = DSpritesImagenetData(visibility=60, mode='bg', data_root=data_root, prepare=True, transform=TransformCls(), in_memory=try_in_memory) - elif name == 'dsprites_imagenet_bg_40': data = DSpritesImagenetData(visibility=40, mode='bg', data_root=data_root, prepare=True, transform=TransformCls(), in_memory=try_in_memory) - elif name == 'dsprites_imagenet_bg_20': data = DSpritesImagenetData(visibility=20, mode='bg', data_root=data_root, prepare=True, transform=TransformCls(), in_memory=try_in_memory) - # --- # - elif name == 'dsprites_imagenet_fg_100': data = DSpritesImagenetData(visibility=100, mode='fg', data_root=data_root, prepare=True, transform=TransformCls(), in_memory=try_in_memory) - elif name == 'dsprites_imagenet_fg_80': data = DSpritesImagenetData(visibility=80, mode='fg', data_root=data_root, prepare=True, transform=TransformCls(), in_memory=try_in_memory) - elif name == 'dsprites_imagenet_fg_60': data = DSpritesImagenetData(visibility=60, mode='fg', data_root=data_root, prepare=True, transform=TransformCls(), in_memory=try_in_memory) - elif name == 'dsprites_imagenet_fg_40': data = DSpritesImagenetData(visibility=40, mode='fg', data_root=data_root, prepare=True, transform=TransformCls(), in_memory=try_in_memory) - elif name == 'dsprites_imagenet_fg_20': data = DSpritesImagenetData(visibility=20, mode='fg', data_root=data_root, prepare=True, transform=TransformCls(), in_memory=try_in_memory) - # DONE - else: raise KeyError(f'invalid data name: {repr(name)}') - # load into memory - if load_into_memory: - old_data, data = data, load_dataset_into_memory(data, dtype=load_memory_dtype, x_shape=(data.img_channels, 64, 64)) - # make dataset - if factors: - raise NotImplementedError('factor returning is not yet implemented in the rewrite! this needs to be fixed!') # TODO! - return data - - -def make_dataset( - name: str = 'xysquares', - factors: bool = False, - data_root: str = 'data/dataset', - try_in_memory: bool = False, - load_into_memory: bool = False, - load_memory_dtype: torch.dtype = torch.float16, - transform_mode: TransformTypeHint = 'float32', - sampler: BaseDisentSampler = None, -) -> DisentDataset: - data = make_data( - name=name, - factors=factors, - data_root=data_root, - try_in_memory=try_in_memory, - load_into_memory=load_into_memory, - load_memory_dtype=load_memory_dtype, - transform_mode=transform_mode, - ) - return DisentDataset( - data, - sampler=GroundTruthSingleSampler() if (sampler is None) else sampler, - return_indices=True - ) - - -def get_single_batch(dataloader, cuda=True): - for batch in dataloader: - (x_targ,) = batch['x_targ'] - break - if cuda: - x_targ = x_targ.cuda() - return x_targ - - -# ========================================================================= # -# sampling helper # -# ========================================================================= # - - -# TODO: clean this up -def sample_factors(gt_data: GroundTruthData, num_obs: int = 1024, factor_mode: str = 'sample_random', factor: Union[int, str] = None): - # sample multiple random factor traversals - if factor_mode == 'sample_traversals': - assert factor is not None, f'factor cannot be None when factor_mode=={repr(factor_mode)}' - # get traversal - f_idx = gt_data.normalise_factor_idx(factor) - # generate traversals - factors = [] - for i in range((num_obs + gt_data.factor_sizes[f_idx] - 1) // gt_data.factor_sizes[f_idx]): - factors.append(gt_data.sample_random_factor_traversal(f_idx=f_idx)) - factors = np.concatenate(factors, axis=0) - elif factor_mode == 'sample_random': - factors = gt_data.sample_factors(num_obs) - else: - raise KeyError - return factors - - -# TODO: move into dataset class -def sample_batch_and_factors(dataset: DisentDataset, num_samples: int, factor_mode: str = 'sample_random', factor: Union[int, str] = None, device=None): - factors = sample_factors(dataset.gt_data, num_obs=num_samples, factor_mode=factor_mode, factor=factor) - batch = dataset.dataset_batch_from_factors(factors, mode='target').to(device=device) - factors = torch.from_numpy(factors).to(dtype=torch.float32, device=device) - return batch, factors - - -# ========================================================================= # -# pair samplers # -# ========================================================================= # - - -def pair_indices_random(max_idx: int, approx_batch_size: Optional[int] = None) -> Tuple[np.ndarray, np.ndarray]: - """ - Generates pairs of indices in corresponding arrays, - returning random permutations - - considers [0, 1] and [1, 0] to be different # TODO: consider them to be the same - - never returns pairs with the same values, eg. [1, 1] - - (default) number of returned values is: `max_idx * sqrt(max_idx) / 2` -- arbitrarily chosen to scale slower than number of combinations - """ - # defaults - if approx_batch_size is None: - approx_batch_size = int(max_idx * (max_idx ** 0.5) / 2) - # sample values - idx_a, idx_b = np.random.randint(0, max_idx, size=(2, approx_batch_size)) - # remove similar - different = (idx_a != idx_b) - idx_a = idx_a[different] - idx_b = idx_b[different] - # return values - return idx_a, idx_b - - -def pair_indices_combinations(max_idx: int) -> Tuple[np.ndarray, np.ndarray]: - """ - Generates pairs of indices in corresponding arrays, - returning all combinations - - considers [0, 1] and [1, 0] to be the same, only returns one of them - - never returns pairs with the same values, eg. [1, 1] - - number of returned values is: `max_idx * (max_idx-1) / 2` - """ - # upper triangle excluding diagonal - # - similar to: `list(itertools.combinations(np.arange(len(t_idxs)), 2))` - idxs_a, idxs_b = np.triu_indices(max_idx, k=1) - return idxs_a, idxs_b - - -def pair_indices_nearby(max_idx: int) -> Tuple[np.ndarray, np.ndarray]: - """ - Generates pairs of indices in corresponding arrays, - returning nearby combinations - - considers [0, 1] and [1, 0] to be the same, only returns one of them - - never returns pairs with the same values, eg. [1, 1] - - number of returned values is: `max_idx` - """ - idxs_a = np.arange(max_idx) # eg. [0 1 2 3 4 5] - idxs_b = np.roll(idxs_a, shift=1, axis=0) # eg. [1 2 3 4 5 0] - return idxs_a, idxs_b - - -_PAIR_INDICES_FNS = { - 'random': pair_indices_random, - 'combinations': pair_indices_combinations, - 'nearby': pair_indices_nearby, -} - - -def pair_indices(max_idx: int, mode: str) -> Tuple[np.ndarray, np.ndarray]: - try: - fn = _PAIR_INDICES_FNS[mode] - except: - raise KeyError(f'invalid mode: {repr(mode)}') - return fn(max_idx=max_idx) - - -# ========================================================================= # -# mask helper # -# ========================================================================= # - - -def make_changed_mask(batch: torch.Tensor, masked=True): - if masked: - mask = torch.zeros_like(batch[0], dtype=torch.bool) - for i in range(len(batch)): - mask |= (batch[0] != batch[i]) - else: - mask = torch.ones_like(batch[0], dtype=torch.bool) - return mask - - -# ========================================================================= # -# dataset indices # -# ========================================================================= # - - -def sample_unique_batch_indices(num_obs: int, num_samples: int) -> np.ndarray: - assert num_obs >= num_samples, 'not enough values to sample' - assert (num_obs - num_samples) / num_obs > 0.5, 'this method might be inefficient' - # get random sample - indices = set() - while len(indices) < num_samples: - indices.update(np.random.randint(low=0, high=num_obs, size=num_samples - len(indices))) - # make sure indices are randomly ordered - indices = np.fromiter(indices, dtype=int) - # indices = np.array(list(indices), dtype=int) - np.random.shuffle(indices) - # return values - return indices - - -def generate_epoch_batch_idxs(num_obs: int, num_batches: int, mode: str = 'shuffle') -> List[np.ndarray]: - """ - Generate `num_batches` batches of indices. - - Each index is in the range [0, num_obs). - - If num_obs is not divisible by num_batches, then batches may not all be the same size. - - eg. [0, 1, 2, 3, 4] -> [[0, 1], [2, 3], [4]] -- num_obs=5, num_batches=3, sample_mode='range' - eg. [0, 1, 2, 3, 4] -> [[1, 4], [2, 0], [3]] -- num_obs=5, num_batches=3, sample_mode='shuffle' - eg. [0, 1, 0, 3, 2] -> [[0, 1], [0, 3], [2]] -- num_obs=5, num_batches=3, sample_mode='random' - """ - # generate indices - if mode == 'range': - idxs = np.arange(num_obs) - elif mode == 'shuffle': - idxs = np.arange(num_obs) - np.random.shuffle(idxs) - elif mode == 'random': - idxs = np.random.randint(0, num_obs, size=(num_obs,)) - else: - raise KeyError(f'invalid mode={repr(mode)}') - # return batches - return np.array_split(idxs, num_batches) - - -def generate_epochs_batch_idxs(num_obs: int, num_epochs: int, num_epoch_batches: int, mode: str = 'shuffle') -> List[np.ndarray]: - """ - Like generate_epoch_batch_idxs, but concatenate the batches of calling the function `num_epochs` times. - - The total number of batches returned is: `num_epochs * num_epoch_batches` - """ - batches = [] - for i in range(num_epochs): - batches.extend(generate_epoch_batch_idxs(num_obs=num_obs, num_batches=num_epoch_batches, mode=mode)) - return batches - - -# ========================================================================= # -# Dataloader Sampler Utilities # -# ========================================================================= # - - -class StochasticSampler(torch.utils.data.Sampler): - """ - Sample random batches, not guaranteed to be unique or cover the entire dataset in one epoch! - """ - - def __init__(self, data_source: Union[Sized, int], batch_size: int = 128): - super().__init__(data_source) - if isinstance(data_source, int): - self._len = data_source - else: - self._len = len(data_source) - self._batch_size = batch_size - assert isinstance(self._len, int) - assert self._len > 0 - assert isinstance(self._batch_size, int) - assert self._batch_size > 0 - - def __iter__(self): - while True: - yield from np.random.randint(0, self._len, size=self._batch_size) - - -def yield_dataloader(dataloader: torch.utils.data.DataLoader, steps: int): - i = 0 - while True: - for it in dataloader: - yield it - i += 1 - if i >= steps: - return - - -def StochasticBatchSampler(data_source: Union[Sized, int], batch_size: int): - return torch.utils.data.BatchSampler( - sampler=StochasticSampler(data_source=data_source, batch_size=batch_size), - batch_size=batch_size, - drop_last=True - ) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/util/_fn_util.py b/research/util/_fn_util.py deleted file mode 100644 index 471bd3fe..00000000 --- a/research/util/_fn_util.py +++ /dev/null @@ -1,114 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import inspect -from typing import Sequence - -from disent.util.deprecate import deprecated - - -# ========================================================================= # -# Function Arguments # -# ========================================================================= # - - -def _get_fn_from_stack(fn_name: str, stack): - # -- do we actually need all of this? - fn = None - for s in stack: - if fn_name in s.frame.f_locals: - fn = s.frame.f_locals[fn_name] - break - if fn is None: - raise RuntimeError(f'could not retrieve function: {repr(fn_name)} from call stack.') - return fn - - -@deprecated('function uses bad mechanics, see commented implementation below') -def get_caller_params(sort: bool = False, exclude: Sequence[str] = None) -> dict: - stack = inspect.stack() - fn_name = stack[1].function - fn_locals = stack[1].frame.f_locals - # get function and params - fn = _get_fn_from_stack(fn_name, stack) - fn_params = inspect.getfullargspec(fn).args - # check excluded - exclude = set() if (exclude is None) else set(exclude) - fn_params = [p for p in fn_params if (p not in exclude)] - # sort values - if sort: - fn_params = sorted(fn_params) - # return dict - return { - k: fn_locals[k] for k in fn_params - } - - -def params_as_string(params: dict, sep: str = '_', names: bool = False): - # get strings - if names: - return sep.join(f"{k}={v}" for k, v in params.items()) - else: - return sep.join(f"{v}" for k, v in params.items()) - - -# ========================================================================= # -# END # -# ========================================================================= # - - -# TODO: replace function above -# -# class DELETED(object): -# def __str__(self): return '' -# def __repr__(self): return str(self) -# -# -# DELETED = DELETED() -# -# -# def get_hparams(exclude: Union[Sequence[str], Set[str]] = None): -# # check values -# if exclude is None: -# exclude = {} -# else: -# exclude = set(exclude) -# # get frame and values -# args = inspect.getargvalues(frame=inspect.currentframe().f_back) -# # sort values -# arg_names = list(args.args) -# if args.varargs is not None: arg_names.append(args.varargs) -# if args.keywords is not None: arg_names.append(args.keywords) -# # filter values -# from argparse import Namespace -# return Namespace(**{ -# name: args.locals.get(name, DELETED) -# for name in arg_names -# if (name not in exclude) -# }) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/util/_io_util.py b/research/util/_io_util.py deleted file mode 100644 index 3c5e8ca8..00000000 --- a/research/util/_io_util.py +++ /dev/null @@ -1,239 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import base64 -import dataclasses -import inspect -import io -import os -from typing import Optional -from typing import Union - -import torch - -from disent.util.inout.paths import ensure_parent_dir_exists - - -# ========================================================================= # -# Github Upload Utility Functions # -# ========================================================================= # - - -def gh_get_repo(repo: str = None): - from github import Github - # get token str - token = os.environ.get('GITHUB_TOKEN', '') - if not token.strip(): - raise ValueError('`GITHUB_TOKEN` env variable has not been set!') - assert isinstance(token, str) - # get repo str - if repo is None: - repo = os.environ.get('GITHUB_REPO', '') - if not repo.strip(): - raise ValueError('`GITHUB_REPO` env variable has not been set!') - assert isinstance(repo, str) - # get repo - return Github(token).get_repo(repo) - - -def gh_get_branch(repo: 'Repository', branch: str = None, source_branch: str = None, allow_new_branch: bool = True) -> 'Branch': - from github import GithubException - # check branch - assert isinstance(branch, str) or (branch is None) - assert isinstance(source_branch, str) or (source_branch is None) - # get default branch - if branch is None: - branch = repo.default_branch - # retrieve branch - try: - return repo.get_branch(branch) - except GithubException as e: - if not allow_new_branch: - raise RuntimeError(f'Creating branch disabled, set `allow_new_branch=True`: {repr(branch)}') - print(f'Creating missing branch: {repr(branch)}') - sb = repo.get_branch(repo.default_branch if (source_branch is None) else source_branch) - repo.create_git_ref(ref='refs/heads/' + branch, sha=sb.commit.sha) - return repo.get_branch(branch) - - -@dataclasses.dataclass -class WriteResult: - commit: 'Commit' - content: 'ContentFile' - - -def gh_write_file(repo: 'Repository', path: str, content: Union[str, bytes], branch: str = None, allow_new_file=True, allow_overwrite_file=False, allow_new_branch=True) -> WriteResult: - from github import UnknownObjectException - # get branch - branch = gh_get_branch(repo, branch, allow_new_branch=allow_new_branch).name - # check that the file exists - try: - sha = repo.get_contents(path, ref=branch).sha - except UnknownObjectException: - sha = None - # handle file exists or not - if sha is None: - if not allow_new_file: - raise RuntimeError(f'Creating file disabled, set `allow_new_file=True`: {repr(path)}') - result = repo.create_file(path=path, message=f'Created File: {path}', content=content, branch=branch) - else: - if not allow_overwrite_file: - raise RuntimeError(f'Overwriting file disabled, `set allow_overwrite_file=True`: {repr(path)}') - result = repo.update_file(path=path, message=f'Updated File: {path}', content=content, branch=branch, sha=sha) - # result is a dict: {'commit': github.Commit, 'content': github.ContentFile} - return WriteResult(**result) - - -# ========================================================================= # -# Github Upload Utility Class # -# ========================================================================= # - - -class GithubWriter(object): - - def __init__(self, repo: str = None, branch: str = None, allow_new_file=True, allow_overwrite_file=True, allow_new_branch=True): - self._kwargs = dict( - repo=gh_get_repo(repo=repo), - branch=branch, - allow_new_file=allow_new_file, - allow_overwrite_file=allow_overwrite_file, - allow_new_branch=allow_new_branch, - ) - - def write_file(self, path: str, content: Union[str, bytes]): - return gh_write_file( - path=path, - content=content, - **self._kwargs, - ) - - -# ========================================================================= # -# Torch Save Utils # -# ========================================================================= # - - -def torch_save_bytes(model) -> bytes: - buffer = io.BytesIO() - torch.save(model, buffer) - buffer.seek(0) - return buffer.read() - - -def torch_save_base64(model) -> str: - b = torch_save_bytes(model) - return base64.b64encode(b).decode('ascii') - - -def torch_load_bytes(b: bytes): - return torch.load(io.BytesIO(b)) - - -def torch_load_base64(s: str): - b = base64.b64decode(s.encode('ascii')) - return torch_load_bytes(b) - - -# ========================================================================= # -# write # -# ========================================================================= # - - -def _split_special_path(path): - if path.startswith('github:'): - # get github repo and path - path = path[len('github:'):] - repo, path = os.path.join(*path.split('/')[:2]), os.path.join(*path.split('/')[2:]) - # check paths - assert repo.strip() and len(repo.split('/')) == 2 - assert path.strip() and len(repo.split('/')) >= 1 - # return components - return 'github', (repo, path) - else: - return 'local', path - - -def torch_write(path: str, model): - path_type, path = _split_special_path(path) - # handle cases - if path_type == 'github': - path, repo = path - # get the name of the path - ghw = GithubWriter(repo) - ghw.write_file(path=path, content=torch_save_bytes(model)) - print(f'Saved in repo: {repr(path)} to file: {repr(repo)}') - elif path_type == 'local': - torch.save(model, ensure_parent_dir_exists(path)) - print(f'Saved to file: {repr(path)}') - else: - raise KeyError(f'unknown path type: {repr(path_type)}') - - -# ========================================================================= # -# Files # -# ========================================================================= # - - -def _make_rel_path(*path_segments, is_file=True, _calldepth=0): - assert not os.path.isabs(os.path.join(*path_segments)), 'path must be relative' - # get source - stack = inspect.stack() - module = inspect.getmodule(stack[_calldepth+1].frame) - reldir = os.path.dirname(module.__file__) - # make everything - path = os.path.join(reldir, *path_segments) - folder_path = os.path.dirname(path) if is_file else path - os.makedirs(folder_path, exist_ok=True) - return path - - -def _make_rel_path_add_ext(*path_segments, ext='.png', _calldepth=0): - # make path - path = _make_rel_path(*path_segments, is_file=True, _calldepth=_calldepth+1) - if not os.path.splitext(path)[1]: - path = f'{path}{ext}' - return path - - -def make_rel_path(*path_segments, is_file=True): - return _make_rel_path(*path_segments, is_file=is_file, _calldepth=1) - - -def make_rel_path_add_ext(*path_segments, ext='.png'): - return _make_rel_path_add_ext(*path_segments, ext=ext, _calldepth=1) - - -def plt_rel_path_savefig(rel_path: Optional[str], save: bool = True, show: bool = True, ext='.png', dpi: Optional[int] = None): - import matplotlib.pyplot as plt - if save and (rel_path is not None): - path = _make_rel_path_add_ext(rel_path, ext=ext, _calldepth=2) - plt.savefig(path, dpi=dpi) - print(f'saved: {repr(path)}') - if show: - plt.show() - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/util/_loss.py b/research/util/_loss.py deleted file mode 100644 index 50cbc1ab..00000000 --- a/research/util/_loss.py +++ /dev/null @@ -1,160 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import inspect -import warnings -from typing import Optional -from typing import Sequence - -import numpy as np -import torch -from torch.nn import functional as F - -from disent import registry -from disent.nn.loss.reduction import batch_loss_reduction - - -# ========================================================================= # -# optimizer # -# ========================================================================= # - - -_SPECIALIZATIONS = {'sgd_m': ('sgd', dict(momentum=0.1))} - - -def make_optimizer(model: torch.nn.Module, name: str = 'sgd', lr=1e-3, weight_decay: Optional[float] = None): - if isinstance(model, torch.nn.Module): - params = model.parameters() - elif isinstance(model, torch.Tensor): - assert model.requires_grad - params = [model] - else: - raise TypeError(f'cannot optimize type: {type(model)}') - # get specializations - kwargs = {} - if name in _SPECIALIZATIONS: - name, kwargs = _SPECIALIZATIONS[name] - # get optimizer class - optimizer_cls = registry.OPTIMIZERS[name] - optimizer_params = set(inspect.signature(optimizer_cls).parameters.keys()) - # add optional arguments - if weight_decay is not None: - if 'weight_decay' in optimizer_params: - kwargs['weight_decay'] = weight_decay - else: - warnings.warn(f'{name}: weight decay cannot be set, optimizer does not have `weight_decay` parameter') - # instantiate - return optimizer_cls(params, lr=lr, **kwargs) - - -def step_optimizer(optimizer, loss): - optimizer.zero_grad() - loss.backward() - optimizer.step() - - -# ========================================================================= # -# Loss # -# ========================================================================= # - - -def _unreduced_mse_loss(pred: torch.Tensor, targ: torch.Tensor) -> torch.Tensor: - return F.mse_loss(pred, targ, reduction='none') - - -def _unreduced_mae_loss(pred: torch.Tensor, targ: torch.Tensor) -> torch.Tensor: - return torch.abs(pred - targ) - - -def unreduced_loss(pred: torch.Tensor, targ: torch.Tensor, mode='mse') -> torch.Tensor: - return _LOSS_FNS[mode](pred, targ) - - -_LOSS_FNS = { - 'mse': _unreduced_mse_loss, - 'mae': _unreduced_mae_loss, -} - - -# ========================================================================= # -# Pairwise Loss # -# ========================================================================= # - - -def pairwise_loss(pred: torch.Tensor, targ: torch.Tensor, mode='mse', mean_dtype=None, mask: Optional[torch.Tensor] = None) -> torch.Tensor: - # check input - assert pred.shape == targ.shape - # mean over final dims - loss = unreduced_loss(pred=pred, targ=targ, mode=mode) - # mask values - if mask is not None: - loss *= mask - # reduce - loss = batch_loss_reduction(loss, reduction_dtype=mean_dtype, reduction='mean') - # check result - assert loss.shape == pred.shape[:1] - # done - return loss - - -def unreduced_overlap(pred: torch.Tensor, targ: torch.Tensor, mode='mse') -> torch.Tensor: - # -ve loss - return - unreduced_loss(pred=pred, targ=targ, mode=mode) - - -def pairwise_overlap(pred: torch.Tensor, targ: torch.Tensor, mode='mse', mean_dtype=None) -> torch.Tensor: - # -ve loss - return - pairwise_loss(pred=pred, targ=targ, mode=mode, mean_dtype=mean_dtype) - - -# ========================================================================= # -# Factor Distances # -# ========================================================================= # - - -def np_factor_dists( - factors_a: np.ndarray, - factors_b: np.ndarray, - factor_sizes: Optional[Sequence[int]] = None, - circular_if_factor_sizes: bool = True, - p: int = 1, -) -> np.ndarray: - assert factors_a.ndim == 2 - assert factors_a.shape == factors_b.shape - # compute factor distances - fdists = np.abs(factors_a - factors_b) # (NUM, FACTOR_SIZE) - # circular distance - if (factor_sizes is not None) and circular_if_factor_sizes: - M = np.array(factor_sizes)[None, :] # (FACTOR_SIZE,) -> (1, FACTOR_SIZE) - assert M.shape == (1, factors_a.shape[-1]) - fdists = np.where(fdists > (M // 2), M - fdists, fdists) # (NUM, FACTOR_SIZE) - # compute final dists - fdists = (fdists ** p).sum(axis=-1) ** (1 / p) - # return values - return fdists # (NUM,) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/wandb_cli.py b/research/wandb_cli.py deleted file mode 100644 index a97dce8a..00000000 --- a/research/wandb_cli.py +++ /dev/null @@ -1,31 +0,0 @@ - -""" -This file is an alias to the wandb cli that first sets the -temporary directory to a different folder `/tmp//tmp`, -in case `/tmp` has been polluted or you don't have the correct -access rights to modify files. - -- I am not sure why we need to do this, it is probably a bug with - wandb (or even tempfile) not respecting the `TMPDIR`, `TEMP` and - `TMP` environment variables which when set should do the same as - below? According to the tempdir docs: - https://docs.python.org/3/library/tempfile.html#tempfile.gettempdir -""" - -# wandb_cli.py -if __name__ == '__main__': - import os - import tempfile - - # generate the temporary directory from the user - temp_dir = f'/tmp/{os.environ["USER"]}/wandb' - print(f'[PATCHING:] tempfile.tempdir={repr(temp_dir)}') - - # we need to patch tempdir before we can import wandb - assert tempfile.tempdir is None - os.makedirs(temp_dir, exist_ok=True) - tempfile.tempdir = temp_dir - - # taken from wandb.__main__ - from wandb.cli.cli import cli - cli(prog_name="python -m wandb") diff --git a/research/working-batch.sh b/research/working-batch.sh deleted file mode 100644 index 53855652..00000000 --- a/research/working-batch.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export PROJECT="N/A" -export USERNAME="N/A" -export PARTITION="batch" -export PARALLELISM=24 - -# source the helper file -source "$(dirname "$(realpath -s "$0")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cuda_nodes "$PARTITION" 43200 "W-disent" # 12 hours diff --git a/research/working-stampede.sh b/research/working-stampede.sh deleted file mode 100644 index c4d7170b..00000000 --- a/research/working-stampede.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export PROJECT="N/A" -export USERNAME="N/A" -export PARTITION="stampede" -export PARALLELISM=24 - -# source the helper file -source "$(dirname "$(realpath -s "$0")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cuda_nodes "$PARTITION" 43200 "W-disent" # 12 hours diff --git a/tests/test_data_similarity.py b/tests/test_data_similarity.py index a22c387b..33169471 100644 --- a/tests/test_data_similarity.py +++ b/tests/test_data_similarity.py @@ -26,8 +26,6 @@ from disent.dataset.data import XYObjectData from disent.dataset.data import XYObjectShadedData -from disent.dataset.data import XYSquaresData # pragma: delete-on-release -from disent.dataset.data import XYSquaresMinimalData # pragma: delete-on-release # ========================================================================= # @@ -35,18 +33,6 @@ # ========================================================================= # -def test_xysquares_similarity(): # pragma: delete-on-release - data_org = XYSquaresData() # pragma: delete-on-release - data_min = XYSquaresMinimalData() # pragma: delete-on-release - # check lengths # pragma: delete-on-release - assert len(data_org) == len(data_min) # pragma: delete-on-release - n = len(data_min) # pragma: delete-on-release - # check items # pragma: delete-on-release - for i in np.random.randint(0, n, size=100): # pragma: delete-on-release - assert np.allclose(data_org[i], data_min[i]) # pragma: delete-on-release - # check bounds # pragma: delete-on-release - assert np.allclose(data_org[0], data_min[0]) # pragma: delete-on-release - assert np.allclose(data_org[n-1], data_min[n-1]) # pragma: delete-on-release def test_xyobject_similarity(): diff --git a/tests/test_frameworks.py b/tests/test_frameworks.py index 438150d1..91c1750c 100644 --- a/tests/test_frameworks.py +++ b/tests/test_frameworks.py @@ -35,9 +35,7 @@ from disent.dataset.sampling import GroundTruthPairSampler from disent.dataset.sampling import GroundTruthTripleSampler from disent.frameworks.ae import * -from disent.frameworks.ae.experimental import * # pragma: delete-on-release from disent.frameworks.vae import * -from disent.frameworks.vae.experimental import * # pragma: delete-on-release from disent.model import AutoEncoder from disent.model.ae import DecoderLinear from disent.model.ae import EncoderLinear @@ -52,16 +50,10 @@ @pytest.mark.parametrize(['Framework', 'cfg_kwargs', 'Data'], [ # AE - unsupervised (Ae, dict(), XYObjectData), - # AE - unsupervised - EXP # pragma: delete-on-release - (DataOverlapTripletAe, dict(overlap_mine_triplet_mode='hard_neg'), XYObjectData), # pragma: delete-on-release # AE - weakly supervised # - # AE - weakly supervised - EXP # pragma: delete-on-release - (AdaAe, dict(), XYObjectData), # pragma: delete-on-release # AE - supervised (TripletAe, dict(), XYObjectData), - # AE - supervised - EXP # pragma: delete-on-release - (AdaNegTripletAe, dict(), XYObjectData), # pragma: delete-on-release # VAE - unsupervised (Vae, dict(), XYObjectData), (BetaVae, dict(), XYObjectData), @@ -71,34 +63,13 @@ (DfcVae, dict(), XYObjectData), (DfcVae, dict(), partial(XYObjectData, rgb=False)), (BetaTcVae, dict(), XYObjectData), - # VAE - unsupervised - EXP # pragma: delete-on-release - (DataOverlapTripletVae,dict(overlap_mine_triplet_mode='none'), XYObjectData), # pragma: delete-on-release - (DataOverlapTripletVae,dict(overlap_mine_triplet_mode='semi_hard_neg'), XYObjectData), # pragma: delete-on-release - (DataOverlapTripletVae,dict(overlap_mine_triplet_mode='hard_neg'), XYObjectData), # pragma: delete-on-release - (DataOverlapTripletVae,dict(overlap_mine_triplet_mode='hard_pos'), XYObjectData), # pragma: delete-on-release - (DataOverlapTripletVae,dict(overlap_mine_triplet_mode='easy_pos'), XYObjectData), # pragma: delete-on-release - (DataOverlapRankVae, dict(), XYObjectData), # pragma: delete-on-release # VAE - weakly supervised (AdaVae, dict(), XYObjectData), (AdaVae, dict(ada_average_mode='ml-vae'), XYObjectData), (AdaGVaeMinimal, dict(), XYObjectData), - # VAE - weakly supervised - EXP # pragma: delete-on-release - (SwappedTargetAdaVae, dict(swap_chance=1.0), XYObjectData), # pragma: delete-on-release - (SwappedTargetBetaVae, dict(swap_chance=1.0), XYObjectData), # pragma: delete-on-release - (AugPosTripletVae, dict(), XYObjectData), # pragma: delete-on-release # VAE - supervised (TripletVae, dict(), XYObjectData), (TripletVae, dict(disable_decoder=True, disable_reg_loss=True, disable_posterior_scale=0.5), XYObjectData), - # VAE - supervised - EXP # pragma: delete-on-release - (BoundedAdaVae, dict(), XYObjectData), # pragma: delete-on-release - (GuidedAdaVae, dict(), XYObjectData), # pragma: delete-on-release - (GuidedAdaVae, dict(gada_anchor_ave_mode='thresh'), XYObjectData), # pragma: delete-on-release - (TripletBoundedAdaVae, dict(), XYObjectData), # pragma: delete-on-release - (TripletGuidedAdaVae, dict(), XYObjectData), # pragma: delete-on-release - (AdaTripletVae, dict(), XYObjectData), # pragma: delete-on-release - (AdaAveTripletVae, dict(adat_share_mask_mode='posterior'), XYObjectData), # pragma: delete-on-release - (AdaAveTripletVae, dict(adat_share_mask_mode='sample'), XYObjectData), # pragma: delete-on-release - (AdaAveTripletVae, dict(adat_share_mask_mode='sample_each'), XYObjectData), # pragma: delete-on-release ]) def test_frameworks(Framework, cfg_kwargs, Data): DataSampler = { diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 5d67b4a0..92b0a9bc 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -43,8 +43,6 @@ wrapped_partial(metric_dci, num_train=7, num_test=7), wrapped_partial(metric_sap, num_train=7, num_test=7), wrapped_partial(metric_factor_vae, num_train=7, num_eval=7, num_variance_estimate=7), - wrapped_partial(metric_flatness, factor_repeats=7), # pragma: delete-on-release - wrapped_partial(metric_flatness_components, factor_repeats=7), # pragma: delete-on-release ]) def test_metrics(metric_fn): z_size = 8 diff --git a/tests/test_registry.py b/tests/test_registry.py index 399c1f62..c173d969 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -44,17 +44,6 @@ } -COUNTS = { # pragma: delete-on-release - 'DATASETS': 10, # pragma: delete-on-release - 'SAMPLERS': 8, # pragma: delete-on-release - 'FRAMEWORKS': 25, # pragma: delete-on-release - 'RECON_LOSSES': 6, # pragma: delete-on-release - 'LATENT_DISTS': 2, # pragma: delete-on-release - 'OPTIMIZERS': 30, # pragma: delete-on-release - 'METRICS': 7, # pragma: delete-on-release - 'SCHEDULES': 5, # pragma: delete-on-release - 'MODELS': 8, # pragma: delete-on-release -} # pragma: delete-on-release def test_registry_loading():