From 6ed30c8d8199e1d31e1ec50325f3041898cd23f5 Mon Sep 17 00:00:00 2001 From: Nathan Michlo Date: Thu, 31 Mar 2022 10:56:05 +0200 Subject: [PATCH] run prepare_release.sh --- disent/dataset/sampling/_groundtruth__dist.py | 2 - disent/dataset/util/stats.py | 15 - prepare_release.sh | 57 - prepare_release_and_commit.sh | 36 - requirements-research-freeze.txt | 121 -- requirements-research.txt | 22 - research/__init__.py | 23 - research/code/__init__.py | 81 -- research/code/dataset/__init__.py | 23 - research/code/dataset/data/__init__.py | 33 - .../data/_groundtruth__dsprites_imagenet.py | 365 ----- .../dataset/data/_groundtruth__xcolumns.py | 66 - .../dataset/data/_groundtruth__xyblocks.py | 160 --- .../dataset/data/_groundtruth__xysquares.py | 260 ---- research/code/dataset/transform/__init__.py | 23 - research/code/dataset/transform/_augment.py | 91 -- research/code/frameworks/__init__.py | 23 - research/code/frameworks/ae/__init__.py | 32 - .../frameworks/ae/_supervised__adaneg_tae.py | 71 - .../frameworks/ae/_unsupervised__dotae.py | 76 - .../frameworks/ae/_weaklysupervised__adaae.py | 81 -- research/code/frameworks/vae/__init__.py | 41 - .../vae/_supervised__adaave_tvae.py | 120 -- .../vae/_supervised__adaneg_tvae.py | 120 -- .../frameworks/vae/_supervised__adatvae.py | 328 ----- .../frameworks/vae/_supervised__badavae.py | 123 -- .../frameworks/vae/_supervised__gadavae.py | 102 -- .../frameworks/vae/_supervised__tbadavae.py | 51 - .../frameworks/vae/_supervised__tgadavae.py | 51 - .../frameworks/vae/_unsupervised__dorvae.py | 169 --- .../frameworks/vae/_unsupervised__dotvae.py | 229 --- .../vae/_weaklysupervised__augpostriplet.py | 82 -- .../vae/_weaklysupervised__st_adavae.py | 63 - .../vae/_weaklysupervised__st_betavae.py | 63 - research/code/metrics/__init__.py | 29 - research/code/metrics/_factored_components.py | 696 --------- research/code/metrics/_flatness.py | 359 ----- research/code/util/__init__.py | 15 - research/code/util/_data.py | 82 -- research/code/util/_dataset.py | 502 ------- research/code/util/_fn_util.py | 114 -- research/code/util/_io_util.py | 239 ---- research/code/util/_loss.py | 160 --- research/code/util/_wandb_plots.py | 165 --- research/code/util/gadfly.mplstyle | 627 --------- research/config/README.md | 10 - research/config/config.yaml | 45 - research/config/config_test.yaml | 45 - .../dataset/X--adv-cars3d--WARNING.yaml | 20 - .../dataset/X--adv-dsprites--WARNING.yaml | 20 - .../dataset/X--adv-shapes3d--WARNING.yaml | 20 - .../dataset/X--adv-smallnorb--WARNING.yaml | 20 - .../dataset/X--dsprites-imagenet-bg-100.yaml | 22 - .../dataset/X--dsprites-imagenet-bg-20.yaml | 22 - .../dataset/X--dsprites-imagenet-bg-25.yaml | 22 - .../dataset/X--dsprites-imagenet-bg-40.yaml | 22 - .../dataset/X--dsprites-imagenet-bg-50.yaml | 22 - .../dataset/X--dsprites-imagenet-bg-60.yaml | 22 - .../dataset/X--dsprites-imagenet-bg-75.yaml | 22 - .../dataset/X--dsprites-imagenet-bg-80.yaml | 22 - .../dataset/X--dsprites-imagenet-fg-100.yaml | 22 - .../dataset/X--dsprites-imagenet-fg-20.yaml | 22 - .../dataset/X--dsprites-imagenet-fg-25.yaml | 22 - .../dataset/X--dsprites-imagenet-fg-40.yaml | 22 - .../dataset/X--dsprites-imagenet-fg-50.yaml | 22 - .../dataset/X--dsprites-imagenet-fg-60.yaml | 22 - .../dataset/X--dsprites-imagenet-fg-75.yaml | 22 - .../dataset/X--dsprites-imagenet-fg-80.yaml | 22 - .../config/dataset/X--dsprites-imagenet.yaml | 81 -- .../config/dataset/X--mask-adv-f-cars3d.yaml | 27 - .../dataset/X--mask-adv-f-dsprites.yaml | 28 - .../dataset/X--mask-adv-f-shapes3d.yaml | 28 - .../dataset/X--mask-adv-f-smallnorb.yaml | 28 - .../config/dataset/X--mask-adv-r-cars3d.yaml | 27 - .../dataset/X--mask-adv-r-dsprites.yaml | 28 - .../dataset/X--mask-adv-r-shapes3d.yaml | 28 - .../dataset/X--mask-adv-r-smallnorb.yaml | 28 - .../config/dataset/X--mask-dthr-cars3d.yaml | 23 - .../config/dataset/X--mask-dthr-dsprites.yaml | 24 - .../config/dataset/X--mask-dthr-shapes3d.yaml | 24 - .../dataset/X--mask-dthr-smallnorb.yaml | 24 - .../config/dataset/X--mask-ran-cars3d.yaml | 27 - .../config/dataset/X--mask-ran-dsprites.yaml | 28 - .../config/dataset/X--mask-ran-shapes3d.yaml | 28 - .../config/dataset/X--mask-ran-smallnorb.yaml | 28 - .../config/dataset/X--monte_rollouts.yaml | 21 - research/config/dataset/X--xyblocks.yaml | 18 - research/config/dataset/X--xyblocks_grey.yaml | 18 - research/config/dataset/X--xysquares.yaml | 17 - .../config/dataset/X--xysquares_grey.yaml | 23 - research/config/dataset/X--xysquares_rgb.yaml | 23 - research/config/framework/X--adaae.yaml | 19 - research/config/framework/X--adaae_os.yaml | 19 - research/config/framework/X--adaavetvae.yaml | 44 - research/config/framework/X--adanegtae.yaml | 27 - research/config/framework/X--adanegtvae.yaml | 36 - research/config/framework/X--adatvae.yaml | 41 - .../config/framework/X--augpos_tvae_os.yaml | 45 - research/config/framework/X--badavae.yaml | 26 - research/config/framework/X--dorvae.yaml | 37 - research/config/framework/X--dorvae_aug.yaml | 42 - research/config/framework/X--dotae.yaml | 35 - research/config/framework/X--dotvae.yaml | 44 - research/config/framework/X--dotvae_aug.yaml | 69 - research/config/framework/X--gadavae.yaml | 28 - research/config/framework/X--st-adavae.yaml | 28 - research/config/framework/X--st-betavae.yaml | 24 - research/config/framework/X--tbadavae.yaml | 32 - research/config/framework/X--tgadavae.yaml | 34 - research/config/metrics/all.yaml | 18 - research/config/metrics/fast.yaml | 12 - research/config/metrics/test.yaml | 21 - research/config/run_location/cluster.yaml | 32 - research/config/run_location/griffin.yaml | 28 - research/config/run_location/heartofgold.yaml | 28 - .../run_location/stampede_rsync_tmp.yaml | 32 - .../config/run_location/stampede_shr.yaml | 32 - .../config/run_location/stampede_tmp.yaml | 32 - research/config/run_plugins/default.yaml | 7 - .../config/schedule/OLD_adavae_down_all.yaml | 27 - .../schedule/OLD_adavae_down_ratio.yaml | 21 - .../schedule/OLD_adavae_down_thresh.yaml | 9 - .../config/schedule/OLD_adavae_up_all.yaml | 27 - .../schedule/OLD_adavae_up_all_full.yaml | 27 - .../config/schedule/OLD_adavae_up_ratio.yaml | 21 - .../schedule/OLD_adavae_up_ratio_full.yaml | 21 - .../config/schedule/OLD_adavae_up_thresh.yaml | 9 - .../config/schedule/adanegtvae_up_all.yaml | 21 - .../schedule/adanegtvae_up_all_full.yaml | 21 - .../schedule/adanegtvae_up_all_weak.yaml | 21 - .../config/schedule/adanegtvae_up_ratio.yaml | 16 - .../schedule/adanegtvae_up_ratio_full.yaml | 16 - .../schedule/adanegtvae_up_ratio_weak.yaml | 16 - .../config/schedule/adanegtvae_up_thresh.yaml | 21 - .../e01_hparam_tuning/submit_param_tuning.sh | 98 -- .../run_basic_checks.py | 72 - .../submit_incr_overlap.sh | 127 -- .../submit_overlap_loss.sh | 125 -- .../animations/.gitignore | 3 - .../plot01_data_traversal/plots/.gitignore | 2 - .../run_01_all_shared_data_prepare.sh | 79 -- .../run_02_plot_data_overlap.py | 185 --- .../run_02_plot_traversals.py | 284 ---- .../run_03_plot_dataset_animation.py | 76 - .../plot02_data_distances/cache/.gitignore | 1 - .../deprecated/run_01_x_z_recon_dists.sh | 38 - .../plot02_data_distances/plots/.gitignore | 2 - .../run_data_correlation.py | 815 ----------- .../run_plot_global_dists.py | 479 ------- .../run_plot_traversal_dists.py | 683 --------- .../util_compute_traversal_dist_pairs.py | 274 ---- .../util_compute_traversal_dists.py | 302 ---- .../plot03_wandb/plot_p01_all_experiments.py | 374 ----- .../plot03_wandb/plots/.gitignore | 2 - .../submit_beta_data_latent_correlation.sh | 49 - .../OLD/submit_01_triplet_hparam_sweep.sh | 85 -- .../OLD/submit_02_check_vae_equivalence.sh | 75 - .../submit_01_triplet_param_tuning.sh | 139 -- .../e02_axis_triplet/OLD/submit_01.sh | 63 - .../e02_axis_triplet/OLD/submit_02.sh | 48 - .../e02_axis_triplet/OLD/submit_03.sh | 78 -- .../e02_axis_triplet/OLD/submit_04.sh | 119 -- .../e02_axis_triplet/OLD/submit_05.sh | 57 - .../submit_01_axis_aligned_triplet.sh | 176 --- .../e03_unsupervised_triplet/OLD/submit_01.sh | 62 - .../e03_unsupervised_triplet/OLD/submit_02.sh | 66 - .../OLD/submit_03_test_softada_vs_ada.sh | 65 - ...1_MSC-p02e03_unsupervised-axis-triplet.txt | 100 -- ...C-p02e03_unsupervised-axis-triplet_ALT.txt | 25 - .../submit_01_unsupervised_axis_triplet.sh | 217 --- .../plot00_all/make_graphs.py | 436 ------ .../plot00_all/plot_p02_wandb_experiments.py | 1240 ----------------- .../plot00_all/plots/.gitignore | 2 - .../config/config_adversarial_dataset.yaml | 60 - .../config_adversarial_dataset_approx.yaml | 121 -- .../deprecated/run_01_gen_adversarial_disk.py | 497 ------- .../deprecated/run_02_adv_dataset.sh | 13 - .../run_02_gen_adversarial_dataset.py | 425 ------ .../deprecated/run_03_check.py | 88 -- .../deprecated/run_04_gen_adversarial_ruck.py | 585 -------- .../run_04_gen_adversarial_ruck_dist_pairs.py | 601 -------- .../submit_02_train_adversarial_data.sh | 58 - .../submit_04_train_dsprites_imagenet.sh | 41 - .../deprecated/submit_04_train_masked_data.sh | 59 - .../submit_04_train_masked_data_dist_pairs.sh | 43 - .../run_02_adv_dataset_approx.sh | 41 - .../run_02_gen_adversarial_dataset_approx.py | 616 -------- .../util_eval_adversarial.py | 348 ----- .../util_eval_adversarial_dist_pairs.py | 290 ---- .../util_gen_adversarial_dataset.py | 447 ------ .../util_load_adversarial_mask.py | 78 -- ...ay_03_MSC-p03e01_kernel-disentangle-xy.txt | 128 -- ...MSC-p03e01_kernel-disentangle-xy_TUNED.txt | 16 - ...earnt_MSC-p03e02_learnt-loss-with-vaes.txt | 80 -- ...C-p03e02_learnt-loss-with-vaes_RETRY-2.txt | 160 --- ...3e02_learnt-loss-with-vaes_RETRY-2_ALT.txt | 120 -- ...C-p03e02_learnt-loss-with-vaes_RETRY-3.txt | 80 -- .../config/config_adversarial_kernel.yaml | 47 - .../config_disentangle_dataset_approx.yaml | 71 - ...0_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt | Bin 4587 -> 0 bytes ...0_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt | Bin 16619 -> 0 bytes ...0_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt | Bin 36843 -> 0 bytes ...0_b512_adam_lr0.001_wd0.0_xysquares_1x1.pt | Bin 65259 -> 0 bytes ...0_b512_adam_lr0.001_wd0.0_xysquares_2x2.pt | Bin 65259 -> 0 bytes ...0_b512_adam_lr0.001_wd0.0_xysquares_4x4.pt | Bin 65259 -> 0 bytes ...0_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt | Bin 65259 -> 0 bytes ...0_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt | Bin 4587 -> 0 bytes ...0_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt | Bin 16619 -> 0 bytes ...0_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt | Bin 36843 -> 0 bytes ...0_b512_adam_lr0.001_wd0.0_xysquares_1x1.pt | Bin 65259 -> 0 bytes ...0_b512_adam_lr0.001_wd0.0_xysquares_2x2.pt | Bin 65259 -> 0 bytes ...0_b512_adam_lr0.001_wd0.0_xysquares_4x4.pt | Bin 65259 -> 0 bytes ...0_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt | Bin 65259 -> 0 bytes ...47-1_s28800_adam_lr0.003_wd0.0001_xy1x1.pt | Bin 36920 -> 0 bytes ...47-1_s28800_adam_lr0.003_wd0.0001_xy8x8.pt | Bin 36920 -> 0 bytes ...D_r47-1_s28800_adam_lr0.003_wd0.0_xy1x1.pt | Bin 36920 -> 0 bytes ...D_r47-1_s28800_adam_lr0.003_wd0.0_xy8x8.pt | Bin 36920 -> 0 bytes .../run_01_sort_loss.py | 80 -- .../run_02_check_aug_gt_dists.py | 168 --- .../run_03_train_disentangle_kernel.py | 386 ----- .../run_04_train_disentangle_model.py | 394 ------ .../submit_03_overlap_loss_experiments.sh | 200 --- .../submit_03_train_kernel.sh | 71 - .../submit_04_train_model.sh | 37 - ...SC-p03e03_different-gt-representations.txt | 48 - .../submit_01.sh | 56 - ..._01_MSC-p03e04_random-external-factors.txt | 54 - ...p03e04_random-external-factors_FIX_ADA.txt | 27 - .../e04_random_external_factors/submit_01.sh | 75 - .../plot00_all/plot_p03_wandb_experiments.py | 556 -------- .../plot00_all/plot_p03e03_kernels.py | 75 - .../plot00_all/plots/.gitignore | 2 - .../e01_learn_xy_representations/_util.py | 63 - .../loading_example.py | 174 --- .../e01_learn_xy_representations/train_rl.py | 138 -- .../e01_learn_xy_representations/train_vae.py | 391 ------ research/scripts/clog-batch.sh | 19 - research/scripts/clog-stampede.sh | 19 - research/scripts/helper.sh | 192 --- research/scripts/permutations.py | 109 -- research/scripts/sbatch_job.sh | 42 - research/scripts/sbatch_submit.sh | 78 -- research/scripts/wandb_cli.py | 31 - research/scripts/working-batch.sh | 19 - research/scripts/working-stampede.sh | 19 - tests/test_data_similarity.py | 14 - tests/test_experiment.py | 7 - tests/test_frameworks.py | 29 - tests/test_metrics.py | 3 - tests/test_registry.py | 15 - 250 files changed, 24345 deletions(-) delete mode 100755 prepare_release.sh delete mode 100755 prepare_release_and_commit.sh delete mode 100644 requirements-research-freeze.txt delete mode 100644 requirements-research.txt delete mode 100644 research/__init__.py delete mode 100644 research/code/__init__.py delete mode 100644 research/code/dataset/__init__.py delete mode 100644 research/code/dataset/data/__init__.py delete mode 100644 research/code/dataset/data/_groundtruth__dsprites_imagenet.py delete mode 100644 research/code/dataset/data/_groundtruth__xcolumns.py delete mode 100644 research/code/dataset/data/_groundtruth__xyblocks.py delete mode 100644 research/code/dataset/data/_groundtruth__xysquares.py delete mode 100644 research/code/dataset/transform/__init__.py delete mode 100644 research/code/dataset/transform/_augment.py delete mode 100644 research/code/frameworks/__init__.py delete mode 100644 research/code/frameworks/ae/__init__.py delete mode 100644 research/code/frameworks/ae/_supervised__adaneg_tae.py delete mode 100644 research/code/frameworks/ae/_unsupervised__dotae.py delete mode 100644 research/code/frameworks/ae/_weaklysupervised__adaae.py delete mode 100644 research/code/frameworks/vae/__init__.py delete mode 100644 research/code/frameworks/vae/_supervised__adaave_tvae.py delete mode 100644 research/code/frameworks/vae/_supervised__adaneg_tvae.py delete mode 100644 research/code/frameworks/vae/_supervised__adatvae.py delete mode 100644 research/code/frameworks/vae/_supervised__badavae.py delete mode 100644 research/code/frameworks/vae/_supervised__gadavae.py delete mode 100644 research/code/frameworks/vae/_supervised__tbadavae.py delete mode 100644 research/code/frameworks/vae/_supervised__tgadavae.py delete mode 100644 research/code/frameworks/vae/_unsupervised__dorvae.py delete mode 100644 research/code/frameworks/vae/_unsupervised__dotvae.py delete mode 100644 research/code/frameworks/vae/_weaklysupervised__augpostriplet.py delete mode 100644 research/code/frameworks/vae/_weaklysupervised__st_adavae.py delete mode 100644 research/code/frameworks/vae/_weaklysupervised__st_betavae.py delete mode 100644 research/code/metrics/__init__.py delete mode 100644 research/code/metrics/_factored_components.py delete mode 100644 research/code/metrics/_flatness.py delete mode 100644 research/code/util/__init__.py delete mode 100644 research/code/util/_data.py delete mode 100644 research/code/util/_dataset.py delete mode 100644 research/code/util/_fn_util.py delete mode 100644 research/code/util/_io_util.py delete mode 100644 research/code/util/_loss.py delete mode 100644 research/code/util/_wandb_plots.py delete mode 100644 research/code/util/gadfly.mplstyle delete mode 100644 research/config/README.md delete mode 100644 research/config/config.yaml delete mode 100644 research/config/config_test.yaml delete mode 100644 research/config/dataset/X--adv-cars3d--WARNING.yaml delete mode 100644 research/config/dataset/X--adv-dsprites--WARNING.yaml delete mode 100644 research/config/dataset/X--adv-shapes3d--WARNING.yaml delete mode 100644 research/config/dataset/X--adv-smallnorb--WARNING.yaml delete mode 100644 research/config/dataset/X--dsprites-imagenet-bg-100.yaml delete mode 100644 research/config/dataset/X--dsprites-imagenet-bg-20.yaml delete mode 100644 research/config/dataset/X--dsprites-imagenet-bg-25.yaml delete mode 100644 research/config/dataset/X--dsprites-imagenet-bg-40.yaml delete mode 100644 research/config/dataset/X--dsprites-imagenet-bg-50.yaml delete mode 100644 research/config/dataset/X--dsprites-imagenet-bg-60.yaml delete mode 100644 research/config/dataset/X--dsprites-imagenet-bg-75.yaml delete mode 100644 research/config/dataset/X--dsprites-imagenet-bg-80.yaml delete mode 100644 research/config/dataset/X--dsprites-imagenet-fg-100.yaml delete mode 100644 research/config/dataset/X--dsprites-imagenet-fg-20.yaml delete mode 100644 research/config/dataset/X--dsprites-imagenet-fg-25.yaml delete mode 100644 research/config/dataset/X--dsprites-imagenet-fg-40.yaml delete mode 100644 research/config/dataset/X--dsprites-imagenet-fg-50.yaml delete mode 100644 research/config/dataset/X--dsprites-imagenet-fg-60.yaml delete mode 100644 research/config/dataset/X--dsprites-imagenet-fg-75.yaml delete mode 100644 research/config/dataset/X--dsprites-imagenet-fg-80.yaml delete mode 100644 research/config/dataset/X--dsprites-imagenet.yaml delete mode 100644 research/config/dataset/X--mask-adv-f-cars3d.yaml delete mode 100644 research/config/dataset/X--mask-adv-f-dsprites.yaml delete mode 100644 research/config/dataset/X--mask-adv-f-shapes3d.yaml delete mode 100644 research/config/dataset/X--mask-adv-f-smallnorb.yaml delete mode 100644 research/config/dataset/X--mask-adv-r-cars3d.yaml delete mode 100644 research/config/dataset/X--mask-adv-r-dsprites.yaml delete mode 100644 research/config/dataset/X--mask-adv-r-shapes3d.yaml delete mode 100644 research/config/dataset/X--mask-adv-r-smallnorb.yaml delete mode 100644 research/config/dataset/X--mask-dthr-cars3d.yaml delete mode 100644 research/config/dataset/X--mask-dthr-dsprites.yaml delete mode 100644 research/config/dataset/X--mask-dthr-shapes3d.yaml delete mode 100644 research/config/dataset/X--mask-dthr-smallnorb.yaml delete mode 100644 research/config/dataset/X--mask-ran-cars3d.yaml delete mode 100644 research/config/dataset/X--mask-ran-dsprites.yaml delete mode 100644 research/config/dataset/X--mask-ran-shapes3d.yaml delete mode 100644 research/config/dataset/X--mask-ran-smallnorb.yaml delete mode 100644 research/config/dataset/X--monte_rollouts.yaml delete mode 100644 research/config/dataset/X--xyblocks.yaml delete mode 100644 research/config/dataset/X--xyblocks_grey.yaml delete mode 100644 research/config/dataset/X--xysquares.yaml delete mode 100644 research/config/dataset/X--xysquares_grey.yaml delete mode 100644 research/config/dataset/X--xysquares_rgb.yaml delete mode 100644 research/config/framework/X--adaae.yaml delete mode 100644 research/config/framework/X--adaae_os.yaml delete mode 100644 research/config/framework/X--adaavetvae.yaml delete mode 100644 research/config/framework/X--adanegtae.yaml delete mode 100644 research/config/framework/X--adanegtvae.yaml delete mode 100644 research/config/framework/X--adatvae.yaml delete mode 100644 research/config/framework/X--augpos_tvae_os.yaml delete mode 100644 research/config/framework/X--badavae.yaml delete mode 100644 research/config/framework/X--dorvae.yaml delete mode 100644 research/config/framework/X--dorvae_aug.yaml delete mode 100644 research/config/framework/X--dotae.yaml delete mode 100644 research/config/framework/X--dotvae.yaml delete mode 100644 research/config/framework/X--dotvae_aug.yaml delete mode 100644 research/config/framework/X--gadavae.yaml delete mode 100644 research/config/framework/X--st-adavae.yaml delete mode 100644 research/config/framework/X--st-betavae.yaml delete mode 100644 research/config/framework/X--tbadavae.yaml delete mode 100644 research/config/framework/X--tgadavae.yaml delete mode 100644 research/config/metrics/all.yaml delete mode 100644 research/config/metrics/fast.yaml delete mode 100644 research/config/metrics/test.yaml delete mode 100644 research/config/run_location/cluster.yaml delete mode 100644 research/config/run_location/griffin.yaml delete mode 100644 research/config/run_location/heartofgold.yaml delete mode 100644 research/config/run_location/stampede_rsync_tmp.yaml delete mode 100644 research/config/run_location/stampede_shr.yaml delete mode 100644 research/config/run_location/stampede_tmp.yaml delete mode 100644 research/config/run_plugins/default.yaml delete mode 100644 research/config/schedule/OLD_adavae_down_all.yaml delete mode 100644 research/config/schedule/OLD_adavae_down_ratio.yaml delete mode 100644 research/config/schedule/OLD_adavae_down_thresh.yaml delete mode 100644 research/config/schedule/OLD_adavae_up_all.yaml delete mode 100644 research/config/schedule/OLD_adavae_up_all_full.yaml delete mode 100644 research/config/schedule/OLD_adavae_up_ratio.yaml delete mode 100644 research/config/schedule/OLD_adavae_up_ratio_full.yaml delete mode 100644 research/config/schedule/OLD_adavae_up_thresh.yaml delete mode 100644 research/config/schedule/adanegtvae_up_all.yaml delete mode 100644 research/config/schedule/adanegtvae_up_all_full.yaml delete mode 100644 research/config/schedule/adanegtvae_up_all_weak.yaml delete mode 100644 research/config/schedule/adanegtvae_up_ratio.yaml delete mode 100644 research/config/schedule/adanegtvae_up_ratio_full.yaml delete mode 100644 research/config/schedule/adanegtvae_up_ratio_weak.yaml delete mode 100644 research/config/schedule/adanegtvae_up_thresh.yaml delete mode 100644 research/part01_data_overlap/e01_hparam_tuning/submit_param_tuning.sh delete mode 100644 research/part01_data_overlap/e02_incr_overlap_xysquares/run_basic_checks.py delete mode 100644 research/part01_data_overlap/e02_incr_overlap_xysquares/submit_incr_overlap.sh delete mode 100644 research/part01_data_overlap/e03_modified_loss_xysquares/submit_overlap_loss.sh delete mode 100644 research/part01_data_overlap/plot01_data_traversal/animations/.gitignore delete mode 100644 research/part01_data_overlap/plot01_data_traversal/plots/.gitignore delete mode 100644 research/part01_data_overlap/plot01_data_traversal/run_01_all_shared_data_prepare.sh delete mode 100644 research/part01_data_overlap/plot01_data_traversal/run_02_plot_data_overlap.py delete mode 100644 research/part01_data_overlap/plot01_data_traversal/run_02_plot_traversals.py delete mode 100644 research/part01_data_overlap/plot01_data_traversal/run_03_plot_dataset_animation.py delete mode 100644 research/part01_data_overlap/plot02_data_distances/cache/.gitignore delete mode 100644 research/part01_data_overlap/plot02_data_distances/deprecated/run_01_x_z_recon_dists.sh delete mode 100644 research/part01_data_overlap/plot02_data_distances/plots/.gitignore delete mode 100644 research/part01_data_overlap/plot02_data_distances/run_data_correlation.py delete mode 100644 research/part01_data_overlap/plot02_data_distances/run_plot_global_dists.py delete mode 100644 research/part01_data_overlap/plot02_data_distances/run_plot_traversal_dists.py delete mode 100644 research/part01_data_overlap/plot02_data_distances/util_compute_traversal_dist_pairs.py delete mode 100644 research/part01_data_overlap/plot02_data_distances/util_compute_traversal_dists.py delete mode 100644 research/part01_data_overlap/plot03_wandb/plot_p01_all_experiments.py delete mode 100644 research/part01_data_overlap/plot03_wandb/plots/.gitignore delete mode 100644 research/part02_metric_learning/e00_metrics/submit_beta_data_latent_correlation.sh delete mode 100644 research/part02_metric_learning/e01_naive_triplet/OLD/submit_01_triplet_hparam_sweep.sh delete mode 100644 research/part02_metric_learning/e01_naive_triplet/OLD/submit_02_check_vae_equivalence.sh delete mode 100644 research/part02_metric_learning/e01_naive_triplet/submit_01_triplet_param_tuning.sh delete mode 100644 research/part02_metric_learning/e02_axis_triplet/OLD/submit_01.sh delete mode 100644 research/part02_metric_learning/e02_axis_triplet/OLD/submit_02.sh delete mode 100644 research/part02_metric_learning/e02_axis_triplet/OLD/submit_03.sh delete mode 100644 research/part02_metric_learning/e02_axis_triplet/OLD/submit_04.sh delete mode 100644 research/part02_metric_learning/e02_axis_triplet/OLD/submit_05.sh delete mode 100644 research/part02_metric_learning/e02_axis_triplet/submit_01_axis_aligned_triplet.sh delete mode 100644 research/part02_metric_learning/e03_unsupervised_triplet/OLD/submit_01.sh delete mode 100644 research/part02_metric_learning/e03_unsupervised_triplet/OLD/submit_02.sh delete mode 100644 research/part02_metric_learning/e03_unsupervised_triplet/OLD/submit_03_test_softada_vs_ada.sh delete mode 100644 research/part02_metric_learning/e03_unsupervised_triplet/array_01_MSC-p02e03_unsupervised-axis-triplet.txt delete mode 100644 research/part02_metric_learning/e03_unsupervised_triplet/array_01_MSC-p02e03_unsupervised-axis-triplet_ALT.txt delete mode 100644 research/part02_metric_learning/e03_unsupervised_triplet/submit_01_unsupervised_axis_triplet.sh delete mode 100644 research/part02_metric_learning/plot00_all/make_graphs.py delete mode 100644 research/part02_metric_learning/plot00_all/plot_p02_wandb_experiments.py delete mode 100644 research/part02_metric_learning/plot00_all/plots/.gitignore delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/config/config_adversarial_dataset.yaml delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/config/config_adversarial_dataset_approx.yaml delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_01_gen_adversarial_disk.py delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_02_adv_dataset.sh delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_02_gen_adversarial_dataset.py delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_03_check.py delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_04_gen_adversarial_ruck.py delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_04_gen_adversarial_ruck_dist_pairs.py delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/submit_02_train_adversarial_data.sh delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/submit_04_train_dsprites_imagenet.sh delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/submit_04_train_masked_data.sh delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/submit_04_train_masked_data_dist_pairs.sh delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/run_02_adv_dataset_approx.sh delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/run_02_gen_adversarial_dataset_approx.py delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/util_eval_adversarial.py delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/util_eval_adversarial_dist_pairs.py delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/util_gen_adversarial_dataset.py delete mode 100644 research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/util_load_adversarial_mask.py delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/array_03_MSC-p03e01_kernel-disentangle-xy.txt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/array_03_MSC-p03e01_kernel-disentangle-xy_TUNED.txt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/array_overlap_learnt_MSC-p03e02_learnt-loss-with-vaes.txt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/array_overlap_learnt_MSC-p03e02_learnt-loss-with-vaes_RETRY-2.txt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/array_overlap_learnt_MSC-p03e02_learnt-loss-with-vaes_RETRY-2_ALT.txt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/array_overlap_learnt_MSC-p03e02_learnt-loss-with-vaes_RETRY-3.txt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/config/config_adversarial_kernel.yaml delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/config/config_disentangle_dataset_approx.yaml delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_abs_r15-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_abs_r31-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_abs_r47-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_abs_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_1x1.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_abs_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_2x2.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_abs_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_4x4.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_abs_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_none_r15-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_none_r31-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_none_r47-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_none_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_1x1.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_none_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_2x2.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_none_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_4x4.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_none_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/OLD_r47-1_s28800_adam_lr0.003_wd0.0001_xy1x1.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/OLD_r47-1_s28800_adam_lr0.003_wd0.0001_xy8x8.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/OLD_r47-1_s28800_adam_lr0.003_wd0.0_xy1x1.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/data/OLD_r47-1_s28800_adam_lr0.003_wd0.0_xy8x8.pt delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/run_01_sort_loss.py delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/run_02_check_aug_gt_dists.py delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/run_03_train_disentangle_kernel.py delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/run_04_train_disentangle_model.py delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/submit_03_overlap_loss_experiments.sh delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/submit_03_train_kernel.sh delete mode 100644 research/part03_learnt_overlap/e01_learn_to_disentangle/submit_04_train_model.sh delete mode 100644 research/part03_learnt_overlap/e03_different_gt_representations/array_01_MSC-p03e03_different-gt-representations.txt delete mode 100644 research/part03_learnt_overlap/e03_different_gt_representations/submit_01.sh delete mode 100644 research/part03_learnt_overlap/e04_random_external_factors/array_01_MSC-p03e04_random-external-factors.txt delete mode 100644 research/part03_learnt_overlap/e04_random_external_factors/array_01_MSC-p03e04_random-external-factors_FIX_ADA.txt delete mode 100644 research/part03_learnt_overlap/e04_random_external_factors/submit_01.sh delete mode 100644 research/part03_learnt_overlap/plot00_all/plot_p03_wandb_experiments.py delete mode 100644 research/part03_learnt_overlap/plot00_all/plot_p03e03_kernels.py delete mode 100644 research/part03_learnt_overlap/plot00_all/plots/.gitignore delete mode 100644 research/part04_application_to_rl/e01_learn_xy_representations/_util.py delete mode 100644 research/part04_application_to_rl/e01_learn_xy_representations/loading_example.py delete mode 100644 research/part04_application_to_rl/e01_learn_xy_representations/train_rl.py delete mode 100644 research/part04_application_to_rl/e01_learn_xy_representations/train_vae.py delete mode 100644 research/scripts/clog-batch.sh delete mode 100644 research/scripts/clog-stampede.sh delete mode 100644 research/scripts/helper.sh delete mode 100644 research/scripts/permutations.py delete mode 100644 research/scripts/sbatch_job.sh delete mode 100644 research/scripts/sbatch_submit.sh delete mode 100644 research/scripts/wandb_cli.py delete mode 100644 research/scripts/working-batch.sh delete mode 100644 research/scripts/working-stampede.sh diff --git a/disent/dataset/sampling/_groundtruth__dist.py b/disent/dataset/sampling/_groundtruth__dist.py index d914e5ee..67c17bec 100644 --- a/disent/dataset/sampling/_groundtruth__dist.py +++ b/disent/dataset/sampling/_groundtruth__dist.py @@ -178,7 +178,6 @@ def main(): from disent.dataset.data import DSpritesData from disent.dataset.data import SmallNorb64Data from disent.util.seeds import TempNumpySeed - from research.code.dataset.data import XYSquaresMinimalData # pragma: delete-on-release from tqdm import tqdm repeats = 1000 @@ -206,7 +205,6 @@ def main(): Shapes3dData, DSpritesData, SmallNorb64Data, - XYSquaresMinimalData, # pragma: delete-on-release XYObjectData, XYObjectShadedData, ]: diff --git a/disent/dataset/util/stats.py b/disent/dataset/util/stats.py index b022f2ce..0ba2985f 100644 --- a/disent/dataset/util/stats.py +++ b/disent/dataset/util/stats.py @@ -87,7 +87,6 @@ def compute_data_mean_std( def main(progress=True, num_workers=0, batch_size=256): # try changing workers to zero on MacOS from disent.dataset import data from disent.dataset.transform import ToImgTensorF32 - from research.code.dataset import data as rdat # pragma: delete-on-release for data_cls in [ # groundtruth -- impl @@ -100,20 +99,6 @@ def main(progress=True, num_workers=0, batch_size=256): # try changing workers # groundtruth -- impl synthetic data.XYObjectData, data.XYObjectShadedData, - rdat.XYBlocksData, # pragma: delete-on-release - rdat.XYSquaresData, # pragma: delete-on-release - rdat.XYSquaresMinimalData, # pragma: delete-on-release - rdat.XColumnsData, # pragma: delete-on-release - # groundtruth -- increasing overlap # pragma: delete-on-release - (rdat.XYSquaresData, dict(grid_size=8, grid_spacing=8)), # pragma: delete-on-release - (rdat.XYSquaresData, dict(grid_size=8, grid_spacing=7)), # pragma: delete-on-release - (rdat.XYSquaresData, dict(grid_size=8, grid_spacing=6)), # pragma: delete-on-release - (rdat.XYSquaresData, dict(grid_size=8, grid_spacing=5)), # pragma: delete-on-release - (rdat.XYSquaresData, dict(grid_size=8, grid_spacing=4)), # pragma: delete-on-release - (rdat.XYSquaresData, dict(grid_size=8, grid_spacing=3)), # pragma: delete-on-release - (rdat.XYSquaresData, dict(grid_size=8, grid_spacing=2)), # pragma: delete-on-release - (rdat.XYSquaresData, dict(grid_size=8, grid_spacing=1)), # pragma: delete-on-release - (rdat.XYSquaresData, dict(rgb=False)), # pragma: delete-on-release # large datasets (data.Mpi3dData, dict(subset='toy', in_memory=True)), (data.Mpi3dData, dict(subset='realistic', in_memory=True)), diff --git a/prepare_release.sh b/prepare_release.sh deleted file mode 100755 index 35be1f58..00000000 --- a/prepare_release.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -# prepare the project for a new release -# removing all the research components -# - yes this is terrible, but at the rate things are changing I -# don't want to rip things out into a separate repo... I will -# do that eventually, but not now. - -# ====== # -# HELPER # -# ====== # - -function remove_delete_commands() { - awk "!/pragma: delete-on-release/" "$1" > "$1.temp" && mv "$1.temp" "$1" -} - -function version_greater_equal() { - printf '%s\n%s\n' "$2" "$1" | sort --check=quiet --version-sort -} - -# check that we have the right version so -# that `shopt -s globstar` does not fail -if ! version_greater_equal "$BASH_VERSION" "4"; then - echo "bash version 4 is required, got: ${BASH_VERSION}" - exit 1 -fi - -# ============ # -# DELETE FILES # -# ============ # - -# RESEARCH: -rm requirements-research.txt -rm requirements-research-freeze.txt -rm -rf research/ - -# ===================== # -# DELETE LINES OF FILES # -# ===================== # - -# enable recursive glob -shopt -s globstar - -# scan for all files that contain 'pragma: delete-on-release' -for file in **/*.{py,yaml}; do - if [ -n "$( grep -m 1 'pragma: delete-on-release' "$file" )" ]; then - echo "preparing: $file" - remove_delete_commands "$file" - fi -done - -# ===================== # -# CLEANUP THIS FILE # -# ===================== # - -rm prepare_release.sh -rm prepare_release_and_commit.sh diff --git a/prepare_release_and_commit.sh b/prepare_release_and_commit.sh deleted file mode 100755 index 1cd513bc..00000000 --- a/prepare_release_and_commit.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - - -# ====== # -# HELPER # -# ====== # - -function version_greater_equal() { - printf '%s\n%s\n' "$2" "$1" | sort --check=quiet --version-sort -} - -# check that we have the right version so -# that `shopt -s globstar` does not fail -if ! version_greater_equal "$BASH_VERSION" "4"; then - echo "bash version 4 is required, got: ${BASH_VERSION}" - exit 1 -fi - -# ====== # -# RUN # -# ====== # - -echo "(1/3) [GIT] Creating Prepare Branch" && \ - git checkout -b xdev-prepare && \ - ( git branch --unset-upstream 2>/dev/null || true ) && \ - \ - echo "(2/3) [PREPARE]" && \ - bash ./prepare_release.sh && \ - \ - echo "(3/3) [GIT] Committing Files" && \ - git add . && \ - git commit -m "run prepare_release.sh" - -# echo "(4/4) [GIT] Merging Changes" && \ -# git checkout dev && \ -# git merge xdev-prepare diff --git a/requirements-research-freeze.txt b/requirements-research-freeze.txt deleted file mode 100644 index 4e7f5b69..00000000 --- a/requirements-research-freeze.txt +++ /dev/null @@ -1,121 +0,0 @@ -# freeze from griffin on 2021-09-29 at 15:20 -# - Python 3.8.11 is used with miniconda-latest installed with pyenv -# - There are lots of unnecessary requirements in this list -# some have been generated with side experiments and other installs -# but experiments are confirmed to work locally with this list -# SLURM, still needs to be tested an might be broken with this. -# - install with: $ pip install --no-deps --ignore-installed -r requirements-research-freeze.txt -absl-py==0.13.0 -aiohttp==3.7.4.post0 -antlr4-python3-runtime==4.8 -argcomplete==1.12.3 -async-timeout==3.0.1 -attrs==21.2.0 -beautifulsoup4==4.10.0 -cachetools==4.2.2 -certifi==2021.5.30 -chardet==4.0.0 -charset-normalizer==2.0.4 -click==8.0.1 -cloudpickle==1.6.0 -colorlog==5.0.1 -configparser==5.0.2 -coverage==5.5 -cycler==0.10.0 -deap==1.3.1 -decorator==4.4.2 -deltas==0.7.0 -Deprecated==1.2.12 -diskcache==5.2.1 -docker-pycreds==0.4.0 -evaluations==0.0.5 -filelock==3.0.12 -fsspec==2021.7.0 -future==0.18.2 -generations==1.3.0 -gitdb==4.0.7 -GitPython==3.1.18 -google-auth==1.35.0 -google-auth-oauthlib==0.4.5 -grpcio==1.39.0 -h5py==3.3.0 -hydra-colorlog==1.0.1 -hydra-core==1.0.7 -hydra-submitit-launcher==1.1.1 -idna==3.2 -imageio==2.9.0 -imageio-ffmpeg==0.4.4 -importlib-resources==5.2.2 -iniconfig==1.1.1 -joblib==1.0.1 -kiwisolver==1.3.1 -llvmlite==0.37.0 -Logbook==1.5.3 -Markdown==3.3.4 -matplotlib==3.4.3 -member==0.0.1 -moviepy==1.0.3 -msgpack==1.0.2 -multidict==5.1.0 -numba==0.54.0 -numpy==1.20.3 -oauthlib==3.1.1 -offspring==0.1.1 -omegaconf==2.0.6 -packaging==21.0 -pathtools==0.1.2 -Pillow==8.3.1 -plotly==5.3.1 -pluggy==0.13.1 -population==0.0.1 -proglog==0.1.9 -promise==2.3 -protobuf==3.17.3 -psutil==5.8.0 -py==1.10.0 -pyasn1==0.4.8 -pyasn1-modules==0.2.8 -pyDeprecate==0.3.1 -pyparsing==2.4.7 -pytest==6.2.4 -pytest-cov==2.12.1 -python-dateutil==2.8.2 -pytorch-lightning==1.4.2 -PyYAML==5.4.1 -ray==1.6.0 -redis==3.5.3 -requests==2.26.0 -requests-oauthlib==1.3.0 -rsa==4.7.2 -ruck==0.2.2 -scikit-learn==0.24.2 -scipy==1.7.1 -sentry-sdk==1.3.1 -shortuuid==1.0.1 -six==1.16.0 -sklearn-genetic==0.4.1 -smmap==4.0.0 -soupsieve==2.2.1 -submitit==1.3.3 -subprocess32==3.5.4 -tenacity==8.0.1 -tensorboard==2.6.0 -tensorboard-data-server==0.6.1 -tensorboard-plugin-wit==1.8.0 -threadpoolctl==2.2.0 -tldr==2.0.0 -toml==0.10.2 -torch==1.9.1 -torchmetrics==0.5.0 -torchsort==0.1.6 -torchvision==0.10.1 -tqdm==4.62.1 -triton==1.0.0 -typing-extensions==3.10.0.0 -urllib3==1.26.6 -wandb==0.12.0 -Werkzeug==2.0.1 -wrapt==1.12.1 -yamlconf==0.2.4 -yarl==1.6.3 -zipp==3.5.0 diff --git a/requirements-research.txt b/requirements-research.txt deleted file mode 100644 index cd7fb323..00000000 --- a/requirements-research.txt +++ /dev/null @@ -1,22 +0,0 @@ - -# TODO: these requirements need to be cleaned up! - -# MISSING DEPS - these are imported in /research, but not included here, in requirements.txt OR in requirements-experiment.txt -# ============= - -# github -# matplotlib -# psutil - -ray>=1.6.0 -ruck==0.2.4 -numba>=0.50.0 # required for ruck optimisations, version could be relaxed? - -seaborn>=0.11.0 -pandas>=1.3.0 -cachier>=1.5.0 - -statsmodels>=0.13.0 # required for seaborn, to estimate outliers in regression plots - -# plotly>=5.0.0 # required for some wandb plotting? -tldr>=3.0.0 # just make our lives easier when we are trying to do things from the command line diff --git a/research/__init__.py b/research/__init__.py deleted file mode 100644 index 9a05a479..00000000 --- a/research/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ diff --git a/research/code/__init__.py b/research/code/__init__.py deleted file mode 100644 index 50fa0868..00000000 --- a/research/code/__init__.py +++ /dev/null @@ -1,81 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import disent.registry as R - - -def register_to_disent(): - # register metrics - R.METRICS.setm['flatness'] = R.LazyImport('research.code.metrics._flatness.metric_flatness') - R.METRICS.setm['factored_components'] = R.LazyImport('research.code.metrics._factored_components.metric_factored_components') - R.METRICS.setm['distances'] = R.LazyImport('research.code.metrics._factored_components.metric_distances') - R.METRICS.setm['linearity'] = R.LazyImport('research.code.metrics._factored_components.metric_linearity') - - # groundtruth -- impl synthetic - R.DATASETS.setm['xyblocks'] = R.LazyImport('research.code.dataset.data._groundtruth__xyblocks') - R.DATASETS.setm['xysquares'] = R.LazyImport('research.code.dataset.data._groundtruth__xysquares') - R.DATASETS.setm['xysquares_minimal'] = R.LazyImport('research.code.dataset.data._groundtruth__xysquares') - R.DATASETS.setm['xcolumns'] = R.LazyImport('research.code.dataset.data._groundtruth__xcolumns') - - # [AE - EXPERIMENTAL] - R.FRAMEWORKS.setm['x__adaneg_tae'] = R.LazyImport('research.code.frameworks.ae._supervised__adaneg_tae.AdaNegTripletAe') - R.FRAMEWORKS.setm['x__dot_ae'] = R.LazyImport('research.code.frameworks.ae._unsupervised__dotae.DataOverlapTripletAe') - R.FRAMEWORKS.setm['x__ada_ae'] = R.LazyImport('research.code.frameworks.ae._weaklysupervised__adaae.AdaAe') - - # [VAE - EXPERIMENTAL] - R.FRAMEWORKS.setm['x__adaave_tvae'] = R.LazyImport('research.code.frameworks.vae._supervised__adaave_tvae.AdaAveTripletVae') - R.FRAMEWORKS.setm['x__adaneg_tvae'] = R.LazyImport('research.code.frameworks.vae._supervised__adaneg_tvae.AdaNegTripletVae') - R.FRAMEWORKS.setm['x__ada_tvae'] = R.LazyImport('research.code.frameworks.vae._supervised__adatvae.AdaTripletVae') - R.FRAMEWORKS.setm['x__bada_vae'] = R.LazyImport('research.code.frameworks.vae._supervised__badavae.BoundedAdaVae') - R.FRAMEWORKS.setm['x__gada_vae'] = R.LazyImport('research.code.frameworks.vae._supervised__gadavae.GuidedAdaVae') - R.FRAMEWORKS.setm['x__tbada_vae'] = R.LazyImport('research.code.frameworks.vae._supervised__tbadavae.TripletBoundedAdaVae') - R.FRAMEWORKS.setm['x__tgada_vae'] = R.LazyImport('research.code.frameworks.vae._supervised__tgadavae.TripletGuidedAdaVae') - R.FRAMEWORKS.setm['x__dor_vae'] = R.LazyImport('research.code.frameworks.vae._unsupervised__dorvae.DataOverlapRankVae') - R.FRAMEWORKS.setm['x__dot_vae'] = R.LazyImport('research.code.frameworks.vae._unsupervised__dotvae.DataOverlapTripletVae') - R.FRAMEWORKS.setm['x__augpos_tvae'] = R.LazyImport('research.code.frameworks.vae._weaklysupervised__augpostriplet.AugPosTripletVae') - R.FRAMEWORKS.setm['x__st_ada_vae'] = R.LazyImport('research.code.frameworks.vae._weaklysupervised__st_adavae.SwappedTargetAdaVae') - R.FRAMEWORKS.setm['x__st_beta_vae'] = R.LazyImport('research.code.frameworks.vae._weaklysupervised__st_betavae.SwappedTargetBetaVae') - - # DEPRECATED -- register the kernels for the loss functions - R.KERNELS.setm.register_regex(pattern=r'^(xy8)_r(47)$', example='xy8_r47', factory_fn='research.code.dataset.transform._augment._make_xy8_r47') # DEPRECATED - R.KERNELS.setm.register_regex(pattern=r'^(xy1)_r(47)$', example='xy1_r47', factory_fn='research.code.dataset.transform._augment._make_xy1_r47') # DEPRECATED - - # register kernels for loss functions - R.KERNELS.setm.register_regex(pattern=r'^(xy1)_abs(63)$', example='xy1_abs63', factory_fn='research.code.dataset.transform._augment._make_xy1_abs63') - R.KERNELS.setm.register_regex(pattern=r'^(xy2)_abs(63)$', example='xy2_abs63', factory_fn='research.code.dataset.transform._augment._make_xy2_abs63') - R.KERNELS.setm.register_regex(pattern=r'^(xy4)_abs(63)$', example='xy4_abs63', factory_fn='research.code.dataset.transform._augment._make_xy4_abs63') - R.KERNELS.setm.register_regex(pattern=r'^(xy8)_abs(63)$', example='xy8_abs63', factory_fn='research.code.dataset.transform._augment._make_xy8_abs63') - - R.KERNELS.setm.register_regex(pattern=r'^(xy8)_abs(15)$', example='xy8_abs15', factory_fn='research.code.dataset.transform._augment._make_xy8_abs15') - R.KERNELS.setm.register_regex(pattern=r'^(xy8)_abs(31)$', example='xy8_abs31', factory_fn='research.code.dataset.transform._augment._make_xy8_abs31') - R.KERNELS.setm.register_regex(pattern=r'^(xy8)_abs(47)$', example='xy8_abs47', factory_fn='research.code.dataset.transform._augment._make_xy8_abs47') - - R.KERNELS.setm.register_regex(pattern=r'^(xy1)_none(63)$', example='xy1_none63', factory_fn='research.code.dataset.transform._augment._make_xy1_none63') - R.KERNELS.setm.register_regex(pattern=r'^(xy2)_none(63)$', example='xy2_none63', factory_fn='research.code.dataset.transform._augment._make_xy2_none63') - R.KERNELS.setm.register_regex(pattern=r'^(xy4)_none(63)$', example='xy4_none63', factory_fn='research.code.dataset.transform._augment._make_xy4_none63') - R.KERNELS.setm.register_regex(pattern=r'^(xy8)_none(63)$', example='xy8_none63', factory_fn='research.code.dataset.transform._augment._make_xy8_none63') - - R.KERNELS.setm.register_regex(pattern=r'^(xy8)_none(15)$', example='xy8_none15', factory_fn='research.code.dataset.transform._augment._make_xy8_none15') - R.KERNELS.setm.register_regex(pattern=r'^(xy8)_none(31)$', example='xy8_none31', factory_fn='research.code.dataset.transform._augment._make_xy8_none31') - R.KERNELS.setm.register_regex(pattern=r'^(xy8)_none(47)$', example='xy8_none47', factory_fn='research.code.dataset.transform._augment._make_xy8_none47') diff --git a/research/code/dataset/__init__.py b/research/code/dataset/__init__.py deleted file mode 100644 index 0725ea39..00000000 --- a/research/code/dataset/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2022 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ diff --git a/research/code/dataset/data/__init__.py b/research/code/dataset/data/__init__.py deleted file mode 100644 index 0c902264..00000000 --- a/research/code/dataset/data/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2022 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -# groundtruth -- impl -from research.code.dataset.data._groundtruth__dsprites_imagenet import DSpritesImagenetData - -# groundtruth -- impl synthetic -from research.code.dataset.data._groundtruth__xyblocks import XYBlocksData -from research.code.dataset.data._groundtruth__xysquares import XYSquaresData -from research.code.dataset.data._groundtruth__xysquares import XYSquaresMinimalData -from research.code.dataset.data._groundtruth__xysquares import XYSingleSquareData -from research.code.dataset.data._groundtruth__xcolumns import XColumnsData diff --git a/research/code/dataset/data/_groundtruth__dsprites_imagenet.py b/research/code/dataset/data/_groundtruth__dsprites_imagenet.py deleted file mode 100644 index d50783a4..00000000 --- a/research/code/dataset/data/_groundtruth__dsprites_imagenet.py +++ /dev/null @@ -1,365 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -import os -import shutil -from tempfile import TemporaryDirectory -from typing import Optional - -import numpy as np -from torch.utils.data import DataLoader -from torch.utils.data import Dataset -from torchvision.datasets import ImageFolder -from tqdm import tqdm - -from disent.dataset.data import GroundTruthData -from disent.dataset.data._groundtruth import _DiskDataMixin -from disent.dataset.data._groundtruth import _Hdf5DataMixin -from disent.dataset.data._groundtruth__dsprites import DSpritesData -from disent.dataset.util.datafile import DataFileHashedDlGen -from disent.dataset.util.hdf5 import H5Builder -from disent.util.inout.files import AtomicSaveFile -from disent.util.iters import LengthIter -from disent.util.math.random import random_choice_prng - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# load imagenet-tiny data # -# ========================================================================= # - - -class NumpyFolder(ImageFolder): - def __getitem__(self, idx): - img, cls = super().__getitem__(idx) - return np.array(img) - - -def _noop(x): - return x - - -def load_imagenet_tiny_data(raw_data_dir): - # load the data - data = NumpyFolder(os.path.join(raw_data_dir, 'train')) - data = DataLoader(data, batch_size=64, num_workers=min(16, os.cpu_count()), shuffle=False, drop_last=False, collate_fn=_noop) - # load data - this is a bit memory inefficient doing it like this instead of with a loop into a pre-allocated array - imgs = np.concatenate(list(tqdm(data, 'loading')), axis=0) - assert imgs.shape == (100_000, 64, 64, 3) - return imgs - - -def resave_imagenet_tiny_archive(orig_zipped_file, new_save_file, overwrite=False, h5_dataset_name: str = 'data'): - """ - Convert a imagenet tiny archive to an hdf5 or numpy file depending on the file extension. - Uncompressing the contents of the archive into a temporary directory in the same folder, - loading the images, then converting. - """ - _, ext = os.path.splitext(new_save_file) - assert ext in {'.npz', '.h5'}, f'unsupported save extension: {repr(ext)}, must be one of: {[".npz", ".h5"]}' - # extract zipfile into temp dir - with TemporaryDirectory(prefix='unzip_imagenet_tiny_', dir=os.path.dirname(orig_zipped_file)) as temp_dir: - log.info(f"Extracting into temporary directory: {temp_dir}") - shutil.unpack_archive(filename=orig_zipped_file, extract_dir=temp_dir) - images = load_imagenet_tiny_data(raw_data_dir=os.path.join(temp_dir, 'tiny-imagenet-200')) - # save the data - with AtomicSaveFile(new_save_file, overwrite=overwrite) as temp_file: - # check the mode - with H5Builder(temp_file, 'atomic_w') as builder: - builder.add_dataset_from_array( - name=h5_dataset_name, - array=images, - chunk_shape='batch', - compression_lvl=4, - attrs=None, - show_progress=True, - ) - - -# ========================================================================= # -# cars3d data object # -# ========================================================================= # - - -class ImageNetTinyDataFile(DataFileHashedDlGen): - """ - download the cars3d dataset and convert it to a hdf5 file. - """ - - dataset_name: str = 'data' - - def _generate(self, inp_file: str, out_file: str): - resave_imagenet_tiny_archive(orig_zipped_file=inp_file, new_save_file=out_file, overwrite=True, h5_dataset_name=self.dataset_name) - - -class ImageNetTinyData(_Hdf5DataMixin, _DiskDataMixin, Dataset, LengthIter): - - name = 'imagenet_tiny' - - datafile_imagenet_h5 = ImageNetTinyDataFile( - uri='http://cs231n.stanford.edu/tiny-imagenet-200.zip', - uri_hash={'fast': '4d97ff8efe3745a3bba9917d6d536559', 'full': '90528d7ca1a48142e341f4ef8d21d0de'}, - file_hash={'fast': '9c23e8ec658b1ec9f3a86afafbdbae51', 'full': '4c32b0b53f257ac04a3afb37e3a4204e'}, - uri_name='tiny-imagenet-200.zip', - file_name='tiny-imagenet-200.h5', - hash_mode='full' - ) - - datafiles = (datafile_imagenet_h5,) - - def __init__(self, data_root: Optional[str] = None, prepare: bool = False, in_memory=False, transform=None): - super().__init__() - self._transform = transform - # initialize mixin - self._mixin_disk_init( - data_root=data_root, - prepare=prepare, - ) - self._mixin_hdf5_init( - h5_path=os.path.join(self.data_dir, self.datafile_imagenet_h5.out_name), - h5_dataset_name=self.datafile_imagenet_h5.dataset_name, - in_memory=in_memory, - ) - - def __getitem__(self, idx: int): - obs = self._data[idx] - if self._transform is not None: - obs = self._transform(obs) - return obs - - -# ========================================================================= # -# dataset_dsprites # -# ========================================================================= # - - -class DSpritesImagenetData(GroundTruthData): - """ - DSprites that has imagenet images in the background. - """ - - name = 'dsprites_imagenet' - - # original dsprites it only (64, 64, 1) imagenet adds the colour channel - img_shape = (64, 64, 3) - factor_names = DSpritesData.factor_names - factor_sizes = DSpritesData.factor_sizes - - def __init__(self, visibility: int = 100, mode: str = 'bg', data_root: Optional[str] = None, prepare: bool = False, in_memory=False, transform=None): - super().__init__(transform=transform) - # check visibility and convert to ratio - assert isinstance(visibility, int), f'incorrect visibility percentage type, expected int, got: {type(visibility)}' - assert 0 <= visibility <= 100, f'incorrect visibility percentage: {repr(visibility)}, must be in range [0, 100]. ' - self._visibility = visibility / 100 - # check mode and convert to foreground boolean - assert mode in {'bg', 'fg'}, f'incorrect mode: {repr(mode)}, must be one of: ["bg", "fg"]' - self._foreground = (mode == 'fg') - # handle the datasets - self._dsprites = DSpritesData(data_root=data_root, prepare=prepare, in_memory=in_memory, transform=None) - self._imagenet = ImageNetTinyData(data_root=data_root, prepare=prepare, in_memory=in_memory, transform=None) - # deterministic randomization of the imagenet order - self._imagenet_order = random_choice_prng( - len(self._imagenet), - size=len(self), - seed=42, - ) - - def _get_observation(self, idx): - # we need to combine the two dataset images - # dsprites contains only {0, 255} for values - # we can directly use these values to mask the imagenet image - bg = self._imagenet[self._imagenet_order[idx]] - fg = self._dsprites[idx].repeat(3, axis=-1) - # compute background - # set foreground - r = self._visibility - if self._foreground: - # lerp content to white, and then insert into fg regions - # r*bg + (1-r)*255 - obs = (r*bg + ((1-r)*255)).astype('uint8') - obs[fg <= 127] = 0 - else: - # lerp content to black, and then insert into bg regions - # r*bg + (1-r)*000 - obs = (r*bg).astype('uint8') - obs[fg > 127] = 255 - # checks - return obs - - -# ========================================================================= # -# STATS # -# ========================================================================= # - - -""" -dsprites_fg_100 - vis_mean: [0.02067051643494642, 0.018688392816012946, 0.01632900510079384] - vis_std: [0.10271307751834059, 0.09390213983525653, 0.08377594259970281] -dsprites_fg_80 - vis_mean: [0.024956427531012196, 0.02336780403840578, 0.021475119672280243] - vis_std: [0.11864125016313823, 0.11137998105649799, 0.10281424917834255] -dsprites_fg_60 - vis_mean: [0.029335176871153983, 0.028145355435322966, 0.026731731769287146] - vis_std: [0.13663242436043319, 0.13114320478634894, 0.1246542727733097] -dsprites_fg_40 - vis_mean: [0.03369999506331255, 0.03290657349801835, 0.03196482946320608] - vis_std: [0.155514074438101, 0.1518464537731621, 0.14750944591836743] -dsprites_fg_20 - vis_mean: [0.038064750024334834, 0.03766780505193579, 0.03719798677641122] - vis_std: [0.17498878664096565, 0.17315570657628315, 0.1709923319496426] - -dsprites_bg_100 - vis_mean: [0.5020433619489952, 0.47206398913310593, 0.42380018909780404] - vis_std: [0.2505510666843685, 0.2500725980366869, 0.2562415603123114] -dsprites_bg_80 - vis_mean: [0.40867981393820857, 0.38468564002021527, 0.34611573047508204] - vis_std: [0.22048328737091344, 0.22102216869942384, 0.22692977053753483] -dsprites_bg_60 - vis_mean: [0.31676960943447674, 0.29877166834408025, 0.2698556821388113] - vis_std: [0.19745897110349, 0.1986606891520453, 0.203808842880044] -dsprites_bg_40 - vis_mean: [0.2248598986983768, 0.21285772298967615, 0.19359577132944206] - vis_std: [0.1841631708032332, 0.18554895825833284, 0.1893568926398198] -dsprites_bg_20 - vis_mean: [0.13294969414492142, 0.12694375140936273, 0.11733572285575933] - vis_std: [0.18311250427586276, 0.1840916474752131, 0.18607373519458442] - -dsprites_fg_75 - vis_mean: [0.02606445677382044, 0.024577082627819637, 0.02280587082174753] - vis_std: [0.12307153238282868, 0.11624914830767437, 0.1081911967745551] -dsprites_fg_50 - vis_mean: [0.031541090790578506, 0.030549541980176148, 0.029368756624861398] - vis_std: [0.14609029304575144, 0.14150919987547408, 0.13607872227034773] -dsprites_fg_25 - vis_mean: [0.03697718115834816, 0.03648095993826591, 0.03589183623762013] - vis_std: [0.17009317531572005, 0.16780075430655303, 0.16508779008691726] -dsprites_fg_0 - vis_mean: [0.042494423521889584, 0.042494423521889584, 0.042494423521889584] - vis_std: [0.1951664588062605, 0.1951664588062605, 0.1951664588062605] -dsprites - vis_mean: [0.042494423521889584] - vis_std: [0.1951664588062605] - -dsprites_bg_75 - vis_mean: [0.38577296742807327, 0.3632825822323436, 0.3271231888851156] - vis_std: [0.21392191050784257, 0.2146731716558466, 0.2204460568339597] -dsprites_bg_50 - vis_mean: [0.271323621109491, 0.25634066038331416, 0.23223046934400662] - vis_std: [0.18930391112143766, 0.19067969524425118, 0.19523218572886117] -dsprites_bg_25 - vis_mean: [0.15596283852200074, 0.14847876264131535, 0.13644703866118635] - vis_std: [0.18208653250875798, 0.18323109038468802, 0.18569624396763393] -dsprites_bg_0 - vis_mean: [0.042494423521889584, 0.042494423521889584, 0.042494423521889584] - vis_std: [0.1951664588062605, 0.1951664588062605, 0.1951664588062605] -dsprites - vis_mean: [0.042494423521889584] - vis_std: [0.1951664588062605] -""" - - -if __name__ == '__main__': - logging.basicConfig(level=logging.INFO) - - def compute_all_stats(): - from disent.dataset.transform import ToImgTensorF32 - from disent.dataset.util.stats import compute_data_mean_std - from disent.util.function import wrapped_partial - from disent.util.visualize.plot import plt_subplots_imshow - - def compute_stats(visibility: Optional[int], mode: Optional[str]): - import psutil - # get class - is_imgnet = (visibility is not None) and (mode is not None) - data_cls = wrapped_partial(DSpritesImagenetData, visibility=visibility, mode=mode) if is_imgnet else DSpritesData - data_title = f'{DSpritesImagenetData.name} visibility={repr(visibility)} mode={repr(mode)}' if is_imgnet else f'{DSpritesData.name}' - data_name = f'dsprites_{mode}_{visibility}' if is_imgnet else f'dsprites' - # plot images - data = data_cls(prepare=True) - grid = np.array([data[i*24733] for i in np.arange(16)]).reshape([4, 4, *data.img_shape]) - plt_subplots_imshow(grid, show=True, title=data_title) - # compute stats - data = data_cls(prepare=True, transform=ToImgTensorF32()) - mean, std = compute_data_mean_std(data, batch_size=256, num_workers=min(psutil.cpu_count(logical=False), 64), progress=True) - print(f'{data_name}\n vis_mean: {mean.tolist()}\n vis_std: {std.tolist()}') - # return stats - return data_name, mean, std - - # compute common stats - stats = [] - for mode in ['fg', 'bg']: - for vis in [ - 100, - 80, 60, 40, 20, - 75, 50, 25, - 0, - ]: - stats.append(compute_stats(vis, mode)) - # 0 above should be the same as this: - stats.append(compute_stats(None, None)) - - # print once at end - for name, mean, std in stats: - print(f'{name}\n vis_mean: {mean.tolist()}\n vis_std: {std.tolist()}') - - compute_all_stats() - - # DEBUG MEMORY USAGE: - - # def debug_mem(): - # import matplotlib.pyplot as plt - # from disent.dataset import DisentDataset - # from disent.dataset.sampling import GroundTruthDistSampler - # from disent.util.profiling import get_memory_usage - # - # data = DSpritesImagenetData(visibility=50, in_memory=True) - # dataset = DisentDataset(data, sampler=GroundTruthDistSampler(num_samples=3)) - # - # dataloader_makers = [ - # lambda dataset: DataLoader(dataset, batch_size=256, shuffle=True, num_workers=os.cpu_count(), pin_memory=True), - # lambda dataset: DataLoader(dataset, batch_size=256, shuffle=True, num_workers=os.cpu_count(), pin_memory=False), - # ] - # - # for i, dataloader_maker in enumerate(dataloader_makers): - # dataloader = dataloader_maker(dataset) - # memory = [] - # for j in range(1): - # for k, batch in enumerate(tqdm(dataloader, desc=f'{i}:{j}')): - # [x.cuda() for x in batch['x_targ']] # memory leak isnt here either... - # if k % 10 == 0: memory.append(get_memory_usage()) - # if k % 100 == 0: tqdm.write(f'{get_memory_usage(pretty=True)}') - # plt.plot(range(len(memory)), memory) - # plt.show() - # - # # entry - # debug_mem() - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/dataset/data/_groundtruth__xcolumns.py b/research/code/dataset/data/_groundtruth__xcolumns.py deleted file mode 100644 index 3942a304..00000000 --- a/research/code/dataset/data/_groundtruth__xcolumns.py +++ /dev/null @@ -1,66 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from typing import Tuple - -import numpy as np - -from research.code.dataset.data._groundtruth__xysquares import XYSquaresData - - -# ========================================================================= # -# xy multi grid data # -# ========================================================================= # - - -class XColumnsData(XYSquaresData): - - name = 'x_columns' - - @property - def factor_names(self) -> Tuple[str, ...]: - return ('x_R', 'x_G', 'x_B')[:self._num_squares] - - @property - def factor_sizes(self) -> Tuple[int, ...]: - return (self._placements,) * self._num_squares - - def _get_observation(self, idx): - # get factors - factors = self.idx_to_pos(idx) - offset, space, size = self._offset, self._spacing, self._square_size - # GENERATE - obs = np.zeros(self.img_shape, dtype=self._dtype) - for i, fx in enumerate(factors): - x = offset + space * fx - if self._rgb: - obs[:, x:x+size, i] = self._fill_value - else: - obs[:, x:x+size, :] = self._fill_value - return obs - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/dataset/data/_groundtruth__xyblocks.py b/research/code/dataset/data/_groundtruth__xyblocks.py deleted file mode 100644 index efeae411..00000000 --- a/research/code/dataset/data/_groundtruth__xyblocks.py +++ /dev/null @@ -1,160 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -from typing import Tuple - -import numpy as np - -from disent.dataset.data._groundtruth import GroundTruthData - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# xy squares data # -# ========================================================================= # - - -class XYBlocksData(GroundTruthData): - - """ - Dataset that generates all possible permutations of xor'd squares of - different scales moving across the grid. - - This dataset is designed not to overlap in the reconstruction loss space, but xor'ing may be too - complex to learn efficiently, and some sizes of factors may be too small (eg. biggest - square moving only has two positions) - """ - - COLOR_PALETTES_1 = { - 'white': [ - [255], - ], - 'greys_halves': [ - [128], - [255], - ], - 'greys_quarters': [ - [64], - [128], - [192], - [255], - ], - # alias for white, so that we can just set `rgb=False` - 'rgb': [ - [255], - ] - } - - COLOR_PALETTES_3 = { - 'white': [ - [255, 255, 255], - ], - # THIS IS IDEAL. - 'rgb': [ - [255, 000, 000], - [000, 255, 000], - [000, 000, 255], - ], - 'colors': [ - [255, 000, 000], [000, 255, 000], [000, 000, 255], - [255, 255, 000], [000, 255, 255], [255, 000, 255], - [255, 255, 255], - ], - } - - @property - def factor_names(self) -> Tuple[str, ...]: - return self._factor_names - - @property - def factor_sizes(self) -> Tuple[int, ...]: - return self._factor_sizes - - @property - def img_shape(self) -> Tuple[int, ...]: - return self._img_shape - - def __init__( - self, - grid_size: int = 64, - grid_levels: Tuple[int, ...] = (1, 2, 3), - rgb: bool = True, - palette: str = 'rgb', - invert_bg: bool = False, - transform=None, - ): - # colors - self._rgb = rgb - if palette != 'rgb': - log.warning('rgb palette is not being used, might overlap for the reconstruction loss.') - if rgb: - assert palette in XYBlocksData.COLOR_PALETTES_3, f'{palette=} must be one of {list(XYBlocksData.COLOR_PALETTES_3.keys())}' - self._colors = np.array(XYBlocksData.COLOR_PALETTES_3[palette]) - else: - assert palette in XYBlocksData.COLOR_PALETTES_1, f'{palette=} must be one of {list(XYBlocksData.COLOR_PALETTES_1.keys())}' - self._colors = np.array(XYBlocksData.COLOR_PALETTES_1[palette]) - - # bg colors - self._bg_color = 255 if invert_bg else 0 # we dont need rgb for this - assert not np.any([np.all(self._bg_color == color) for color in self._colors]), f'Color conflict with background: {self._bg_color} ({invert_bg=}) in {self._colors}' - - # grid - grid_levels = np.arange(1, grid_levels+1) if isinstance(grid_levels, int) else np.array(grid_levels) - assert np.all(grid_size % (2 ** grid_levels) == 0), f'{grid_size=} is not divisible by pow(2, {grid_levels=})' - assert np.all(grid_levels[:-1] <= grid_levels[1:]) - self._grid_size = grid_size - self._grid_levels = grid_levels - self._grid_dims = len(grid_levels) - - # axis sizes - self._axis_divisions = 2 ** self._grid_levels - assert len(self._axis_divisions) == self._grid_dims and np.all(grid_size % self._axis_divisions) == 0, 'This should never happen' - self._axis_division_sizes = grid_size // self._axis_divisions - - # info - self._factor_names = tuple([f'{prefix}-{d}' for prefix in ['color', 'x', 'y'] for d in self._axis_divisions]) - self._factor_sizes = tuple([len(self._colors)] * self._grid_dims + list(self._axis_divisions) * 2) - self._img_shape = (grid_size, grid_size, 3 if self._rgb else 1) - - # initialise - super().__init__(transform=transform) - - def _get_observation(self, idx): - positions = self.idx_to_pos(idx) - cs, xs, ys = positions[:self._grid_dims*1], positions[self._grid_dims*1:self._grid_dims*2], positions[self._grid_dims*2:] - assert len(xs) == len(ys) == len(cs) - # GENERATE - obs = np.full(self.img_shape, self._bg_color, dtype=np.uint8) - for i, (x, y, s, c) in enumerate(zip(xs, ys, self._axis_division_sizes, cs)): - obs[y*s:(y+1)*s, x*s:(x+1)*s, :] = self._colors[c] if np.any(obs[y*s, x*s, :] != self._colors[c]) else self._bg_color - # RETURN - return obs - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/dataset/data/_groundtruth__xysquares.py b/research/code/dataset/data/_groundtruth__xysquares.py deleted file mode 100644 index 92221525..00000000 --- a/research/code/dataset/data/_groundtruth__xysquares.py +++ /dev/null @@ -1,260 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -from typing import Optional -from typing import Tuple -from typing import Union - -import numpy as np - -from disent.dataset.data._groundtruth import GroundTruthData -from disent.util.iters import iter_chunks - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# xy multi grid data # -# ========================================================================= # - - -class XYSquaresMinimalData(GroundTruthData): - """ - Dataset that generates all possible permutations of 3 (R, G, B) coloured - squares placed on a square grid. This dataset is designed to not overlap - in the reconstruction loss space. - - If you use this in your work, please cite: https://github.com/nmichlo/disent - - NOTE: Unlike XYSquaresData, XYSquaresMinimalData is the bare-minimum class - to generate the same results as the default values for XYSquaresData, - this class is a fair bit faster (~0.8x)! - - All 3 squares are returned, in RGB, each square is size 8, with - non-overlapping grid spacing set to 8 pixels, in total leaving - 8*8*8*8*8*8 factors. Images are uint8 with fill values 0 (bg) - and 255 (fg). - """ - - name = 'xy_squares_minimal' - - @property - def factor_names(self) -> Tuple[str, ...]: - return 'x_R', 'y_R', 'x_G', 'y_G', 'x_B', 'y_B' - - @property - def factor_sizes(self) -> Tuple[int, ...]: - return 8, 8, 8, 8, 8, 8 # R, G, B squares - - @property - def img_shape(self) -> Tuple[int, ...]: - return 64, 64, 3 - - def _get_observation(self, idx): - # get factors - factors = np.reshape(np.unravel_index(idx, self.factor_sizes), (-1, 2)) - # GENERATE - obs = np.zeros(self.img_shape, dtype=np.uint8) - for i, (fx, fy) in enumerate(factors): - x, y = 8 * fx, 8 * fy - obs[y:y+8, x:x+8, i] = 255 - return obs - - -# ========================================================================= # -# xy multi grid data # -# ========================================================================= # - - -class XYSquaresData(GroundTruthData): - - """ - Dataset that generates all possible permutations of 3 (R, G, B) coloured - squares placed on a square grid. This dataset is designed to not overlap - in the reconstruction loss space. (if the spacing is set correctly.) - - If you use this in your work, please cite: https://github.com/nmichlo/disent - - NOTE: Unlike XYSquaresMinimalData, XYSquaresData allows adjusting various aspects - of the data that is generated, but the generation process is slower (~1.25x). - """ - - name = 'xy_squares' - - @property - def factor_names(self) -> Tuple[str, ...]: - return ('x_R', 'y_R', 'x_G', 'y_G', 'x_B', 'y_B')[:self._num_squares*2] - - @property - def factor_sizes(self) -> Tuple[int, ...]: - return (self._placements, self._placements) * self._num_squares # R, G, B squares - - @property - def img_shape(self) -> Tuple[int, ...]: - return self._width, self._width, (3 if self._rgb else 1) - - def __init__( - self, - square_size: int = 8, - image_size: int = 64, - grid_size: Optional[int] = None, - grid_spacing: Optional[int] = None, - num_squares: int = 3, - rgb: bool = True, - fill_value: Optional[Union[float, int]] = None, - dtype: Union[np.dtype, str] = np.uint8, - no_warnings: bool = False, - transform=None, - ): - """ - :param square_size: the size of the individual squares in pixels - :param image_size: the image size in pixels - :param grid_spacing: the step size between square positions on the grid. By - default this is set to square_size which results in non-overlapping - data if `grid_spacing >= square_size` Reducing this value such that - `grid_spacing < square_size` results in overlapping data. - :param num_squares: The number of squares drawn. `1 <= num_squares <= 3` - :param rgb: Image has 3 channels if True, otherwise it is greyscale with 1 channel. - :param no_warnings: If warnings should be disabled if overlapping. - :param fill_value: The foreground value to use for filling squares, the default background value is 0. - :param grid_size: The number of grid positions available for the square to be placed in. The square is centered if this is less than - :param dtype: - """ - if grid_spacing is None: - grid_spacing = square_size - if (grid_spacing < square_size) and not no_warnings: - log.warning(f'overlap between squares for reconstruction loss, {grid_spacing} < {square_size}') - # color - self._rgb = rgb - self._dtype = np.dtype(dtype) - # check fill values - if self._dtype.kind == 'u': - self._fill_value = 255 if (fill_value is None) else fill_value - assert isinstance(self._fill_value, int) - assert 0 < self._fill_value <= 255, f'0 < {self._fill_value} <= 255' - elif self._dtype.kind == 'f': - self._fill_value = 1.0 if (fill_value is None) else fill_value - assert isinstance(self._fill_value, (int, float)) - assert 0.0 < self._fill_value <= 1.0, f'0.0 < {self._fill_value} <= 1.0' - else: - raise TypeError(f'invalid dtype: {self._dtype}, must be float or unsigned integer') - # image sizes - self._width = image_size - # number of squares - self._num_squares = num_squares - assert 1 <= num_squares <= 3, 'Only 1, 2 or 3 squares are supported!' - # square scales - self._square_size = square_size - # x, y - self._spacing = grid_spacing - self._placements = (self._width - self._square_size) // grid_spacing + 1 - # maximum placements - if grid_size is not None: - assert isinstance(grid_size, int) - assert grid_size > 0 - if (grid_size > self._placements) and not no_warnings: - log.warning(f'number of possible placements: {self._placements} is less than the given grid size: {grid_size}, reduced grid size from: {grid_size} -> {self._placements}') - self._placements = min(self._placements, grid_size) - # center elements - self._offset = (self._width - (self._square_size + (self._placements-1)*self._spacing)) // 2 - # initialise parents -- they depend on self.factors - super().__init__(transform=transform) - - def _get_observation(self, idx): - # get factors - factors = self.idx_to_pos(idx) - offset, space, size = self._offset, self._spacing, self._square_size - # GENERATE - obs = np.zeros(self.img_shape, dtype=self._dtype) - for i, (fx, fy) in enumerate(iter_chunks(factors, 2)): - x, y = offset + space * fx, offset + space * fy - if self._rgb: - obs[y:y+size, x:x+size, i] = self._fill_value - else: - obs[y:y+size, x:x+size, :] = self._fill_value - return obs - - -# ========================================================================= # -# xy minimal single square dataset # -# ========================================================================= # - - -class XYSingleSquareData(GroundTruthData): - - name = 'xy_single_square' - - factor_names = ('x', 'y') - - @property - def factor_sizes(self) -> Tuple[int, ...]: - return (self._placements, self._placements) - - @property - def img_shape(self) -> Tuple[int, ...]: - return self._width, self._width, 1 - - def __init__( - self, - square_size: int = 8, # square width and height - image_size: int = 64, # image width and height - grid_size: Optional[int] = None, # limit the number of square placements along an axis, automatically set as the maximum valid - grid_spacing: Optional[int] = None, # how far apart the square is spaced, buy default this is the square size, meaning no overlap! - no_warnings: bool = False, - transform=None, - ): - if grid_spacing is None: - grid_spacing = square_size - if (grid_spacing < square_size) and not no_warnings: - log.warning(f'overlap between squares for reconstruction loss, {grid_spacing} < {square_size}') - # vars - self._width = image_size # image width and height - self._square_size = square_size # square width and height - self._spacing = grid_spacing # spacing between square positions - self._placements = (self._width - self._square_size) // grid_spacing + 1 # number of positions the square can be in along an axis - # maximum placements - if grid_size is not None: - if (grid_size > self._placements): - log.warning(f'number of possible placements: {self._placements} is less than the given grid size: {grid_size}, reduced grid size from: {grid_size} -> {self._placements}') - self._placements = min(self._placements, grid_size) - # center elements - self._offset = (self._width - (self._square_size + (self._placements-1)*self._spacing)) // 2 - # initialise parents -- they depend on self.factors - super().__init__(transform=transform) - - def _get_observation(self, idx): - # get factors == grid position/index - fx, fy = self.idx_to_pos(idx) - offset, space, size = self._offset, self._spacing, self._square_size - # draw square onto image - obs = np.zeros(self.img_shape, dtype='uint8') - x, y = offset + space * fx, offset + space * fy - obs[y:y+size, x:x+size, :] = 255 - return obs - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/dataset/transform/__init__.py b/research/code/dataset/transform/__init__.py deleted file mode 100644 index 0725ea39..00000000 --- a/research/code/dataset/transform/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2022 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ diff --git a/research/code/dataset/transform/_augment.py b/research/code/dataset/transform/_augment.py deleted file mode 100644 index 4b687038..00000000 --- a/research/code/dataset/transform/_augment.py +++ /dev/null @@ -1,91 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2022 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -import os -from typing import Callable - -import torch -import research -from disent.dataset.transform._augment import _scale_kernel -from disent.util.deprecate import deprecated - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Helper # -# -- import this file to register the functions! # -# ========================================================================= # - - -_KERNEL_DIR = os.path.abspath(os.path.join(research.__file__, '..', 'part03_learnt_overlap/e01_learn_to_disentangle/data')) - - -def _get_make_kernel_fn(file_name: str, *, normalize_mode: str, root_dir: str = _KERNEL_DIR) -> Callable[[], torch.Tensor]: - def _make_kernel(kern: str = None, radius: str = None) -> torch.Tensor: - path = os.path.join(root_dir, file_name) - log.debug(f'Loading kernel from: {path}') - with torch.no_grad(): - kernel = torch.load(path) - return _scale_kernel(kernel, mode=normalize_mode) - return _make_kernel - - -# ========================================================================= # -# Kernel Registry # -# -- import this file to register the functions! # -# ========================================================================= # - - -# old kernels, with negative values -- these should be removed -_make_xy1_r47 = deprecated(fn=_get_make_kernel_fn('OLD_r47-1_s28800_adam_lr0.003_wd0.0_xy1x1.pt', normalize_mode='none'), msg='kernel `xy1_r47` has been deprecated! It is not correctly scaled, please use `xy1_abs63` instead!') -_make_xy8_r47 = deprecated(fn=_get_make_kernel_fn('OLD_r47-1_s28800_adam_lr0.003_wd0.0_xy8x8.pt', normalize_mode='none'), msg='kernel `xy8_r47` has been deprecated! It is not correctly scaled, please use `xy8_abs63` instead!') - -# kernels learnt with `kernel = abs(params)` parameterization -# - no negative values -_make_xy1_abs63 = _get_make_kernel_fn('MSC_abs_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_1x1.pt', normalize_mode='none') -_make_xy2_abs63 = _get_make_kernel_fn('MSC_abs_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_2x2.pt', normalize_mode='none') -_make_xy4_abs63 = _get_make_kernel_fn('MSC_abs_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_4x4.pt', normalize_mode='none') -_make_xy8_abs63 = _get_make_kernel_fn('MSC_abs_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt', normalize_mode='none') - -_make_xy8_abs15 = _get_make_kernel_fn('MSC_abs_r15-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt', normalize_mode='none') -_make_xy8_abs31 = _get_make_kernel_fn('MSC_abs_r31-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt', normalize_mode='none') -_make_xy8_abs47 = _get_make_kernel_fn('MSC_abs_r47-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt', normalize_mode='none') - -# kernels learnt with `kernel = params` parameterization -# - has negative values -_make_xy1_none63 = _get_make_kernel_fn('MSC_none_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_1x1.pt', normalize_mode='none') -_make_xy2_none63 = _get_make_kernel_fn('MSC_none_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_2x2.pt', normalize_mode='none') -_make_xy4_none63 = _get_make_kernel_fn('MSC_none_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_4x4.pt', normalize_mode='none') -_make_xy8_none63 = _get_make_kernel_fn('MSC_none_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt', normalize_mode='none') - -_make_xy8_none15 = _get_make_kernel_fn('MSC_none_r15-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt', normalize_mode='none') -_make_xy8_none31 = _get_make_kernel_fn('MSC_none_r31-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt', normalize_mode='none') -_make_xy8_none47 = _get_make_kernel_fn('MSC_none_r47-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt', normalize_mode='none') - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/frameworks/__init__.py b/research/code/frameworks/__init__.py deleted file mode 100644 index 0725ea39..00000000 --- a/research/code/frameworks/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2022 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ diff --git a/research/code/frameworks/ae/__init__.py b/research/code/frameworks/ae/__init__.py deleted file mode 100644 index 95b7de41..00000000 --- a/research/code/frameworks/ae/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -# supervised frameworks -from research.code.frameworks.ae._supervised__adaneg_tae import AdaNegTripletAe - -# unsupervised frameworks -from research.code.frameworks.ae._unsupervised__dotae import DataOverlapTripletAe - -# weakly supervised frameworks -from research.code.frameworks.ae._weaklysupervised__adaae import AdaAe diff --git a/research/code/frameworks/ae/_supervised__adaneg_tae.py b/research/code/frameworks/ae/_supervised__adaneg_tae.py deleted file mode 100644 index 61abfa31..00000000 --- a/research/code/frameworks/ae/_supervised__adaneg_tae.py +++ /dev/null @@ -1,71 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -from dataclasses import dataclass -from numbers import Number -from typing import Any -from typing import Dict -from typing import Sequence -from typing import Tuple -from typing import Union - -import torch - -from disent.frameworks.ae._supervised__tae import TripletAe -from research.code.frameworks.ae._weaklysupervised__adaae import AdaAe -from research.code.frameworks.vae._supervised__adaneg_tvae import AdaNegTripletVae - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Guided Ada Vae # -# ========================================================================= # - - -class AdaNegTripletAe(TripletAe): - """ - This is a condensed version of the ada_tvae and adaave_tvae, - using approximately the best settings and loss... - """ - - REQUIRED_OBS = 3 - - @dataclass - class cfg(TripletAe.cfg, AdaAe.cfg): - # ada_tvae - loss - adat_triplet_share_scale: float = 0.95 - - def hook_ae_compute_ave_aug_loss(self, zs: Sequence[torch.Tensor], xs_partial_recon: Sequence[torch.Tensor], xs_targ: Sequence[torch.Tensor]) -> Tuple[Union[torch.Tensor, Number], Dict[str, Any]]: - return AdaNegTripletVae.estimate_ada_triplet_loss_from_zs( - zs=zs, - cfg=self.cfg, - ) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/frameworks/ae/_unsupervised__dotae.py b/research/code/frameworks/ae/_unsupervised__dotae.py deleted file mode 100644 index 8435d55a..00000000 --- a/research/code/frameworks/ae/_unsupervised__dotae.py +++ /dev/null @@ -1,76 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -from dataclasses import dataclass -from typing import Any -from typing import Dict -from typing import Sequence -from typing import Tuple - -import torch - -from research.code.frameworks.ae._supervised__adaneg_tae import AdaNegTripletAe -from research.code.frameworks.vae._supervised__adaneg_tvae import AdaNegTripletVae -from research.code.frameworks.vae._unsupervised__dotvae import DataOverlapMixin - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Data Overlap Triplet AE # -# ========================================================================= # - - -class DataOverlapTripletAe(AdaNegTripletAe, DataOverlapMixin): - - REQUIRED_OBS = 1 - - @dataclass - class cfg(AdaNegTripletAe.cfg, DataOverlapMixin.cfg): - pass - - def __init__(self, model: 'AutoEncoder', cfg: cfg = None, batch_augment=None): - super().__init__(model=model, cfg=cfg, batch_augment=batch_augment) - # initialise mixin - self.init_data_overlap_mixin() - - def hook_ae_compute_ave_aug_loss(self, zs: Sequence[torch.Tensor], xs_partial_recon: Sequence[torch.Tensor], xs_targ: Sequence[torch.Tensor]) -> Tuple[torch.Tensor, Dict[str, Any]]: - [z], [x_targ_orig] = zs, xs_targ - # 1. randomly generate and mine triplets using augmented versions of the inputs - a_idxs, p_idxs, n_idxs = self.random_mined_triplets(x_targ_orig=x_targ_orig) - # 2. compute triplet loss - loss, loss_log = AdaNegTripletVae.estimate_ada_triplet_loss_from_zs( - zs=[z[idxs] for idxs in (a_idxs, p_idxs, n_idxs)], - cfg=self.cfg, - ) - return loss, { - **loss_log, - } - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/frameworks/ae/_weaklysupervised__adaae.py b/research/code/frameworks/ae/_weaklysupervised__adaae.py deleted file mode 100644 index f38690a5..00000000 --- a/research/code/frameworks/ae/_weaklysupervised__adaae.py +++ /dev/null @@ -1,81 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from typing import Any -from typing import Dict -from typing import Sequence -from typing import Tuple - -import torch -from dataclasses import dataclass - -from disent.frameworks.ae._unsupervised__ae import Ae -from disent.frameworks.vae._weaklysupervised__adavae import AdaVae - - -# ========================================================================= # -# Ada-GVAE # -# ========================================================================= # - - -class AdaAe(Ae): - """ - Custom implementation, removing Variational Auto-Encoder components of: - Weakly Supervised Disentanglement Learning Without Compromises: https://arxiv.org/abs/2002.02886 - - MODIFICATION: - - L1 distance for deltas instead of KL divergence - - adjustable threshold value - """ - - REQUIRED_OBS = 2 - - @dataclass - class cfg(Ae.cfg): - ada_thresh_ratio: float = 0.5 - - def hook_ae_intercept_zs(self, zs: Sequence[torch.Tensor]) -> Tuple[Sequence[torch.Tensor], Dict[str, Any]]: - """ - Adaptive VAE Glue Method, putting the various components together - 1. find differences between deltas - 2. estimate a threshold for differences - 3. compute a shared mask from this threshold - 4. average together elements that should be considered shared - - TODO: the methods used in this function should probably be moved here - TODO: this function could be turned into a torch.nn.Module! - """ - z0, z1 = zs - # shared elements that need to be averaged, computed per pair in the batch. - share_mask = AdaVae.compute_shared_mask_from_zs(z0, z1, ratio=self.cfg.ada_thresh_ratio) - # compute average posteriors - new_zs = AdaVae.make_shared_zs(z0, z1, share_mask) - # return new args & generate logs - return new_zs, { - 'shared': share_mask.sum(dim=1).float().mean() - } - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/frameworks/vae/__init__.py b/research/code/frameworks/vae/__init__.py deleted file mode 100644 index 68db38fe..00000000 --- a/research/code/frameworks/vae/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -# supervised frameworks -from research.code.frameworks.vae._supervised__adaave_tvae import AdaAveTripletVae -from research.code.frameworks.vae._supervised__adaneg_tvae import AdaNegTripletVae -from research.code.frameworks.vae._supervised__adatvae import AdaTripletVae -from research.code.frameworks.vae._supervised__badavae import BoundedAdaVae -from research.code.frameworks.vae._supervised__gadavae import GuidedAdaVae -from research.code.frameworks.vae._supervised__tbadavae import TripletBoundedAdaVae -from research.code.frameworks.vae._supervised__tgadavae import TripletGuidedAdaVae - -# unsupervised frameworks -from research.code.frameworks.vae._unsupervised__dorvae import DataOverlapRankVae -from research.code.frameworks.vae._unsupervised__dotvae import DataOverlapTripletVae - -# weakly supervised frameworks -from research.code.frameworks.vae._weaklysupervised__augpostriplet import AugPosTripletVae -from research.code.frameworks.vae._weaklysupervised__st_adavae import SwappedTargetAdaVae -from research.code.frameworks.vae._weaklysupervised__st_betavae import SwappedTargetBetaVae diff --git a/research/code/frameworks/vae/_supervised__adaave_tvae.py b/research/code/frameworks/vae/_supervised__adaave_tvae.py deleted file mode 100644 index c7aa619d..00000000 --- a/research/code/frameworks/vae/_supervised__adaave_tvae.py +++ /dev/null @@ -1,120 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -import warnings -from dataclasses import dataclass -from typing import Any -from typing import Dict -from typing import Sequence -from typing import Tuple - -from disent.util.deprecate import deprecated -from torch.distributions import Distribution -from torch.distributions import Normal - -from research.code.frameworks.vae._supervised__adatvae import AdaTripletVae -from research.code.frameworks.vae._supervised__adatvae import compute_ave_shared_distributions -from research.code.frameworks.vae._supervised__adatvae import compute_triplet_shared_masks - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Guided Ada Vae # -# ========================================================================= # - - -@deprecated('Rather use the AdaNegTripletVae') -class AdaAveTripletVae(AdaTripletVae): - """ - This was a more general attempt of the ada-tvae, - that also averages representations passed to the decoder. - - just averaging in this way without augmenting the loss with - triplet, or ada_triplet is too weak of a supervision signal. - """ - - REQUIRED_OBS = 3 - - @dataclass - class cfg(AdaTripletVae.cfg): - # adavae - ada_thresh_mode: str = 'dist' # RESET OVERRIDEN VALUE - # adaave_tvae - adaave_augment_orig: bool = True # triplet over original OR averaged embeddings - adaave_decode_orig: bool = True # decode & regularize original OR averaged embeddings - - def __init__(self, model: 'AutoEncoder', cfg: cfg = None, batch_augment=None): - super().__init__(model=model, cfg=cfg, batch_augment=batch_augment) - # checks - if self.cfg.ada_thresh_mode != 'dist': - warnings.warn(f'cfg.ada_thresh_mode == {repr(self.cfg.ada_thresh_mode)}. Modes other than "dist" do not work well!') - val = (self.cfg.adat_triplet_loss != 'triplet_hard_ave_all') - if self.cfg.adaave_augment_orig == val: - warnings.warn(f'cfg.adaave_augment_orig == {repr(self.cfg.adaave_augment_orig)}. Modes other than {repr(val)} do not work well!') - if self.cfg.adaave_decode_orig == False: - warnings.warn(f'cfg.adaave_decode_orig == {repr(self.cfg.adaave_decode_orig)}. Modes other than True do not work well!') - - def hook_intercept_ds(self, ds_posterior: Sequence[Normal], ds_prior: Sequence[Normal]) -> Tuple[Sequence[Distribution], Sequence[Distribution], Dict[str, Any]]: - # triplet vae intercept -- in case detached - ds_posterior, ds_prior, intercept_logs = super().hook_intercept_ds(ds_posterior, ds_prior) - - # compute shared masks, shared embeddings & averages over shared embeddings - share_masks, share_logs = compute_triplet_shared_masks(ds_posterior, cfg=self.cfg) - ds_posterior_shared, ds_posterior_shared_ave = compute_ave_shared_distributions(ds_posterior, share_masks, cfg=self.cfg) - - # DIFFERENCE FROM ADAVAE | get return values - # adavae: adaave_augment_orig == True, adaave_decode_orig == False - ds_posterior_augment = (ds_posterior if self.cfg.adaave_augment_orig else ds_posterior_shared_ave) - ds_posterior_return = (ds_posterior if self.cfg.adaave_decode_orig else ds_posterior_shared_ave) - - # save params for aug_loss hook step - self._curr_ada_loss_kwargs = dict( - share_masks=share_masks, - zs=[d.mean for d in ds_posterior], - zs_shared=[d.mean for d in ds_posterior_shared], - zs_shared_ave=[d.mean for d in ds_posterior_augment], # USUALLY: zs_params_shared_ave - ) - - return ds_posterior_return, ds_prior, { - **intercept_logs, - **share_logs, - } - - def hook_compute_ave_aug_loss(self, ds_posterior, ds_prior, zs_sampled, xs_partial_recon, xs_targ): - """ - NOTE: we don't use input parameters here, this function will only work - if called as part of training_step or do_training_step - """ - # compute triplet loss - result = AdaTripletVae.compute_ada_triplet_loss(**self._curr_ada_loss_kwargs, cfg=self.cfg) - # cleanup temporary variables - del self._curr_ada_loss_kwargs - # we are done - return result - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/frameworks/vae/_supervised__adaneg_tvae.py b/research/code/frameworks/vae/_supervised__adaneg_tvae.py deleted file mode 100644 index 50e600ea..00000000 --- a/research/code/frameworks/vae/_supervised__adaneg_tvae.py +++ /dev/null @@ -1,120 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -from dataclasses import dataclass -from typing import Sequence - -import torch -from torch.distributions import Normal - -from disent.nn.loss.triplet import configured_dist_triplet -from disent.nn.loss.triplet import configured_triplet -from disent.frameworks.vae._supervised__tvae import TripletVae -from disent.frameworks.vae._weaklysupervised__adavae import AdaVae -from research.code.frameworks.vae._supervised__adatvae import compute_triplet_shared_masks -from research.code.frameworks.vae._supervised__adatvae import compute_triplet_shared_masks_from_zs - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Guided Ada Vae # -# ========================================================================= # - - -class AdaNegTripletVae(TripletVae): - - """ - This is a condensed version of the ada_tvae and adaave_tvae, - using approximately the best settings and loss... - """ - - REQUIRED_OBS = 3 - - @dataclass - class cfg(TripletVae.cfg, AdaVae.cfg): - # adavae - ada_thresh_mode: str = 'dist' # only works for: adat_share_mask_mode == "posterior" - # ada_tvae - loss - # * this should be used with a schedule, slowly decrease from 1.0 down to 0.5 or less - # * a similar schedule should also be used on `ada_thresh_ratio`, slowly increasing from 0.0 to 0.5 - adat_triplet_share_scale: float = 0.95 - # ada_tvae - averaging - adat_share_mask_mode: str = 'posterior' - - def hook_compute_ave_aug_loss(self, ds_posterior: Sequence[Normal], ds_prior: Sequence[Normal], zs_sampled: Sequence[torch.Tensor], xs_partial_recon: Sequence[torch.Tensor], xs_targ: Sequence[torch.Tensor]): - return self.estimate_ada_triplet_loss( - ds_posterior=ds_posterior, - cfg=self.cfg, - ) - - @staticmethod - def estimate_ada_triplet_loss_from_zs(zs: Sequence[torch.Tensor], cfg: cfg): - # compute shared masks, shared embeddings & averages over shared embeddings - share_masks, share_logs = compute_triplet_shared_masks_from_zs(zs=zs, cfg=cfg) - # compute loss - ada_triplet_loss, ada_triplet_logs = AdaNegTripletVae.compute_ada_triplet_loss(share_masks=share_masks, zs=zs, cfg=cfg) - # merge logs & return loss - return ada_triplet_loss, { - **ada_triplet_logs, - **share_logs, - } - - @staticmethod - def estimate_ada_triplet_loss(ds_posterior: Sequence[Normal], cfg: cfg): - # compute shared masks, shared embeddings & averages over shared embeddings - share_masks, share_logs = compute_triplet_shared_masks(ds_posterior, cfg=cfg) - # compute loss - ada_triplet_loss, ada_triplet_logs = AdaNegTripletVae.compute_ada_triplet_loss(share_masks=share_masks, zs=(d.mean for d in ds_posterior), cfg=cfg) - # merge logs & return loss - return ada_triplet_loss, { - **ada_triplet_logs, - **share_logs, - } - - @staticmethod - def compute_ada_triplet_loss(share_masks, zs, cfg: cfg): - # Normal Triplet Loss - (a_z, p_z, n_z) = zs - trip_loss = configured_triplet(a_z, p_z, n_z, cfg=cfg) - - # Soft Scaled Negative Triplet - (ap_share_mask, an_share_mask, pn_share_mask) = share_masks - triplet_hard_neg_ave_scaled = configured_dist_triplet( - pos_delta=a_z - p_z, - neg_delta=torch.where(an_share_mask, cfg.adat_triplet_share_scale * (a_z - n_z), (a_z - n_z)), - cfg=cfg, - ) - - return triplet_hard_neg_ave_scaled, { - 'triplet': trip_loss, - 'triplet_chosen': triplet_hard_neg_ave_scaled, - } - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/frameworks/vae/_supervised__adatvae.py b/research/code/frameworks/vae/_supervised__adatvae.py deleted file mode 100644 index 4fa02ba9..00000000 --- a/research/code/frameworks/vae/_supervised__adatvae.py +++ /dev/null @@ -1,328 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -from dataclasses import dataclass -from typing import Sequence -from typing import Tuple - -import torch -from disent.util.deprecate import deprecated -from torch.distributions import Distribution -from torch.distributions import Normal - -from disent.nn.loss.triplet import configured_dist_triplet -from disent.nn.loss.triplet import configured_triplet -from disent.frameworks.vae._supervised__tvae import TripletVae -from disent.frameworks.vae._weaklysupervised__adavae import AdaVae - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Guided Ada Vae # -# ========================================================================= # - - -@deprecated('Rather use the AdaNegTripletVae') -class AdaTripletVae(TripletVae): - - REQUIRED_OBS = 3 - - @dataclass - class cfg(TripletVae.cfg, AdaVae.cfg): - # adavae - ada_thresh_mode: str = 'dist' # only works for: adat_share_mask_mode == "posterior" - # ada_tvae - loss - adat_triplet_loss: str = 'triplet_hard_neg_ave' # should be used with a schedule! - adat_triplet_ratio: float = 1.0 - adat_triplet_soft_scale: float = 1.0 - adat_triplet_pull_weight: float = 0.1 # only works for: adat_triplet_loss == "triplet_hard_neg_ave_pull" - adat_triplet_share_scale: float = 0.95 # only works for: adat_triplet_loss == "triplet_hard_neg_ave_scaled" - # ada_tvae - averaging - adat_share_mask_mode: str = 'posterior' - adat_share_ave_mode: str = 'all' # only works for: adat_triplet_loss == "triplet_hard_ave_all" - - def hook_compute_ave_aug_loss(self, ds_posterior: Sequence[Normal], ds_prior: Sequence[Normal], zs_sampled: Sequence[torch.Tensor], xs_partial_recon: Sequence[torch.Tensor], xs_targ: Sequence[torch.Tensor]): - return self.estimate_ada_triplet_loss( - ds_posterior=ds_posterior, - cfg=self.cfg, - ) - - @staticmethod - def estimate_ada_triplet_loss(ds_posterior: Sequence[Normal], cfg: cfg): - """ - zs_params and ds_posterior are convenience variables here. - - they should contain the same values - - in practice we only need one of them and can compute the other! - """ - # compute shared masks, shared embeddings & averages over shared embeddings - share_masks, share_logs = compute_triplet_shared_masks(ds_posterior, cfg=cfg) - ds_posterior_shared, ds_posterior_shared_ave = compute_ave_shared_distributions(ds_posterior, share_masks, cfg=cfg) - - # compute loss - ada_triplet_loss, ada_triplet_logs = AdaTripletVae.compute_ada_triplet_loss( - share_masks=share_masks, - zs=[d.mean for d in ds_posterior], - zs_shared=[d.mean for d in ds_posterior_shared], - zs_shared_ave=[d.mean for d in ds_posterior_shared_ave], - cfg=cfg, - ) - - return ada_triplet_loss, { - **ada_triplet_logs, - **share_logs, - } - - @staticmethod - def compute_ada_triplet_loss(share_masks: Sequence[torch.Tensor], zs: Sequence[Normal], zs_shared: Sequence[Normal], zs_shared_ave: Sequence[Normal], cfg: cfg): - - # Normal Triplet Loss - (a_z, p_z, n_z) = zs - trip_loss = configured_triplet(a_z, p_z, n_z, cfg=cfg) - - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # Hard Losses - zs_shared - # TODO: implement triplet over KL divergence rather than l1/l2 distance? - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # Hard Averaging Before Triplet - (ap_a_ave, ap_p_ave, an_a_ave, an_n_ave, pn_p_ave, pn_n_ave) = zs_shared - triplet_hard_ave = configured_dist_triplet(pos_delta=ap_a_ave - ap_p_ave, neg_delta=an_a_ave - an_n_ave, cfg=cfg) - triplet_hard_ave_neg = configured_dist_triplet(pos_delta=a_z - p_z, neg_delta=an_a_ave - an_n_ave, cfg=cfg) - - # Hard Averaging Before Triplet - PULLING PUSHING - (ap_share_mask, an_share_mask, pn_share_mask) = share_masks - neg_delta_push = torch.where(~an_share_mask, a_z - n_z, torch.zeros_like(a_z)) # this is the same as: an_a_ave - an_n_ave - neg_delta_pull = torch.where( an_share_mask, a_z - n_z, torch.zeros_like(a_z)) - triplet_hard_ave_neg_pull = configured_dist_push_pull_triplet(pos_delta=a_z - p_z, neg_delta=neg_delta_push, neg_delta_pull=neg_delta_pull, cfg=cfg) - - # Hard All Averaging Before Triplet - (a_ave, p_ave, n_ave) = zs_shared_ave - triplet_all_hard_ave = configured_dist_triplet(pos_delta=a_ave-p_ave, neg_delta=a_ave-n_ave, cfg=cfg) - - # Soft Scaled Negative Triplet - triplet_hard_neg_ave_scaled = configured_dist_triplet( - pos_delta=a_z - p_z, - neg_delta=torch.where(an_share_mask, cfg.adat_triplet_share_scale * (a_z - n_z), (a_z - n_z)), - cfg=cfg, - ) - - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # Soft Losses - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # Individual Pair Averaging Losses - _soft_ap_loss = configured_soft_ave_loss(share_mask=ap_share_mask, delta=a_z - p_z, cfg=cfg) - _soft_an_loss = configured_soft_ave_loss(share_mask=an_share_mask, delta=a_z - n_z, cfg=cfg) - _soft_pn_loss = configured_soft_ave_loss(share_mask=pn_share_mask, delta=p_z - n_z, cfg=cfg) - - # soft losses - soft_loss_an = (_soft_an_loss) - soft_loss_an_ap = (_soft_an_loss + _soft_ap_loss) / 2 - soft_loss_an_ap_pn = (_soft_an_loss + _soft_ap_loss + _soft_pn_loss) / 3 - - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # Return - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - losses = { - 'triplet': trip_loss, - # soft ave - 'triplet_soft_ave_neg': trip_loss + soft_loss_an, - 'triplet_soft_ave_p_n': trip_loss + soft_loss_an_ap, - 'triplet_soft_ave_all': trip_loss + soft_loss_an_ap_pn, - # hard ave - 'triplet_hard_ave': torch.lerp(trip_loss, triplet_hard_ave, weight=cfg.adat_triplet_ratio), - 'triplet_hard_neg_ave': torch.lerp(trip_loss, triplet_hard_ave_neg, weight=cfg.adat_triplet_ratio), - 'triplet_hard_neg_ave_pull': torch.lerp(trip_loss, triplet_hard_ave_neg_pull, weight=cfg.adat_triplet_ratio), - 'triplet_hard_ave_all': torch.lerp(trip_loss, triplet_all_hard_ave, weight=cfg.adat_triplet_ratio), - # scaled - 'triplet_hard_neg_ave_scaled': torch.lerp(trip_loss, triplet_hard_neg_ave_scaled, weight=cfg.adat_triplet_ratio), - } - - return losses[cfg.adat_triplet_loss], { - 'triplet': trip_loss, - 'triplet_chosen': losses[cfg.adat_triplet_loss], - } - - -# ========================================================================= # -# Ada-TVae # -# ========================================================================= # - - -def dist_push_pull_triplet(pos_delta, neg_delta, neg_delta_pull, margin_max=1., p=1, pull_weight=1.): - """ - Pushing Pulling Triplet Loss - - should match standard triplet loss if pull_weight=0. - """ - p_dist = torch.norm(pos_delta, p=p, dim=-1) - n_dist = torch.norm(neg_delta, p=p, dim=-1) - n_dist_pull = torch.norm(neg_delta_pull, p=p, dim=-1) - loss = torch.clamp_min(p_dist - n_dist + margin_max + pull_weight * n_dist_pull, 0) - return loss.mean() - - -def configured_dist_push_pull_triplet(pos_delta, neg_delta, neg_delta_pull, cfg: AdaTripletVae.cfg): - """ - required config params: - - cfg.triplet_margin_max: (0, inf) - - cfg.triplet_p: 1 or 2 - - cfg.triplet_scale: [0, inf) - - cfg.adat_triplet_pull_weight: [0, 1] - """ - return dist_push_pull_triplet( - pos_delta=pos_delta, neg_delta=neg_delta, neg_delta_pull=neg_delta_pull, - margin_max=cfg.triplet_margin_max, p=cfg.triplet_p, pull_weight=cfg.adat_triplet_pull_weight, - ) * cfg.triplet_scale - - -def soft_ave_loss(share_mask, delta): - return torch.norm(torch.where(share_mask, delta, torch.zeros_like(delta)), p=2, dim=-1).mean() - - -def configured_soft_ave_loss(share_mask, delta, cfg: AdaTripletVae.cfg): - """ - required config params: - - cfg.triplet_scale: [0, inf) - - cfg.adat_triplet_soft_scale: [0, inf) - """ - return soft_ave_loss(share_mask=share_mask, delta=delta) * (cfg.adat_triplet_soft_scale * cfg.triplet_scale) - - -# ========================================================================= # -# AveAda-TVAE # -# ========================================================================= # - - -def compute_triplet_shared_masks_from_zs(zs: Sequence[torch.Tensor], cfg): - """ - required config params: - - cfg.ada_thresh_ratio: - """ - a_z, p_z, n_z = zs - # shared elements that need to be averaged, computed per pair in the batch. - ap_share_mask = AdaVae.compute_shared_mask_from_zs(a_z, p_z, ratio=cfg.ada_thresh_ratio) - an_share_mask = AdaVae.compute_shared_mask_from_zs(a_z, n_z, ratio=cfg.ada_thresh_ratio) - pn_share_mask = AdaVae.compute_shared_mask_from_zs(p_z, n_z, ratio=cfg.ada_thresh_ratio) - # return values - share_masks = (ap_share_mask, an_share_mask, pn_share_mask) - return share_masks, { - 'ap_shared': ap_share_mask.sum(dim=1).float().mean(), - 'an_shared': an_share_mask.sum(dim=1).float().mean(), - 'pn_shared': pn_share_mask.sum(dim=1).float().mean(), - } - - -def compute_triplet_shared_masks(ds_posterior: Sequence[Distribution], cfg: AdaTripletVae.cfg): - """ - required config params: - - cfg.ada_thresh_ratio: - - cfg.ada_thresh_mode: "kl", "symmetric_kl", "dist", "sampled_dist" - : only applies if cfg.ada_share_mask_mode=="posterior" - - cfg.adat_share_mask_mode: "posterior", "sample", "sample_each" - """ - a_posterior, p_posterior, n_posterior = ds_posterior - - # shared elements that need to be averaged, computed per pair in the batch. - if cfg.adat_share_mask_mode == 'posterior': - ap_share_mask = AdaVae.compute_shared_mask_from_posteriors(a_posterior, p_posterior, thresh_mode=cfg.ada_thresh_mode, ratio=cfg.ada_thresh_ratio) - an_share_mask = AdaVae.compute_shared_mask_from_posteriors(a_posterior, n_posterior, thresh_mode=cfg.ada_thresh_mode, ratio=cfg.ada_thresh_ratio) - pn_share_mask = AdaVae.compute_shared_mask_from_posteriors(p_posterior, n_posterior, thresh_mode=cfg.ada_thresh_mode, ratio=cfg.ada_thresh_ratio) - elif cfg.adat_share_mask_mode == 'sample': - a_z_sample, p_z_sample, n_z_sample = a_posterior.rsample(), p_posterior.rsample(), n_posterior.rsample() - ap_share_mask = AdaVae.compute_shared_mask_from_zs(a_z_sample, p_z_sample, ratio=cfg.ada_thresh_ratio) - an_share_mask = AdaVae.compute_shared_mask_from_zs(a_z_sample, n_z_sample, ratio=cfg.ada_thresh_ratio) - pn_share_mask = AdaVae.compute_shared_mask_from_zs(p_z_sample, n_z_sample, ratio=cfg.ada_thresh_ratio) - elif cfg.adat_share_mask_mode == 'sample_each': - ap_share_mask = AdaVae.compute_shared_mask_from_zs(a_posterior.rsample(), p_posterior.rsample(), ratio=cfg.ada_thresh_ratio) - an_share_mask = AdaVae.compute_shared_mask_from_zs(a_posterior.rsample(), n_posterior.rsample(), ratio=cfg.ada_thresh_ratio) - pn_share_mask = AdaVae.compute_shared_mask_from_zs(p_posterior.rsample(), n_posterior.rsample(), ratio=cfg.ada_thresh_ratio) - else: - raise KeyError(f'Invalid cfg.adat_share_mask_mode={repr(cfg.adat_share_mask_mode)}') - - # return values - share_masks = (ap_share_mask, an_share_mask, pn_share_mask) - return share_masks, { - 'ap_shared': ap_share_mask.sum(dim=1).float().mean(), - 'an_shared': an_share_mask.sum(dim=1).float().mean(), - 'pn_shared': pn_share_mask.sum(dim=1).float().mean(), - } - - -def compute_ave_shared_distributions(ds_posterior: Sequence[Normal], share_masks: Sequence[torch.Tensor], cfg: AdaTripletVae.cfg) -> Tuple[Sequence[Normal], Sequence[Normal]]: - """ - required config params: - - cfg.ada_average_mode: "gvae", "ml-vae" - - cfg.adat_share_ave_mode: "all", "pos_neg", "pos", "neg" - """ - a_posterior, p_posterior, n_posterior = ds_posterior - ap_share_mask, an_share_mask, pn_share_mask = share_masks - - # compute shared embeddings - ave_ap_a_posterior, ave_ap_p_posterior = AdaVae.make_shared_posteriors(a_posterior, p_posterior, ap_share_mask, average_mode=cfg.ada_average_mode) - ave_an_a_posterior, ave_an_n_posterior = AdaVae.make_shared_posteriors(a_posterior, n_posterior, an_share_mask, average_mode=cfg.ada_average_mode) - ave_pn_p_posterior, ave_pn_n_posterior = AdaVae.make_shared_posteriors(p_posterior, n_posterior, pn_share_mask, average_mode=cfg.ada_average_mode) - - # compute averaged shared embeddings - if cfg.adat_share_ave_mode == 'all': - ave_a_posterior = AdaVae.compute_average_distribution(ave_ap_a_posterior, ave_an_a_posterior, average_mode=cfg.ada_average_mode) - ave_p_posterior = AdaVae.compute_average_distribution(ave_ap_p_posterior, ave_pn_p_posterior, average_mode=cfg.ada_average_mode) - ave_n_posterior = AdaVae.compute_average_distribution(ave_an_n_posterior, ave_pn_n_posterior, average_mode=cfg.ada_average_mode) - elif cfg.adat_share_ave_mode == 'pos_neg': - ave_a_posterior = AdaVae.compute_average_distribution(ave_ap_a_posterior, ave_an_a_posterior, average_mode=cfg.ada_average_mode) - ave_p_posterior = ave_ap_p_posterior - ave_n_posterior = ave_an_n_posterior - elif cfg.adat_share_ave_mode == 'pos': - ave_a_posterior = ave_ap_a_posterior - ave_p_posterior = ave_ap_p_posterior - ave_n_posterior = n_posterior - elif cfg.adat_share_ave_mode == 'neg': - ave_a_posterior = ave_an_a_posterior - ave_p_posterior = p_posterior - ave_n_posterior = ave_an_n_posterior - else: - raise KeyError(f'Invalid cfg.adat_share_ave_mode={repr(cfg.adat_share_ave_mode)}') - - ds_posterior_shared = ( - ave_ap_a_posterior, ave_ap_p_posterior, # a & p - ave_an_a_posterior, ave_an_n_posterior, # a & n - ave_pn_p_posterior, ave_pn_n_posterior, # p & n - ) - - ds_posterior_shared_ave = ( - ave_a_posterior, - ave_p_posterior, - ave_n_posterior - ) - - # return values - return ds_posterior_shared, ds_posterior_shared_ave - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/frameworks/vae/_supervised__badavae.py b/research/code/frameworks/vae/_supervised__badavae.py deleted file mode 100644 index 1ba1e563..00000000 --- a/research/code/frameworks/vae/_supervised__badavae.py +++ /dev/null @@ -1,123 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from dataclasses import dataclass -from typing import Any -from typing import Dict -from typing import Sequence -from typing import Tuple - -import torch -from torch.distributions import Distribution - -from disent.frameworks.vae._weaklysupervised__adavae import AdaVae - - -# ========================================================================= # -# Guided Ada Vae # -# ========================================================================= # - - -class BoundedAdaVae(AdaVae): - - REQUIRED_OBS = 3 - - @dataclass - class cfg(AdaVae.cfg): - pass - - def hook_intercept_ds(self, ds_posterior: Sequence[Distribution], ds_prior: Sequence[Distribution]) -> Tuple[Sequence[Distribution], Sequence[Distribution], Dict[str, Any]]: - a_posterior, p_posterior, n_posterior = ds_posterior - - # get deltas - a_p_deltas = AdaVae.compute_deltas_from_posteriors(a_posterior, p_posterior, thresh_mode=self.cfg.ada_thresh_mode) - a_n_deltas = AdaVae.compute_deltas_from_posteriors(a_posterior, n_posterior, thresh_mode=self.cfg.ada_thresh_mode) - - # shared elements that need to be averaged, computed per pair in the batch. - old_p_shared_mask = AdaVae.estimate_shared_mask(a_p_deltas, ratio=self.cfg.ada_thresh_ratio) - old_n_shared_mask = AdaVae.estimate_shared_mask(a_n_deltas, ratio=self.cfg.ada_thresh_ratio) - - # modify threshold based on criterion and recompute if necessary - # CORE of this approach! - p_shared_mask, n_shared_mask = compute_constrained_masks(a_p_deltas, old_p_shared_mask, a_n_deltas, old_n_shared_mask) - - # make averaged variables - # TODO: this will probably be better if it is the negative involed - # TODO: this can be merged with the gadavae/badavae - ave_ap_a_posterior, ave_ap_p_posterior = AdaVae.make_shared_posteriors(a_posterior, p_posterior, p_shared_mask, average_mode=self.cfg.ada_average_mode) - - # TODO: n_z_params should not be here! this does not match the original version - # number of loss elements is not 2 like the original - # - recons gets 2 items, p & a only - # - reg gets 2 items, p & a only - new_ds_posterior = (ave_ap_a_posterior, ave_ap_p_posterior, n_posterior) - - # return new args & generate logs - # -- we only return 2 parameters a & p, not n - return new_ds_posterior, ds_prior, { - 'p_shared_before': old_p_shared_mask.sum(dim=1).float().mean(), - 'p_shared_after': p_shared_mask.sum(dim=1).float().mean(), - 'n_shared_before': old_n_shared_mask.sum(dim=1).float().mean(), - 'n_shared_after': n_shared_mask.sum(dim=1).float().mean(), - } - - -# ========================================================================= # -# HELPER # -# ========================================================================= # - - -# TODO: rerun experiments that used this because ELEMENTWISE may have been wrong -# maximum and minimum, used to be max and min which may have given wrong results? -def compute_constrained_masks(p_kl_deltas, p_shared_mask, n_kl_deltas, n_shared_mask): - # number of changed factors - p_shared_num = torch.sum(p_shared_mask, dim=1, keepdim=True) - n_shared_num = torch.sum(n_shared_mask, dim=1, keepdim=True) - - # POSITIVE SHARED MASK - # order from smallest to largest - p_sort_indices = torch.argsort(p_kl_deltas, dim=1) - # p_shared should be at least n_shared - new_p_shared_num = torch.maximum(p_shared_num, n_shared_num) # ELEMENTWISE - - # NEGATIVE SHARED MASK - # order from smallest to largest - n_sort_indices = torch.argsort(n_kl_deltas, dim=1) - # n_shared should be at most p_shared - new_n_shared_num = torch.minimum(p_shared_num, n_shared_num) # ELEMENTWISE - - # COMPUTE NEW MASKS - new_p_shared_mask = torch.zeros_like(p_shared_mask) - new_n_shared_mask = torch.zeros_like(n_shared_mask) - for i, (new_shared_p, new_shared_n) in enumerate(zip(new_p_shared_num, new_n_shared_num)): - new_p_shared_mask[i, p_sort_indices[i, :new_shared_p]] = True - new_n_shared_mask[i, n_sort_indices[i, :new_shared_n]] = True - - # return masks - return new_p_shared_mask, new_n_shared_mask - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/frameworks/vae/_supervised__gadavae.py b/research/code/frameworks/vae/_supervised__gadavae.py deleted file mode 100644 index de5e6d05..00000000 --- a/research/code/frameworks/vae/_supervised__gadavae.py +++ /dev/null @@ -1,102 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from dataclasses import dataclass -from typing import Any -from typing import Dict -from typing import Sequence -from typing import Tuple - -from torch.distributions import Distribution - -from disent.frameworks.vae._weaklysupervised__adavae import AdaVae -from research.code.frameworks.vae._supervised__badavae import compute_constrained_masks - - -# ========================================================================= # -# Guided Ada Vae # -# ========================================================================= # - - -class GuidedAdaVae(AdaVae): - - REQUIRED_OBS = 3 - - @dataclass - class cfg(AdaVae.cfg): - gada_anchor_ave_mode: str = 'average' - - def __init__(self, model: 'AutoEncoder', cfg: cfg = None, batch_augment=None): - super().__init__(model=model, cfg=cfg, batch_augment=batch_augment) - # how the anchor is averaged - assert cfg.gada_anchor_ave_mode in {'thresh', 'average'} - - def hook_intercept_ds(self, ds_posterior: Sequence[Distribution], ds_prior: Sequence[Distribution]) -> Tuple[Sequence[Distribution], Sequence[Distribution], Dict[str, Any]]: - """ - *NB* arguments must satisfy: d(l, l2) < d(l, l3) [positive dist < negative dist] - - This function assumes that the distance between labels l, l2, l3 - corresponding to z, z2, z3 satisfy the criteria d(l, l2) < d(l, l3) - ie. l2 is the positive sample, l3 is the negative sample - """ - a_posterior, p_posterior, n_posterior = ds_posterior - - # get deltas - a_p_deltas = AdaVae.compute_deltas_from_posteriors(a_posterior, p_posterior, thresh_mode=self.cfg.ada_thresh_mode) - a_n_deltas = AdaVae.compute_deltas_from_posteriors(a_posterior, n_posterior, thresh_mode=self.cfg.ada_thresh_mode) - - # shared elements that need to be averaged, computed per pair in the batch. - old_p_shared_mask = AdaVae.estimate_shared_mask(a_p_deltas, ratio=self.cfg.ada_thresh_ratio) - old_n_shared_mask = AdaVae.estimate_shared_mask(a_n_deltas, ratio=self.cfg.ada_thresh_ratio) - - # modify threshold based on criterion and recompute if necessary - # CORE of this approach! - p_shared_mask, n_shared_mask = compute_constrained_masks(a_p_deltas, old_p_shared_mask, a_n_deltas, old_n_shared_mask) - - # make averaged variables - # TODO: this can be merged with the gadavae/badavae - ave_ap_a_posterior, ave_ap_p_posterior = AdaVae.make_shared_posteriors(a_posterior, p_posterior, p_shared_mask, average_mode=self.cfg.ada_average_mode) - ave_an_a_posterior, ave_an_n_posterior = AdaVae.make_shared_posteriors(a_posterior, n_posterior, n_shared_mask, average_mode=self.cfg.ada_average_mode) - ave_a_posterior = AdaVae.compute_average_distribution(ave_ap_a_posterior, ave_an_a_posterior, average_mode=self.cfg.ada_average_mode) - - # compute anchor average using the adaptive threshold | TODO: this doesn't really make sense - anchor_ave_logs = {} - if self.cfg.gada_anchor_ave_mode == 'thresh': - ave_shared_mask = p_shared_mask * n_shared_mask - ave_params, _ = AdaVae.make_shared_posteriors(a_posterior, ave_a_posterior, ave_shared_mask, average_mode=self.cfg.ada_average_mode) - anchor_ave_logs['ave_shared'] = ave_shared_mask.sum(dim=1).float().mean() - - new_ds_posterior = ave_a_posterior, ave_ap_p_posterior, ave_an_n_posterior - - return new_ds_posterior, ds_prior, { - 'p_shared_before': old_p_shared_mask.sum(dim=1).float().mean(), - 'p_shared_after': p_shared_mask.sum(dim=1).float().mean(), - 'n_shared_before': old_n_shared_mask.sum(dim=1).float().mean(), - 'n_shared_after': n_shared_mask.sum(dim=1).float().mean(), - **anchor_ave_logs, - } - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/frameworks/vae/_supervised__tbadavae.py b/research/code/frameworks/vae/_supervised__tbadavae.py deleted file mode 100644 index 93c1cba9..00000000 --- a/research/code/frameworks/vae/_supervised__tbadavae.py +++ /dev/null @@ -1,51 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from dataclasses import dataclass - -from research.code.frameworks.vae._supervised__badavae import BoundedAdaVae -from disent.nn.loss.triplet import compute_triplet_loss -from disent.nn.loss.triplet import TripletLossConfig - - -# ========================================================================= # -# tbadavae # -# ========================================================================= # - - -class TripletBoundedAdaVae(BoundedAdaVae): - - REQUIRED_OBS = 3 - - @dataclass - class cfg(BoundedAdaVae.cfg, TripletLossConfig): - pass - - def hook_compute_ave_aug_loss(self, ds_posterior, ds_prior, zs_sampled, xs_partial_recon, xs_targ): - return compute_triplet_loss(zs=[d.mean for d in ds_posterior], cfg=self.cfg) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/frameworks/vae/_supervised__tgadavae.py b/research/code/frameworks/vae/_supervised__tgadavae.py deleted file mode 100644 index 5a2c150e..00000000 --- a/research/code/frameworks/vae/_supervised__tgadavae.py +++ /dev/null @@ -1,51 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from dataclasses import dataclass - -from disent.nn.loss.triplet import compute_triplet_loss -from disent.nn.loss.triplet import TripletLossConfig -from research.code.frameworks.vae._supervised__gadavae import GuidedAdaVae - - -# ========================================================================= # -# tgadavae # -# ========================================================================= # - - -class TripletGuidedAdaVae(GuidedAdaVae): - - REQUIRED_OBS = 3 - - @dataclass - class cfg(GuidedAdaVae.cfg, TripletLossConfig): - pass - - def hook_compute_ave_aug_loss(self, ds_posterior, ds_prior, zs_sampled, xs_partial_recon, xs_targ): - return compute_triplet_loss(zs=[d.mean for d in ds_posterior], cfg=self.cfg) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/frameworks/vae/_unsupervised__dorvae.py b/research/code/frameworks/vae/_unsupervised__dorvae.py deleted file mode 100644 index d8b139f4..00000000 --- a/research/code/frameworks/vae/_unsupervised__dorvae.py +++ /dev/null @@ -1,169 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from dataclasses import dataclass -from typing import final -from typing import Optional -from typing import Sequence - -import torch -from torch.distributions import Normal - -from disent.frameworks.helper.reconstructions import make_reconstruction_loss -from disent.frameworks.helper.reconstructions import ReconLossHandler -from disent.frameworks.vae._supervised__tvae import TripletVae -from disent.frameworks.vae._weaklysupervised__adavae import AdaVae -from disent.nn.loss.softsort import torch_mse_rank_loss -from disent.nn.loss.softsort import spearman_rank_loss - - -# ========================================================================= # -# tvae # -# ========================================================================= # - - -class DataOverlapRankVae(TripletVae): - """ - This converges really well! - - but doesn't introduce axis alignment as well if there is no additional - inward pressure term like triplet to move representations closer together - """ - - REQUIRED_OBS = 1 - - @dataclass - class cfg(TripletVae.cfg): - # compatibility - ada_thresh_mode: str = 'dist' # kl, symmetric_kl, dist, sampled_dist - ada_thresh_ratio: float = 0.5 - adat_triplet_share_scale: float = 0.95 - # OVERLAP VAE - overlap_loss: Optional[str] = None - overlap_num: int = 1024 - # AUGMENT - overlap_augment_mode: str = 'none' - overlap_augment: Optional[dict] = None - # REPRESENTATIONS - overlap_repr: str = 'deterministic' # deterministic, stochastic - overlap_rank_mode: str = 'spearman_rank' # spearman_rank, mse_rank - overlap_inward_pressure_masked: bool = False - overlap_inward_pressure_scale: float = 0.1 - - def __init__(self, model: 'AutoEncoder', cfg: cfg = None, batch_augment=None): - # TODO: duplicate code - super().__init__(model=model, cfg=cfg, batch_augment=batch_augment) - # initialise - if self.cfg.overlap_augment_mode != 'none': - assert self.cfg.overlap_augment is not None, 'if cfg.overlap_augment_mode is not "none", then cfg.overlap_augment must be defined.' - # set augment and instantiate if needed - self._augment = None - if isinstance(self._augment, dict): - import hydra - self._augment = hydra.utils.instantiate(self.cfg.overlap_augment) - assert callable(self._augment), f'augment is not callable: {repr(self._augment)}' - # get overlap loss - overlap_loss = self.cfg.overlap_loss if (self.cfg.overlap_loss is not None) else self.cfg.recon_loss - self.__overlap_handler: ReconLossHandler = make_reconstruction_loss(overlap_loss, reduction='mean') - - @final - @property - def overlap_handler(self) -> ReconLossHandler: - return self.__overlap_handler - - def hook_compute_ave_aug_loss(self, ds_posterior: Sequence[Normal], ds_prior, zs_sampled, xs_partial_recon, xs_targ: Sequence[torch.Tensor]): - # ++++++++++++++++++++++++++++++++++++++++++ # - # 1. augment batch - (x_targ_orig,) = xs_targ - with torch.no_grad(): - (x_targ,) = self.augment_triplet_targets(xs_targ) - (d_posterior,) = ds_posterior - (z_sampled,) = zs_sampled - # 2. generate random pairs -- this does not generate unique pairs - a_idxs, p_idxs = torch.randint(len(x_targ), size=(2, self.cfg.overlap_num), device=x_targ.device) - # ++++++++++++++++++++++++++++++++++++++++++ # - # compute image distances - with torch.no_grad(): - ap_recon_dists = self.overlap_handler.compute_pairwise_loss(x_targ[a_idxs], x_targ[p_idxs]) - # ++++++++++++++++++++++++++++++++++++++++++ # - # get representations - if self.cfg.overlap_repr == 'deterministic': - a_z, p_z = d_posterior.loc[a_idxs], d_posterior.loc[p_idxs] - elif self.cfg.overlap_repr == 'stochastic': - a_z, p_z = z_sampled[a_idxs], z_sampled[p_idxs] - else: - raise KeyError(f'invalid overlap_repr mode: {repr(self.cfg.overlap_repr)}') - # DISENTANGLE! - # compute adaptive mask & weight deltas - a_posterior = Normal(d_posterior.loc[a_idxs], d_posterior.scale[a_idxs]) - p_posterior = Normal(d_posterior.loc[p_idxs], d_posterior.scale[p_idxs]) - share_mask = AdaVae.compute_shared_mask_from_posteriors(a_posterior, p_posterior, thresh_mode=self.cfg.ada_thresh_mode, ratio=self.cfg.ada_thresh_ratio) - deltas = torch.where(share_mask, self.cfg.adat_triplet_share_scale * (a_z - p_z), (a_z - p_z)) - # compute representation distances - ap_repr_dists = torch.abs(deltas).sum(dim=-1) - # ++++++++++++++++++++++++++++++++++++++++++ # - if self.cfg.overlap_rank_mode == 'mse_rank': - loss = torch_mse_rank_loss(ap_repr_dists, ap_recon_dists.detach(), dims=-1, reduction='mean') - loss_logs = {'mse_rank_loss': loss} - elif self.cfg.overlap_rank_mode == 'spearman_rank': - loss = - spearman_rank_loss(ap_repr_dists, ap_recon_dists.detach(), nan_to_num=True) - loss_logs = {'spearman_rank_loss': loss} - else: - raise KeyError(f'invalid overlap_rank_mode: {repr(self.cfg.overlap_repr)}') - # ++++++++++++++++++++++++++++++++++++++++++ # - # inward pressure - if self.cfg.overlap_inward_pressure_masked: - in_deltas = torch.abs(deltas) * share_mask - else: - in_deltas = torch.abs(deltas) - # compute inward pressure - inward_pressure = self.cfg.overlap_inward_pressure_scale * in_deltas.mean() - loss += inward_pressure - # ++++++++++++++++++++++++++++++++++++++++++ # - # return the loss - return loss, { - **loss_logs, - 'inward_pressure': inward_pressure, - } - - def augment_triplet_targets(self, xs_targ): - # TODO: duplicate code - if self.cfg.overlap_augment_mode == 'none': - aug_xs_targ = xs_targ - elif (self.cfg.overlap_augment_mode == 'augment') or (self.cfg.overlap_augment_mode == 'augment_each'): - # recreate augment each time - if self.cfg.overlap_augment_mode == 'augment_each': - import hydra - self._augment = hydra.utils.instantiate(self.cfg.overlap_augment) - # augment on correct device - aug_xs_targ = [self._augment(x_targ) for x_targ in xs_targ] - # checks - assert all(a.shape == b.shape for a, b in zip(xs_targ, aug_xs_targ)) - else: - raise KeyError(f'invalid cfg.overlap_augment_mode={repr(self.cfg.overlap_augment_mode)}') - return aug_xs_targ - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/frameworks/vae/_unsupervised__dotvae.py b/research/code/frameworks/vae/_unsupervised__dotvae.py deleted file mode 100644 index a5d65ae6..00000000 --- a/research/code/frameworks/vae/_unsupervised__dotvae.py +++ /dev/null @@ -1,229 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -from dataclasses import dataclass -from typing import final -from typing import Optional -from typing import Sequence - -import torch -from torch.distributions import Normal - -from disent.frameworks.helper.reconstructions import make_reconstruction_loss -from disent.frameworks.helper.reconstructions import ReconLossHandler -from disent.nn.loss.triplet_mining import configured_idx_mine -from research.code.frameworks.vae import AdaNegTripletVae - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Mixin # -# ========================================================================= # - - -class DataOverlapMixin(object): - - # should be inherited by the config on the child class - @dataclass - class cfg: - # override from AE - recon_loss: str = 'mse' - # OVERLAP VAE - overlap_loss: Optional[str] = None # if None, use the value from recon_loss - overlap_num: int = 1024 - overlap_mine_ratio: float = 0.1 - overlap_mine_triplet_mode: str = 'none' - # AUGMENT - overlap_augment_mode: str = 'augment' - overlap_augment: Optional[dict] = None - - # private properties - # - since this class does not have a constructor, it - # provides the `init_data_overlap_mixin` method, which - # should be called inside the constructor of the child class - _augment: callable - _overlap_handler: ReconLossHandler - _init: bool - - def init_data_overlap_mixin(self): - if hasattr(self, '_init'): - raise RuntimeError(f'{DataOverlapMixin.__name__} on {self.__class__.__name__} was initialised more than once!') - self._init = True - # set augment and instantiate if needed - if self.cfg.overlap_augment is not None: - import hydra - self._augment = hydra.utils.instantiate(self.cfg.overlap_augment) - assert (self._augment is None) or callable(self._augment), f'augment is not None or callable: {repr(self._augment)}, obtained from `overlap_augment={repr(self.cfg.overlap_augment)}`' - else: - self._augment = None - # get overlap loss - if self.cfg.overlap_loss is None: - log.info(f'`overlap_loss` not specified for {repr(self.__class__.__name__)}, using `recon_loss` instead: {repr(self.cfg.recon_loss)}') - overlap_loss = self.cfg.recon_loss - else: - overlap_loss = self.cfg.overlap_loss - # construct the overlap handler - self._overlap_handler: ReconLossHandler = make_reconstruction_loss(name=overlap_loss, reduction='mean') - - @final - @property - def overlap_handler(self) -> ReconLossHandler: - return self._overlap_handler - - def overlap_swap_triplet_idxs(self, x_targ, a_idxs, p_idxs, n_idxs): - xs_targ = [x_targ[idxs] for idxs in (a_idxs, p_idxs, n_idxs)] - # CORE: order the latent variables for triplet - swap_mask = self.overlap_swap_mask(xs_targ=xs_targ) - # swap all idxs - swapped_a_idxs = a_idxs - swapped_p_idxs = torch.where(swap_mask, n_idxs, p_idxs) - swapped_n_idxs = torch.where(swap_mask, p_idxs, n_idxs) - # return values - return swapped_a_idxs, swapped_p_idxs, swapped_n_idxs - - def overlap_swap_mask(self, xs_targ: Sequence[torch.Tensor]) -> torch.Tensor: - # get variables - a_x_targ_OLD, p_x_targ_OLD, n_x_targ_OLD = xs_targ - # CORE OF THIS APPROACH - # ++++++++++++++++++++++++++++++++++++++++++ # - # calculate which are wrong! - # TODO: add more loss functions, like perceptual & others - with torch.no_grad(): - a_p_losses = self.overlap_handler.compute_pairwise_loss(a_x_targ_OLD, p_x_targ_OLD) # (B, C, H, W) -> (B,) - a_n_losses = self.overlap_handler.compute_pairwise_loss(a_x_targ_OLD, n_x_targ_OLD) # (B, C, H, W) -> (B,) - swap_mask = (a_p_losses > a_n_losses) # (B,) - # ++++++++++++++++++++++++++++++++++++++++++ # - return swap_mask - - @torch.no_grad() - def augment_batch(self, x_targ): - # ++++++++++++++++++++++++++++++++++++++++++ # - # perform the augments - if self.cfg.overlap_augment_mode in ('augment', 'augment_each'): - # recreate augment each time - if self.cfg.overlap_augment_mode == 'augment_each': - import hydra - self._augment = hydra.utils.instantiate(self.cfg.overlap_augment) - # augment on correct device, but skip if not defined! - aug_x_targ = x_targ if (self._augment is None) else self._augment(x_targ) - elif self.cfg.overlap_augment_mode == 'none': - # no augment - aug_x_targ = x_targ - else: - raise KeyError(f'invalid cfg.overlap_augment_mode={repr(self.cfg.overlap_augment_mode)}') - # ++++++++++++++++++++++++++++++++++++++++++ # - # checks - assert x_targ.shape == aug_x_targ.shape - return aug_x_targ - - def mine_triplets(self, x_targ, a_idxs, p_idxs, n_idxs): - return configured_idx_mine( - x_targ=x_targ, - a_idxs=a_idxs, - p_idxs=p_idxs, - n_idxs=n_idxs, - cfg=self.cfg, - pairwise_loss_fn=self.overlap_handler.compute_pairwise_loss, - ) - - def random_mined_triplets(self, x_targ_orig: torch.Tensor): - # ++++++++++++++++++++++++++++++++++++++++++ # - # 1. augment batch - aug_x_targ = self.augment_batch(x_targ_orig) - # 2. generate random triples -- this does not generate unique pairs - a_idxs, p_idxs, n_idxs = torch.randint(len(aug_x_targ), size=(3, min(self.cfg.overlap_num, len(aug_x_targ)**3)), device=aug_x_targ.device) - # ++++++++++++++++++++++++++++++++++++++++++ # - # self.debug(x_targ_orig, x_targ, a_idxs, p_idxs, n_idxs) - # ++++++++++++++++++++++++++++++++++++++++++ # - # TODO: this can be merged into a single function -- inefficient currently with deltas computed twice - # 3. reorder random triples - a_idxs, p_idxs, n_idxs = self.overlap_swap_triplet_idxs(aug_x_targ, a_idxs, p_idxs, n_idxs) - # 4. mine random triples - a_idxs, p_idxs, n_idxs = self.mine_triplets(aug_x_targ, a_idxs, p_idxs, n_idxs) - # ++++++++++++++++++++++++++++++++++++++++++ # - return a_idxs, p_idxs, n_idxs - - # def debug(self, x_targ_orig, x_targ, a_idxs, p_idxs, n_idxs): - # a_p_overlap_orig = - self.recon_handler.compute_unreduced_loss(x_targ_orig[a_idxs], x_targ_orig[p_idxs]).mean(dim=(-3, -2, -1)) # (B, C, H, W) -> (B,) - # a_n_overlap_orig = - self.recon_handler.compute_unreduced_loss(x_targ_orig[a_idxs], x_targ_orig[n_idxs]).mean(dim=(-3, -2, -1)) # (B, C, H, W) -> (B,) - # a_p_overlap = - self.recon_handler.compute_unreduced_loss(x_targ[a_idxs], x_targ[p_idxs]).mean(dim=(-3, -2, -1)) # (B, C, H, W) -> (B,) - # a_n_overlap = - self.recon_handler.compute_unreduced_loss(x_targ[a_idxs], x_targ[n_idxs]).mean(dim=(-3, -2, -1)) # (B, C, H, W) -> (B,) - # a_p_overlap_mul = - (a_p_overlap_orig * a_p_overlap) - # a_n_overlap_mul = - (a_n_overlap_orig * a_n_overlap) - # # check number of things - # (up_values_orig, up_counts_orig) = torch.unique(a_p_overlap_orig, sorted=True, return_inverse=False, return_counts=True) - # (un_values_orig, un_counts_orig) = torch.unique(a_n_overlap_orig, sorted=True, return_inverse=False, return_counts=True) - # (up_values, up_counts) = torch.unique(a_p_overlap, sorted=True, return_inverse=False, return_counts=True) - # (un_values, un_counts) = torch.unique(a_n_overlap, sorted=True, return_inverse=False, return_counts=True) - # (up_values_mul, up_counts_mul) = torch.unique(a_p_overlap_mul, sorted=True, return_inverse=False, return_counts=True) - # (un_values_mul, un_counts_mul) = torch.unique(a_n_overlap_mul, sorted=True, return_inverse=False, return_counts=True) - # # plot! - # plt.scatter(up_values_orig.detach().cpu(), torch.cumsum(up_counts_orig, dim=-1).detach().cpu()) - # plt.scatter(un_values_orig.detach().cpu(), torch.cumsum(un_counts_orig, dim=-1).detach().cpu()) - # plt.scatter(up_values.detach().cpu(), torch.cumsum(up_counts, dim=-1).detach().cpu()) - # plt.scatter(un_values.detach().cpu(), torch.cumsum(un_counts, dim=-1).detach().cpu()) - # plt.scatter(up_values_mul.detach().cpu(), torch.cumsum(up_counts_mul, dim=-1).detach().cpu()) - # plt.scatter(un_values_mul.detach().cpu(), torch.cumsum(un_counts_mul, dim=-1).detach().cpu()) - # plt.show() - # time.sleep(10) - - -# ========================================================================= # -# Data Overlap Triplet VAE # -# ========================================================================= # - - -class DataOverlapTripletVae(AdaNegTripletVae, DataOverlapMixin): - - REQUIRED_OBS = 1 - - @dataclass - class cfg(AdaNegTripletVae.cfg, DataOverlapMixin.cfg): - pass - - def __init__(self, model: 'AutoEncoder', cfg: cfg = None, batch_augment=None): - super().__init__(model=model, cfg=cfg, batch_augment=batch_augment) - # initialise mixin - self.init_data_overlap_mixin() - - def hook_compute_ave_aug_loss(self, ds_posterior: Sequence[Normal], ds_prior, zs_sampled, xs_partial_recon, xs_targ: Sequence[torch.Tensor]): - [d_posterior], [x_targ_orig] = ds_posterior, xs_targ - # 1. randomly generate and mine triplets using augmented versions of the inputs - a_idxs, p_idxs, n_idxs = self.random_mined_triplets(x_targ_orig=x_targ_orig) - # 2. compute triplet loss - loss, loss_log = AdaNegTripletVae.estimate_ada_triplet_loss( - ds_posterior=[Normal(d_posterior.loc[idxs], d_posterior.scale[idxs]) for idxs in (a_idxs, p_idxs, n_idxs)], - cfg=self.cfg, - ) - return loss, { - **loss_log, - } - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/frameworks/vae/_weaklysupervised__augpostriplet.py b/research/code/frameworks/vae/_weaklysupervised__augpostriplet.py deleted file mode 100644 index bad6354a..00000000 --- a/research/code/frameworks/vae/_weaklysupervised__augpostriplet.py +++ /dev/null @@ -1,82 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -import warnings -from dataclasses import dataclass -from typing import Union - -import torch - -from disent.frameworks.vae._supervised__tvae import TripletVae - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Guided Ada Vae # -# ========================================================================= # - - -class AugPosTripletVae(TripletVae): - - REQUIRED_OBS = 2 # third obs is generated from augmentations - - @dataclass - class cfg(TripletVae.cfg): - overlap_augment: Union[dict, callable] = None - - def __init__(self, model: 'AutoEncoder', cfg: cfg = None, batch_augment=None): - super().__init__(model=model, cfg=cfg, batch_augment=batch_augment) - # set augment and instantiate if needed - self._augment = self.cfg.overlap_augment - if isinstance(self._augment, dict): - import hydra - self._augment = hydra.utils.instantiate(self._augment) - # get default if needed - if self._augment is None: - self._augment = torch.nn.Identity() - warnings.warn(f'{self.__class__.__name__}, no overlap_augment was specified, defaulting to nn.Identity which WILL break things!') - # checks! - assert callable(self._augment), f'augment is not callable: {repr(self._augment)}' - - def do_training_step(self, batch, batch_idx): - (a_x, n_x), (a_x_targ, n_x_targ) = self._get_xs_and_targs(batch, batch_idx) - - # generate augmented items - with torch.no_grad(): - p_x_targ = a_x_targ - p_x = self._augment(a_x) - # a_x = self._aug(a_x) - # n_x = self._aug(n_x) - - batch['x'], batch['x_targ'] = (a_x, p_x, n_x), (a_x_targ, p_x_targ, n_x_targ) - # compute! - return super().do_training_step(batch, batch_idx) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/frameworks/vae/_weaklysupervised__st_adavae.py b/research/code/frameworks/vae/_weaklysupervised__st_adavae.py deleted file mode 100644 index deddced4..00000000 --- a/research/code/frameworks/vae/_weaklysupervised__st_adavae.py +++ /dev/null @@ -1,63 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from dataclasses import dataclass - -import numpy as np -from disent.frameworks.vae._weaklysupervised__adavae import AdaVae - - -# ========================================================================= # -# Swapped Target AdaVae # -# ========================================================================= # - - -class SwappedTargetAdaVae(AdaVae): - - REQUIRED_OBS = 2 - - @dataclass - class cfg(AdaVae.cfg): - swap_chance: float = 0.1 - - def __init__(self, model: 'AutoEncoder', cfg: cfg = None, batch_augment=None): - super().__init__(model=model, cfg=cfg, batch_augment=batch_augment) - assert cfg.swap_chance >= 0 - - def do_training_step(self, batch, batch_idx): - (x0, x1), (x0_targ, x1_targ) = self._get_xs_and_targs(batch, batch_idx) - - # random change for the target not to be equal to the input - if np.random.random() < self.cfg.swap_chance: - x0_targ, x1_targ = x1_targ, x0_targ - - return super(SwappedTargetAdaVae, self).do_training_step({ - 'x': (x0, x1), - 'x_targ': (x0_targ, x1_targ), - }, batch_idx) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/frameworks/vae/_weaklysupervised__st_betavae.py b/research/code/frameworks/vae/_weaklysupervised__st_betavae.py deleted file mode 100644 index c3042059..00000000 --- a/research/code/frameworks/vae/_weaklysupervised__st_betavae.py +++ /dev/null @@ -1,63 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from dataclasses import dataclass - -import numpy as np -from disent.frameworks.vae._unsupervised__betavae import BetaVae - - -# ========================================================================= # -# Swapped Target BetaVAE # -# ========================================================================= # - - -class SwappedTargetBetaVae(BetaVae): - - REQUIRED_OBS = 2 - - @dataclass - class cfg(BetaVae.cfg): - swap_chance: float = 0.1 - - def __init__(self, model: 'AutoEncoder', cfg: cfg = None, batch_augment=None): - super().__init__(model=model, cfg=cfg, batch_augment=batch_augment) - assert cfg.swap_chance >= 0 - - def do_training_step(self, batch, batch_idx): - (x0, x1), (x0_targ, x1_targ) = self._get_xs_and_targs(batch, batch_idx) - - # random change for the target not to be equal to the input - if np.random.random() < self.cfg.swap_chance: - x0_targ, x1_targ = x1_targ, x0_targ - - return super(SwappedTargetBetaVae, self).do_training_step({ - 'x': (x0,), - 'x_targ': (x0_targ,), - }, batch_idx) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/metrics/__init__.py b/research/code/metrics/__init__.py deleted file mode 100644 index c7aabe87..00000000 --- a/research/code/metrics/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -# Nathan Michlo et. al -from research.code.metrics._flatness import metric_flatness -from research.code.metrics._factored_components import metric_factored_components -from research.code.metrics._factored_components import metric_distances -from research.code.metrics._factored_components import metric_linearity diff --git a/research/code/metrics/_factored_components.py b/research/code/metrics/_factored_components.py deleted file mode 100644 index e8c91073..00000000 --- a/research/code/metrics/_factored_components.py +++ /dev/null @@ -1,696 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2022 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -""" -Factored Components Metric -- Nathan Michlo 2021 (Unpublished) -- Cite disent -""" - -import logging -from typing import Dict -from typing import List -from typing import Optional -from typing import Tuple -from typing import Union - -import numpy as np -import torch -import torch.nn.functional as F -from disent.frameworks.helper.reconstructions import ReconLossHandler -from scipy.stats import pearsonr -from scipy.stats import spearmanr - -from disent.dataset import DisentDataset -from disent.metrics.utils import make_metric -from disent.nn.functional import torch_mean_generalized -from disent.nn.functional import torch_pca -from disent.nn.loss.reduction import batch_loss_reduction -from disent.util import to_numpy -from research.code.metrics._flatness import encode_all_along_factor -from research.code.metrics._flatness import encode_all_factors -from research.code.metrics._flatness import filter_inactive_factors - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# flatness # -# ========================================================================= # - - -_SAMPLES_MULTIPLIER_GLOBAL = 4 -_SAMPLES_MULTIPLIER_FACTOR = 2 - - -# ========================================================================= # -# flatness # -# ========================================================================= # - - -def _metric_factored_components( - dataset: DisentDataset, - representation_function: callable, - num_samples: int = 64, - global_subset_size: int = 32, - repeats: int = 1024, - batch_size: int = 64, - compute_distances: bool = True, - compute_linearity: bool = True, -): - """ - Computes the factored components metric (ordering, linearity & axis alignment): - - # Distances: - rcorr_factor_data: rank correlation between ground-truth factor dists and MSE distances between data points - rcorr_latent_data: rank correlation between l2 latent dists and MSE distances between data points - rcorr_factor_latent: rank correlation between ground-truth factor dists and latent dists - - lcorr_factor_data: linear correlation between ground-truth factor dists and MSE distances between data points - lcorr_latent_data: linear correlation between l2 latent dists and MSE distances between data points - lcorr_factor_latent: linear correlation between ground-truth factor dists and latent dists - - -- only active if `compute_swap_ratio=True` - rsame_factor_data: how similar ground-truth factor dists are compared to MSE distances between data points MEAN: ((aA)&(b>B)) - rsame_latent_data: how similar l2 latent dists are compared to MSE distances between data points MEAN: ((aA)&(b>B)) - rsame_factor_latent: how similar ground-truth factor dists are compared to latent dists MEAN: ((aA)&(b>B)) - - * modifiers: - - .global | computed using random global samples - - .factor | computed using random values along a ground-truth factor traversal - - .l1 | computed using l1 distance - - .l2 | computed using l2 distance -- (if an .l1 or .l2 tag is missing, then it is .l2 by default) - - # Linearity & Axis Alignment - linear_ratio: average (largest singular value over sum of singular values) - axis_ratio: average (largest std/variance over sum of std/variances) - axis_alignment: axis ratio is bounded by linear ratio - compute: axis / linear - - linear_ratio_ave: [INVALID] average (largest singular value) over average (sum of singular values) - axis_ratio_ave: average (largest std/variance) over average (sum of std/variances) - axis_alignment_ave: [INVALID] axis ratio is bounded by linear ratio - compute: axis / linear - - * modifiers: - - .var | computed using the variance - - .std | computed using the standard deviation - - Args: - dataset: DisentDataset to be sampled from. - representation_function: Function that takes observations as input and outputs a dim_representation sized representation for each observation. - num_samples: How many random triplets are sampled from a single factor-traversal or global ranom mini-batch - global_subset_size: Controls the size of the global random minibatch, for individual factors this is usually the size of the factor. Triplets are randomly sampled from this. - repeats: how many times to repeat a traversal along each factors, these are then averaged together. - batch_size: Batch size to process at any time while generating representations, should not effect metric results. - compute_distances: If the distance components of the metric should be computed. - compute_linearity: If the linearity components of the metric should be computed. - Returns: - Dictionary with metrics - """ - # checks - if not (compute_distances or compute_linearity): - raise ValueError(f'Metric will not compute any values! At least one of: `compute_distances` or `compute_linearity` must be `True`') - - # compute actual metric values - factor_scores, global_scores = _compute_factored_metric_components( - dataset, - representation_function, - num_samples=num_samples, - global_subset_size=global_subset_size, - repeats=repeats, - batch_size=batch_size, - compute_distances=compute_distances, - compute_linearity=compute_linearity, - - ) - - # convert values from torch - scores = { - **global_scores, - **factor_scores, - } - - # sorted - return {k: scores[k] for k in sorted(scores.keys())} - - -# EXPORT: metric_factored_components -metric_factored_components = make_metric( - 'factored_components', - default_kwargs=dict(compute_distances=True, compute_linearity=True), - fast_kwargs=dict(compute_distances=True, compute_linearity=True, repeats=128), -)(_metric_factored_components) - -# EXPORT: metric_distances -metric_distances = make_metric( - 'distances', - default_kwargs=dict(compute_distances=True, compute_linearity=False), - fast_kwargs=dict(compute_distances=True, compute_linearity=False, repeats=128), -)(_metric_factored_components) - -# EXPORT: metric_linearity -metric_linearity = make_metric( - 'linearity', - default_kwargs=dict(compute_distances=False, compute_linearity=True), - fast_kwargs=dict(compute_distances=False, compute_linearity=True, repeats=128), -)(_metric_factored_components) - - -# ========================================================================= # -# flatness # -# ========================================================================= # - - -def _filtered_mean(values: torch.Tensor, p: Union[str, int], factor_sizes: Tuple[int, ...]): - # increase precision - values = values.to(torch.float64) - # check size - assert values.shape == (len(factor_sizes),) - # filter - # -- filter out factor dimensions that are incorrect. ie. size <= 1 - values = filter_inactive_factors(values, factor_sizes) - # compute mean - mean = torch_mean_generalized(values, dim=0, p=p) - # return decreased precision - return to_numpy(mean.to(torch.float32)) - - -@torch.no_grad() -def _compute_factored_metric_components( - dataset: DisentDataset, - representation_function, - num_samples: int, - global_subset_size: int, - repeats: int, - batch_size: int, - compute_distances: bool, - compute_linearity: bool, -) -> (dict, dict): - - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - # COMPUTE FOR EACH FACTOR - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - - # shapes: (num_factors,) - factor_values: Dict[str, np.ndarray] = _numpy_stack_all_dicts([ - _compute_factored_metric_components_along_factor( - dataset, - representation_function, - f_idx=f_idx, - num_samples=num_samples, - repeats=repeats, - batch_size=batch_size, - compute_distances=compute_distances, - compute_linearity=compute_linearity, - ) - for f_idx in range(dataset.gt_data.num_factors) - ]) - - # aggregate for each factor - # -- filter out factor dimensions that are incorrect. ie. size <= 1 - factor_scores = { - k: float(_filtered_mean(torch.from_numpy(v), p='geometric', factor_sizes=dataset.gt_data.factor_sizes)) - for k, v in factor_values.items() - } - - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - # RANDOM GLOBAL SAMPLES - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - - if compute_distances: - # storage - distance_measures: List[Dict[str, np.ndarray]] = [] - - # was: `iter_chunks(range(int(repeats * np.mean(dataset.gt_data.factor_sizes))), batch_size)` - for _ in range(repeats): - # sample random factors - factors = dataset.gt_data.sample_factors(size=global_subset_size) - # encode factors - zs, xs = encode_all_factors(dataset, representation_function, factors, batch_size=batch_size, return_batch=True) - zs, xs, factors = zs.cpu(), xs.cpu(), torch.from_numpy(factors).to(torch.float32) - # [COMPUTE SAME RATIO & CORRELATION]: was `_SAMPLES_MULTIPLIER_GLOBAL*len(zs)` - computed_dists = _compute_dists(num_triplets=num_samples, zs_traversal=zs, xs_traversal=xs, factors=factors) - # [STORE DISTANCES] - distance_measures.append(computed_dists) - - # [AGGREGATE] - # concatenate all into arrays: - # then aggregate over first dimension: - distance_measures: Dict[str, np.ndarray] = _numpy_concat_all_dicts(distance_measures) - distance_measures: Dict[str, float] = _compute_scores_from_dists(distance_measures) - distance_measures: Dict[str, float] = {f'distances.{k}.global': v for k, v in distance_measures.items()} - else: - distance_measures: Dict[str, float] = {} - - # update global scores - global_scores = distance_measures - - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - # RETURN - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - - return factor_scores, global_scores - - -# ========================================================================= # -# CORE # -# -- using variance instead of standard deviation makes it easier to # -# obtain high scores. # -# ========================================================================= # - - -def _compute_unsorted_axis_values(zs_traversal, use_std: bool = True): - # CORRELATIONS -- SORTED IN DESCENDING ORDER: - # correlation with standard basis (1, 0, 0, ...), (0, 1, 0, ...), ... - axis_values = torch.var(zs_traversal, dim=0) # (z_size,) - if use_std: - axis_values = torch.sqrt(axis_values) - return axis_values - - -def _compute_unsorted_linear_values(zs_traversal, use_std: bool = True): - # CORRELATIONS -- SORTED IN DESCENDING ORDER: - # correlation along arbitrary orthogonal basis - # -- note pca_mode='svd' returns the number of values equal to: min(factor_size, z_size) !!! this may lower scores on average - # -- note pca_mode='eig' returns the number of values equal to: z_size - _, linear_values = torch_pca(zs_traversal, center=True, mode='eig') - if use_std: - linear_values = torch.sqrt(linear_values) - return linear_values - - -def _score_from_sorted(sorted_vars: torch.Tensor, top_2: bool = False, norm: bool = True) -> torch.Tensor: - if top_2: - # use two max values - # this is more like mig - sorted_vars = sorted_vars[:2] - # sum all values - n = len(sorted_vars) - r = sorted_vars[0] / torch.sum(sorted_vars) - # get norm if needed - if norm: - # for: x/(x+a) - # normalised = (x/(x+a) - (1/n)) / (1 - (1/n)) - # normalised = (x - 1/(n-1) * a) / (x + a) - r = (r - (1/n)) / (1 - (1/n)) - # done! - return r - - -def _score_from_unsorted(unsorted_values: torch.Tensor, top_2: bool = False, norm: bool = True): - assert unsorted_values.ndim == 1 - # sort in descending order - sorted_values = torch.sort(unsorted_values, dim=-1, descending=True).values - # compute score - return _score_from_sorted(sorted_values, top_2=top_2, norm=norm) - - -def compute_axis_score(zs_traversal: torch.Tensor, use_std: bool = True, top_2: bool = False, norm: bool = True): - unsorted_values = _compute_unsorted_axis_values(zs_traversal, use_std=use_std) - score = _score_from_unsorted(unsorted_values, top_2=top_2, norm=norm) - return score - - -def compute_linear_score(zs_traversal: torch.Tensor, use_std: bool = True, top_2: bool = False, norm: bool = True): - unsorted_values = _compute_unsorted_linear_values(zs_traversal, use_std=use_std) - score = _score_from_unsorted(unsorted_values, top_2=top_2, norm=norm) - return score - - -# ========================================================================= # -# Collate Dict - Helper Functions - More specific than `default_collate()` # -# ========================================================================= # - - -def _torch_concat_all_dicts(dists_list: List[Dict[str, torch.Tensor]]) -> Dict[str, torch.Tensor]: - return {k: torch.cat([dists_dict[k] for dists_dict in dists_list], dim=0) for k in dists_list[0].keys()} - - -def _torch_stack_all_dicts(dists_list: List[Dict[str, torch.Tensor]]) -> Dict[str, torch.Tensor]: - return {k: torch.stack([dists_dict[k] for dists_dict in dists_list], dim=0) for k in dists_list[0].keys()} - - -def _numpy_concat_all_dicts(dists_list: List[Dict[str, Union[np.ndarray, float, int]]]) -> Dict[str, np.ndarray]: - return {k: np.concatenate([dists_dict[k] for dists_dict in dists_list], axis=0) for k in dists_list[0].keys()} - - -def _numpy_stack_all_dicts(dists_list: List[Dict[str, Union[np.ndarray, float, int]]]) -> Dict[str, np.ndarray]: - return {k: np.stack([dists_dict[k] for dists_dict in dists_list], axis=0) for k in dists_list[0].keys()} - - -# ========================================================================= # -# Distance Helper Functions # -# ========================================================================= # - - -def _same_mask(ap0, an0, ap1, an1): - return ((ap0 < an0) & (ap1 < an1)) | ((ap0 == an0) & (ap1 == an1)) | ((ap0 > an0) & (ap1 > an1)) - - -def _unswapped_ratio_torch(ap0: torch.Tensor, an0: torch.Tensor, ap1: torch.Tensor, an1: torch.Tensor) -> float: - # values must correspond - same_mask = _same_mask(ap0=ap0, an0=an0, ap1=ap1, an1=an1) - # num values - return float(same_mask.to(torch.float32).mean()) - - -def _unswapped_ratio_numpy(ap0: np.ndarray, an0: np.ndarray, ap1: np.ndarray, an1: np.ndarray): - # values must correspond - same_mask = _same_mask(ap0=ap0, an0=an0, ap1=ap1, an1=an1) - # num values - return np.mean(same_mask, dtype='float32') - - -def _compute_dists(num_triplets: int, zs_traversal: Optional[torch.Tensor], xs_traversal: torch.Tensor, factors: Optional[torch.Tensor], recon_loss_fn=F.mse_loss) -> Dict[str, np.ndarray]: - assert (factors is None) or (len(factors) == len(xs_traversal)) - assert (zs_traversal is None) or (len(zs_traversal) == len(xs_traversal)) - - # get the recon loss function - def _unreduced_loss(input, target): - if isinstance(recon_loss_fn, ReconLossHandler): - return recon_loss_fn.compute_unreduced_loss(input, target) - else: - return recon_loss_fn(input, target, reduction='none') - - # compute! - with torch.no_grad(): - # generate random triplets - # - {p, n} indices do not need to be sorted like triplets, these can be random. - # This metric is symmetric for swapped p & n values. - idxs_a, idxs_p, idxs_n = torch.randint(0, len(xs_traversal), size=(3, num_triplets), device=xs_traversal.device) - # compute distances -- shape: (num,) - distances = { - 'ap_ground_dists': (torch.norm(factors[idxs_a, :] - factors[idxs_p, :], p=1, dim=-1) if (factors is not None) else torch.abs(idxs_a - idxs_p)).cpu().numpy(), - 'an_ground_dists': (torch.norm(factors[idxs_a, :] - factors[idxs_n, :], p=1, dim=-1) if (factors is not None) else torch.abs(idxs_a - idxs_n)).cpu().numpy(), - 'ap_data_dists': batch_loss_reduction(_unreduced_loss(xs_traversal[idxs_a, ...], xs_traversal[idxs_p, ...]), reduction_dtype=torch.float32, reduction='mean').cpu().numpy(), - 'an_data_dists': batch_loss_reduction(_unreduced_loss(xs_traversal[idxs_a, ...], xs_traversal[idxs_n, ...]), reduction_dtype=torch.float32, reduction='mean').cpu().numpy(), - } - # compute distances -- shape: (num,) - if zs_traversal is not None: - distances.update({ - 'ap_latent_dists.l1': torch.norm(zs_traversal[idxs_a, :] - zs_traversal[idxs_p, :], dim=-1, p=1).cpu().numpy(), - 'an_latent_dists.l1': torch.norm(zs_traversal[idxs_a, :] - zs_traversal[idxs_n, :], dim=-1, p=1).cpu().numpy(), - 'ap_latent_dists.l2': torch.norm(zs_traversal[idxs_a, :] - zs_traversal[idxs_p, :], dim=-1, p=2).cpu().numpy(), - 'an_latent_dists.l2': torch.norm(zs_traversal[idxs_a, :] - zs_traversal[idxs_n, :], dim=-1, p=2).cpu().numpy(), - }) - # return values -- shape: (num,) - return distances - - -def _compute_scores_from_dists(dists: Dict[str, np.array]) -> Dict[str, float]: - # [DATA & GROUND DISTS]: - # extract the distances -- shape: (num,) - ap_ground_dists = dists['ap_ground_dists'] - an_ground_dists = dists['an_ground_dists'] - ap_data_dists = dists['ap_data_dists'] - an_data_dists = dists['an_data_dists'] - # concatenate values -- shape: (2 * num,) - ground_dists = np.concatenate([ap_ground_dists, an_ground_dists], axis=0) - data_dists = np.concatenate([ap_data_dists, an_data_dists], axis=0) - # compute the scores - # - check the number of swapped elements along a factor for random triplets. - # - compute the spearman rank correlation coefficient over the concatenated distances - # - compute the pearman correlation coefficient over the concatenated distances - scores = { - 'rsame_ground_data': _unswapped_ratio_numpy(ap0=ap_ground_dists, an0=an_ground_dists, ap1=ap_data_dists, an1=an_data_dists), # simplifies to: (ap_data_dists > an_data_dists).to(torch.float32).mean() - 'rcorr_ground_data': spearmanr(ground_dists, data_dists)[0], - 'lcorr_ground_data': pearsonr(ground_dists, data_dists)[0], - } - - # [RETURN EARLY]: - if 'ap_latent_dists.l1' not in dists: - return scores - - # [LATENT DISTS]: - # extract the distances -- shape: (num,) - ap_latent_dists_l1 = dists['ap_latent_dists.l1'] - an_latent_dists_l1 = dists['an_latent_dists.l1'] - ap_latent_dists_l2 = dists['ap_latent_dists.l2'] - an_latent_dists_l2 = dists['an_latent_dists.l2'] - # concatenate values -- shape: (2 * num,) - latent_dists_l1 = np.concatenate([ap_latent_dists_l1, an_latent_dists_l1], axis=0) - latent_dists_l2 = np.concatenate([ap_latent_dists_l2, an_latent_dists_l2], axis=0) - # compute the scores - scores.update({ - # - check the number of swapped elements along a factor for random triplets. - 'rsame_ground_latent.l1': _unswapped_ratio_numpy(ap0=ap_ground_dists, an0=an_ground_dists, ap1=ap_latent_dists_l1, an1=an_latent_dists_l1), # simplifies to: (ap_latent_dists > an_latent_dists).to(torch.float32).mean() - 'rsame_latent_data.l1': _unswapped_ratio_numpy(ap0=ap_latent_dists_l1, an0=an_latent_dists_l1, ap1=ap_data_dists, an1=an_data_dists), - 'rsame_ground_latent.l2': _unswapped_ratio_numpy(ap0=ap_ground_dists, an0=an_ground_dists, ap1=ap_latent_dists_l2, an1=an_latent_dists_l2), # simplifies to: (ap_latent_dists > an_latent_dists).to(torch.float32).mean() - 'rsame_latent_data.l2': _unswapped_ratio_numpy(ap0=ap_latent_dists_l2, an0=an_latent_dists_l2, ap1=ap_data_dists, an1=an_data_dists), - # - compute the spearman rank correlation coefficient over the concatenated distances - 'rcorr_ground_latent.l1': spearmanr(ground_dists, latent_dists_l1)[0], - 'rcorr_latent_data.l1': spearmanr(latent_dists_l1, data_dists)[0], - 'rcorr_ground_latent.l2': spearmanr(ground_dists, latent_dists_l2)[0], - 'rcorr_latent_data.l2': spearmanr(latent_dists_l2, data_dists)[0], - # - compute the pearman correlation coefficient over the concatenated distances - 'lcorr_ground_latent.l1': pearsonr(ground_dists, latent_dists_l1)[0], - 'lcorr_latent_data.l1': pearsonr(latent_dists_l1, data_dists)[0], - 'lcorr_ground_latent.l2': pearsonr(ground_dists, latent_dists_l2)[0], - 'lcorr_latent_data.l2': pearsonr(latent_dists_l2, data_dists)[0], - }) - - # [DONE] - return scores - - -# ========================================================================= # -# TRAVERSAL FLATNESS # -# ========================================================================= # - - -def _compute_factored_metric_components_along_factor( - dataset: DisentDataset, - representation_function, - f_idx: int, - num_samples: int, - repeats: int, - batch_size: int, - compute_distances: bool, - compute_linearity: bool, -) -> Dict[str, float]: - # NOTE: what to do if the factor size is too small? - - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - # FEED FORWARD, COMPUTE ALL - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - - distance_measures: List[Dict[str, np.ndarray]] = [] - linear_measures: List[Dict[str, torch.Tensor]] = [] - - for i in range(repeats): - # [ENCODE TRAVERSAL]: - # - generate repeated factors, varying one factor over the entire range - # - shape: (factor_size, z_size) - zs_traversal, xs_traversal = encode_all_along_factor(dataset, representation_function, f_idx=f_idx, batch_size=batch_size, return_batch=True) - zs_traversal = zs_traversal.cpu() - - if compute_distances: - xs_traversal = xs_traversal.cpu() - # [COMPUTE SAME RATIO & CORRELATION] | was: `num_triplets=_SAMPLES_MULTIPLIER_FACTOR*len(zs_traversal)` - computed_dists = _compute_dists(num_triplets=num_samples, zs_traversal=zs_traversal, xs_traversal=xs_traversal, factors=None) - # [STORE DISTANCES] - distance_measures.append(computed_dists) - - if compute_linearity: - # [VARIANCE ALONG DIFFERING AXES]: - # 1. axis: correlation with standard basis (1, 0, 0, ...), (0, 1, 0, ...), ... - # 2. linear: correlation along arbitrary orthogonal bases - axis_values_var = _compute_unsorted_axis_values(zs_traversal, use_std=False) # shape: (z_size,) - linear_values_var = _compute_unsorted_linear_values(zs_traversal, use_std=False) # shape: (z_size,) - # [COMPUTE LINEARITY SCORES]: - axis_ratio_var = _score_from_unsorted(axis_values_var, top_2=False, norm=True) # shape: () - linear_ratio_var = _score_from_unsorted(linear_values_var, top_2=False, norm=True) # shape: () - # [STORE SCORES] - linear_measures.append({ - 'linearity.axis_ratio.var': axis_ratio_var, - 'linearity.linear_ratio.var': linear_ratio_var, - # aggregating linear values outside this function does not make sense, values do not correspond between repeats. - 'linearity.axis_alignment.var': axis_ratio_var / (linear_ratio_var + 1e-20), - # temp values - '_TEMP_.axis_values.var': axis_values_var, - }) - - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - # AGGREGATE DATA - For each distance measure - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - - if compute_distances: - # concatenate all into arrays: - # then aggregate over first dimension: - distance_measures: Dict[str, np.ndarray] = _numpy_concat_all_dicts(distance_measures) - distance_measures: Dict[str, float] = _compute_scores_from_dists(distance_measures) - distance_measures: Dict[str, float] = {f'distances.{k}.factor': v for k, v in distance_measures.items()} - else: - distance_measures: Dict[str, float] = {} - - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - # AGGREGATE DATA - For each linearity measure - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - - if compute_linearity: - # stack all into arrays: - # then aggregate over first dimension: - # - eg: axis_ratio (repeats,) -> () - # - eg: axis_values (repeats, z_size) -> (z_size,) - linear_measures: Dict[str, torch.Tensor] = _torch_stack_all_dicts(linear_measures) - linear_measures: Dict[str, torch.Tensor] = {k: v.mean(dim=0) for k, v in linear_measures.items()} - # compute average scores & remove keys - linear_measures['linearity.axis_ratio_ave.var'] = _score_from_unsorted(linear_measures.pop('_TEMP_.axis_values.var'), top_2=False, norm=True) # shape: (z_size,) -> () - # convert values - linear_measures: Dict[str, float] = {k: float(v) for k, v in linear_measures.items()} - else: - linear_measures: Dict[str, float] = {} - - # done! - return { - **distance_measures, - **linear_measures, - } - - -# ========================================================================= # -# END # -# ========================================================================= # - - -# if __name__ == '__main__': -# -# def main(): -# import pytorch_lightning as pl -# from torch.utils.data import DataLoader -# from disent.frameworks.vae import BetaVae -# from disent.frameworks.vae import TripletVae -# from disent.model import AutoEncoder -# from disent.model.ae import EncoderConv64, DecoderConv64 -# from disent.util.strings import colors -# from disent.util.profiling import Timer -# from disent.dataset.data import XYObjectData -# from research.code.dataset.data import XYSquaresData -# from disent.dataset.sampling import RandomSampler -# from disent.dataset.sampling import GroundTruthDistSampler -# from disent.dataset.transform import ToImgTensorF32 -# from disent.nn.weights import init_model_weights -# from disent.util.seeds import seed -# import logging -# -# logging.basicConfig(level=logging.INFO) -# -# def get_str(r): -# return ', '.join(f'{k}={v:6.4f}' for k, v in r.items()) -# -# def print_r(name, steps, result, clr=colors.lYLW, t: Timer = None): -# print(f'{clr}{name:<13} ({steps:>04}){f" {colors.GRY}[{t.pretty}]{clr}" if t else ""}: {get_str(result)}{colors.RST}') -# -# def calculate(name, steps, dataset, get_repr): -# with Timer() as t: -# r = { -# **metric_factored_components.compute_fast(dataset, get_repr), -# } -# results.append((name, steps, r)) -# print_r(name, steps, r, colors.lRED, t=t) -# print(colors.GRY, '='*100, colors.RST, sep='') -# return r -# -# class XYOverlapData(XYSquaresData): -# def __init__(self, square_size=8, image_size=64, grid_spacing=None, num_squares=3, rgb=True): -# if grid_spacing is None: -# grid_spacing = (square_size+1) // 2 -# super().__init__(square_size=square_size, image_size=image_size, grid_spacing=grid_spacing, num_squares=num_squares, rgb=rgb) -# -# # datasets = [XYObjectData(rgb=False, palette='white'), XYSquaresData(), XYOverlapData(), XYObjectData()] -# datasets = [XYObjectData()] -# -# # TODO: fix for dead dimensions -# # datasets = [XYObjectData(rgb=False, palette='white')] -# -# results = [] -# for data in datasets: -# seed(7777) -# -# # dataset = DisentDataset(data, sampler=RandomSampler(num_samples=1), transform=ToImgTensorF32()) -# # dataloader = DataLoader(dataset=dataset, batch_size=32, shuffle=True, pin_memory=True) -# # module = init_model_weights(BetaVae( -# # model=AutoEncoder( -# # encoder=EncoderConv64(x_shape=data.x_shape, z_size=9, z_multiplier=2), -# # decoder=DecoderConv64(x_shape=data.x_shape, z_size=9), -# # ), -# # cfg=BetaVae.cfg(beta=0.0001, loss_reduction='mean', optimizer='adam', optimizer_kwargs=dict(lr=1e-3)) -# # ), mode='xavier_normal') -# -# dataset = DisentDataset(data, sampler=GroundTruthDistSampler(num_samples=3), transform=ToImgTensorF32()) -# dataloader = DataLoader(dataset=dataset, batch_size=32, shuffle=True, pin_memory=True) -# module = init_model_weights(TripletVae( -# model=AutoEncoder( -# encoder=EncoderConv64(x_shape=data.x_shape, z_size=9, z_multiplier=2), -# decoder=DecoderConv64(x_shape=data.x_shape, z_size=9), -# ), -# cfg=TripletVae.cfg(beta=0.0001, loss_reduction='mean', optimizer='adam', optimizer_kwargs=dict(lr=2e-4), triplet_p=1, triplet_loss='triplet_soft', triplet_scale=1) -# ), mode='xavier_normal') -# -# gpus = 1 if torch.cuda.is_available() else 0 -# -# # we cannot guarantee which device the representation is on -# get_repr = lambda x: module.encode(x.to(module.device)) -# # PHASE 1, UNTRAINED -# pl.Trainer(logger=False, checkpoint_callback=False, fast_dev_run=True, gpus=gpus, weights_summary=None).fit(module, dataloader) -# if torch.cuda.is_available(): module = module.to('cuda') -# calculate(data.__class__.__name__, 0, dataset, get_repr) -# # PHASE 2, LITTLE TRAINING -# pl.Trainer(logger=False, checkpoint_callback=False, max_steps=256, gpus=gpus, weights_summary=None).fit(module, dataloader) -# if torch.cuda.is_available(): module = module.to('cuda') -# calculate(data.__class__.__name__, 256, dataset, get_repr) -# # PHASE 3, MORE TRAINING -# pl.Trainer(logger=False, checkpoint_callback=False, max_steps=2048, gpus=gpus, weights_summary=None).fit(module, dataloader) -# if torch.cuda.is_available(): module = module.to('cuda') -# calculate(data.__class__.__name__, 256+2048, dataset, get_repr) -# results.append(None) -# -# for result in results: -# if result is None: -# print() -# continue -# (name, steps, result) = result -# print_r(name, steps, result, colors.lYLW) -# -# main() - - -# if __name__ == '__main__': -# -# def _same(ap0, an0, ap1, an1): -# return ((ap0 < an0) & (ap1 < an1)) | ((ap0 == an0) & (ap1 == an1)) | ((ap0 > an0) & (ap1 > an1)) -# -# def main(): -# ap0 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2]) -# an0 = np.array([1, 1, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 2, 2, 2, 2]) -# ap1 = np.array([1, 1, 2, 2, 1, 1, 2, 2, 1, 1, 2, 2, 1, 1, 2, 2]) -# an1 = np.array([1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2]) -# print(ap0) -# print(an0) -# print(ap1) -# print(an1) -# -# print(_same(ap0, an0, ap1, an1).astype('int')) -# -# main() diff --git a/research/code/metrics/_flatness.py b/research/code/metrics/_flatness.py deleted file mode 100644 index 0e1c81fb..00000000 --- a/research/code/metrics/_flatness.py +++ /dev/null @@ -1,359 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -""" -Flatness Metric -- Nathan Michlo 2021 (Unpublished) -- Cite disent -""" - -import logging -import math -from typing import Iterable -from typing import Tuple -from typing import Union - -import torch - -from disent.metrics.utils import make_metric -from disent.util.deprecate import deprecated -from torch.utils.data.dataloader import default_collate - -from disent.dataset import DisentDataset -from disent.util.iters import iter_chunks - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# flatness # -# ========================================================================= # - - -@make_metric('flatness', fast_kwargs=dict(repeats=128)) -def metric_flatness( - dataset: DisentDataset, - representation_function: callable, - repeats: int = 1024, - batch_size: int = 64, -): - """ - Computes the flatness metric: - approximately equal to: total_dim_width / (ave_point_dist_along_dim * num_points_along_dim) - - Complexity of this metric is: - O(num_factors * ave_factor_size * repeats) - eg. 9 factors * 64 indices on ave * 128 repeats = 73728 observations loaded from the dataset - - repeats: - - can go all the way down to about 64 and still get decent results. - - 64 is accurate to about +- 0.01 - - 128 is accurate to about +- 0.003 - - 1024 is accurate to about +- 0.001 - - Args: - dataset: DisentDataset to be sampled from. - representation_function: Function that takes observations as input and outputs a dim_representation sized representation for each observation. - repeats: how many times to repeat a traversal along each factors, these are then averaged together. - batch_size: Batch size to process at any time while generating representations, should not effect metric results. - p: how to calculate distances in the latent space, see torch.norm - Returns: - Dictionary with average disentanglement score, completeness and - informativeness (train and test). - """ - p_fs_measures = aggregate_measure_distances_along_all_factors(dataset, representation_function, repeats=repeats, batch_size=batch_size, ps=(1, 2)) - # get info - factor_sizes = dataset.gt_data.factor_sizes - # aggregate data - results = { - 'flatness.ave_flatness': compute_flatness(widths=p_fs_measures[2]['fs_ave_widths'], lengths=p_fs_measures[1]['fs_ave_lengths'], factor_sizes=factor_sizes), - 'flatness.ave_flatness_l1': compute_flatness(widths=p_fs_measures[1]['fs_ave_widths'], lengths=p_fs_measures[1]['fs_ave_lengths'], factor_sizes=factor_sizes), - 'flatness.ave_flatness_l2': compute_flatness(widths=p_fs_measures[2]['fs_ave_widths'], lengths=p_fs_measures[2]['fs_ave_lengths'], factor_sizes=factor_sizes), - # distances - 'flatness.ave_width_l1': torch.mean(filter_inactive_factors(p_fs_measures[1]['fs_ave_widths'], factor_sizes=factor_sizes)), - 'flatness.ave_width_l2': torch.mean(filter_inactive_factors(p_fs_measures[2]['fs_ave_widths'], factor_sizes=factor_sizes)), - 'flatness.ave_length_l1': torch.mean(filter_inactive_factors(p_fs_measures[1]['fs_ave_lengths'], factor_sizes=factor_sizes)), - 'flatness.ave_length_l2': torch.mean(filter_inactive_factors(p_fs_measures[2]['fs_ave_lengths'], factor_sizes=factor_sizes)), - # angles - 'flatness.cosine_angles': (1 / math.pi) * torch.mean(filter_inactive_factors(p_fs_measures[1]['fs_ave_angles'], factor_sizes=factor_sizes)), - } - # convert values from torch - return {k: float(v) for k, v in results.items()} - - -def compute_flatness(widths, lengths, factor_sizes): - widths = filter_inactive_factors(widths, factor_sizes) - lengths = filter_inactive_factors(lengths, factor_sizes) - # checks - assert torch.all(widths >= 0) - assert torch.all(lengths >= 0) - assert torch.all(torch.eq(widths == 0, lengths == 0)) - # update scores - widths[lengths == 0] = 0 - lengths[lengths == 0] = 1 - # compute flatness - return (widths / lengths).mean() - - -def filter_inactive_factors(tensor, factor_sizes): - factor_sizes = torch.tensor(factor_sizes, device=tensor.device) - assert torch.all(factor_sizes >= 1) - # remove - active_factors = torch.nonzero(factor_sizes-1, as_tuple=True) - return tensor[active_factors] - - -def aggregate_measure_distances_along_all_factors( - dataset: DisentDataset, - representation_function, - repeats: int, - batch_size: int, - ps: Iterable[Union[str, int]] = (1, 2), -) -> dict: - # COMPUTE AGGREGATES FOR EACH FACTOR - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - fs_p_measures = [ - aggregate_measure_distances_along_factor(dataset, representation_function, f_idx=f_idx, repeats=repeats, batch_size=batch_size, ps=ps) - for f_idx in range(dataset.gt_data.num_factors) - ] - - # FINALIZE FOR EACH FACTOR - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - p_fs_measures = {} - for p, fs_measures in default_collate(fs_p_measures).items(): - fs_ave_widths = fs_measures['ave_width'] - # get number of spaces deltas (number of points minus 1) - # compute length: estimated version of factors_ave_width = factors_num_deltas * factors_ave_delta - _fs_num_deltas = torch.as_tensor(dataset.gt_data.factor_sizes, device=fs_ave_widths.device) - 1 - _fs_ave_deltas = fs_measures['ave_delta'] - fs_ave_lengths = _fs_num_deltas * _fs_ave_deltas - # angles - fs_ave_angles = fs_measures['ave_angle'] - # update - p_fs_measures[p] = {'fs_ave_widths': fs_ave_widths, 'fs_ave_lengths': fs_ave_lengths, 'fs_ave_angles': fs_ave_angles} - return p_fs_measures - - -def aggregate_measure_distances_along_factor( - dataset: DisentDataset, - representation_function, - f_idx: int, - repeats: int, - batch_size: int, - ps: Iterable[Union[str, int]] = (1, 2), - cycle_fail: bool = False, -) -> dict: - f_size = dataset.gt_data.factor_sizes[f_idx] - - if f_size == 1: - if cycle_fail: - raise ValueError(f'dataset factor size is too small for flatness metric with cycle_normalize enabled! size={f_size} < 2') - zero = torch.as_tensor(0., device=get_device(dataset, representation_function)) - return {p: {'ave_width': zero.clone(), 'ave_delta': zero.clone(), 'ave_angle': zero.clone()} for p in ps} - - # FEED FORWARD, COMPUTE ALL DELTAS & WIDTHS - For each distance measure - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - p_measures: list = [{} for _ in range(repeats)] - for measures in p_measures: - # generate repeated factors, varying one factor over the entire range - zs_traversal = encode_all_along_factor(dataset, representation_function, f_idx=f_idx, batch_size=batch_size) - # for each distance measure compute everything - # - width: calculate the distance between the furthest two points - # - deltas: calculating the distances of their representations to the next values. - # - cycle_normalize: we cant get the ave next dist directly because of cycles, so we remove the largest dist - for p in ps: - deltas_next = torch.norm(torch.roll(zs_traversal, -1, dims=0) - zs_traversal, dim=-1, p=p) # next | shape: (factor_size, z_size) - deltas_prev = torch.norm(torch.roll(zs_traversal, 1, dims=0) - zs_traversal, dim=-1, p=p) # prev | shape: (factor_size, z_size) - # values needed for flatness - width = knn(x=zs_traversal, y=zs_traversal, k=1, largest=True, p=p).values.max() # shape: (,) - min_deltas = torch.topk(deltas_next, k=f_size-1, dim=-1, largest=False, sorted=False) # shape: (factor_size-1, z_size) - # values needed for cosine angles - # TODO: this should not be calculated per p - # TODO: should we filter the cyclic value? - # a. if the point is an endpoint we set its value to pi indicating that it is flat - # b. [THIS] we do not allow less than 3 points, ie. a factor_size of at least 3, otherwise - # we set the angle to pi (considered flat) and filter the factor from the metric - angles = angles_between(deltas_next, deltas_prev, dim=-1, nan_to_angle=0) # shape: (factor_size,) - # TODO: other measures can be added: - # 1. multivariate skewness - # 2. normality measure - # 3. independence - # 4. menger curvature (Cayley-Menger Determinant?) - # save variables - measures[p] = {'widths': width, 'deltas': min_deltas.values, 'angles': angles} - - # AGGREGATE DATA - For each distance measure - # -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # - return { - p: { - 'ave_width': measures['widths'].mean(dim=0), # shape: (repeats,) -> () - 'ave_delta': measures['deltas'].mean(dim=[0, 1]), # shape: (repeats, factor_size - 1) -> () - 'ave_angle': measures['angles'].mean(dim=0), # shape: (repeats,) -> () - } for p, measures in default_collate(p_measures).items() - } - - -# ========================================================================= # -# ENCODE # -# ========================================================================= # - - -def encode_all_along_factor(dataset: DisentDataset, representation_function, f_idx: int, batch_size: int, return_batch: bool = False): - # generate repeated factors, varying one factor over a range (f_size, f_dims) - factors = dataset.gt_data.sample_random_factor_traversal(f_idx=f_idx) - # get the representations of all the factors (f_size, z_size) - # * if return_batch is False: return sequential_zs - # * if return_batch is True: return (sequential_zs, sequential_batch) - return encode_all_factors(dataset, representation_function, factors=factors, batch_size=batch_size, return_batch=return_batch) - - -def encode_all_factors(dataset: DisentDataset, representation_function, factors, batch_size: int, return_batch: bool = False) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: - zs = [] - xs = [] - with torch.no_grad(): - for batch_factors in iter_chunks(factors, chunk_size=batch_size): - batch = dataset.dataset_batch_from_factors(batch_factors, mode='input') - z = representation_function(batch) - zs.append(z) - if return_batch: - xs.append(batch) - # handle case - if return_batch: - # might be on different devices - return torch.cat(zs, dim=0), torch.cat(xs, dim=0) - else: - return torch.cat(zs, dim=0) - - -def get_device(dataset: DisentDataset, representation_function): - # this is a hack... - return representation_function(dataset.dataset_sample_batch(1, mode='input')).device - - -# ========================================================================= # -# DISTANCES # -# ========================================================================= # - - -def knn(x, y, k: int = None, largest=False, p='fro'): - assert 0 < k <= y.shape[0] - # check input vectors, must be array of vectors - assert 2 == x.ndim == y.ndim - assert x.shape[1:] == y.shape[1:] - # compute distances between each and every pair - dist_mat = x[:, None, ...] - y[None, :, ...] - dist_mat = torch.norm(dist_mat, dim=-1, p=p) - # return closest distances - return torch.topk(dist_mat, k=k, dim=-1, largest=largest, sorted=True) - - -# ========================================================================= # -# ANGLES # -# ========================================================================= # - - -def angles_between(a, b, dim=-1, nan_to_angle=None): - a = a / torch.norm(a, dim=dim, keepdim=True) - b = b / torch.norm(b, dim=dim, keepdim=True) - dot = torch.sum(a * b, dim=dim) - angles = torch.acos(torch.clamp(dot, -1.0, 1.0)) - if nan_to_angle is not None: - return torch.where(torch.isnan(angles), torch.full_like(angles, fill_value=nan_to_angle), angles) - return angles - - -# ========================================================================= # -# END # -# ========================================================================= # - - -# if __name__ == '__main__': -# import pytorch_lightning as pl -# from torch.optim import Adam -# from torch.utils.data import DataLoader -# from disent.data.groundtruth import XYObjectData, XYSquaresData -# from disent.dataset.groundtruth import GroundTruthDataset, GroundTruthDatasetPairs -# from disent.frameworks.vae import BetaVae -# from disent.frameworks.vae import AdaVae -# from disent.model.ae import EncoderConv64, DecoderConv64, AutoEncoder -# from disent.transform import ToImgTensorF32 -# from disent.util import colors -# from disent.util import Timer -# -# def get_str(r): -# return ', '.join(f'{k}={v:6.4f}' for k, v in r.items()) -# -# def print_r(name, steps, result, clr=colors.lYLW, t: Timer = None): -# print(f'{clr}{name:<13} ({steps:>04}){f" {colors.GRY}[{t.pretty}]{clr}" if t else ""}: {get_str(result)}{colors.RST}') -# -# def calculate(name, steps, dataset, get_repr): -# global aggregate_measure_distances_along_factor -# with Timer() as t: -# r = metric_flatness(dataset, get_repr, repeats=64, batch_size=64) -# results.append((name, steps, r)) -# print_r(name, steps, r, colors.lRED, t=t) -# print(colors.GRY, '='*100, colors.RST, sep='') -# return r -# -# class XYOverlapData(XYSquaresData): -# def __init__(self, square_size=8, image_size=64, grid_spacing=None, num_squares=3, rgb=True): -# if grid_spacing is None: -# grid_spacing = (square_size+1) // 2 -# super().__init__(square_size=square_size, image_size=image_size, grid_spacing=grid_spacing, num_squares=num_squares, rgb=rgb) -# -# # datasets = [XYObjectData(rgb=False, palette='white'), XYSquaresData(), XYOverlapData(), XYObjectData()] -# datasets = [XYObjectData()] -# -# results = [] -# for data in datasets: -# dataset = GroundTruthDatasetPairs(data, transform=ToImgTensorF32()) -# dataloader = DataLoader(dataset=dataset, batch_size=32, shuffle=True, pin_memory=True) -# module = AdaVae( -# model=AutoEncoder( -# encoder=EncoderConv64(x_shape=data.x_shape, z_size=6, z_multiplier=2), -# decoder=DecoderConv64(x_shape=data.x_shape, z_size=6), -# ), -# cfg=AdaVae.cfg(beta=0.001, loss_reduction='mean', optimizer=torch.optim.Adam, optimizer_kwargs=dict(lr=5e-4)) -# ) -# # we cannot guarantee which device the representation is on -# get_repr = lambda x: module.encode(x.to(module.device)) -# # PHASE 1, UNTRAINED -# pl.Trainer(logger=False, checkpoint_callback=False, fast_dev_run=True, gpus=1, weights_summary=None).fit(module, dataloader) -# module = module.to('cuda') -# calculate(data.__class__.__name__, 0, dataset, get_repr) -# # PHASE 2, LITTLE TRAINING -# pl.Trainer(logger=False, checkpoint_callback=False, max_steps=256, gpus=1, weights_summary=None).fit(module, dataloader) -# calculate(data.__class__.__name__, 256, dataset, get_repr) -# # PHASE 3, MORE TRAINING -# pl.Trainer(logger=False, checkpoint_callback=False, max_steps=2048, gpus=1, weights_summary=None).fit(module, dataloader) -# calculate(data.__class__.__name__, 256+2048, dataset, get_repr) -# results.append(None) -# -# for result in results: -# if result is None: -# print() -# continue -# (name, steps, result) = result -# print_r(name, steps, result, colors.lYLW) diff --git a/research/code/util/__init__.py b/research/code/util/__init__.py deleted file mode 100644 index e02c2205..00000000 --- a/research/code/util/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ - -from ._fn_util import * -from ._dataset import * -from ._io_util import * -from ._loss import * - -# disent exports to make life easy -from disent.util.visualize.plot import to_img -from disent.util.visualize.plot import to_imgs -from disent.util.visualize.plot import plt_imshow -from disent.util.visualize.plot import plt_subplots -from disent.util.visualize.plot import plt_subplots_imshow -from disent.util.visualize.plot import plt_hide_axis -from disent.util.visualize.plot import visualize_dataset_traversal -from disent.util.visualize.plot import plt_2d_density diff --git a/research/code/util/_data.py b/research/code/util/_data.py deleted file mode 100644 index cce196e9..00000000 --- a/research/code/util/_data.py +++ /dev/null @@ -1,82 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from typing import Tuple - -import numpy as np - -from disent.dataset.data import GroundTruthData -from disent.dataset.data._raw import Hdf5Dataset - - -# TODO: these classes are old... -# TODO: these classes are old... -# TODO: these classes are old... - - -class TransformDataset(GroundTruthData): - - # TODO: all data should be datasets - # TODO: file preparation should be separate from datasets - # TODO: disent/data should be datasets, and disent/datasets should be samplers that wrap disent/data - - def __init__(self, base_data: GroundTruthData, transform=None): - self.base_data = base_data - super().__init__(transform=transform) - - @property - def factor_names(self) -> Tuple[str, ...]: - return self.base_data.factor_names - - @property - def factor_sizes(self) -> Tuple[int, ...]: - return self.base_data.factor_sizes - - @property - def img_shape(self) -> Tuple[int, ...]: - return self.base_data.img_shape - - def _get_observation(self, idx): - return self.base_data[idx] - - -class AdversarialOptimizedData(TransformDataset): - - def __init__(self, h5_path: str, base_data: GroundTruthData, transform=None): - # normalize hd5f data - def _normalize_hdf5(x): - c, h, w = x.shape - if c in (1, 3): - return np.moveaxis(x, 0, -1) - return x - # get the data - self.hdf5_data = Hdf5Dataset(h5_path, transform=_normalize_hdf5) - # checks - assert isinstance(base_data, GroundTruthData), f'base_data must be an instance of {repr(GroundTruthData.__name__)}, got: {repr(base_data)}' - assert len(base_data) == len(self.hdf5_data), f'length of base_data: {len(base_data)} does not match length of hd5f data: {len(self.hdf5_data)}' - # initialize - super().__init__(base_data=base_data, transform=transform) - - def _get_observation(self, idx): - return self.hdf5_data[idx] diff --git a/research/code/util/_dataset.py b/research/code/util/_dataset.py deleted file mode 100644 index 5f076b2a..00000000 --- a/research/code/util/_dataset.py +++ /dev/null @@ -1,502 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import os -import warnings -from typing import List -from typing import Literal -from typing import Optional -from typing import Sequence -from typing import Sized -from typing import Tuple -from typing import Union - -import numpy as np -import torch -import torch.utils.data - -from disent.dataset import DisentDataset -from disent.dataset.data import Cars3d64Data -from disent.dataset.data import DSpritesData -from disent.dataset.data import SmallNorb64Data -from disent.dataset.data import XYObjectShadedData -from research.code.dataset.data import DSpritesImagenetData -from disent.dataset.data import GroundTruthData -from disent.dataset.data import Shapes3dData -from research.code.dataset.data import XColumnsData -from research.code.dataset.data import XYBlocksData -from disent.dataset.data import XYObjectData -from research.code.dataset.data import XYSquaresData -from disent.dataset.sampling import BaseDisentSampler -from disent.dataset.sampling import GroundTruthSingleSampler -from disent.dataset.transform import Noop -from disent.dataset.transform import ToImgTensorF32 -from disent.dataset.transform import ToImgTensorU8 - - -# ========================================================================= # -# dataset io # -# ========================================================================= # - - -# TODO: this is much faster! -# -# import psutil -# import multiprocessing as mp -# -# def copy_batch_into(src: GroundTruthData, dst: torch.Tensor, i: int, j: int): -# for k in range(i, min(j, len(dst))): -# dst[k, ...] = src[k] -# return (i, j) -# -# def load_dataset_into_memory( -# gt_data: GroundTruthData, -# workers: int = min(psutil.cpu_count(logical=False), 16), -# ) -> ArrayGroundTruthData: -# # make data and tensors -# tensor = torch.zeros(len(gt_data), *gt_data.obs_shape, dtype=gt_data[0].dtype).share_memory_() -# # compute batch size -# n = len(gt_data) -# batch_size = (n + workers - 1) // workers -# # load in batches -# with mp.Pool(processes=workers) as POOL: -# POOL.starmap( -# copy_batch_into, [ -# (gt_data, tensor, i, i + batch_size) -# for i in range(0, n, batch_size) -# ] -# ) -# # return array -# return ArrayGroundTruthData.new_like(tensor, gt_data, array_chn_is_last=False) - - -def load_dataset_into_memory(gt_data: GroundTruthData, x_shape: Optional[Tuple[int, ...]] = None, batch_size=64, num_workers=min(os.cpu_count(), 16), dtype=torch.float32, raw_array=False): - assert dtype in {torch.float16, torch.float32} - # TODO: this should be part of disent? - from torch.utils.data import DataLoader - from tqdm import tqdm - from disent.dataset.data import ArrayGroundTruthData - # get observation shape - # - manually specify this if the gt_data has a transform applied that resizes the observations for example! - if x_shape is None: - x_shape = gt_data.x_shape - # load dataset into memory manually! - data = torch.zeros(len(gt_data), *x_shape, dtype=dtype) - # load all batches - dataloader = DataLoader(gt_data, batch_size=batch_size, shuffle=False, num_workers=num_workers, drop_last=False) - idx = 0 - for batch in tqdm(dataloader, desc='loading dataset into memory'): - data[idx:idx+len(batch)] = batch.to(dtype) - idx += len(batch) - # done! - if raw_array: - return data - else: - # channels get swapped by the below ToImgTensorF32(), maybe allow `array_chn_is_last` as param - return ArrayGroundTruthData.new_like(array=data, gt_data=gt_data, array_chn_is_last=False) - - -# ========================================================================= # -# dataset # -# ========================================================================= # - - -TransformTypeHint = Union[Literal['uint8'], Literal['float32'], Literal['none']] - - -def make_transform(mode: Optional[str]) -> Optional[callable]: - if mode == 'uint8': - return ToImgTensorU8() - elif mode == 'float32': - return ToImgTensorF32() - elif mode in ('none', None): - return None - else: - raise KeyError(f'invalid transform mode: {repr(mode)}') - - -def make_data( - name: str = 'xysquares', - factors: bool = False, - data_root: str = 'data/dataset', - try_in_memory: bool = False, - load_into_memory: bool = False, - load_memory_dtype: torch.dtype = torch.float16, - transform_mode: TransformTypeHint = 'float32' -) -> GroundTruthData: - # override values - if load_into_memory and try_in_memory: - warnings.warn('`load_into_memory==True` is incompatible with `try_in_memory==True`, setting `try_in_memory=False`!') - try_in_memory = False - # make the transform - transform = make_transform(mode=transform_mode) - # make data - if name == 'xysquares': data = XYSquaresData(transform=transform) # equivalent: [xysquares, xysquares_8x8, xysquares_8x8_s8] - elif name == 'xysquares_1x1': data = XYSquaresData(square_size=1, transform=transform) - elif name == 'xysquares_2x2': data = XYSquaresData(square_size=2, transform=transform) - elif name == 'xysquares_4x4': data = XYSquaresData(square_size=4, transform=transform) - elif name == 'xysquares_8x8': data = XYSquaresData(square_size=8, transform=transform) # 8x8x8x8x8x8 = 262144 # equivalent: [xysquares, xysquares_8x8, xysquares_8x8_s8] - elif name == 'xysquares_8x8_mini': data = XYSquaresData(square_size=8, grid_spacing=14, transform=transform) # 5x5x5x5x5x5 = 15625 - # TOY DATASETS - elif name == 'xysquares_8x8_toy': data = XYSquaresData(square_size=8, grid_spacing=8, rgb=False, num_squares=1, transform=transform) # 8x8 = ? - elif name == 'xysquares_8x8_toy_s1': data = XYSquaresData(square_size=8, grid_spacing=1, rgb=False, num_squares=1, transform=transform) # ?x? = ? - elif name == 'xysquares_8x8_toy_s2': data = XYSquaresData(square_size=8, grid_spacing=2, rgb=False, num_squares=1, transform=transform) # ?x? = ? - elif name == 'xysquares_8x8_toy_s4': data = XYSquaresData(square_size=8, grid_spacing=4, rgb=False, num_squares=1, transform=transform) # ?x? = ? - elif name == 'xysquares_8x8_toy_s8': data = XYSquaresData(square_size=8, grid_spacing=8, rgb=False, num_squares=1, transform=transform) # 8x8 = ? - # TOY DATASETS ALT - elif name == 'xcolumns_8x_toy': data = XColumnsData(square_size=8, grid_spacing=8, rgb=False, num_squares=1, transform=transform) # 8 = ? - elif name == 'xcolumns_8x_toy_s1': data = XColumnsData(square_size=8, grid_spacing=1, rgb=False, num_squares=1, transform=transform) # ? = ? - elif name == 'xcolumns_8x_toy_s2': data = XColumnsData(square_size=8, grid_spacing=2, rgb=False, num_squares=1, transform=transform) # ? = ? - elif name == 'xcolumns_8x_toy_s4': data = XColumnsData(square_size=8, grid_spacing=4, rgb=False, num_squares=1, transform=transform) # ? = ? - elif name == 'xcolumns_8x_toy_s8': data = XColumnsData(square_size=8, grid_spacing=8, rgb=False, num_squares=1, transform=transform) # 8 = ? - # OVERLAPPING DATASETS - elif name == 'xysquares_8x8_s1': data = XYSquaresData(square_size=8, grid_size=8, grid_spacing=1, transform=transform) # ?x?x?x?x?x? = ? - elif name == 'xysquares_8x8_s2': data = XYSquaresData(square_size=8, grid_size=8, grid_spacing=2, transform=transform) # ?x?x?x?x?x? = ? - elif name == 'xysquares_8x8_s3': data = XYSquaresData(square_size=8, grid_size=8, grid_spacing=3, transform=transform) # ?x?x?x?x?x? = ? - elif name == 'xysquares_8x8_s4': data = XYSquaresData(square_size=8, grid_size=8, grid_spacing=4, transform=transform) # ?x?x?x?x?x? = ? - elif name == 'xysquares_8x8_s5': data = XYSquaresData(square_size=8, grid_size=8, grid_spacing=5, transform=transform) # ?x?x?x?x?x? = ? - elif name == 'xysquares_8x8_s6': data = XYSquaresData(square_size=8, grid_size=8, grid_spacing=6, transform=transform) # ?x?x?x?x?x? = ? - elif name == 'xysquares_8x8_s7': data = XYSquaresData(square_size=8, grid_size=8, grid_spacing=7, transform=transform) # ?x?x?x?x?x? = ? - elif name == 'xysquares_8x8_s8': data = XYSquaresData(square_size=8, grid_size=8, grid_spacing=8, transform=transform) # 8x8x8x8x8x8 = 262144 # equivalent: [xysquares, xysquares_8x8, xysquares_8x8_s8] - # OTHER SYNTHETIC DATASETS - elif name == 'xyobject': data = XYObjectData(transform=transform) - elif name == 'xyobject_shaded': data = XYObjectShadedData(transform=transform) - elif name == 'xyblocks': data = XYBlocksData(transform=transform) - # NORMAL DATASETS - elif name == 'cars3d': data = Cars3d64Data(data_root=data_root, prepare=True, transform=transform) - elif name == 'smallnorb': data = SmallNorb64Data(data_root=data_root, prepare=True, transform=transform) - elif name == 'shapes3d': data = Shapes3dData(data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - elif name == 'dsprites': data = DSpritesData(data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - # CUSTOM DATASETS - elif name == 'dsprites_imagenet_bg_100': data = DSpritesImagenetData(visibility=100, mode='bg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - elif name == 'dsprites_imagenet_bg_80': data = DSpritesImagenetData(visibility=80, mode='bg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - elif name == 'dsprites_imagenet_bg_60': data = DSpritesImagenetData(visibility=60, mode='bg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - elif name == 'dsprites_imagenet_bg_40': data = DSpritesImagenetData(visibility=40, mode='bg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - elif name == 'dsprites_imagenet_bg_20': data = DSpritesImagenetData(visibility=20, mode='bg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - # - elif name == 'dsprites_imagenet_bg_75': data = DSpritesImagenetData(visibility=75, mode='bg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - elif name == 'dsprites_imagenet_bg_50': data = DSpritesImagenetData(visibility=50, mode='bg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - elif name == 'dsprites_imagenet_bg_25': data = DSpritesImagenetData(visibility=25, mode='bg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - elif name == 'dsprites_imagenet_bg_0': data = DSpritesImagenetData(visibility=0, mode='bg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) # same as normal dsprites, but with 3 channels - # --- # - elif name == 'dsprites_imagenet_fg_100': data = DSpritesImagenetData(visibility=100, mode='fg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - elif name == 'dsprites_imagenet_fg_80': data = DSpritesImagenetData(visibility=80, mode='fg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - elif name == 'dsprites_imagenet_fg_60': data = DSpritesImagenetData(visibility=60, mode='fg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - elif name == 'dsprites_imagenet_fg_40': data = DSpritesImagenetData(visibility=40, mode='fg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - elif name == 'dsprites_imagenet_fg_20': data = DSpritesImagenetData(visibility=20, mode='fg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - # - elif name == 'dsprites_imagenet_fg_75': data = DSpritesImagenetData(visibility=75, mode='fg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - elif name == 'dsprites_imagenet_fg_50': data = DSpritesImagenetData(visibility=50, mode='fg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - elif name == 'dsprites_imagenet_fg_25': data = DSpritesImagenetData(visibility=25, mode='fg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) - elif name == 'dsprites_imagenet_fg_0': data = DSpritesImagenetData(visibility=0, mode='fg', data_root=data_root, prepare=True, transform=transform, in_memory=try_in_memory) # same as normal dsprites, but with 3 channels - # DONE - else: raise KeyError(f'invalid data name: {repr(name)}') - # load into memory - if load_into_memory: - old_data, data = data, load_dataset_into_memory(data, dtype=load_memory_dtype, x_shape=(data.img_channels, 64, 64)) - # make dataset - if factors: - raise NotImplementedError('factor returning is not yet implemented in the rewrite! this needs to be fixed!') # TODO! - return data - - -def make_dataset( - name: str = 'xysquares', - factors: bool = False, - data_root: str = 'data/dataset', - try_in_memory: bool = False, - load_into_memory: bool = False, - load_memory_dtype: torch.dtype = torch.float16, - transform_mode: TransformTypeHint = 'float32', - sampler: BaseDisentSampler = None, -) -> DisentDataset: - # make data - data = make_data( - name=name, - data_root=data_root, - try_in_memory=try_in_memory, - load_into_memory=load_into_memory, - load_memory_dtype=load_memory_dtype, - transform_mode='none', # we move the transform over to the DisentDataset instead! - ) - return DisentDataset( - data, - sampler=GroundTruthSingleSampler() if (sampler is None) else sampler, - return_indices=True, - return_factors=factors, - transform=make_transform(mode=transform_mode), - ) - - -def get_single_batch(dataloader, cuda=True): - for batch in dataloader: - (x_targ,) = batch['x_targ'] - break - if cuda: - x_targ = x_targ.cuda() - return x_targ - - -# ========================================================================= # -# sampling helper # -# ========================================================================= # - - -# TODO: clean this up -def sample_factors(gt_data: GroundTruthData, num_obs: int = 1024, factor_mode: str = 'sample_random', factor: Union[int, str] = None): - # sample multiple random factor traversals - if factor_mode == 'sample_traversals': - assert factor is not None, f'factor cannot be None when factor_mode=={repr(factor_mode)}' - # get traversal - f_idx = gt_data.normalise_factor_idx(factor) - # generate traversals - factors = [] - for i in range((num_obs + gt_data.factor_sizes[f_idx] - 1) // gt_data.factor_sizes[f_idx]): - factors.append(gt_data.sample_random_factor_traversal(f_idx=f_idx)) - factors = np.concatenate(factors, axis=0) - elif factor_mode == 'sample_random': - factors = gt_data.sample_factors(num_obs) - else: - raise KeyError - return factors - - -# TODO: move into dataset class -def sample_batch_and_factors(dataset: DisentDataset, num_samples: int, factor_mode: str = 'sample_random', factor: Union[int, str] = None, device=None): - factors = sample_factors(dataset.gt_data, num_obs=num_samples, factor_mode=factor_mode, factor=factor) - batch = dataset.dataset_batch_from_factors(factors, mode='target').to(device=device) - factors = torch.from_numpy(factors).to(dtype=torch.float32, device=device) - return batch, factors - - -# ========================================================================= # -# pair samplers # -# ========================================================================= # - - -def pair_indices_random(max_idx: int, approx_batch_size: Optional[int] = None) -> Tuple[np.ndarray, np.ndarray]: - """ - Generates pairs of indices in corresponding arrays, - returning random permutations - - considers [0, 1] and [1, 0] to be different # TODO: consider them to be the same - - never returns pairs with the same values, eg. [1, 1] - - (default) number of returned values is: `max_idx * sqrt(max_idx) / 2` -- arbitrarily chosen to scale slower than number of combinations - """ - # defaults - if approx_batch_size is None: - approx_batch_size = int(max_idx * (max_idx ** 0.5) / 2) - # sample values - idx_a, idx_b = np.random.randint(0, max_idx, size=(2, approx_batch_size)) - # remove similar - different = (idx_a != idx_b) - idx_a = idx_a[different] - idx_b = idx_b[different] - # return values - return idx_a, idx_b - - -def pair_indices_combinations(max_idx: int) -> Tuple[np.ndarray, np.ndarray]: - """ - Generates pairs of indices in corresponding arrays, - returning all combinations - - considers [0, 1] and [1, 0] to be the same, only returns one of them - - never returns pairs with the same values, eg. [1, 1] - - number of returned values is: `max_idx * (max_idx-1) / 2` - """ - # upper triangle excluding diagonal - # - similar to: `list(itertools.combinations(np.arange(len(t_idxs)), 2))` - idxs_a, idxs_b = np.triu_indices(max_idx, k=1) - return idxs_a, idxs_b - - -def pair_indices_nearby(max_idx: int) -> Tuple[np.ndarray, np.ndarray]: - """ - Generates pairs of indices in corresponding arrays, - returning nearby combinations - - considers [0, 1] and [1, 0] to be the same, only returns one of them - - never returns pairs with the same values, eg. [1, 1] - - number of returned values is: `max_idx` - """ - idxs_a = np.arange(max_idx) # eg. [0 1 2 3 4 5] - idxs_b = np.roll(idxs_a, shift=1, axis=0) # eg. [1 2 3 4 5 0] - return idxs_a, idxs_b - - -_PAIR_INDICES_FNS = { - 'random': pair_indices_random, - 'combinations': pair_indices_combinations, - 'nearby': pair_indices_nearby, -} - - -def pair_indices(max_idx: int, mode: str) -> Tuple[np.ndarray, np.ndarray]: - try: - fn = _PAIR_INDICES_FNS[mode] - except: - raise KeyError(f'invalid mode: {repr(mode)}') - return fn(max_idx=max_idx) - - -# ========================================================================= # -# mask helper # -# ========================================================================= # - - -def make_changed_mask(batch: torch.Tensor, masked=True): - if masked: - mask = torch.zeros_like(batch[0], dtype=torch.bool) - for i in range(len(batch)): - mask |= (batch[0] != batch[i]) - else: - mask = torch.ones_like(batch[0], dtype=torch.bool) - return mask - - -# ========================================================================= # -# dataset indices # -# ========================================================================= # - - -def sample_unique_batch_indices(num_obs: int, num_samples: int) -> np.ndarray: - assert num_obs >= num_samples, 'not enough values to sample' - assert (num_obs - num_samples) / num_obs > 0.5, 'this method might be inefficient' - # get random sample - indices = set() - while len(indices) < num_samples: - indices.update(np.random.randint(low=0, high=num_obs, size=num_samples - len(indices))) - # make sure indices are randomly ordered - indices = np.fromiter(indices, dtype=int) - # indices = np.array(list(indices), dtype=int) - np.random.shuffle(indices) - # return values - return indices - - -def generate_epoch_batch_idxs(num_obs: int, num_batches: int, mode: str = 'shuffle') -> List[np.ndarray]: - """ - Generate `num_batches` batches of indices. - - Each index is in the range [0, num_obs). - - If num_obs is not divisible by num_batches, then batches may not all be the same size. - - eg. [0, 1, 2, 3, 4] -> [[0, 1], [2, 3], [4]] -- num_obs=5, num_batches=3, sample_mode='range' - eg. [0, 1, 2, 3, 4] -> [[1, 4], [2, 0], [3]] -- num_obs=5, num_batches=3, sample_mode='shuffle' - eg. [0, 1, 0, 3, 2] -> [[0, 1], [0, 3], [2]] -- num_obs=5, num_batches=3, sample_mode='random' - """ - # generate indices - if mode == 'range': - idxs = np.arange(num_obs) - elif mode == 'shuffle': - idxs = np.arange(num_obs) - np.random.shuffle(idxs) - elif mode == 'random': - idxs = np.random.randint(0, num_obs, size=(num_obs,)) - else: - raise KeyError(f'invalid mode={repr(mode)}') - # return batches - return np.array_split(idxs, num_batches) - - -def generate_epochs_batch_idxs(num_obs: int, num_epochs: int, num_epoch_batches: int, mode: str = 'shuffle') -> List[np.ndarray]: - """ - Like generate_epoch_batch_idxs, but concatenate the batches of calling the function `num_epochs` times. - - The total number of batches returned is: `num_epochs * num_epoch_batches` - """ - batches = [] - for i in range(num_epochs): - batches.extend(generate_epoch_batch_idxs(num_obs=num_obs, num_batches=num_epoch_batches, mode=mode)) - return batches - - -# ========================================================================= # -# Dataloader Sampler Utilities # -# ========================================================================= # - - -class StochasticSampler(torch.utils.data.Sampler): - """ - Sample random batches, not guaranteed to be unique or cover the entire dataset in one epoch! - """ - - def __init__(self, data_source: Union[Sized, int], batch_size: int = 128): - super().__init__(data_source) - if isinstance(data_source, int): - self._len = data_source - else: - self._len = len(data_source) - self._batch_size = batch_size - assert isinstance(self._len, int) - assert self._len > 0 - assert isinstance(self._batch_size, int) - assert self._batch_size > 0 - - def __iter__(self): - while True: - yield from np.random.randint(0, self._len, size=self._batch_size) - - -class ProgressiveStochasticSampler(StochasticSampler): - - def __init__(self, data_source: Union[Sized, int], batch_size: int = 128, progression_rate: float = 0.05): - super().__init__(data_source=data_source, batch_size=batch_size) - self._available = 0 - self._progression_rate = progression_rate - - def __iter__(self): - while True: - self._available += self._batch_size - M = int(self._available * self._progression_rate) - M = min(self._len, max(M, self._batch_size)) - yield from np.random.randint(0, M, size=self._batch_size) - - - -def yield_dataloader(dataloader: torch.utils.data.DataLoader, steps: int): - i = 0 - while True: - for it in dataloader: - yield it - i += 1 - if i >= steps: - return - - -def StochasticBatchSampler(data_source: Union[Sized, int], batch_size: int): - return torch.utils.data.BatchSampler( - sampler=StochasticSampler(data_source=data_source, batch_size=batch_size), - batch_size=batch_size, - drop_last=True - ) - - -def ProgressiveStochasticBatchSampler(data_source: Union[Sized, int], batch_size: int, progression_rate: float = 0.05): - return torch.utils.data.BatchSampler( - sampler=ProgressiveStochasticSampler(data_source=data_source, batch_size=batch_size, progression_rate=progression_rate), - batch_size=batch_size, - drop_last=True - ) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/util/_fn_util.py b/research/code/util/_fn_util.py deleted file mode 100644 index 471bd3fe..00000000 --- a/research/code/util/_fn_util.py +++ /dev/null @@ -1,114 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import inspect -from typing import Sequence - -from disent.util.deprecate import deprecated - - -# ========================================================================= # -# Function Arguments # -# ========================================================================= # - - -def _get_fn_from_stack(fn_name: str, stack): - # -- do we actually need all of this? - fn = None - for s in stack: - if fn_name in s.frame.f_locals: - fn = s.frame.f_locals[fn_name] - break - if fn is None: - raise RuntimeError(f'could not retrieve function: {repr(fn_name)} from call stack.') - return fn - - -@deprecated('function uses bad mechanics, see commented implementation below') -def get_caller_params(sort: bool = False, exclude: Sequence[str] = None) -> dict: - stack = inspect.stack() - fn_name = stack[1].function - fn_locals = stack[1].frame.f_locals - # get function and params - fn = _get_fn_from_stack(fn_name, stack) - fn_params = inspect.getfullargspec(fn).args - # check excluded - exclude = set() if (exclude is None) else set(exclude) - fn_params = [p for p in fn_params if (p not in exclude)] - # sort values - if sort: - fn_params = sorted(fn_params) - # return dict - return { - k: fn_locals[k] for k in fn_params - } - - -def params_as_string(params: dict, sep: str = '_', names: bool = False): - # get strings - if names: - return sep.join(f"{k}={v}" for k, v in params.items()) - else: - return sep.join(f"{v}" for k, v in params.items()) - - -# ========================================================================= # -# END # -# ========================================================================= # - - -# TODO: replace function above -# -# class DELETED(object): -# def __str__(self): return '' -# def __repr__(self): return str(self) -# -# -# DELETED = DELETED() -# -# -# def get_hparams(exclude: Union[Sequence[str], Set[str]] = None): -# # check values -# if exclude is None: -# exclude = {} -# else: -# exclude = set(exclude) -# # get frame and values -# args = inspect.getargvalues(frame=inspect.currentframe().f_back) -# # sort values -# arg_names = list(args.args) -# if args.varargs is not None: arg_names.append(args.varargs) -# if args.keywords is not None: arg_names.append(args.keywords) -# # filter values -# from argparse import Namespace -# return Namespace(**{ -# name: args.locals.get(name, DELETED) -# for name in arg_names -# if (name not in exclude) -# }) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/util/_io_util.py b/research/code/util/_io_util.py deleted file mode 100644 index e9f345dd..00000000 --- a/research/code/util/_io_util.py +++ /dev/null @@ -1,239 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import base64 -import dataclasses -import inspect -import io -import os -from typing import Optional -from typing import Union - -import torch - -from disent.util.inout.paths import ensure_parent_dir_exists - - -# ========================================================================= # -# Github Upload Utility Functions # -# ========================================================================= # - - -def gh_get_repo(repo: str = None): - from github import Github - # get token str - token = os.environ.get('GITHUB_TOKEN', '') - if not token.strip(): - raise ValueError('`GITHUB_TOKEN` env variable has not been set!') - assert isinstance(token, str) - # get repo str - if repo is None: - repo = os.environ.get('GITHUB_REPO', '') - if not repo.strip(): - raise ValueError('`GITHUB_REPO` env variable has not been set!') - assert isinstance(repo, str) - # get repo - return Github(token).get_repo(repo) - - -def gh_get_branch(repo: 'Repository', branch: str = None, source_branch: str = None, allow_new_branch: bool = True) -> 'Branch': - from github import GithubException - # check branch - assert isinstance(branch, str) or (branch is None) - assert isinstance(source_branch, str) or (source_branch is None) - # get default branch - if branch is None: - branch = repo.default_branch - # retrieve branch - try: - return repo.get_branch(branch) - except GithubException as e: - if not allow_new_branch: - raise RuntimeError(f'Creating branch disabled, set `allow_new_branch=True`: {repr(branch)}') - print(f'Creating missing branch: {repr(branch)}') - sb = repo.get_branch(repo.default_branch if (source_branch is None) else source_branch) - repo.create_git_ref(ref='refs/heads/' + branch, sha=sb.commit.sha) - return repo.get_branch(branch) - - -@dataclasses.dataclass -class WriteResult: - commit: 'Commit' - content: 'ContentFile' - - -def gh_write_file(repo: 'Repository', path: str, content: Union[str, bytes], branch: str = None, allow_new_file=True, allow_overwrite_file=False, allow_new_branch=True) -> WriteResult: - from github import UnknownObjectException - # get branch - branch = gh_get_branch(repo, branch, allow_new_branch=allow_new_branch).name - # check that the file exists - try: - sha = repo.get_contents(path, ref=branch).sha - except UnknownObjectException: - sha = None - # handle file exists or not - if sha is None: - if not allow_new_file: - raise RuntimeError(f'Creating file disabled, set `allow_new_file=True`: {repr(path)}') - result = repo.create_file(path=path, message=f'Created File: {path}', content=content, branch=branch) - else: - if not allow_overwrite_file: - raise RuntimeError(f'Overwriting file disabled, `set allow_overwrite_file=True`: {repr(path)}') - result = repo.update_file(path=path, message=f'Updated File: {path}', content=content, branch=branch, sha=sha) - # result is a dict: {'commit': github.Commit, 'content': github.ContentFile} - return WriteResult(**result) - - -# ========================================================================= # -# Github Upload Utility Class # -# ========================================================================= # - - -class GithubWriter(object): - - def __init__(self, repo: str = None, branch: str = None, allow_new_file=True, allow_overwrite_file=True, allow_new_branch=True): - self._kwargs = dict( - repo=gh_get_repo(repo=repo), - branch=branch, - allow_new_file=allow_new_file, - allow_overwrite_file=allow_overwrite_file, - allow_new_branch=allow_new_branch, - ) - - def write_file(self, path: str, content: Union[str, bytes]): - return gh_write_file( - path=path, - content=content, - **self._kwargs, - ) - - -# ========================================================================= # -# Torch Save Utils # -# ========================================================================= # - - -def torch_save_bytes(model) -> bytes: - buffer = io.BytesIO() - torch.save(model, buffer) - buffer.seek(0) - return buffer.read() - - -def torch_save_base64(model) -> str: - b = torch_save_bytes(model) - return base64.b64encode(b).decode('ascii') - - -def torch_load_bytes(b: bytes): - return torch.load(io.BytesIO(b)) - - -def torch_load_base64(s: str): - b = base64.b64decode(s.encode('ascii')) - return torch_load_bytes(b) - - -# ========================================================================= # -# write # -# ========================================================================= # - - -def _split_special_path(path): - if path.startswith('github:'): - # get github repo and path - path = path[len('github:'):] - repo, path = os.path.join(*path.split('/')[:2]), os.path.join(*path.split('/')[2:]) - # check paths - assert repo.strip() and len(repo.split('/')) == 2 - assert path.strip() and len(repo.split('/')) >= 1 - # return components - return 'github', (repo, path) - else: - return 'local', path - - -def torch_write(path: str, model): - path_type, path = _split_special_path(path) - # handle cases - if path_type == 'github': - path, repo = path - # get the name of the path - ghw = GithubWriter(repo) - ghw.write_file(path=path, content=torch_save_bytes(model)) - print(f'Saved in repo: {repr(path)} to file: {repr(repo)}') - elif path_type == 'local': - torch.save(model, ensure_parent_dir_exists(path)) - print(f'Saved to file: {repr(path)}') - else: - raise KeyError(f'unknown path type: {repr(path_type)}') - - -# ========================================================================= # -# Files # -# ========================================================================= # - - -def _make_rel_path(*path_segments, is_file=True, _calldepth=0): - assert not os.path.isabs(os.path.join(*path_segments)), 'path must be relative' - # get source - stack = inspect.stack() - module = inspect.getmodule(stack[_calldepth+1].frame) - reldir = os.path.dirname(module.__file__) - # make everything - path = os.path.join(reldir, *path_segments) - folder_path = os.path.dirname(path) if is_file else path - os.makedirs(folder_path, exist_ok=True) - return path - - -def _make_rel_path_add_ext(*path_segments, ext='.png', _calldepth=0): - # make path - path = _make_rel_path(*path_segments, is_file=True, _calldepth=_calldepth+1) - if not os.path.splitext(path)[1]: - path = f'{path}{ext}' - return path - - -def make_rel_path(*path_segments, is_file=True): - return _make_rel_path(*path_segments, is_file=is_file, _calldepth=1) - - -def make_rel_path_add_ext(*path_segments, ext='.png'): - return _make_rel_path_add_ext(*path_segments, ext=ext, _calldepth=1) - - -def plt_rel_path_savefig(rel_path: Optional[str], save: bool = True, show: bool = True, ext='.png', dpi: Optional[int] = None, **kwargs): - import matplotlib.pyplot as plt - if save and (rel_path is not None): - path = _make_rel_path_add_ext(rel_path, ext=ext, _calldepth=2) - plt.savefig(path, dpi=dpi, **kwargs) - print(f'saved: {repr(path)}') - if show: - plt.show(**kwargs) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/util/_loss.py b/research/code/util/_loss.py deleted file mode 100644 index 50cbc1ab..00000000 --- a/research/code/util/_loss.py +++ /dev/null @@ -1,160 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import inspect -import warnings -from typing import Optional -from typing import Sequence - -import numpy as np -import torch -from torch.nn import functional as F - -from disent import registry -from disent.nn.loss.reduction import batch_loss_reduction - - -# ========================================================================= # -# optimizer # -# ========================================================================= # - - -_SPECIALIZATIONS = {'sgd_m': ('sgd', dict(momentum=0.1))} - - -def make_optimizer(model: torch.nn.Module, name: str = 'sgd', lr=1e-3, weight_decay: Optional[float] = None): - if isinstance(model, torch.nn.Module): - params = model.parameters() - elif isinstance(model, torch.Tensor): - assert model.requires_grad - params = [model] - else: - raise TypeError(f'cannot optimize type: {type(model)}') - # get specializations - kwargs = {} - if name in _SPECIALIZATIONS: - name, kwargs = _SPECIALIZATIONS[name] - # get optimizer class - optimizer_cls = registry.OPTIMIZERS[name] - optimizer_params = set(inspect.signature(optimizer_cls).parameters.keys()) - # add optional arguments - if weight_decay is not None: - if 'weight_decay' in optimizer_params: - kwargs['weight_decay'] = weight_decay - else: - warnings.warn(f'{name}: weight decay cannot be set, optimizer does not have `weight_decay` parameter') - # instantiate - return optimizer_cls(params, lr=lr, **kwargs) - - -def step_optimizer(optimizer, loss): - optimizer.zero_grad() - loss.backward() - optimizer.step() - - -# ========================================================================= # -# Loss # -# ========================================================================= # - - -def _unreduced_mse_loss(pred: torch.Tensor, targ: torch.Tensor) -> torch.Tensor: - return F.mse_loss(pred, targ, reduction='none') - - -def _unreduced_mae_loss(pred: torch.Tensor, targ: torch.Tensor) -> torch.Tensor: - return torch.abs(pred - targ) - - -def unreduced_loss(pred: torch.Tensor, targ: torch.Tensor, mode='mse') -> torch.Tensor: - return _LOSS_FNS[mode](pred, targ) - - -_LOSS_FNS = { - 'mse': _unreduced_mse_loss, - 'mae': _unreduced_mae_loss, -} - - -# ========================================================================= # -# Pairwise Loss # -# ========================================================================= # - - -def pairwise_loss(pred: torch.Tensor, targ: torch.Tensor, mode='mse', mean_dtype=None, mask: Optional[torch.Tensor] = None) -> torch.Tensor: - # check input - assert pred.shape == targ.shape - # mean over final dims - loss = unreduced_loss(pred=pred, targ=targ, mode=mode) - # mask values - if mask is not None: - loss *= mask - # reduce - loss = batch_loss_reduction(loss, reduction_dtype=mean_dtype, reduction='mean') - # check result - assert loss.shape == pred.shape[:1] - # done - return loss - - -def unreduced_overlap(pred: torch.Tensor, targ: torch.Tensor, mode='mse') -> torch.Tensor: - # -ve loss - return - unreduced_loss(pred=pred, targ=targ, mode=mode) - - -def pairwise_overlap(pred: torch.Tensor, targ: torch.Tensor, mode='mse', mean_dtype=None) -> torch.Tensor: - # -ve loss - return - pairwise_loss(pred=pred, targ=targ, mode=mode, mean_dtype=mean_dtype) - - -# ========================================================================= # -# Factor Distances # -# ========================================================================= # - - -def np_factor_dists( - factors_a: np.ndarray, - factors_b: np.ndarray, - factor_sizes: Optional[Sequence[int]] = None, - circular_if_factor_sizes: bool = True, - p: int = 1, -) -> np.ndarray: - assert factors_a.ndim == 2 - assert factors_a.shape == factors_b.shape - # compute factor distances - fdists = np.abs(factors_a - factors_b) # (NUM, FACTOR_SIZE) - # circular distance - if (factor_sizes is not None) and circular_if_factor_sizes: - M = np.array(factor_sizes)[None, :] # (FACTOR_SIZE,) -> (1, FACTOR_SIZE) - assert M.shape == (1, factors_a.shape[-1]) - fdists = np.where(fdists > (M // 2), M - fdists, fdists) # (NUM, FACTOR_SIZE) - # compute final dists - fdists = (fdists ** p).sum(axis=-1) ** (1 / p) - # return values - return fdists # (NUM,) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/code/util/_wandb_plots.py b/research/code/util/_wandb_plots.py deleted file mode 100644 index 7e22c86e..00000000 --- a/research/code/util/_wandb_plots.py +++ /dev/null @@ -1,165 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2022 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import os.path -import warnings -from typing import List -from typing import Sequence - -import pandas as pd -import wandb -from cachier import cachier as _cachier -from tqdm import tqdm - -from disent.util.function import wrapped_partial - - -# ========================================================================= # -# Helper # -# ========================================================================= # - - -# `research/code/util/.cache` -CACHE_DIR = os.path.join(os.path.dirname(__file__), '.cache') - -# cachier instance -CACHIER: _cachier = wrapped_partial(_cachier, cache_dir=CACHE_DIR) - - -def clear_runs_cache(): - _load_runs_data.clear_cache() - - -# ========================================================================= # -# Load WANDB Data # -# ========================================================================= # - - -@CACHIER() -def _load_runs_data(project: str, include_history: bool = False) -> pd.DataFrame: - api = wandb.Api() - - runs = api.runs(project) - - info_list, summary_list, config_list, name_list, history_list = [], [], [], [], [] - for run in tqdm(runs, desc=f'loading: {project}'): - info_list.append({ - 'id': run.id, - 'name': run.name, - 'state': run.state, - 'storage_id': run.storage_id, - 'url': run.url, - }) - summary_list.append(run.summary._json_dict) - config_list.append({k: v for k, v in run.config.items() if not k.startswith('_')}) - name_list.append(run.name) - if include_history: - history_list.append(run.history()) - - data = { - "info": info_list, - "summary": summary_list, - "config": config_list, - "name": name_list - } - - if include_history: - data['history'] = history_list - - return pd.DataFrame(data) - - -def load_runs(project: str, include_history: bool = False) -> pd.DataFrame: - # load the data - df_runs_data: pd.DataFrame = _load_runs_data(project, include_history=include_history) - # expand the dictionaries - df_info: pd.DataFrame = df_runs_data['info'].apply(pd.Series) - df_summary: pd.DataFrame = df_runs_data['summary'].apply(pd.Series) - df_config: pd.DataFrame = df_runs_data['config'].apply(pd.Series) - # merge the data - df: pd.DataFrame = df_config.join(df_summary).join(df_info) - assert len(df.columns) == len(df_info.columns) + len(df_summary.columns) + len(df_config.columns) - # add history - if include_history: - assert 'history' not in df.columns - df = df.join(df_runs_data['history']) # history is already a series - assert 'history' in df.columns - assert len(df.columns) == len(df_info.columns) + len(df_summary.columns) + len(df_config.columns) + 1 - # done! - return df - - -# ========================================================================= # -# Run Filtering # -# ========================================================================= # - - -def drop_unhashable_cols(df: pd.DataFrame, inplace: bool = False, skip: Sequence[str] = None) -> (pd.DataFrame, List[str]): - """ - Drop all the columns of a dataframe that cannot be hashed - -- this will remove media or other usually unnecessary content from the wandb api - """ - # keep these items - if skip is None: - skip = [] - skip = set(skip) - if skip - set(df.columns): - warnings.warn(f'some skipped column names are not contained in the df: {skip - set(df.columns)}') - # drop columns - dropped = [] - for col in df.columns: - if col in skip: - continue - try: - df[col].unique() - except: - dropped.append(col) - df = df.drop(col, inplace=inplace, axis=1) - return df, dropped - - -def drop_non_unique_cols(df: pd.DataFrame, inplace: bool = False, skip: Sequence[str] = None) -> (pd.DataFrame, List[str]): - """ - Drop all the columns of a dataframe where all the values are the same! - """ - # keep these items - if skip is None: - skip = [] - skip = set(skip) - if skip - set(df.columns): - warnings.warn(f'some skipped column names are not contained in the df: {skip - set(df.columns)}') - # drop columns - dropped = [] - for col in df.columns: - if col in skip: - continue - if len(df[col].unique()) == 1: - dropped.append(col) - df = df.drop(col, inplace=inplace, axis=1) - return df, dropped - - -# ========================================================================= # -# Prepare Data # -# ========================================================================= # diff --git a/research/code/util/gadfly.mplstyle b/research/code/util/gadfly.mplstyle deleted file mode 100644 index 00b215ac..00000000 --- a/research/code/util/gadfly.mplstyle +++ /dev/null @@ -1,627 +0,0 @@ -#### MATPLOTLIBRC FORMAT - -# FROM: https://towardsdatascience.com/a-new-plot-theme-for-matplotlib-gadfly-2cffc745ff84 - -## This is a sample matplotlib configuration file - you can find a copy -## of it on your system in -## site-packages/matplotlib/mpl-data/matplotlibrc. If you edit it -## there, please note that it will be overwritten in your next install. -## If you want to keep a permanent local copy that will not be -## overwritten, place it in the following location: -## unix/linux: -## $HOME/.config/matplotlib/matplotlibrc or -## $XDG_CONFIG_HOME/matplotlib/matplotlibrc (if $XDG_CONFIG_HOME is set) -## other platforms: -## $HOME/.matplotlib/matplotlibrc -## -## See http://matplotlib.org/users/customizing.html#the-matplotlibrc-file for -## more details on the paths which are checked for the configuration file. -## -## This file is best viewed in a editor which supports python mode -## syntax highlighting. Blank lines, or lines starting with a comment -## symbol, are ignored, as are trailing comments. Other lines must -## have the format -## key : val ## optional comment -## -## Colors: for the color values below, you can either use - a -## matplotlib color string, such as r, k, or b - an rgb tuple, such as -## (1.0, 0.5, 0.0) - a hex string, such as ff00ff - a scalar -## grayscale intensity such as 0.75 - a legal html color name, e.g., red, -## blue, darkslategray - -##### CONFIGURATION BEGINS HERE - -## The default backend; one of GTK3Agg GTK3Cairo MacOSX Qt4Agg Qt5Agg TkAgg -## WX WXAgg Agg Cairo PS PDF SVG Template. -## You can also deploy your own backend outside of matplotlib by -## referring to the module name (which must be in the PYTHONPATH) as -## 'module://my_backend'. -## -## If you omit this parameter, the backend will be determined by fallback. -#backend : Agg - -## Note that this can be overridden by the environment variable -## QT_API used by Enthought Tool Suite (ETS); valid values are -## "pyqt" and "pyside". The "pyqt" setting has the side effect of -## forcing the use of Version 2 API for QString and QVariant. - -## The port to use for the web server in the WebAgg backend. -#webagg.port : 8988 - -## The address on which the WebAgg web server should be reachable -#webagg.address : 127.0.0.1 - -## If webagg.port is unavailable, a number of other random ports will -## be tried until one that is available is found. -#webagg.port_retries : 50 - -## When True, open the webbrowser to the plot that is shown -#webagg.open_in_browser : True - -## if you are running pyplot inside a GUI and your backend choice -## conflicts, we will automatically try to find a compatible one for -## you if backend_fallback is True -#backend_fallback: True - -#interactive : False -#toolbar : toolbar2 ## None | toolbar2 ("classic" is deprecated) -#timezone : UTC ## a pytz timezone string, e.g., US/Central or Europe/Paris - -## Where your matplotlib data lives if you installed to a non-default -## location. This is where the matplotlib fonts, bitmaps, etc reside -#datapath : /home/jdhunter/mpldata - - -#### LINES -## See http://matplotlib.org/api/artist_api.html#module-matplotlib.lines for more -## information on line properties. -lines.linewidth : 2 ## line width in points -#lines.linestyle : - ## solid line -#lines.color : C0 ## has no affect on plot(); see axes.prop_cycle -#lines.marker : None ## the default marker -# lines.markerfacecolor : auto ## the default markerfacecolor -lines.markeredgecolor : white ## the default markeredgecolor -lines.markeredgewidth : 1 ## the line width around the marker symbol -lines.markersize : 7 ## markersize, in points -#lines.dash_joinstyle : round ## miter|round|bevel -#lines.dash_capstyle : butt ## butt|round|projecting -#lines.solid_joinstyle : round ## miter|round|bevel -#lines.solid_capstyle : projecting ## butt|round|projecting -#lines.antialiased : True ## render lines in antialiased (no jaggies) - -## The three standard dash patterns. These are scaled by the linewidth. -#lines.dashed_pattern : 3.7, 1.6 -#lines.dashdot_pattern : 6.4, 1.6, 1, 1.6 -#lines.dotted_pattern : 1, 1.65 -#lines.scale_dashes : True - -#markers.fillstyle: full ## full|left|right|bottom|top|none - -#### PATCHES -## Patches are graphical objects that fill 2D space, like polygons or -## circles. See -## http://matplotlib.org/api/artist_api.html#module-matplotlib.patches -## information on patch properties -patch.linewidth : 1 ## edge width in points. -patch.facecolor : C0 -patch.edgecolor : black ## if forced, or patch is not filled -#patch.force_edgecolor : False ## True to always use edgecolor -#patch.antialiased : True ## render patches in antialiased (no jaggies) - -#### HATCHES -#hatch.color : black -#hatch.linewidth : 1.0 - -#### Boxplot -#boxplot.notch : False -#boxplot.vertical : True -#boxplot.whiskers : 1.5 -# boxplot.bootstrap : None -boxplot.patchartist : True -#boxplot.showmeans : False -#boxplot.showcaps : True -#boxplot.showbox : True -#boxplot.showfliers : True -#boxplot.meanline : False - -boxplot.flierprops.color : C0 -boxplot.flierprops.marker : o -boxplot.flierprops.markerfacecolor : auto -boxplot.flierprops.markeredgecolor : white -boxplot.flierprops.markersize : 7 -boxplot.flierprops.linestyle : none -boxplot.flierprops.linewidth : 1.0 - -boxplot.boxprops.color : 9ae1f9 -boxplot.boxprops.linewidth : 0 -boxplot.boxprops.linestyle : - - -boxplot.whiskerprops.color : C0 -boxplot.whiskerprops.linewidth : 1.0 -boxplot.whiskerprops.linestyle : - - -boxplot.capprops.color : C0 -boxplot.capprops.linewidth : 1.0 -boxplot.capprops.linestyle : - - -boxplot.medianprops.color : 9ae1f9 -boxplot.medianprops.linewidth : 1 -boxplot.medianprops.linestyle : - - -boxplot.meanprops.color : C1 -boxplot.meanprops.marker : ^ -boxplot.meanprops.markerfacecolor : C1 -boxplot.meanprops.markeredgecolor : C1 -boxplot.meanprops.markersize : 7 -boxplot.meanprops.linestyle : -- -boxplot.meanprops.linewidth : 1.0 - - -#### FONT - -## font properties used by text.Text. See -## http://matplotlib.org/api/font_manager_api.html for more -## information on font properties. The 6 font properties used for font -## matching are given below with their default values. -## -## The font.family property has five values: 'serif' (e.g., Times), -## 'sans-serif' (e.g., Helvetica), 'cursive' (e.g., Zapf-Chancery), -## 'fantasy' (e.g., Western), and 'monospace' (e.g., Courier). Each of -## these font families has a default list of font names in decreasing -## order of priority associated with them. When text.usetex is False, -## font.family may also be one or more concrete font names. -## -## The font.style property has three values: normal (or roman), italic -## or oblique. The oblique style will be used for italic, if it is not -## present. -## -## The font.variant property has two values: normal or small-caps. For -## TrueType fonts, which are scalable fonts, small-caps is equivalent -## to using a font size of 'smaller', or about 83%% of the current font -## size. -## -## The font.weight property has effectively 13 values: normal, bold, -## bolder, lighter, 100, 200, 300, ..., 900. Normal is the same as -## 400, and bold is 700. bolder and lighter are relative values with -## respect to the current weight. -## -## The font.stretch property has 11 values: ultra-condensed, -## extra-condensed, condensed, semi-condensed, normal, semi-expanded, -## expanded, extra-expanded, ultra-expanded, wider, and narrower. This -## property is not currently implemented. -## -## The font.size property is the default font size for text, given in pts. -## 10 pt is the standard value. - -#font.family : sans-serif -#font.style : normal -#font.variant : normal -#font.weight : normal -#font.stretch : normal -## note that font.size controls default text sizes. To configure -## special text sizes tick labels, axes, labels, title, etc, see the rc -## settings for axes and ticks. Special text sizes can be defined -## relative to font.size, using the following values: xx-small, x-small, -## small, medium, large, x-large, xx-large, larger, or smaller -#font.size : 10.0 -#font.serif : DejaVu Serif, Bitstream Vera Serif, Computer Modern Roman, New Century Schoolbook, Century Schoolbook L, Utopia, ITC Bookman, Bookman, Nimbus Roman No9 L, Times New Roman, Times, Palatino, Charter, serif -#font.sans-serif : DejaVu Sans, Bitstream Vera Sans, Computer Modern Sans Serif, Lucida Grande, Verdana, Geneva, Lucid, Arial, Helvetica, Avant Garde, sans-serif -#font.cursive : Apple Chancery, Textile, Zapf Chancery, Sand, Script MT, Felipa, cursive -#font.fantasy : Comic Sans MS, Chicago, Charcoal, ImpactWestern, Humor Sans, xkcd, fantasy -#font.monospace : DejaVu Sans Mono, Bitstream Vera Sans Mono, Computer Modern Typewriter, Andale Mono, Nimbus Mono L, Courier New, Courier, Fixed, Terminal, monospace - -#### TEXT -## text properties used by text.Text. See -## http://matplotlib.org/api/artist_api.html#module-matplotlib.text for more -## information on text properties -text.color : 707074 - -#### LaTeX customizations. See http://wiki.scipy.org/Cookbook/Matplotlib/UsingTex -#text.usetex : False ## use latex for all text handling. The following fonts - ## are supported through the usual rc parameter settings: - ## new century schoolbook, bookman, times, palatino, - ## zapf chancery, charter, serif, sans-serif, helvetica, - ## avant garde, courier, monospace, computer modern roman, - ## computer modern sans serif, computer modern typewriter - ## If another font is desired which can loaded using the - ## LaTeX \usepackage command, please inquire at the - ## matplotlib mailing list -#text.latex.preamble : ## IMPROPER USE OF THIS FEATURE WILL LEAD TO LATEX FAILURES - ## AND IS THEREFORE UNSUPPORTED. PLEASE DO NOT ASK FOR HELP - ## IF THIS FEATURE DOES NOT DO WHAT YOU EXPECT IT TO. - ## preamble is a comma separated list of LaTeX statements - ## that are included in the LaTeX document preamble. - ## An example: - ## text.latex.preamble : \usepackage{bm},\usepackage{euler} - ## The following packages are always loaded with usetex, so - ## beware of package collisions: color, geometry, graphicx, - ## type1cm, textcomp. Adobe Postscript (PSSNFS) font packages - ## may also be loaded, depending on your font settings -#text.latex.preview : False - -#text.hinting : auto ## May be one of the following: - ## none: Perform no hinting - ## auto: Use FreeType's autohinter - ## native: Use the hinting information in the - # font file, if available, and if your - # FreeType library supports it - ## either: Use the native hinting information, - # or the autohinter if none is available. - ## For backward compatibility, this value may also be - ## True === 'auto' or False === 'none'. -#text.hinting_factor : 8 ## Specifies the amount of softness for hinting in the - ## horizontal direction. A value of 1 will hint to full - ## pixels. A value of 2 will hint to half pixels etc. -#text.antialiased : True ## If True (default), the text will be antialiased. - ## This only affects the Agg backend. - -## The following settings allow you to select the fonts in math mode. -## They map from a TeX font name to a fontconfig font pattern. -## These settings are only used if mathtext.fontset is 'custom'. -## Note that this "custom" mode is unsupported and may go away in the -## future. -#mathtext.cal : cursive -#mathtext.rm : sans -#mathtext.tt : monospace -#mathtext.it : sans:italic -#mathtext.bf : sans:bold -#mathtext.sf : sans -#mathtext.fontset : dejavusans ## Should be 'dejavusans' (default), - ## 'dejavuserif', 'cm' (Computer Modern), 'stix', - ## 'stixsans' or 'custom' -#mathtext.fallback_to_cm : True ## When True, use symbols from the Computer Modern - ## fonts when a symbol can not be found in one of - ## the custom math fonts. -#mathtext.default : it ## The default font to use for math. - ## Can be any of the LaTeX font names, including - ## the special name "regular" for the same font - ## used in regular text. - -#### AXES -## default face and edge color, default tick sizes, -## default fontsizes for ticklabels, and so on. See -## http://matplotlib.org/api/axes_api.html#module-matplotlib.axes -#axes.facecolor : white ## axes background color -axes.edgecolor : D0D0E0 ## axes edge color -#axes.linewidth : 0.8 ## edge linewidth -axes.grid : True ## display grid or not -axes.grid.axis : both ## which axis the grid should apply to -#axes.grid.which : major ## gridlines at major, minor or both ticks -axes.titlesize : 18 ## fontsize of the axes title -#axes.titleweight : normal ## font weight of title -#axes.titlepad : 6.0 ## pad between axes and title in points -axes.labelsize : 14 ## fontsize of the x any y labels -#axes.labelpad : 4.0 ## space between label and axis -#axes.labelweight : normal ## weight of the x and y labels -axes.labelcolor : 707074 -#axes.axisbelow : line ## draw axis gridlines and ticks below - ## patches (True); above patches but below - ## lines ('line'); or above all (False) -#axes.formatter.limits : -7, 7 ## use scientific notation if log10 - ## of the axis range is smaller than the - ## first or larger than the second -#axes.formatter.use_locale : False ## When True, format tick labels - ## according to the user's locale. - ## For example, use ',' as a decimal - ## separator in the fr_FR locale. -#axes.formatter.use_mathtext : False ## When True, use mathtext for scientific - ## notation. -#axes.formatter.min_exponent: 0 ## minimum exponent to format in scientific notation -#axes.formatter.useoffset : True ## If True, the tick label formatter - ## will default to labeling ticks relative - ## to an offset when the data range is - ## small compared to the minimum absolute - ## value of the data. -#axes.formatter.offset_threshold : 4 ## When useoffset is True, the offset - ## will be used when it can remove - ## at least this number of significant - ## digits from tick labels. -axes.spines.left : False ## display axis spines -axes.spines.bottom : False -axes.spines.top : False -axes.spines.right : False -#axes.unicode_minus : True ## use unicode for the minus symbol - ## rather than hyphen. See - ## http://en.wikipedia.org/wiki/Plus_and_minus_signs#Character_codes -## ========================================================================================== ## -## ========================================================================================== ## -## ========================================================================================== ## -## COLOR PALETTE -# v1 https://coolors.co/2364aa-3da5d9-4ebc93-b4da1b-fbcc23-ec8232-e40066-df26cf-ae5ce6-9b899f -# v2 https://coolors.co/3482d5-66b8e1-5cc19c-b9d548-fbc737-f2822c-ff338f-d54ee4-a072e9-9b899f -axes.prop_cycle : cycler('color', ['3482d5', '66b8e1', '5cc19c', 'b9d548', 'fbc737', 'f2822c', 'ff338f', 'd54ee4', 'a072e9', '9b899f']) ## CUSTOM -## axes.prop_cycle : cycler('color', ['00BEFF', 'D4CA3A', 'FF6DAE', '67E1B5', 'EBACFA', '9E9E9E', 'F1988E', '5DB15A', 'E28544', '52B8AA']) ## ORIG - ## color cycle for plot lines as list of string - ## colorspecs: single letter, long name, or web-style hex - ## Note the use of string escapes here ('1f77b4', instead of 1f77b4) - ## as opposed to the rest of this file. -## ========================================================================================== ## -## ========================================================================================== ## -## ========================================================================================== ## -#axes.autolimit_mode : data ## How to scale axes limits to the data. - ## Use "data" to use data limits, plus some margin - ## Use "round_number" move to the nearest "round" number -#axes.xmargin : .05 ## x margin. See `axes.Axes.margins` -#axes.ymargin : .05 ## y margin See `axes.Axes.margins` -#polaraxes.grid : True ## display grid on polar axes -#axes3d.grid : True ## display grid on 3d axes - -#### DATES -## These control the default format strings used in AutoDateFormatter. -## Any valid format datetime format string can be used (see the python -## `datetime` for details). For example using '%%x' will use the locale date representation -## '%%X' will use the locale time representation and '%%c' will use the full locale datetime -## representation. -## These values map to the scales: -## {'year': 365, 'month': 30, 'day': 1, 'hour': 1/24, 'minute': 1 / (24 * 60)} - -#date.autoformatter.year : %Y -#date.autoformatter.month : %Y-%m -#date.autoformatter.day : %Y-%m-%d -#date.autoformatter.hour : %m-%d %H -#date.autoformatter.minute : %d %H:%M -#date.autoformatter.second : %H:%M:%S -#date.autoformatter.microsecond : %M:%S.%f - -#### TICKS -## see http://matplotlib.org/api/axis_api.html#matplotlib.axis.Tick -#xtick.top : False ## draw ticks on the top side -#xtick.bottom : True ## draw ticks on the bottom side -#xtick.labeltop : False ## draw label on the top -#xtick.labelbottom : True ## draw label on the bottom -#xtick.major.size : 3.5 ## major tick size in points -#xtick.minor.size : 2 ## minor tick size in points -#xtick.major.width : 0.8 ## major tick width in points -#xtick.minor.width : 0.6 ## minor tick width in points -#xtick.major.pad : 3.5 ## distance to major tick label in points -#xtick.minor.pad : 3.4 ## distance to the minor tick label in points -xtick.color : 707074 ## color of the tick labels -xtick.labelsize : 12 ## fontsize of the tick labels -#xtick.direction : out ## direction: in, out, or inout -#xtick.minor.visible : False ## visibility of minor ticks on x-axis -#xtick.major.top : True ## draw x axis top major ticks -#xtick.major.bottom : True ## draw x axis bottom major ticks -#xtick.minor.top : True ## draw x axis top minor ticks -#xtick.minor.bottom : True ## draw x axis bottom minor ticks -#xtick.alignment : center ## alignment of xticks - -#ytick.left : True ## draw ticks on the left side -#ytick.right : False ## draw ticks on the right side -#ytick.labelleft : True ## draw tick labels on the left side -#ytick.labelright : False ## draw tick labels on the right side -#ytick.major.size : 3.5 ## major tick size in points -#ytick.minor.size : 2 ## minor tick size in points -#ytick.major.width : 0.8 ## major tick width in points -#ytick.minor.width : 0.6 ## minor tick width in points -#ytick.major.pad : 3.5 ## distance to major tick label in points -#ytick.minor.pad : 3.4 ## distance to the minor tick label in points -ytick.color : 707074 ## color of the tick labels -ytick.labelsize : 12 ## fontsize of the tick labels -#ytick.direction : out ## direction: in, out, or inout -#ytick.minor.visible : False ## visibility of minor ticks on y-axis -#ytick.major.left : True ## draw y axis left major ticks -#ytick.major.right : True ## draw y axis right major ticks -#ytick.minor.left : True ## draw y axis left minor ticks -#ytick.minor.right : True ## draw y axis right minor ticks -#ytick.alignment : center_baseline ## alignment of yticks - -#### GRIDS -grid.color : 93939c ## grid color -grid.linestyle : -- ## solid -#grid.linewidth : 0.8 ## in points -grid.alpha : 0.2 ## transparency, between 0.0 and 1.0 - -#### Legend -#legend.loc : best -#legend.frameon : True ## if True, draw the legend on a background patch -#legend.framealpha : 0.8 ## legend patch transparency -#legend.facecolor : inherit ## inherit from axes.facecolor; or color spec -#legend.edgecolor : 0.8 ## background patch boundary color -#legend.fancybox : True ## if True, use a rounded box for the - ## legend background, else a rectangle -#legend.shadow : False ## if True, give background a shadow effect -#legend.numpoints : 1 ## the number of marker points in the legend line -#legend.scatterpoints : 1 ## number of scatter points -#legend.markerscale : 1.0 ## the relative size of legend markers vs. original -#legend.fontsize : medium -#legend.title_fontsize : None ## None sets to the same as the default axes. -## Dimensions as fraction of fontsize: -#legend.borderpad : 0.4 ## border whitespace -#legend.labelspacing : 0.5 ## the vertical space between the legend entries -#legend.handlelength : 2.0 ## the length of the legend lines -#legend.handleheight : 0.7 ## the height of the legend handle -#legend.handletextpad : 0.8 ## the space between the legend line and legend text -#legend.borderaxespad : 0.5 ## the border between the axes and legend edge -#legend.columnspacing : 2.0 ## column separation - -#### FIGURE -## See http://matplotlib.org/api/figure_api.html#matplotlib.figure.Figure -#figure.titlesize : large ## size of the figure title (Figure.suptitle()) -#figure.titleweight : normal ## weight of the figure title -#figure.figsize : 6.4, 4.8 ## figure size in inches -#figure.dpi : 100 ## figure dots per inch -#figure.facecolor : white ## figure facecolor -#figure.edgecolor : white ## figure edgecolor -#figure.frameon : True ## enable figure frame -#figure.max_open_warning : 20 ## The maximum number of figures to open through - ## the pyplot interface before emitting a warning. - ## If less than one this feature is disabled. -## The figure subplot parameters. All dimensions are a fraction of the -#figure.subplot.left : 0.125 ## the left side of the subplots of the figure -#figure.subplot.right : 0.9 ## the right side of the subplots of the figure -#figure.subplot.bottom : 0.11 ## the bottom of the subplots of the figure -#figure.subplot.top : 0.88 ## the top of the subplots of the figure -#figure.subplot.wspace : 0.2 ## the amount of width reserved for space between subplots, - ## expressed as a fraction of the average axis width -#figure.subplot.hspace : 0.2 ## the amount of height reserved for space between subplots, - ## expressed as a fraction of the average axis height - -## Figure layout -#figure.autolayout : False ## When True, automatically adjust subplot - ## parameters to make the plot fit the figure - ## using `tight_layout` -#figure.constrained_layout.use: False ## When True, automatically make plot - ## elements fit on the figure. (Not compatible - ## with `autolayout`, above). -#figure.constrained_layout.h_pad : 0.04167 ## Padding around axes objects. Float representing -#figure.constrained_layout.w_pad : 0.04167 ## inches. Default is 3./72. inches (3 pts) -#figure.constrained_layout.hspace : 0.02 ## Space between subplot groups. Float representing -#figure.constrained_layout.wspace : 0.02 ## a fraction of the subplot widths being separated. - -#### IMAGES -#image.aspect : equal ## equal | auto | a number -#image.interpolation : nearest ## see help(imshow) for options -#image.cmap : viridis ## A colormap name, gray etc... -#image.lut : 256 ## the size of the colormap lookup table -#image.origin : upper ## lower | upper -#image.resample : True -#image.composite_image : True ## When True, all the images on a set of axes are - ## combined into a single composite image before - ## saving a figure as a vector graphics file, - ## such as a PDF. - -#### CONTOUR PLOTS -#contour.negative_linestyle : dashed ## string or on-off ink sequence -#contour.corner_mask : True ## True | False | legacy - -#### ERRORBAR PLOTS -#errorbar.capsize : 0 ## length of end cap on error bars in pixels - -#### HISTOGRAM PLOTS -#hist.bins : 10 ## The default number of histogram bins. - ## If Numpy 1.11 or later is - ## installed, may also be `auto` - -#### SCATTER PLOTS -#scatter.marker : o ## The default marker type for scatter plots. - -#### Agg rendering -#### Warning: experimental, 2008/10/10 -#agg.path.chunksize : 0 ## 0 to disable; values in the range - ## 10000 to 100000 can improve speed slightly - ## and prevent an Agg rendering failure - ## when plotting very large data sets, - ## especially if they are very gappy. - ## It may cause minor artifacts, though. - ## A value of 20000 is probably a good - ## starting point. -#### PATHS -#path.simplify : True ## When True, simplify paths by removing "invisible" - ## points to reduce file size and increase rendering - ## speed -#path.simplify_threshold : 0.111111111111 ## The threshold of similarity below which - ## vertices will be removed in the - ## simplification process -#path.snap : True ## When True, rectilinear axis-aligned paths will be snapped to - ## the nearest pixel when certain criteria are met. When False, - ## paths will never be snapped. -#path.sketch : None ## May be none, or a 3-tuple of the form (scale, length, - ## randomness). - ## *scale* is the amplitude of the wiggle - ## perpendicular to the line (in pixels). *length* - ## is the length of the wiggle along the line (in - ## pixels). *randomness* is the factor by which - ## the length is randomly scaled. -#path.effects : [] ## - -#### SAVING FIGURES -## the default savefig params can be different from the display params -## e.g., you may want a higher resolution, or to make the figure -## background white -#savefig.dpi : figure ## figure dots per inch or 'figure' -#savefig.facecolor : white ## figure facecolor when saving -#savefig.edgecolor : white ## figure edgecolor when saving -#savefig.format : png ## png, ps, pdf, svg -#savefig.bbox : standard ## 'tight' or 'standard'. - ## 'tight' is incompatible with pipe-based animation - ## backends but will workd with temporary file based ones: - ## e.g. setting animation.writer to ffmpeg will not work, - ## use ffmpeg_file instead -#savefig.pad_inches : 0.1 ## Padding to be used when bbox is set to 'tight' -#savefig.jpeg_quality: 95 ## when a jpeg is saved, the default quality parameter. -#savefig.directory : ~ ## default directory in savefig dialog box, - ## leave empty to always use current working directory -#savefig.transparent : False ## setting that controls whether figures are saved with a - ## transparent background by default -#savefig.frameon : True ## enable frame of figure when saving -#savefig.orientation : portrait ## Orientation of saved figure - -### tk backend params -#tk.window_focus : False ## Maintain shell focus for TkAgg - -### ps backend params -#ps.papersize : letter ## auto, letter, legal, ledger, A0-A10, B0-B10 -#ps.useafm : False ## use of afm fonts, results in small files -#ps.usedistiller : False ## can be: None, ghostscript or xpdf - ## Experimental: may produce smaller files. - ## xpdf intended for production of publication quality files, - ## but requires ghostscript, xpdf and ps2eps -#ps.distiller.res : 6000 ## dpi -#ps.fonttype : 3 ## Output Type 3 (Type3) or Type 42 (TrueType) - -### pdf backend params -#pdf.compression : 6 ## integer from 0 to 9 - ## 0 disables compression (good for debugging) -#pdf.fonttype : 3 ## Output Type 3 (Type3) or Type 42 (TrueType) -#pdf.use14corefonts : False -#pdf.inheritcolor : False - -### svg backend params -#svg.image_inline : True ## write raster image data directly into the svg file -#svg.fonttype : path ## How to handle SVG fonts: - ## none: Assume fonts are installed on the machine where the SVG will be viewed. - ## path: Embed characters as paths -- supported by most SVG renderers - ## svgfont: Embed characters as SVG fonts -- supported only by Chrome, - ## Opera and Safari -#svg.hashsalt : None ## if not None, use this string as hash salt - ## instead of uuid4 -### pgf parameter -#pgf.rcfonts : True -#pgf.preamble : -#pgf.texsystem : xelatex - -### docstring params -##docstring.hardcopy = False ## set this when you want to generate hardcopy docstring - -## Event keys to interact with figures/plots via keyboard. -## Customize these settings according to your needs. -## Leave the field(s) empty if you don't need a key-map. (i.e., fullscreen : '') -#keymap.fullscreen : f, ctrl+f ## toggling -#keymap.home : h, r, home ## home or reset mnemonic -#keymap.back : left, c, backspace ## forward / backward keys to enable -#keymap.forward : right, v ## left handed quick navigation -#keymap.pan : p ## pan mnemonic -#keymap.zoom : o ## zoom mnemonic -#keymap.save : s, ctrl+s ## saving current figure -#keymap.help : f1 ## display help about active tools -#keymap.quit : ctrl+w, cmd+w, q ## close the current figure -#keymap.quit_all : W, cmd+W, Q ## close all figures -#keymap.grid : g ## switching on/off major grids in current axes -#keymap.grid_minor : G ## switching on/off minor grids in current axes -#keymap.yscale : l ## toggle scaling of y-axes ('log'/'linear') -#keymap.xscale : k, L ## toggle scaling of x-axes ('log'/'linear') -#keymap.all_axes : a ## enable all axes -#keymap.copy : ctrl+c, cmd+c ## Copy figure to clipboard - -###ANIMATION settings -#animation.html : none ## How to display the animation as HTML in - ## the IPython notebook. 'html5' uses - ## HTML5 video tag; 'jshtml' creates a - ## Javascript animation -#animation.writer : ffmpeg ## MovieWriter 'backend' to use -#animation.codec : h264 ## Codec to use for writing movie -#animation.bitrate: -1 ## Controls size/quality tradeoff for movie. - ## -1 implies let utility auto-determine -#animation.frame_format: png ## Controls frame format used by temp files -#animation.html_args: ## Additional arguments to pass to html writer -#animation.ffmpeg_path: ffmpeg ## Path to ffmpeg binary. Without full path - ## $PATH is searched -#animation.ffmpeg_args: ## Additional arguments to pass to ffmpeg -#animation.avconv_path: avconv ## Path to avconv binary. Without full path - ## $PATH is searched -#animation.avconv_args: ## Additional arguments to pass to avconv -#animation.convert_path: convert ## Path to ImageMagick's convert binary. - ## On Windows use the full path since convert - ## is also the name of a system tool. -#animation.convert_args: ## Additional arguments to pass to convert -#animation.embed_limit : 20.0 \ No newline at end of file diff --git a/research/config/README.md b/research/config/README.md deleted file mode 100644 index d8427ca0..00000000 --- a/research/config/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Research - Config - -- The research config is added to the experiment search path such that any - files are found and read before that of the default experiment config. - This means that if a file has the same name, it will overwrite the default file! - The search path is overridden by setting the `DISENT_CONFIGS_PREPEND` environment variable. - -- Additionally, we expose the research code by registering it with disent using the experiment - plugin functionality. See `config/run_plugins`. The plugin will register each metric, framework - or dataset with the `disent.registry`. Allowing easy use elsewhere through config entries. diff --git a/research/config/config.yaml b/research/config/config.yaml deleted file mode 100644 index 30c8583f..00000000 --- a/research/config/config.yaml +++ /dev/null @@ -1,45 +0,0 @@ -defaults: - - _self_ # defaults lists override entries from this file! - # data - - sampling: default__bb - - dataset: xyobject - - augment: none - # system - - framework: betavae - - model: vae_conv64 - # training - - optimizer: adam - - schedule: none - - metrics: all - - run_length: long - # logs - - run_callbacks: vis - - run_logging: wandb - # runtime - - run_location: stampede_shr - - run_launcher: slurm - - run_action: train - # experiment - - run_plugins: default - -settings: - job: - user: '${oc.env:USER}' - project: 'DELETE' - name: '${framework.name}:${settings.framework.recon_loss}|${dataset.name}:${sampling.name}|${trainer.max_steps}' - seed: NULL - framework: - beta: 0.0316 - recon_loss: mse - loss_reduction: mean # beta scaling - framework_opt: - latent_distribution: normal # only used by VAEs - overlap_loss: NULL # only used for experimental dotvae and dorvae - usage_ratio: 0.5 # only used by adversarial masked datasets - model: - z_size: 25 - weight_init: 'xavier_normal' # xavier_normal, default - dataset: - batch_size: 256 - optimizer: - lr: 1e-3 diff --git a/research/config/config_test.yaml b/research/config/config_test.yaml deleted file mode 100644 index 35b6e65e..00000000 --- a/research/config/config_test.yaml +++ /dev/null @@ -1,45 +0,0 @@ -defaults: - - _self_ # defaults lists override entries from this file! - # data - - sampling: default__bb - - dataset: xyobject - - augment: example - # system - - framework: betavae - - model: linear - # training - - optimizer: adam - - schedule: beta_cyclic - - metrics: test - - run_length: test - # logs - - run_callbacks: test - - run_logging: none - # runtime - - run_location: local_cpu - - run_launcher: local - - run_action: train - # experiment - - run_plugins: default - -settings: - job: - user: 'invalid' - project: 'invalid' - name: '${framework.name}:${settings.framework.recon_loss}|${dataset.name}:${sampling.name}|${trainer.max_steps}' - seed: NULL - framework: - beta: 0.0316 - recon_loss: mse - loss_reduction: mean # beta scaling - framework_opt: - latent_distribution: normal # only used by VAEs - overlap_loss: NULL # only used for experimental dotvae and dorvae - usage_ratio: 0.5 # only used by adversarial masked datasets - model: - z_size: 25 - weight_init: 'xavier_normal' # xavier_normal, default - dataset: - batch_size: 5 - optimizer: - lr: 1e-3 diff --git a/research/config/dataset/X--adv-cars3d--WARNING.yaml b/research/config/dataset/X--adv-cars3d--WARNING.yaml deleted file mode 100644 index ac8d26a5..00000000 --- a/research/config/dataset/X--adv-cars3d--WARNING.yaml +++ /dev/null @@ -1,20 +0,0 @@ -defaults: - - _data_type_: gt - -name: adv_cars3d - -data: - _target_: disent.dataset.data.SelfContainedHdf5GroundTruthData - h5_path: '${oc.env:HOME}/workspace/research/disent/out/adversarial_data_approx/2021-09-06--05-42-06_INVERT-VSTRONG-cars3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06/data.h5' - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.76418207, 0.75554032, 0.75075393] - vis_std: [0.31892905, 0.32751031, 0.33319886] - -# TODO: this does not yet copy the data to /tmp/ and thus if run on a cluster of a network drive, this will hammer the network disk. Fix this! diff --git a/research/config/dataset/X--adv-dsprites--WARNING.yaml b/research/config/dataset/X--adv-dsprites--WARNING.yaml deleted file mode 100644 index 3965bf84..00000000 --- a/research/config/dataset/X--adv-dsprites--WARNING.yaml +++ /dev/null @@ -1,20 +0,0 @@ -defaults: - - _data_type_: gt - -name: adv_dsprites - -data: - _target_: disent.dataset.data.SelfContainedHdf5GroundTruthData - h5_path: '${oc.env:HOME}/workspace/research/disent/out/adversarial_data_approx/2021-09-06--03-17-28_INVERT-VSTRONG-dsprites_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06/data.h5' - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.20482841] - vis_std: [0.33634909] - -# TODO: this does not yet copy the data to /tmp/ and thus if run on a cluster of a network drive, this will hammer the network disk. Fix this! diff --git a/research/config/dataset/X--adv-shapes3d--WARNING.yaml b/research/config/dataset/X--adv-shapes3d--WARNING.yaml deleted file mode 100644 index 5983845a..00000000 --- a/research/config/dataset/X--adv-shapes3d--WARNING.yaml +++ /dev/null @@ -1,20 +0,0 @@ -defaults: - - _data_type_: gt - -name: adv_shapes3d - -data: - _target_: disent.dataset.data.SelfContainedHdf5GroundTruthData - h5_path: '${oc.env:HOME}/workspace/research/disent/out/adversarial_data_approx/2021-09-06--00-29-23_INVERT-VSTRONG-shapes3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06/data.h5' - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.47992192, 0.51311111, 0.54627272] - vis_std: [0.28653814, 0.29201543, 0.27395435] - -# TODO: this does not yet copy the data to /tmp/ and thus if run on a cluster of a network drive, this will hammer the network disk. Fix this! diff --git a/research/config/dataset/X--adv-smallnorb--WARNING.yaml b/research/config/dataset/X--adv-smallnorb--WARNING.yaml deleted file mode 100644 index fa483e82..00000000 --- a/research/config/dataset/X--adv-smallnorb--WARNING.yaml +++ /dev/null @@ -1,20 +0,0 @@ -defaults: - - _data_type_: gt - -name: adv_smallnorb - -data: - _target_: disent.dataset.data.SelfContainedHdf5GroundTruthData - h5_path: '${oc.env:HOME}/workspace/research/disent/out/adversarial_data_approx/2021-09-06--09-10-59_INVERT-VSTRONG-smallnorb_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06/data.h5' - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.69691603] - vis_std: [0.21310608] - -# TODO: this does not yet copy the data to /tmp/ and thus if run on a cluster of a network drive, this will hammer the network disk. Fix this! diff --git a/research/config/dataset/X--dsprites-imagenet-bg-100.yaml b/research/config/dataset/X--dsprites-imagenet-bg-100.yaml deleted file mode 100644 index d6ef5922..00000000 --- a/research/config/dataset/X--dsprites-imagenet-bg-100.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_bg_100 - -data: - _target_: research.code.dataset.data.DSpritesImagenetData - visibility: 100 - mode: bg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.5020433619489952, 0.47206398913310593, 0.42380018909780404] - vis_std: [0.2505510666843685, 0.2500725980366869, 0.2562415603123114] diff --git a/research/config/dataset/X--dsprites-imagenet-bg-20.yaml b/research/config/dataset/X--dsprites-imagenet-bg-20.yaml deleted file mode 100644 index 7b315dd8..00000000 --- a/research/config/dataset/X--dsprites-imagenet-bg-20.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_bg_20 - -data: - _target_: research.code.dataset.data.DSpritesImagenetData - visibility: 20 - mode: bg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.13294969414492142, 0.12694375140936273, 0.11733572285575933] - vis_std: [0.18311250427586276, 0.1840916474752131, 0.18607373519458442] diff --git a/research/config/dataset/X--dsprites-imagenet-bg-25.yaml b/research/config/dataset/X--dsprites-imagenet-bg-25.yaml deleted file mode 100644 index dafb54e7..00000000 --- a/research/config/dataset/X--dsprites-imagenet-bg-25.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_bg_25 - -data: - _target_: research.code.dataset.data.DSpritesImagenetData - visibility: 25 - mode: bg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.15596283852200074, 0.14847876264131535, 0.13644703866118635] - vis_std: [0.18208653250875798, 0.18323109038468802, 0.18569624396763393] diff --git a/research/config/dataset/X--dsprites-imagenet-bg-40.yaml b/research/config/dataset/X--dsprites-imagenet-bg-40.yaml deleted file mode 100644 index b33e5c8f..00000000 --- a/research/config/dataset/X--dsprites-imagenet-bg-40.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_bg_40 - -data: - _target_: research.code.dataset.data.DSpritesImagenetData - visibility: 40 - mode: bg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.2248598986983768, 0.21285772298967615, 0.19359577132944206] - vis_std: [0.1841631708032332, 0.18554895825833284, 0.1893568926398198] diff --git a/research/config/dataset/X--dsprites-imagenet-bg-50.yaml b/research/config/dataset/X--dsprites-imagenet-bg-50.yaml deleted file mode 100644 index 74b515f0..00000000 --- a/research/config/dataset/X--dsprites-imagenet-bg-50.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_bg_50 - -data: - _target_: research.code.dataset.data.DSpritesImagenetData - visibility: 50 - mode: bg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.271323621109491, 0.25634066038331416, 0.23223046934400662] - vis_std: [0.18930391112143766, 0.19067969524425118, 0.19523218572886117] diff --git a/research/config/dataset/X--dsprites-imagenet-bg-60.yaml b/research/config/dataset/X--dsprites-imagenet-bg-60.yaml deleted file mode 100644 index e8b29814..00000000 --- a/research/config/dataset/X--dsprites-imagenet-bg-60.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_bg_60 - -data: - _target_: research.code.dataset.data.DSpritesImagenetData - visibility: 60 - mode: bg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.31676960943447674, 0.29877166834408025, 0.2698556821388113] - vis_std: [0.19745897110349, 0.1986606891520453, 0.203808842880044] diff --git a/research/config/dataset/X--dsprites-imagenet-bg-75.yaml b/research/config/dataset/X--dsprites-imagenet-bg-75.yaml deleted file mode 100644 index 4e899a29..00000000 --- a/research/config/dataset/X--dsprites-imagenet-bg-75.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_bg_75 - -data: - _target_: research.code.dataset.data.DSpritesImagenetData - visibility: 75 - mode: bg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.38577296742807327, 0.3632825822323436, 0.3271231888851156] - vis_std: [0.21392191050784257, 0.2146731716558466, 0.2204460568339597] diff --git a/research/config/dataset/X--dsprites-imagenet-bg-80.yaml b/research/config/dataset/X--dsprites-imagenet-bg-80.yaml deleted file mode 100644 index 2bdb13ee..00000000 --- a/research/config/dataset/X--dsprites-imagenet-bg-80.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_bg_80 - -data: - _target_: research.code.dataset.data.DSpritesImagenetData - visibility: 80 - mode: bg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.40867981393820857, 0.38468564002021527, 0.34611573047508204] - vis_std: [0.22048328737091344, 0.22102216869942384, 0.22692977053753483] diff --git a/research/config/dataset/X--dsprites-imagenet-fg-100.yaml b/research/config/dataset/X--dsprites-imagenet-fg-100.yaml deleted file mode 100644 index 5f121c01..00000000 --- a/research/config/dataset/X--dsprites-imagenet-fg-100.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_fg_100 - -data: - _target_: research.code.dataset.data.DSpritesImagenetData - visibility: 100 - mode: fg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.02067051643494642, 0.018688392816012946, 0.01632900510079384] - vis_std: [0.10271307751834059, 0.09390213983525653, 0.08377594259970281] diff --git a/research/config/dataset/X--dsprites-imagenet-fg-20.yaml b/research/config/dataset/X--dsprites-imagenet-fg-20.yaml deleted file mode 100644 index 4020d934..00000000 --- a/research/config/dataset/X--dsprites-imagenet-fg-20.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_fg_20 - -data: - _target_: research.code.dataset.data.DSpritesImagenetData - visibility: 20 - mode: fg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.038064750024334834, 0.03766780505193579, 0.03719798677641122] - vis_std: [0.17498878664096565, 0.17315570657628315, 0.1709923319496426] diff --git a/research/config/dataset/X--dsprites-imagenet-fg-25.yaml b/research/config/dataset/X--dsprites-imagenet-fg-25.yaml deleted file mode 100644 index 6bb549da..00000000 --- a/research/config/dataset/X--dsprites-imagenet-fg-25.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_fg_25 - -data: - _target_: research.code.dataset.data.DSpritesImagenetData - visibility: 25 - mode: fg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.03697718115834816, 0.03648095993826591, 0.03589183623762013] - vis_std: [0.17009317531572005, 0.16780075430655303, 0.16508779008691726] diff --git a/research/config/dataset/X--dsprites-imagenet-fg-40.yaml b/research/config/dataset/X--dsprites-imagenet-fg-40.yaml deleted file mode 100644 index d1b7d47c..00000000 --- a/research/config/dataset/X--dsprites-imagenet-fg-40.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_fg_40 - -data: - _target_: research.code.dataset.data.DSpritesImagenetData - visibility: 40 - mode: fg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.03369999506331255, 0.03290657349801835, 0.03196482946320608] - vis_std: [0.155514074438101, 0.1518464537731621, 0.14750944591836743] diff --git a/research/config/dataset/X--dsprites-imagenet-fg-50.yaml b/research/config/dataset/X--dsprites-imagenet-fg-50.yaml deleted file mode 100644 index 05c9ccfc..00000000 --- a/research/config/dataset/X--dsprites-imagenet-fg-50.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_fg_50 - -data: - _target_: research.code.dataset.data.DSpritesImagenetData - visibility: 50 - mode: fg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.031541090790578506, 0.030549541980176148, 0.029368756624861398] - vis_std: [0.14609029304575144, 0.14150919987547408, 0.13607872227034773] diff --git a/research/config/dataset/X--dsprites-imagenet-fg-60.yaml b/research/config/dataset/X--dsprites-imagenet-fg-60.yaml deleted file mode 100644 index 968aed7c..00000000 --- a/research/config/dataset/X--dsprites-imagenet-fg-60.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_fg_60 - -data: - _target_: research.code.dataset.data.DSpritesImagenetData - visibility: 60 - mode: fg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.029335176871153983, 0.028145355435322966, 0.026731731769287146] - vis_std: [0.13663242436043319, 0.13114320478634894, 0.1246542727733097] diff --git a/research/config/dataset/X--dsprites-imagenet-fg-75.yaml b/research/config/dataset/X--dsprites-imagenet-fg-75.yaml deleted file mode 100644 index 3bd07e74..00000000 --- a/research/config/dataset/X--dsprites-imagenet-fg-75.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_fg_75 - -data: - _target_: research.code.dataset.data.DSpritesImagenetData - visibility: 75 - mode: fg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.02606445677382044, 0.024577082627819637, 0.02280587082174753] - vis_std: [0.12307153238282868, 0.11624914830767437, 0.1081911967745551] diff --git a/research/config/dataset/X--dsprites-imagenet-fg-80.yaml b/research/config/dataset/X--dsprites-imagenet-fg-80.yaml deleted file mode 100644 index a2dd5176..00000000 --- a/research/config/dataset/X--dsprites-imagenet-fg-80.yaml +++ /dev/null @@ -1,22 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_fg_80 - -data: - _target_: research.code.dataset.data.DSpritesImagenetData - visibility: 80 - mode: fg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.024956427531012196, 0.02336780403840578, 0.021475119672280243] - vis_std: [0.11864125016313823, 0.11137998105649799, 0.10281424917834255] diff --git a/research/config/dataset/X--dsprites-imagenet.yaml b/research/config/dataset/X--dsprites-imagenet.yaml deleted file mode 100644 index 7c23ff87..00000000 --- a/research/config/dataset/X--dsprites-imagenet.yaml +++ /dev/null @@ -1,81 +0,0 @@ -defaults: - - _data_type_: gt - -name: dsprites_imagenet_${dataset.mode}_${dataset.visibility} - -data: - _target_: research.code.dataset.data.DSpritesImagenetData - visibility: 100 - mode: bg - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: ${exit:EXITING... dsprites-imagenet has been disabled} # ${dataset.__STATS.${dataset.name}.vis_mean} - vis_std: ${exit:EXITING... dsprites-imagenet has been disabled} # ${dataset.__STATS.${dataset.name}.vis_std} - -__STATS: - dsprites_imagenet_fg_100: - vis_mean: [0.02067051643494642, 0.018688392816012946, 0.01632900510079384] - vis_std: [0.10271307751834059, 0.09390213983525653, 0.08377594259970281] - dsprites_imagenet_fg_80: - vis_mean: [0.024956427531012196, 0.02336780403840578, 0.021475119672280243] - vis_std: [0.11864125016313823, 0.11137998105649799, 0.10281424917834255] - dsprites_imagenet_fg_60: - vis_mean: [0.029335176871153983, 0.028145355435322966, 0.026731731769287146] - vis_std: [0.13663242436043319, 0.13114320478634894, 0.1246542727733097] - dsprites_imagenet_fg_40: - vis_mean: [0.03369999506331255, 0.03290657349801835, 0.03196482946320608] - vis_std: [0.155514074438101, 0.1518464537731621, 0.14750944591836743] - dsprites_imagenet_fg_20: - vis_mean: [0.038064750024334834, 0.03766780505193579, 0.03719798677641122] - vis_std: [0.17498878664096565, 0.17315570657628315, 0.1709923319496426] - - dsprites_imagenet_bg_100: - vis_mean: [0.5020433619489952, 0.47206398913310593, 0.42380018909780404] - vis_std: [0.2505510666843685, 0.2500725980366869, 0.2562415603123114] - dsprites_imagenet_bg_80: - vis_mean: [0.40867981393820857, 0.38468564002021527, 0.34611573047508204] - vis_std: [0.22048328737091344, 0.22102216869942384, 0.22692977053753483] - dsprites_imagenet_bg_60: - vis_mean: [0.31676960943447674, 0.29877166834408025, 0.2698556821388113] - vis_std: [0.19745897110349, 0.1986606891520453, 0.203808842880044] - dsprites_imagenet_bg_40: - vis_mean: [0.2248598986983768, 0.21285772298967615, 0.19359577132944206] - vis_std: [0.1841631708032332, 0.18554895825833284, 0.1893568926398198] - dsprites_imagenet_bg_20: - vis_mean: [0.13294969414492142, 0.12694375140936273, 0.11733572285575933] - vis_std: [0.18311250427586276, 0.1840916474752131, 0.18607373519458442] - - dsprites_imagenet_fg_75: - vis_mean: [0.02606445677382044, 0.024577082627819637, 0.02280587082174753] - vis_std: [0.12307153238282868, 0.11624914830767437, 0.1081911967745551] - dsprites_imagenet_fg_50: - vis_mean: [0.031541090790578506, 0.030549541980176148, 0.029368756624861398] - vis_std: [0.14609029304575144, 0.14150919987547408, 0.13607872227034773] - dsprites_imagenet_fg_25: - vis_mean: [0.03697718115834816, 0.03648095993826591, 0.03589183623762013] - vis_std: [0.17009317531572005, 0.16780075430655303, 0.16508779008691726] - dsprites_imagenet_fg_0: - vis_mean: [0.042494423521889584, 0.042494423521889584, 0.042494423521889584] - vis_std: [0.1951664588062605, 0.1951664588062605, 0.1951664588062605] - - dsprites_imagenet_bg_75: - vis_mean: [0.38577296742807327, 0.3632825822323436, 0.3271231888851156] - vis_std: [0.21392191050784257, 0.2146731716558466, 0.2204460568339597] - dsprites_imagenet_bg_50: - vis_mean: [0.271323621109491, 0.25634066038331416, 0.23223046934400662] - vis_std: [0.18930391112143766, 0.19067969524425118, 0.19523218572886117] - dsprites_imagenet_bg_25: - vis_mean: [0.15596283852200074, 0.14847876264131535, 0.13644703866118635] - vis_std: [0.18208653250875798, 0.18323109038468802, 0.18569624396763393] - dsprites_imagenet_bg_0: - vis_mean: [0.042494423521889584, 0.042494423521889584, 0.042494423521889584] - vis_std: [0.1951664588062605, 0.1951664588062605, 0.1951664588062605] diff --git a/research/config/dataset/X--mask-adv-f-cars3d.yaml b/research/config/dataset/X--mask-adv-f-cars3d.yaml deleted file mode 100644 index 0bc10894..00000000 --- a/research/config/dataset/X--mask-adv-f-cars3d.yaml +++ /dev/null @@ -1,27 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_adv_f_cars3d - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.part03_learnt_overlap.e02_learn_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--21-23-27_EXP_cars3d_1000x256_all_std_mean/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--22-58-24_EXP_cars3d_1000x256_all_std_gmean/data.pkl.gz' - randomize: FALSE - data: - _target_: disent.dataset.data.Cars3d64Data - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.8976676149976628, 0.8891658020067508, 0.885147515814868] - vis_std: [0.22503195531503034, 0.2399461278981261, 0.24792106319684404] diff --git a/research/config/dataset/X--mask-adv-f-dsprites.yaml b/research/config/dataset/X--mask-adv-f-dsprites.yaml deleted file mode 100644 index dcd6d378..00000000 --- a/research/config/dataset/X--mask-adv-f-dsprites.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_adv_f_dsprites - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.part03_learnt_overlap.e02_learn_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--21-45-46_EXP_dsprites_1000x256_all_std_mean/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--23-21-51_EXP_dsprites_1000x256_all_std_gmean/data.pkl.gz' - randomize: FALSE - data: - _target_: disent.dataset.data.DSpritesData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.042494423521889584] - vis_std: [0.19516645880626055] diff --git a/research/config/dataset/X--mask-adv-f-shapes3d.yaml b/research/config/dataset/X--mask-adv-f-shapes3d.yaml deleted file mode 100644 index 6f6eb69a..00000000 --- a/research/config/dataset/X--mask-adv-f-shapes3d.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_adv_f_shapes3d - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.part03_learnt_overlap.e02_learn_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--21-33-57_EXP_shapes3d_1000x256_all_std_mean/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--23-09-05_EXP_shapes3d_1000x256_all_std_gmean/data.pkl.gz' - randomize: FALSE - data: - _target_: disent.dataset.data.Shapes3dData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.502584966788819, 0.5787597566089667, 0.6034499731859578] - vis_std: [0.2940814043555559, 0.3443979087517214, 0.3661685981524748] diff --git a/research/config/dataset/X--mask-adv-f-smallnorb.yaml b/research/config/dataset/X--mask-adv-f-smallnorb.yaml deleted file mode 100644 index fea26a2d..00000000 --- a/research/config/dataset/X--mask-adv-f-smallnorb.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_adv_f_smallnorb - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.part03_learnt_overlap.e02_learn_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--21-28-42_EXP_smallnorb_1000x256_all_std_mean/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--23-03-51_EXP_smallnorb_1000x256_all_std_gmean/data.pkl.gz' - randomize: FALSE - data: - _target_: disent.dataset.data.SmallNorb64Data - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - is_test: False - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.7520918401088603] - vis_std: [0.09563879016827262] diff --git a/research/config/dataset/X--mask-adv-r-cars3d.yaml b/research/config/dataset/X--mask-adv-r-cars3d.yaml deleted file mode 100644 index 27942eee..00000000 --- a/research/config/dataset/X--mask-adv-r-cars3d.yaml +++ /dev/null @@ -1,27 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_adv_r_cars3d - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.part03_learnt_overlap.e02_learn_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-10-19--14-49-26_DISTS-SCALED_cars3d_1000x384_random_256_True_std_False/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-10-19--18-41-14_DISTS-SCALED_cars3d_1000x384_random_256_True_range_False/data.pkl.gz' - randomize: FALSE - data: - _target_: disent.dataset.data.Cars3d64Data - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.8976676149976628, 0.8891658020067508, 0.885147515814868] - vis_std: [0.22503195531503034, 0.2399461278981261, 0.24792106319684404] diff --git a/research/config/dataset/X--mask-adv-r-dsprites.yaml b/research/config/dataset/X--mask-adv-r-dsprites.yaml deleted file mode 100644 index 9ef237b9..00000000 --- a/research/config/dataset/X--mask-adv-r-dsprites.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_adv_r_dsprites - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.part03_learnt_overlap.e02_learn_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-10-19--16-31-56_DISTS-SCALED_dsprites_1000x384_random_256_True_std_False/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-10-19--19-58-39_DISTS-SCALED_dsprites_1000x384_random_256_True_range_False/data.pkl.gz' - randomize: FALSE - data: - _target_: disent.dataset.data.DSpritesData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.042494423521889584] - vis_std: [0.19516645880626055] diff --git a/research/config/dataset/X--mask-adv-r-shapes3d.yaml b/research/config/dataset/X--mask-adv-r-shapes3d.yaml deleted file mode 100644 index d2988716..00000000 --- a/research/config/dataset/X--mask-adv-r-shapes3d.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_adv_r_shapes3d - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.part03_learnt_overlap.e02_learn_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-10-19--15-20-48_DISTS-SCALED_shapes3d_1000x384_random_256_True_std_False/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-10-19--19-04-26_DISTS-SCALED_shapes3d_1000x384_random_256_True_range_False/data.pkl.gz' - randomize: FALSE - data: - _target_: disent.dataset.data.Shapes3dData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.502584966788819, 0.5787597566089667, 0.6034499731859578] - vis_std: [0.2940814043555559, 0.3443979087517214, 0.3661685981524748] diff --git a/research/config/dataset/X--mask-adv-r-smallnorb.yaml b/research/config/dataset/X--mask-adv-r-smallnorb.yaml deleted file mode 100644 index e39656db..00000000 --- a/research/config/dataset/X--mask-adv-r-smallnorb.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_adv_r_smallnorb - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.part03_learnt_overlap.e02_learn_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-10-19--15-10-07_DISTS-SCALED_smallnorb_1000x384_random_256_True_std_False/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-10-19--18-53-52_DISTS-SCALED_smallnorb_1000x384_random_256_True_range_False/data.pkl.gz' - randomize: FALSE - data: - _target_: disent.dataset.data.SmallNorb64Data - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - is_test: False - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.7520918401088603] - vis_std: [0.09563879016827262] diff --git a/research/config/dataset/X--mask-dthr-cars3d.yaml b/research/config/dataset/X--mask-dthr-cars3d.yaml deleted file mode 100644 index e181ded0..00000000 --- a/research/config/dataset/X--mask-dthr-cars3d.yaml +++ /dev/null @@ -1,23 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_dthr_cars3d - -data: - _target_: disent.dataset.wrapper.DitheredDataset - dither_n: 2 - keep_ratio: ${settings.framework_opt.usage_ratio} - gt_data: - _target_: disent.dataset.data.Cars3d64Data - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.8976676149976628, 0.8891658020067508, 0.885147515814868] - vis_std: [0.22503195531503034, 0.2399461278981261, 0.24792106319684404] diff --git a/research/config/dataset/X--mask-dthr-dsprites.yaml b/research/config/dataset/X--mask-dthr-dsprites.yaml deleted file mode 100644 index 03000f9b..00000000 --- a/research/config/dataset/X--mask-dthr-dsprites.yaml +++ /dev/null @@ -1,24 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_dthr_dsprites - -data: - _target_: disent.dataset.wrapper.DitheredDataset - dither_n: 2 - keep_ratio: ${settings.framework_opt.usage_ratio} - gt_data: - _target_: disent.dataset.data.DSpritesData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.042494423521889584] - vis_std: [0.19516645880626055] diff --git a/research/config/dataset/X--mask-dthr-shapes3d.yaml b/research/config/dataset/X--mask-dthr-shapes3d.yaml deleted file mode 100644 index 9aa229da..00000000 --- a/research/config/dataset/X--mask-dthr-shapes3d.yaml +++ /dev/null @@ -1,24 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_dthr_shapes3d - -data: - _target_: disent.dataset.wrapper.DitheredDataset - dither_n: 2 - keep_ratio: ${settings.framework_opt.usage_ratio} - gt_data: - _target_: disent.dataset.data.Shapes3dData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.502584966788819, 0.5787597566089667, 0.6034499731859578] - vis_std: [0.2940814043555559, 0.3443979087517214, 0.3661685981524748] diff --git a/research/config/dataset/X--mask-dthr-smallnorb.yaml b/research/config/dataset/X--mask-dthr-smallnorb.yaml deleted file mode 100644 index 96508980..00000000 --- a/research/config/dataset/X--mask-dthr-smallnorb.yaml +++ /dev/null @@ -1,24 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_dthr_smallnorb - -data: - _target_: disent.dataset.wrapper.DitheredDataset - dither_n: 2 - keep_ratio: ${settings.framework_opt.usage_ratio} - gt_data: - _target_: disent.dataset.data.SmallNorb64Data - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - is_test: False - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.7520918401088603] - vis_std: [0.09563879016827262] diff --git a/research/config/dataset/X--mask-ran-cars3d.yaml b/research/config/dataset/X--mask-ran-cars3d.yaml deleted file mode 100644 index db6fa252..00000000 --- a/research/config/dataset/X--mask-ran-cars3d.yaml +++ /dev/null @@ -1,27 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_ran_cars3d - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.part03_learnt_overlap.e02_learn_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--21-23-27_EXP_cars3d_1000x256_all_std_mean/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--22-58-24_EXP_cars3d_1000x256_all_std_gmean/data.pkl.gz' - randomize: TRUE - data: - _target_: disent.dataset.data.Cars3d64Data - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.8976676149976628, 0.8891658020067508, 0.885147515814868] - vis_std: [0.22503195531503034, 0.2399461278981261, 0.24792106319684404] diff --git a/research/config/dataset/X--mask-ran-dsprites.yaml b/research/config/dataset/X--mask-ran-dsprites.yaml deleted file mode 100644 index e4ec1c0f..00000000 --- a/research/config/dataset/X--mask-ran-dsprites.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_ran_dsprites - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.part03_learnt_overlap.e02_learn_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--21-45-46_EXP_dsprites_1000x256_all_std_mean/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--23-21-51_EXP_dsprites_1000x256_all_std_gmean/data.pkl.gz' - randomize: TRUE - data: - _target_: disent.dataset.data.DSpritesData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.042494423521889584] - vis_std: [0.19516645880626055] diff --git a/research/config/dataset/X--mask-ran-shapes3d.yaml b/research/config/dataset/X--mask-ran-shapes3d.yaml deleted file mode 100644 index 5cfd617f..00000000 --- a/research/config/dataset/X--mask-ran-shapes3d.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_ran_shapes3d - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.part03_learnt_overlap.e02_learn_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--21-33-57_EXP_shapes3d_1000x256_all_std_mean/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--23-09-05_EXP_shapes3d_1000x256_all_std_gmean/data.pkl.gz' - randomize: TRUE - data: - _target_: disent.dataset.data.Shapes3dData - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - in_memory: ${dsettings.dataset.try_in_memory} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.502584966788819, 0.5787597566089667, 0.6034499731859578] - vis_std: [0.2940814043555559, 0.3443979087517214, 0.3661685981524748] diff --git a/research/config/dataset/X--mask-ran-smallnorb.yaml b/research/config/dataset/X--mask-ran-smallnorb.yaml deleted file mode 100644 index c0f2fb9a..00000000 --- a/research/config/dataset/X--mask-ran-smallnorb.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _data_type_: random - -name: mask_ran_smallnorb - -data: - _target_: disent.dataset.wrapper.MaskedDataset - mask: - _target_: research.part03_learnt_overlap.e02_learn_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: ${settings.framework_opt.usage_ratio} - # pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--21-28-42_EXP_smallnorb_1000x256_all_std_mean/data.pkl.gz' - pickle_file: '${oc.env:HOME}/workspace/research/disent/out/adversarial_mask/2021-09-27--23-03-51_EXP_smallnorb_1000x256_all_std_gmean/data.pkl.gz' - randomize: TRUE - data: - _target_: disent.dataset.data.SmallNorb64Data - data_root: ${dsettings.storage.data_root} - prepare: ${dsettings.dataset.prepare} - is_test: False - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.7520918401088603] - vis_std: [0.09563879016827262] diff --git a/research/config/dataset/X--monte_rollouts.yaml b/research/config/dataset/X--monte_rollouts.yaml deleted file mode 100644 index 682eeb54..00000000 --- a/research/config/dataset/X--monte_rollouts.yaml +++ /dev/null @@ -1,21 +0,0 @@ -defaults: - - _data_type_: episodes - -name: monte_rollouts - -data: - _target_: disent.dataset.data.EpisodesDownloadZippedPickledData - required_file: ${dsettings.storage.data_root}/episodes/monte.pkl - download_url: 'https://raw.githubusercontent.com/nmichlo/uploads/main/monte_key.tar.xz' - prepare: ${dsettings.dataset.prepare} - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - size: [64, 64] # slightly squashed? - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] # [3, 210, 160] - vis_mean: "${exit:EXITING... please compute the vis_mean and vis_std}" - vis_std: "${exit:EXITING... please compute the vis_mean and vis_std}" diff --git a/research/config/dataset/X--xyblocks.yaml b/research/config/dataset/X--xyblocks.yaml deleted file mode 100644 index 435267a9..00000000 --- a/research/config/dataset/X--xyblocks.yaml +++ /dev/null @@ -1,18 +0,0 @@ -defaults: - - _data_type_: gt - -name: xyblocks - -data: - _target_: research.code.dataset.data.XYBlocksData - rgb: TRUE - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.10040509259259259, 0.10040509259259259, 0.10040509259259259] - vis_std: [0.21689087652106678, 0.21689087652106676, 0.21689087652106678] diff --git a/research/config/dataset/X--xyblocks_grey.yaml b/research/config/dataset/X--xyblocks_grey.yaml deleted file mode 100644 index c6ca546f..00000000 --- a/research/config/dataset/X--xyblocks_grey.yaml +++ /dev/null @@ -1,18 +0,0 @@ -defaults: - - _data_type_: gt - -name: xyblocks_grey - -data: - _target_: research.code.dataset.data.XYBlocksData - rgb: FALSE - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: "${exit:EXITING... please compute the vis_mean and vis_std}" - vis_std: "${exit:EXITING... please compute the vis_mean and vis_std}" diff --git a/research/config/dataset/X--xysquares.yaml b/research/config/dataset/X--xysquares.yaml deleted file mode 100644 index ab6a0d60..00000000 --- a/research/config/dataset/X--xysquares.yaml +++ /dev/null @@ -1,17 +0,0 @@ -defaults: - - _data_type_: gt - -name: xysquares_minimal - -data: - _target_: research.code.dataset.data.XYSquaresMinimalData - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.015625, 0.015625, 0.015625] - vis_std: [0.12403473458920855, 0.12403473458920854, 0.12403473458920854] diff --git a/research/config/dataset/X--xysquares_grey.yaml b/research/config/dataset/X--xysquares_grey.yaml deleted file mode 100644 index 729634d4..00000000 --- a/research/config/dataset/X--xysquares_grey.yaml +++ /dev/null @@ -1,23 +0,0 @@ -defaults: - - _data_type_: gt - -name: xysquares_grey - -data: - _target_: research.code.dataset.data.XYSquaresData - square_size: 8 # AFFECTS: mean and std - image_size: 64 # usually ok to adjust - grid_size: 8 # usually ok to adjust - grid_spacing: 8 # usually ok to adjust - num_squares: 3 # AFFECTS: mean and std - rgb: FALSE # AFFECTS: mean and std - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [1, 64, 64] - vis_mean: [0.046146392822265625] - vis_std: [0.2096506119375896] diff --git a/research/config/dataset/X--xysquares_rgb.yaml b/research/config/dataset/X--xysquares_rgb.yaml deleted file mode 100644 index c56957da..00000000 --- a/research/config/dataset/X--xysquares_rgb.yaml +++ /dev/null @@ -1,23 +0,0 @@ -defaults: - - _data_type_: gt - -name: xysquares_rgb - -data: - _target_: research.code.dataset.data.XYSquaresData - square_size: 8 # AFFECTS: mean and std - image_size: 64 # usually ok to adjust - grid_size: 8 # usually ok to adjust - grid_spacing: 8 # usually ok to adjust - num_squares: 3 # AFFECTS: mean and std - rgb: TRUE # AFFECTS: mean and std - -transform: - _target_: disent.dataset.transform.ToImgTensorF32 - mean: ${dataset.meta.vis_mean} - std: ${dataset.meta.vis_std} - -meta: - x_shape: [3, 64, 64] - vis_mean: [0.015625, 0.015625, 0.015625] - vis_std: [0.12403473458920855, 0.12403473458920854, 0.12403473458920854] diff --git a/research/config/framework/X--adaae.yaml b/research/config/framework/X--adaae.yaml deleted file mode 100644 index a132af12..00000000 --- a/research/config/framework/X--adaae.yaml +++ /dev/null @@ -1,19 +0,0 @@ -defaults: - - _input_mode_: pair - -name: adaae - -cfg: - _target_: research.code.frameworks.ae.AdaAe.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # disable various components - detach_decoder: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # adavae - ada_thresh_ratio: 0.5 - -meta: - model_z_multiplier: 1 diff --git a/research/config/framework/X--adaae_os.yaml b/research/config/framework/X--adaae_os.yaml deleted file mode 100644 index 57e0ea75..00000000 --- a/research/config/framework/X--adaae_os.yaml +++ /dev/null @@ -1,19 +0,0 @@ -defaults: - - _input_mode_: weak_pair - -name: adaae - -cfg: - _target_: research.code.frameworks.ae.AdaAe.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # disable various components - detach_decoder: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # adavae - ada_thresh_ratio: 0.5 - -meta: - model_z_multiplier: 1 diff --git a/research/config/framework/X--adaavetvae.yaml b/research/config/framework/X--adaavetvae.yaml deleted file mode 100644 index 7925f711..00000000 --- a/research/config/framework/X--adaavetvae.yaml +++ /dev/null @@ -1,44 +0,0 @@ -defaults: - - _input_mode_: triplet - -name: adaave_tvae - -cfg: - _target_: research.code.frameworks.vae.AdaAveTripletVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - detach_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # Beta-VAE - beta: ${settings.framework.beta} - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - # adavae - ada_average_mode: gvae - ada_thresh_mode: symmetric_kl # Only works for: adat_share_mask_mode == "posterior" --- kl, symmetric_kl, dist, sampled_dist - ada_thresh_ratio: 0.5 # >> USE WITH A SCHEDULE << - # ada_tvae - loss - adat_triplet_loss: triplet_soft_ave_all - adat_triplet_ratio: 1.0 # >> USE WITH A SCHEDULE << 0.5 is half of triplet and ada-triplet, 1.0 is all ada-triplet - adat_triplet_soft_scale: 1.0 # >> USE WITH A SCHEDULE << - adat_triplet_pull_weight: 0.1 # Only works for: adat_triplet_loss == "triplet_hard_neg_ave_pull" - adat_triplet_share_scale: 0.95 # >> USE WITH A SCHEDULE << only works for: adat_triplet_loss == "triplet_hard_neg_ave_scaled" - # ada_tvae - averaging - adat_share_mask_mode: posterior - adat_share_ave_mode: all # Only works for: adat_triplet_loss == "triplet_hard_ave_all" - # adaave_tvae - adaave_augment_orig: FALSE # triplet over original OR averaged embeddings - adaave_decode_orig: FALSE # decode & regularize original OR averaged embeddings - -meta: - model_z_multiplier: 2 diff --git a/research/config/framework/X--adanegtae.yaml b/research/config/framework/X--adanegtae.yaml deleted file mode 100644 index 253ce8c8..00000000 --- a/research/config/framework/X--adanegtae.yaml +++ /dev/null @@ -1,27 +0,0 @@ -defaults: - - _input_mode_: triplet - -name: adanegtae - -cfg: - _target_: research.code.frameworks.ae.AdaNegTripletAe.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # disable various components - detach_decoder: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - # adavae - ada_thresh_ratio: 0.5 # >> USE WITH A SCHEDULE << - # ada_tvae - loss - adat_triplet_share_scale: 0.95 # >> USE WITH A SCHEDULE << only works for: adat_triplet_loss == "triplet_hard_neg_ave_scaled" - -meta: - model_z_multiplier: 1 diff --git a/research/config/framework/X--adanegtvae.yaml b/research/config/framework/X--adanegtvae.yaml deleted file mode 100644 index 37a8d8be..00000000 --- a/research/config/framework/X--adanegtvae.yaml +++ /dev/null @@ -1,36 +0,0 @@ -defaults: - - _input_mode_: triplet - -name: adanegtvae - -cfg: - _target_: research.code.frameworks.vae.AdaNegTripletVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - detach_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # Beta-VAE - beta: ${settings.framework.beta} - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - # adavae - ada_average_mode: gvae - ada_thresh_mode: dist # Only works for: adat_share_mask_mode == "posterior" --- kl, symmetric_kl, dist, sampled_dist - ada_thresh_ratio: 0.5 # >> USE WITH A SCHEDULE << - # ada_tvae - loss - adat_triplet_share_scale: 0.95 # >> USE WITH A SCHEDULE << only works for: adat_triplet_loss == "triplet_hard_neg_ave_scaled" - # ada_tvae - averaging - adat_share_mask_mode: posterior - -meta: - model_z_multiplier: 2 diff --git a/research/config/framework/X--adatvae.yaml b/research/config/framework/X--adatvae.yaml deleted file mode 100644 index 53b1c2bd..00000000 --- a/research/config/framework/X--adatvae.yaml +++ /dev/null @@ -1,41 +0,0 @@ -defaults: - - _input_mode_: triplet - -name: adatvae - -cfg: - _target_: research.code.frameworks.vae.AdaTripletVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - detach_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # Beta-VAE - beta: ${settings.framework.beta} - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - # adavae - ada_average_mode: gvae - ada_thresh_mode: dist # Only works for: adat_share_mask_mode == "posterior" --- kl, symmetric_kl, dist, sampled_dist - ada_thresh_ratio: 0.5 # >> USE WITH A SCHEDULE << - # ada_tvae - loss - adat_triplet_loss: triplet_soft_ave_all - adat_triplet_ratio: 1.0 # >> USE WITH A SCHEDULE << 0.5 is half of triplet and ada-triplet, 1.0 is all ada-triplet - adat_triplet_soft_scale: 1.0 # >> USE WITH A SCHEDULE << - adat_triplet_pull_weight: 0.1 # Only works for: adat_triplet_loss == "triplet_hard_neg_ave_pull" - adat_triplet_share_scale: 0.95 # >> USE WITH A SCHEDULE << only works for: adat_triplet_loss == "triplet_hard_neg_ave_scaled" - # ada_tvae - averaging - adat_share_mask_mode: posterior - adat_share_ave_mode: all # Only works for: adat_triplet_loss == "triplet_hard_ave_all" - -meta: - model_z_multiplier: 2 diff --git a/research/config/framework/X--augpos_tvae_os.yaml b/research/config/framework/X--augpos_tvae_os.yaml deleted file mode 100644 index 6ad30ac5..00000000 --- a/research/config/framework/X--augpos_tvae_os.yaml +++ /dev/null @@ -1,45 +0,0 @@ -defaults: - - _input_mode_: weak_pair - -name: augpos_tvae_os - -cfg: - _target_: research.code.frameworks.vae.AugPosTripletVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - detach_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # Beta-VAE - beta: ${settings.framework.beta} - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - # overlap - overlap_augment: - _target_: disent.transform.FftBoxBlur - p: 1.0 - radius: [ 16, 16 ] - random_mode: "batch" - random_same_xy: TRUE - - # TODO: try original - # overlap_augment: - # size = a_x.shape[2:4] - # self._augment = torchvision.transforms.RandomOrder([ - # kornia.augmentation.ColorJitter(brightness=0.25, contrast=0.25, saturation=0, hue=0.15), - # kornia.augmentation.RandomCrop(size=size, padding=8), - # # kornia.augmentation.RandomPerspective(distortion_scale=0.05, p=1.0), - # # kornia.augmentation.RandomRotation(degrees=4), - # ]) - -meta: - model_z_multiplier: 2 diff --git a/research/config/framework/X--badavae.yaml b/research/config/framework/X--badavae.yaml deleted file mode 100644 index ea6534c4..00000000 --- a/research/config/framework/X--badavae.yaml +++ /dev/null @@ -1,26 +0,0 @@ -defaults: - - _input_mode_: triplet - -name: badavae - -cfg: - _target_: research.code.frameworks.vae.BoundedAdaVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - detach_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # Beta-VAE - beta: ${settings.framework.beta} - # adavae - ada_average_mode: gvae # gvae or ml-vae - ada_thresh_mode: symmetric_kl - ada_thresh_ratio: 0.5 - -meta: - model_z_multiplier: 2 diff --git a/research/config/framework/X--dorvae.yaml b/research/config/framework/X--dorvae.yaml deleted file mode 100644 index a70ccce1..00000000 --- a/research/config/framework/X--dorvae.yaml +++ /dev/null @@ -1,37 +0,0 @@ -defaults: - - _input_mode_: single - -name: dor_vae - -cfg: - _target_: research.code.frameworks.vae.DataOverlapRankVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - detach_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # Beta-VAE - beta: ${settings.framework.beta} - # compatibility - ada_thresh_mode: dist # kl, symmetric_kl, dist, sampled_dist - ada_thresh_ratio: 0.5 - adat_triplet_share_scale: 0.95 - # dorvae - overlap_loss: ${settings.framework_opt.overlap_loss} # any of the recon_loss values, or NULL to use the recon_loss value - overlap_num: 512 - # dorvae -- representation loss - overlap_repr: deterministic # deterministic, stochastic - overlap_rank_mode: spearman_rank # spearman_rank, mse_rank - overlap_inward_pressure_masked: FALSE - overlap_inward_pressure_scale: 0.01 - # dorvae -- augment - overlap_augment_mode: 'none' - overlap_augment: NULL - -meta: - model_z_multiplier: 2 diff --git a/research/config/framework/X--dorvae_aug.yaml b/research/config/framework/X--dorvae_aug.yaml deleted file mode 100644 index 1b919a35..00000000 --- a/research/config/framework/X--dorvae_aug.yaml +++ /dev/null @@ -1,42 +0,0 @@ -defaults: - - _input_mode_: single - -name: dor_vae_aug - -cfg: - _target_: research.code.frameworks.vae.DataOverlapRankVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - detach_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # Beta-VAE - beta: ${settings.framework.beta} - # compatibility - ada_thresh_mode: dist # kl, symmetric_kl, dist, sampled_dist - ada_thresh_ratio: 0.5 - adat_triplet_share_scale: 0.95 - # dorvae - overlap_loss: ${settings.framework_opt.overlap_loss} # any of the recon_loss values, or NULL to use the recon_loss value - overlap_num: 512 - # dorvae -- representation loss - overlap_repr: deterministic # deterministic, stochastic - overlap_rank_mode: spearman_rank # spearman_rank, mse_rank - overlap_inward_pressure_masked: FALSE - overlap_inward_pressure_scale: 0.01 - # dorvae -- augment - overlap_augment_mode: 'augment' - overlap_augment: - _target_: disent.transform.FftBoxBlur - p: 1.0 - radius: [16, 16] - random_mode: "batch" - random_same_xy: TRUE - -meta: - model_z_multiplier: 2 diff --git a/research/config/framework/X--dotae.yaml b/research/config/framework/X--dotae.yaml deleted file mode 100644 index d57d81e3..00000000 --- a/research/config/framework/X--dotae.yaml +++ /dev/null @@ -1,35 +0,0 @@ -defaults: - - _input_mode_: single - -name: dotae - -cfg: - _target_: research.code.frameworks.ae.DataOverlapTripletAe.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # disable various components - detach_decoder: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - # adavae - ada_thresh_ratio: 0.5 # >> USE WITH A SCHEDULE << - # ada_tvae - loss - adat_triplet_share_scale: 0.95 # >> USE WITH A SCHEDULE << only works for: adat_triplet_loss == "triplet_hard_neg_ave_scaled" - # dotvae - overlap_loss: ${settings.framework_opt.overlap_loss} # any of the recon_loss values, or NULL to use the recon_loss value - overlap_num: 512 - overlap_mine_ratio: 0.1 - overlap_mine_triplet_mode: 'none' # none, hard_neg, semi_hard_neg, hard_pos, easy_pos, ran:hard_neg+hard_pos <- etc, dynamically evaluated, can chain multiple "+"s - # dotvae -- augment - overlap_augment_mode: 'none' - overlap_augment: NULL - -meta: - model_z_multiplier: 2 diff --git a/research/config/framework/X--dotvae.yaml b/research/config/framework/X--dotvae.yaml deleted file mode 100644 index b6d5fc01..00000000 --- a/research/config/framework/X--dotvae.yaml +++ /dev/null @@ -1,44 +0,0 @@ -defaults: - - _input_mode_: single - -name: do_tvae - -cfg: - _target_: research.code.frameworks.vae.DataOverlapTripletVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - detach_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # Beta-VAE - beta: ${settings.framework.beta} - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - # adavae - ada_average_mode: gvae - ada_thresh_mode: dist # Only works for: adat_share_mask_mode == "posterior" --- kl, symmetric_kl, dist, sampled_dist - ada_thresh_ratio: 0.5 # >> USE WITH A SCHEDULE << - # ada_tvae - loss - adat_triplet_share_scale: 0.95 # >> USE WITH A SCHEDULE << only works for: adat_triplet_loss == "triplet_hard_neg_ave_scaled" - # ada_tvae - averaging - adat_share_mask_mode: posterior - # dotvae - overlap_loss: ${settings.framework_opt.overlap_loss} # any of the recon_loss values, or NULL to use the recon_loss value - overlap_num: 512 - overlap_mine_ratio: 0.1 - overlap_mine_triplet_mode: 'none' # none, hard_neg, semi_hard_neg, hard_pos, easy_pos, ran:hard_neg+hard_pos <- etc, dynamically evaluated, can chain multiple "+"s - # dotvae -- augment - overlap_augment_mode: 'augment' # none, augment, augment_each (if overlap_augment is NULL, then it is the same as setting this to "none") - overlap_augment: NULL - -meta: - model_z_multiplier: 2 diff --git a/research/config/framework/X--dotvae_aug.yaml b/research/config/framework/X--dotvae_aug.yaml deleted file mode 100644 index 7cbc955f..00000000 --- a/research/config/framework/X--dotvae_aug.yaml +++ /dev/null @@ -1,69 +0,0 @@ -defaults: - - _input_mode_: single - -name: do_tvae_aug - -cfg: - _target_: research.code.frameworks.vae.DataOverlapTripletVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - detach_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # Beta-VAE - beta: ${settings.framework.beta} - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - # adavae - ada_average_mode: gvae - ada_thresh_mode: dist # Only works for: adat_share_mask_mode == "posterior" --- kl, symmetric_kl, dist, sampled_dist - ada_thresh_ratio: 0.5 # >> USE WITH A SCHEDULE << - # ada_tvae - loss - adat_triplet_share_scale: 0.95 # >> USE WITH A SCHEDULE << only works for: adat_triplet_loss == "triplet_hard_neg_ave_scaled" - # ada_tvae - averaging - adat_share_mask_mode: posterior - # dotvae - overlap_loss: ${settings.framework_opt.overlap_loss} # any of the recon_loss values, or NULL to use the recon_loss value - overlap_num: 512 - overlap_mine_ratio: 0.1 - overlap_mine_triplet_mode: 'ran:hard_neg+easy_pos' # none, hard_neg, semi_hard_neg, hard_pos, easy_pos, ran:hard_neg+hard_pos <- etc, dynamically evaluated, can chain multiple "+"s - # dotvae -- augment - overlap_augment_mode: 'augment' # none, augment, augment_each (if overlap_augment is NULL, then it is the same as setting this to "none") - overlap_augment: - _target_: disent.transform.FftKernel - kernel: xy1_r47 - -# overlap_augment: -# _target_: disent.transform.FftBoxBlur -# p: 1.0 -# radius: [16, 16] -# random_mode: "batch" -# random_same_xy: TRUE -# - _target_: disent.transform.FftGaussianBlur -# p: 1.0 -# sigma: [0.1, 10.0] -# truncate: 3.0 -# random_mode: "batch" -# random_same_xy: FALSE -# - _target_: kornia.augmentation.RandomCrop -# p: 1.0 -# size: [64, 64] -# padding: 7 -# - _target_: kornia.augmentation.RandomPerspective -# p: 0.5 -# distortion_scale: 0.15 -# - _target_: kornia.augmentation.RandomRotation -# p: 0.5 -# degrees: 9 - -meta: - model_z_multiplier: 2 diff --git a/research/config/framework/X--gadavae.yaml b/research/config/framework/X--gadavae.yaml deleted file mode 100644 index b79a2bdf..00000000 --- a/research/config/framework/X--gadavae.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _input_mode_: triplet - -name: gadavae - -cfg: - _target_: research.code.frameworks.vae.GuidedAdaVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - detach_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # Beta-VAE - beta: ${settings.framework.beta} - # adavae - ada_average_mode: gvae # gvae or ml-vae - ada_thresh_mode: symmetric_kl - ada_thresh_ratio: 0.5 - # guided adavae - gada_anchor_ave_mode: 'average' # [average, thresh] - -meta: - model_z_multiplier: 2 diff --git a/research/config/framework/X--st-adavae.yaml b/research/config/framework/X--st-adavae.yaml deleted file mode 100644 index 01f79193..00000000 --- a/research/config/framework/X--st-adavae.yaml +++ /dev/null @@ -1,28 +0,0 @@ -defaults: - - _input_mode_: pair - -name: st-adavae - -cfg: - _target_: research.code.frameworks.vae.SwappedTargetAdaVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - detach_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # Beta-VAE - beta: ${settings.framework.beta} - # adavae - ada_average_mode: gvae # gvae or ml-vae - ada_thresh_mode: symmetric_kl - ada_thresh_ratio: 0.5 - # swapped target - swap_chance: 0.1 - -meta: - model_z_multiplier: 2 diff --git a/research/config/framework/X--st-betavae.yaml b/research/config/framework/X--st-betavae.yaml deleted file mode 100644 index 7fdd9459..00000000 --- a/research/config/framework/X--st-betavae.yaml +++ /dev/null @@ -1,24 +0,0 @@ -defaults: - - _input_mode_: pair - -name: st-betavae - -cfg: - _target_: research.code.frameworks.vae.SwappedTargetBetaVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - detach_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # Beta-VAE - beta: ${settings.framework.beta} - # swapped target - swap_chance: 0.1 - -meta: - model_z_multiplier: 2 diff --git a/research/config/framework/X--tbadavae.yaml b/research/config/framework/X--tbadavae.yaml deleted file mode 100644 index 0b0c5c98..00000000 --- a/research/config/framework/X--tbadavae.yaml +++ /dev/null @@ -1,32 +0,0 @@ -defaults: - - _input_mode_: triplet - -name: tbadavae - -cfg: - _target_: research.code.frameworks.vae.TripletBoundedAdaVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - detach_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # Beta-VAE - beta: ${settings.framework.beta} - # adavae - ada_average_mode: gvae # gvae or ml-vae - ada_thresh_mode: symmetric_kl - ada_thresh_ratio: 0.5 - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - -meta: - model_z_multiplier: 2 diff --git a/research/config/framework/X--tgadavae.yaml b/research/config/framework/X--tgadavae.yaml deleted file mode 100644 index 559e9f21..00000000 --- a/research/config/framework/X--tgadavae.yaml +++ /dev/null @@ -1,34 +0,0 @@ -defaults: - - _input_mode_: triplet - -name: tgadavae - -cfg: - _target_: research.code.frameworks.vae.TripletGuidedAdaVae.cfg - # base ae - recon_loss: ${settings.framework.recon_loss} - loss_reduction: ${settings.framework.loss_reduction} - # base vae - latent_distribution: ${settings.framework_opt.latent_distribution} - # disable various components - detach_decoder: FALSE - disable_reg_loss: FALSE - disable_rec_loss: FALSE - disable_aug_loss: FALSE - # Beta-VAE - beta: ${settings.framework.beta} - # adavae - ada_average_mode: gvae # gvae or ml-vae - ada_thresh_mode: symmetric_kl - ada_thresh_ratio: 0.5 - # guided adavae - gada_anchor_ave_mode: 'average' # [average, thresh] - # tvae: triplet stuffs - triplet_loss: triplet - triplet_margin_min: 0.001 - triplet_margin_max: 1 - triplet_scale: 0.1 - triplet_p: 1 - -meta: - model_z_multiplier: 2 diff --git a/research/config/metrics/all.yaml b/research/config/metrics/all.yaml deleted file mode 100644 index f455d1c0..00000000 --- a/research/config/metrics/all.yaml +++ /dev/null @@ -1,18 +0,0 @@ -metric_list: - - flatness: {} - - factored_components: {} - - mig: {} - - sap: {} - - dci: - every_n_steps: 7200 - on_final: TRUE - - factor_vae: - every_n_steps: 7200 - on_final: TRUE - - unsupervised: {} - -# these are the default settings, these can be placed in the list above -default_on_final: TRUE -default_on_train: TRUE -default_every_n_steps: 2400 -default_begin_first_step: FALSE diff --git a/research/config/metrics/fast.yaml b/research/config/metrics/fast.yaml deleted file mode 100644 index 8ab4efc1..00000000 --- a/research/config/metrics/fast.yaml +++ /dev/null @@ -1,12 +0,0 @@ -metric_list: - - flatness: {} - - factored_components: {} - - mig: {} - - sap: {} - - unsupervised: {} - -# these are the default settings, these can be placed in the list above -default_on_final: TRUE -default_on_train: TRUE -default_every_n_steps: 2400 -default_begin_first_step: FALSE diff --git a/research/config/metrics/test.yaml b/research/config/metrics/test.yaml deleted file mode 100644 index 0ec06370..00000000 --- a/research/config/metrics/test.yaml +++ /dev/null @@ -1,21 +0,0 @@ -metric_list: - - flatness: - every_n_steps: 110 - - factored_components: - every_n_steps: 111 - - mig: - every_n_steps: 112 - - sap: - every_n_steps: 113 - - unsupervised: - every_n_steps: 114 - - dci: - every_n_steps: 115 - - factor_vae: - every_n_steps: 116 - -# these are the default settings, these can be placed in the list above -default_on_final: FALSE -default_on_train: TRUE -default_every_n_steps: 200 -default_begin_first_step: FALSE diff --git a/research/config/run_location/cluster.yaml b/research/config/run_location/cluster.yaml deleted file mode 100644 index c2ae13b9..00000000 --- a/research/config/run_location/cluster.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# @package _global_ - -dsettings: - trainer: - cuda: NULL # auto-detect cuda, some nodes may be configured incorrectly - storage: - logs_dir: 'logs' - data_root: '/tmp/${oc.env:USER}/datasets' - dataset: - prepare: TRUE - try_in_memory: TRUE - launcher: - partition: batch - array_parallelism: 16 - exclude: "mscluster93,mscluster94,mscluster97,mscluster99" - -datamodule: - gpu_augment: FALSE - prepare_data_per_node: TRUE - dataloader: - num_workers: 8 - pin_memory: ${dsettings.trainer.cuda} # uses more memory, but faster! - batch_size: ${settings.dataset.batch_size} - -hydra: - job: - name: 'disent' - run: - dir: '${dsettings.storage.logs_dir}/hydra_run/${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.name}' - sweep: - dir: '${dsettings.storage.logs_dir}/hydra_sweep/${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.name}' - subdir: '${hydra.job.id}' # hydra.job.id is not available for dir diff --git a/research/config/run_location/griffin.yaml b/research/config/run_location/griffin.yaml deleted file mode 100644 index a234f9f5..00000000 --- a/research/config/run_location/griffin.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# @package _global_ - -dsettings: - trainer: - cuda: TRUE - storage: - logs_dir: 'logs' - data_root: '${oc.env:HOME}/workspace/research/disent/data/dataset' - dataset: - prepare: TRUE - try_in_memory: TRUE - -datamodule: - gpu_augment: FALSE - prepare_data_per_node: TRUE - dataloader: - num_workers: 32 # max 128, more than 16 doesn't really seem to help (tested on batch size 256*3)? - pin_memory: ${dsettings.trainer.cuda} # uses more memory, but faster! - batch_size: ${settings.dataset.batch_size} - -hydra: - job: - name: 'disent' - run: - dir: '${dsettings.storage.logs_dir}/hydra_run/${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.name}' - sweep: - dir: '${dsettings.storage.logs_dir}/hydra_sweep/${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.name}' - subdir: '${hydra.job.id}' # hydra.job.id is not available for dir diff --git a/research/config/run_location/heartofgold.yaml b/research/config/run_location/heartofgold.yaml deleted file mode 100644 index 4aa55fbd..00000000 --- a/research/config/run_location/heartofgold.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# @package _global_ - -dsettings: - trainer: - cuda: TRUE - storage: - logs_dir: 'logs' - data_root: '${oc.env:HOME}/workspace/research/disent/data/dataset' - dataset: - prepare: TRUE - try_in_memory: TRUE - -datamodule: - gpu_augment: FALSE - prepare_data_per_node: TRUE - dataloader: - num_workers: 12 - pin_memory: ${dsettings.trainer.cuda} # uses more memory, but faster! - batch_size: ${settings.dataset.batch_size} - -hydra: - job: - name: 'disent' - run: - dir: '${dsettings.storage.logs_dir}/hydra_run/${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.name}' - sweep: - dir: '${dsettings.storage.logs_dir}/hydra_sweep/${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.name}' - subdir: '${hydra.job.id}' # hydra.job.id is not available for dir diff --git a/research/config/run_location/stampede_rsync_tmp.yaml b/research/config/run_location/stampede_rsync_tmp.yaml deleted file mode 100644 index 3b392611..00000000 --- a/research/config/run_location/stampede_rsync_tmp.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# @package _global_ - -dsettings: - trainer: - cuda: NULL # auto-detect cuda, some nodes are not configured correctly - storage: - logs_dir: 'logs' - data_root: ${rsync_dir:'${oc.env:HOME}/downloads/datasets','/tmp/${oc.env:USER}/datasets'} - dataset: - prepare: TRUE - try_in_memory: FALSE - launcher: - partition: stampede - array_parallelism: 16 - exclude: "mscluster92,mscluster94,mscluster96" - -datamodule: - gpu_augment: FALSE - prepare_data_per_node: TRUE - dataloader: - num_workers: 16 - pin_memory: ${dsettings.trainer.cuda} # uses more memory, but faster! - batch_size: ${settings.dataset.batch_size} - -hydra: - job: - name: 'disent' - run: - dir: '${dsettings.storage.logs_dir}/hydra_run/${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.name}' - sweep: - dir: '${dsettings.storage.logs_dir}/hydra_sweep/${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.name}' - subdir: '${hydra.job.id}' # hydra.job.id is not available for dir diff --git a/research/config/run_location/stampede_shr.yaml b/research/config/run_location/stampede_shr.yaml deleted file mode 100644 index 488d9662..00000000 --- a/research/config/run_location/stampede_shr.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# @package _global_ - -dsettings: - trainer: - cuda: NULL # auto-detect cuda, some nodes are not configured correctly - storage: - logs_dir: 'logs' - data_root: '${oc.env:HOME}/downloads/datasets' # WE NEED TO BE VERY CAREFUL ABOUT USING A SHARED DRIVE - dataset: - prepare: FALSE # WE MUST PREPARE DATA MANUALLY BEFOREHAND - try_in_memory: TRUE - launcher: - partition: stampede - array_parallelism: 16 - exclude: "mscluster92,mscluster94,mscluster96" - -datamodule: - gpu_augment: FALSE - prepare_data_per_node: TRUE - dataloader: - num_workers: 16 - pin_memory: ${dsettings.trainer.cuda} # uses more memory, but faster! - batch_size: ${settings.dataset.batch_size} - -hydra: - job: - name: 'disent' - run: - dir: '${dsettings.storage.logs_dir}/hydra_run/${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.name}' - sweep: - dir: '${dsettings.storage.logs_dir}/hydra_sweep/${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.name}' - subdir: '${hydra.job.id}' # hydra.job.id is not available for dir diff --git a/research/config/run_location/stampede_tmp.yaml b/research/config/run_location/stampede_tmp.yaml deleted file mode 100644 index 5b6127df..00000000 --- a/research/config/run_location/stampede_tmp.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# @package _global_ - -dsettings: - trainer: - cuda: NULL # auto-detect cuda, some nodes are not configured correctly - storage: - logs_dir: 'logs' - data_root: '/tmp/${oc.env:USER}/datasets' - dataset: - prepare: TRUE - try_in_memory: TRUE - launcher: - partition: stampede - array_parallelism: 16 - exclude: "mscluster92,mscluster94,mscluster96" - -datamodule: - gpu_augment: FALSE - prepare_data_per_node: TRUE - dataloader: - num_workers: 16 - pin_memory: ${dsettings.trainer.cuda} # uses more memory, but faster! - batch_size: ${settings.dataset.batch_size} - -hydra: - job: - name: 'disent' - run: - dir: '${dsettings.storage.logs_dir}/hydra_run/${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.name}' - sweep: - dir: '${dsettings.storage.logs_dir}/hydra_sweep/${now:%Y-%m-%d_%H-%M-%S}_${hydra.job.name}' - subdir: '${hydra.job.id}' # hydra.job.id is not available for dir diff --git a/research/config/run_plugins/default.yaml b/research/config/run_plugins/default.yaml deleted file mode 100644 index 60ffc051..00000000 --- a/research/config/run_plugins/default.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# @package _global_ - -# call the listed functions here before the experiment is started -# - make sure we register the research code to disent so that it can be used! -experiment: - plugins: - - research.code.register_to_disent diff --git a/research/config/schedule/OLD_adavae_down_all.yaml b/research/config/schedule/OLD_adavae_down_all.yaml deleted file mode 100644 index cad8d017..00000000 --- a/research/config/schedule/OLD_adavae_down_all.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: OLD_adavae_down_all - -schedule_items: - adat_triplet_ratio: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 1.0 # ada triplet - r_end: 0.0 # triplet - adat_triplet_soft_scale: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 1.0 # loss active - r_end: 0.0 # loss inactive - adat_triplet_share_scale: # reversed compared to others - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 0.5 # ada weighted triplet - r_end: 1.0 # normal triplet - ada_thresh_ratio: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 1.0 # all averaged, should this not be 0.5 the recommended value - r_end: 0.0 # none averaged diff --git a/research/config/schedule/OLD_adavae_down_ratio.yaml b/research/config/schedule/OLD_adavae_down_ratio.yaml deleted file mode 100644 index eba316ae..00000000 --- a/research/config/schedule/OLD_adavae_down_ratio.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: OLD_adavae_down_ratio - -schedule_items: - adat_triplet_ratio: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 1.0 # ada triplet - r_end: 0.0 # triplet - adat_triplet_soft_scale: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 1.0 # loss active - r_end: 0.0 # loss inactive - adat_triplet_share_scale: # reversed compared to others - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 0.5 # ada weighted triplet - r_end: 1.0 # normal triplet diff --git a/research/config/schedule/OLD_adavae_down_thresh.yaml b/research/config/schedule/OLD_adavae_down_thresh.yaml deleted file mode 100644 index a72e5dd9..00000000 --- a/research/config/schedule/OLD_adavae_down_thresh.yaml +++ /dev/null @@ -1,9 +0,0 @@ -name: OLD_adavae_down_thresh - -schedule_items: - ada_thresh_ratio: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 1.0 # all averaged, should this not be 0.5 the recommended value - r_end: 0.0 # none averaged diff --git a/research/config/schedule/OLD_adavae_up_all.yaml b/research/config/schedule/OLD_adavae_up_all.yaml deleted file mode 100644 index 76a11bf5..00000000 --- a/research/config/schedule/OLD_adavae_up_all.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: OLD_adavae_up_all - -schedule_items: - adat_triplet_ratio: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 0.0 # triplet - r_end: 1.0 # ada triplet - adat_triplet_soft_scale: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 0.0 # loss inactive - r_end: 1.0 # loss active - adat_triplet_share_scale: # reversed compared to others - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 1.0 # normal triplet - r_end: 0.5 # ada weighted triplet - ada_thresh_ratio: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 0.0 # none averaged - r_end: 1.0 # all averaged, should this not be 0.5 the recommended value diff --git a/research/config/schedule/OLD_adavae_up_all_full.yaml b/research/config/schedule/OLD_adavae_up_all_full.yaml deleted file mode 100644 index 04925dd8..00000000 --- a/research/config/schedule/OLD_adavae_up_all_full.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: OLD_adavae_up_all_full - -schedule_items: - adat_triplet_ratio: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 0.0 # triplet - r_end: 1.0 # ada triplet - adat_triplet_soft_scale: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 0.0 # loss inactive - r_end: 1.0 # loss active - adat_triplet_share_scale: # reversed compared to others - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 1.0 # normal triplet - r_end: 0.0 # ada weighted triplet - ada_thresh_ratio: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 0.0 # none averaged - r_end: 1.0 # all averaged, should this not be 0.5 the recommended value diff --git a/research/config/schedule/OLD_adavae_up_ratio.yaml b/research/config/schedule/OLD_adavae_up_ratio.yaml deleted file mode 100644 index 66196525..00000000 --- a/research/config/schedule/OLD_adavae_up_ratio.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: OLD_adavae_up_ratio - -schedule_items: - adat_triplet_ratio: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 0.0 # triplet - r_end: 1.0 # ada triplet - adat_triplet_soft_scale: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 0.0 # loss inactive - r_end: 1.0 # loss active - adat_triplet_share_scale: # reversed compared to others - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 1.0 # normal triplet - r_end: 0.5 # ada weighted triplet diff --git a/research/config/schedule/OLD_adavae_up_ratio_full.yaml b/research/config/schedule/OLD_adavae_up_ratio_full.yaml deleted file mode 100644 index a9d94dba..00000000 --- a/research/config/schedule/OLD_adavae_up_ratio_full.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: OLD_adavae_up_ratio_full - -schedule_items: - adat_triplet_ratio: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 0.0 # triplet - r_end: 1.0 # ada triplet - adat_triplet_soft_scale: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 0.0 # loss inactive - r_end: 1.0 # loss active - adat_triplet_share_scale: # reversed compared to others - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 1.0 # normal triplet - r_end: 0.0 # ada weighted triplet diff --git a/research/config/schedule/OLD_adavae_up_thresh.yaml b/research/config/schedule/OLD_adavae_up_thresh.yaml deleted file mode 100644 index e868d56c..00000000 --- a/research/config/schedule/OLD_adavae_up_thresh.yaml +++ /dev/null @@ -1,9 +0,0 @@ -name: OLD_adavae_up_thresh - -schedule_items: - ada_thresh_ratio: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 0.0 # none averaged - r_end: 1.0 # all averaged, should this not be 0.5 the recommended value diff --git a/research/config/schedule/adanegtvae_up_all.yaml b/research/config/schedule/adanegtvae_up_all.yaml deleted file mode 100644 index 3980d206..00000000 --- a/research/config/schedule/adanegtvae_up_all.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: adanegtvae_up_all - -schedule_items: - adat_triplet_share_scale: - _target_: disent.schedule.FixedValueSchedule - value: 1.0 - schedule: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 1.0 # normal triplet - r_end: 0.5 # ada weighted triplet - ada_thresh_ratio: - _target_: disent.schedule.FixedValueSchedule - value: 0.5 - schedule: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 0.0 # none averaged - r_end: 1.0 # all averaged, should this not be 0.5 the recommended value diff --git a/research/config/schedule/adanegtvae_up_all_full.yaml b/research/config/schedule/adanegtvae_up_all_full.yaml deleted file mode 100644 index 027046c4..00000000 --- a/research/config/schedule/adanegtvae_up_all_full.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: adanegtvae_up_all_full - -schedule_items: - adat_triplet_share_scale: - _target_: disent.schedule.FixedValueSchedule - value: 1.0 - schedule: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 1.0 # normal triplet - r_end: 0.0 # ada weighted triplet - ada_thresh_ratio: - _target_: disent.schedule.FixedValueSchedule - value: 0.5 - schedule: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 0.0 # none averaged - r_end: 1.0 # all averaged, should this not be 0.5 the recommended value diff --git a/research/config/schedule/adanegtvae_up_all_weak.yaml b/research/config/schedule/adanegtvae_up_all_weak.yaml deleted file mode 100644 index d3bec05b..00000000 --- a/research/config/schedule/adanegtvae_up_all_weak.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: adanegtvae_up_all_weak - -schedule_items: - adat_triplet_share_scale: - _target_: disent.schedule.FixedValueSchedule - value: 1.0 - schedule: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 1.0 # normal triplet - r_end: 0.75 # ada weighted triplet - ada_thresh_ratio: - _target_: disent.schedule.FixedValueSchedule - value: 0.5 - schedule: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 0.0 # none averaged - r_end: 1.0 # all averaged, should this not be 0.5 the recommended value diff --git a/research/config/schedule/adanegtvae_up_ratio.yaml b/research/config/schedule/adanegtvae_up_ratio.yaml deleted file mode 100644 index b36c62e7..00000000 --- a/research/config/schedule/adanegtvae_up_ratio.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: adanegtvae_up_ratio - -schedule_items: - adat_triplet_share_scale: - _target_: disent.schedule.FixedValueSchedule - value: 1.0 - schedule: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 1.0 # normal triplet - r_end: 0.5 # ada weighted triplet - ada_thresh_ratio: - _target_: disent.schedule.FixedValueSchedule - value: 0.5 - schedule: NULL diff --git a/research/config/schedule/adanegtvae_up_ratio_full.yaml b/research/config/schedule/adanegtvae_up_ratio_full.yaml deleted file mode 100644 index 4f766506..00000000 --- a/research/config/schedule/adanegtvae_up_ratio_full.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: adanegtvae_up_ratio_full - -schedule_items: - adat_triplet_share_scale: - _target_: disent.schedule.FixedValueSchedule - value: 1.0 - schedule: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 1.0 # normal triplet - r_end: 0.0 # ada weighted triplet - ada_thresh_ratio: - _target_: disent.schedule.FixedValueSchedule - value: 0.5 - schedule: NULL diff --git a/research/config/schedule/adanegtvae_up_ratio_weak.yaml b/research/config/schedule/adanegtvae_up_ratio_weak.yaml deleted file mode 100644 index 18cc8d6b..00000000 --- a/research/config/schedule/adanegtvae_up_ratio_weak.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: adanegtvae_up_ratio_weak - -schedule_items: - adat_triplet_share_scale: - _target_: disent.schedule.FixedValueSchedule - value: 1.0 - schedule: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 1.0 # normal triplet - r_end: 0.75 # ada weighted triplet - ada_thresh_ratio: - _target_: disent.schedule.FixedValueSchedule - value: 0.5 - schedule: NULL diff --git a/research/config/schedule/adanegtvae_up_thresh.yaml b/research/config/schedule/adanegtvae_up_thresh.yaml deleted file mode 100644 index e3a3757d..00000000 --- a/research/config/schedule/adanegtvae_up_thresh.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: adanegtvae_up_thresh - -schedule_items: - adat_triplet_share_scale: - _target_: disent.schedule.FixedValueSchedule - value: 0.5 - schedule: NULL - # | - # | - # | - # | - # | - ada_thresh_ratio: - _target_: disent.schedule.FixedValueSchedule - value: 0.5 - schedule: - _target_: disent.schedule.LinearSchedule - start_step: 0 - end_step: ${trainer.max_steps} - r_start: 0.0 # none averaged - r_end: 1.0 # all averaged, should this not be 0.5 the recommended value diff --git a/research/part01_data_overlap/e01_hparam_tuning/submit_param_tuning.sh b/research/part01_data_overlap/e01_hparam_tuning/submit_param_tuning.sh deleted file mode 100644 index f823ed62..00000000 --- a/research/part01_data_overlap/e01_hparam_tuning/submit_param_tuning.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash - - -# OVERVIEW: -# - this experiment is designed to find the optimal hyper-parameters for disentanglement, as well as investigate the -# effect of the adversarial XYSquares dataset against existing approaches. - - -# OUTCOMES: -# - Existing frameworks fail on the adversarial dataset -# - Much lower beta is required for adversarial dataset - - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="CVPR-00__basic-hparam-tuning" -export PARTITION="stampede" -export PARALLELISM=28 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 129600 "C-disent" # 36 hours - -# RUN SWEEP FOR GOOD BETA VALUES -# - beta: 0.01, 0.0316 seem good, 0.1 starts getting too strong, 0.00316 is a bit weak -# - z_size: higher means you can increase beta, eg. 25: beta=0.1 and 9: beta=0.01 -# - framework: adavae needs lower beta, eg. betavae: 0.1, adavae25: 0.0316, adavae9: 0.00316 -# - xy_squares really struggles to learn when non-overlapping, beta needs to be very low. -# might be worth using a warmup schedule -# betavae with zsize=25 and beta<=0.00316 -# betavae with zsize=09 and beta<=0.000316 -# adavae with zsize=25 does not work -# adavae with zsize=09 and beta<=0.001 (must get very lucky) - -# TODO: I should try lower the learning rate to 1e-3 from 1e-4, this might help with xysquares -# 1 * (2 * 8 * 2 * 5) = 160 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_beta' \ - hydra.job.name="vae_hparams" \ - \ - run_length=long \ - metrics=all \ - \ - settings.framework.beta=0.000316,0.001,0.00316,0.01,0.0316,0.1,0.316,1.0 \ - framework=betavae,adavae_os \ - schedule=none \ - settings.model.z_size=9,25 \ - \ - dataset=dsprites,shapes3d,cars3d,smallnorb,X--xysquares \ - sampling=default__bb - - -# TEST DISTANCES IN AEs VS VAEs -# -- supplementary material -# 3 * (1 * 5 = 2) = 15 -submit_sweep \ - +DUMMY.repeat=1,2,3 \ - +EXTRA.tags='sweep_ae' \ - hydra.job.name="ae_test" \ - \ - run_length=medium \ - metrics=all \ - \ - settings.framework.beta=0.0001 \ - framework=ae \ - schedule=none \ - settings.model.z_size=25 \ - \ - dataset=dsprites,shapes3d,cars3d,smallnorb,X--xysquares \ - sampling=default__bb - - -# RUN SWEEP FOR GOOD SCHEDULES -# -- unused -# 1 * (3 * 2 * 4 * 5) = 120 -#submit_sweep \ -# +DUMMY.repeat=1 \ -# +EXTRA.tags='sweep_schedule' \ -# \ -# run_length=long \ -# metrics=all \ -# \ -# settings.framework.beta=0.1,0.316,1.0 \ -# framework=betavae,adavae_os \ -# schedule=beta_cyclic,beta_cyclic_slow,beta_cyclic_fast,beta_decrease \ -# settings.model.z_size=25 \ -# \ -# dataset=dsprites,shapes3d,cars3d,smallnorb,X--xysquares \ -# sampling=default__bb diff --git a/research/part01_data_overlap/e02_incr_overlap_xysquares/run_basic_checks.py b/research/part01_data_overlap/e02_incr_overlap_xysquares/run_basic_checks.py deleted file mode 100644 index a479d8ce..00000000 --- a/research/part01_data_overlap/e02_incr_overlap_xysquares/run_basic_checks.py +++ /dev/null @@ -1,72 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - - -import numpy as np -from research.code.dataset.data import XYSquaresData - - -class XYSquaresSampler(XYSquaresData): - - def sample_1d_boxes(self, size=None): - size = (2,) if (size is None) else ((size, 2) if isinstance(size, int) else (*size, 2)) - # sample x0, y0 - s0 = self._offset + self._spacing * np.random.randint(0, self._placements, size=size) - # sample x1, y1 - s1 = s0 + self._square_size - # return (x0, y0), (x1, y1) - return s0, s1 - - def sample_1d_overlap(self, size=None): - s0, s1 = self.sample_1d_boxes(size=size) - # compute overlap - return np.maximum(np.min(s1, axis=-1) - np.max(s0, axis=-1), 0) - - def sample_1d_delta(self, size=None): - s0, s1 = self.sample_1d_boxes(size=size) - # compute differences - l_delta = np.max(s0, axis=-1) - np.min(s0, axis=-1) - r_delta = np.max(s1, axis=-1) - np.min(s1, axis=-1) - # return delta - return np.minimum(l_delta + r_delta, self._square_size * 2) - - -if __name__ == '__main__': - - print('\nDecreasing Spacing & Increasing Size') - for ss, gs in [(8, 8), (9, 7), (17, 6), (25, 5), (33, 4), (41, 3), (49, 2), (57, 1)][::-1]: - d = XYSquaresSampler(square_size=ss, grid_spacing=gs, max_placements=8, no_warnings=True) - print('ss={:2d} gs={:1d} overlap={:7.4f} delta={:7.4f}'.format(ss, gs, d.sample_1d_overlap(size=1_000_000).mean(), d.sample_1d_delta(size=1_000_000).mean())) - - print('\nDecreasing Spacing') - for i in range(8): - ss, gs = 8, 8-i - d = XYSquaresSampler(square_size=ss, grid_spacing=gs, max_placements=8, no_warnings=True) - print('ss={:2d} gs={:1d} overlap={:7.4f} delta={:7.4f}'.format(ss, gs, d.sample_1d_overlap(size=1_000_000).mean(), d.sample_1d_delta(size=1_000_000).mean())) - - print('\nDecreasing Spacing & Keeping Dimension Size Constant') - for i in range(8): - ss, gs = 8, 8-i - d = XYSquaresSampler(square_size=ss, grid_spacing=gs, max_placements=None, no_warnings=True) - print('ss={:2d} gs={:1d} overlap={:7.4f} delta={:7.4f}'.format(ss, gs, d.sample_1d_overlap(size=1_000_000).mean(), d.sample_1d_delta(size=1_000_000).mean())) diff --git a/research/part01_data_overlap/e02_incr_overlap_xysquares/submit_incr_overlap.sh b/research/part01_data_overlap/e02_incr_overlap_xysquares/submit_incr_overlap.sh deleted file mode 100644 index 2f0b00d5..00000000 --- a/research/part01_data_overlap/e02_incr_overlap_xysquares/submit_incr_overlap.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/bin/bash - - -# OVERVIEW: -# - this experiment is designed to check how increasing overlap (reducing -# the spacing between square positions on XYSquares) affects learning. - - -# OUTCOMES: -# - increasing overlap improves disentanglement & ability for the -# neural network to learn values. -# - decreasing overlap worsens disentanglement, but it also becomes -# very hard for the neural net to learn specific values needed. The -# average image does not correspond well to individual samples. -# Disentanglement performance is also a result of this fact, as -# the network can't always learn the dataset effectively. - - -# ========================================================================= # -# Settings # -# ========================================================================= # - - -export USERNAME="n_michlo" -export PROJECT="CVPR-01__incr_overlap" -export PARTITION="stampede" -export PARALLELISM=28 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - - -# ========================================================================= # -# Experiment # -# ========================================================================= # - - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - - -# background launch various xysquares -# -- original experiment also had dfcvae -# -- beta is too high for adavae -# 5 * (2*2*8 = 32) = 160 -submit_sweep \ - +DUMMY.repeat=1,2,3,4,5 \ - +EXTRA.tags='sweep_xy_squares_overlap' \ - hydra.job.name="incr_ovlp" \ - \ - run_length=medium \ - metrics=all \ - \ - settings.framework.beta=0.001,0.00316 \ - framework=betavae,adavae_os \ - settings.model.z_size=9 \ - \ - sampling=default__bb \ - dataset=X--xysquares_rgb \ - dataset.data.grid_spacing=8,7,6,5,4,3,2,1 - - -# background launch various xysquares -# -- original experiment also had dfcvae -# -- beta is too high for adavae -# 5 * (2*8 = 16) = 80 -submit_sweep \ - +DUMMY.repeat=1,2,3,4,5 \ - +EXTRA.tags='sweep_xy_squares_overlap_small_beta' \ - hydra.job.name="sb_incr_ovlp" \ - \ - run_length=medium \ - metrics=all \ - \ - settings.framework.beta=0.0001,0.00001 \ - framework=adavae_os \ - settings.model.z_size=9 \ - \ - sampling=default__bb \ - dataset=X--xysquares_rgb \ - dataset.data.grid_spacing=8,7,6,5,4,3,2,1 - - -# background launch various xysquares -# - this time we try delay beta, so that it can learn properly... -# - NOTE: this doesn't actually work, the VAE loss often becomes -# NAN because the values are too small. -# 3 * (2*2*8 = 32) = 96 -# submit_sweep \ -# +DUMMY.repeat=1,2,3 \ -# +EXTRA.tags='sweep_xy_squares_overlap_delay' \ -# hydra.job.name="schd_incr_ovlp" \ -# \ -# schedule=beta_delay_long \ -# \ -# run_length=medium \ -# metrics=all \ -# \ -# settings.framework.beta=0.001 \ -# framework=betavae,adavae_os \ -# settings.model.z_size=9,25 \ -# \ -# sampling=default__bb \ -# dataset=X--xysquares_rgb \ -# dataset.data.grid_spacing=8,7,6,5,4,3,2,1 - - -# background launch traditional datasets -# -- original experiment also had dfcvae -# 5 * (2*2*4 = 16) = 80 -#submit_sweep \ -# +DUMMY.repeat=1,2,3,4,5 \ -# +EXTRA.tags='sweep_other' \ -# \ -# run_length=medium \ -# metrics=all \ -# \ -# settings.framework.beta=0.01,0.0316 \ -# framework=betavae,adavae_os \ -# settings.model.z_size=9 \ -# \ -# sampling=default__bb \ -# dataset=cars3d,shapes3d,dsprites,smallnorb - - -# ========================================================================= # -# DONE # -# ========================================================================= # diff --git a/research/part01_data_overlap/e03_modified_loss_xysquares/submit_overlap_loss.sh b/research/part01_data_overlap/e03_modified_loss_xysquares/submit_overlap_loss.sh deleted file mode 100644 index 71dd1f34..00000000 --- a/research/part01_data_overlap/e03_modified_loss_xysquares/submit_overlap_loss.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash - -# OVERVIEW: -# - this experiment is designed to test how changing the reconstruction loss to match the -# ground-truth distances allows datasets to be disentangled. - - -# OUTCOMES: -# - When the reconstruction loss is used as a distance function between observations, and those -# distances match the ground truth, it enables disentanglement. -# - Loss must still be able to reconstruct the inputs correctly. -# - AEs have no incentive to learn the same distances as VAEs - - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="CVPR-09__vae_overlap_loss" -export PARTITION="stampede" -export PARALLELISM=28 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TEST MSE vs BoxBlur MSE (with different beta values over different datasets) -# - mse boxblur weight is too strong, need to lower significantly -# 1 * (5 * 2*4*2) = 80 -#submit_sweep \ -# +DUMMY.repeat=1 \ -# +EXTRA.tags='sweep_overlap_boxblur' \ -# hydra.job.name="ovlp_loss" \ -# \ -# +VAR.recon_loss_weight=1.0 \ -# +VAR.kernel_loss_weight=3969.0 \ -# +VAR.kernel_radius=31 \ -# \ -# run_length=medium \ -# metrics=all \ -# \ -# dataset=X--xysquares,dsprites,shapes3d,smallnorb,cars3d \ -# \ -# framework=betavae,adavae_os \ -# settings.framework.beta=0.0316,0.316,0.1,0.01 \ -# settings.model.z_size=25,9 \ -# settings.framework.recon_loss='mse_box_r${VAR.kernel_radius}_l${VAR.recon_loss_weight}_k${VAR.kernel_loss_weight}_norm_sum' \ -# \ -# sampling=default__bb - - -# TEST MSE vs BoxBlur MSE -# - changing the reconstruction loss enables disentanglement -# 5 * (2*2*2 = 8) = 40 -submit_sweep \ - +DUMMY.repeat=1,2,3,4,5 \ - +EXTRA.tags='sweep_overlap_boxblur_specific' \ - hydra.job.name="s_ovlp_loss" \ - \ - +VAR.recon_loss_weight=1.0 \ - +VAR.kernel_loss_weight=3969.0 \ - +VAR.kernel_radius=31 \ - \ - run_length=medium \ - metrics=all \ - \ - dataset=X--xysquares \ - \ - framework=betavae,adavae_os \ - settings.framework.beta=0.0316,0.0001 \ - settings.model.z_size=25 \ - settings.framework.recon_loss=mse,'mse_box_r${VAR.kernel_radius}_l${VAR.recon_loss_weight}_k${VAR.kernel_loss_weight}_norm_sum' \ - \ - sampling=default__bb - - -# TEST DISTANCES IN AEs VS VAEs -# -- supplementary material -# 3 * (1 * 2 = 2) = 6 -submit_sweep \ - +DUMMY.repeat=1,2,3 \ - +EXTRA.tags='sweep_overlap_boxblur_autoencoders' \ - hydra.job.name="e_ovlp_loss" \ - \ - +VAR.recon_loss_weight=1.0 \ - +VAR.kernel_loss_weight=3969.0 \ - +VAR.kernel_radius=31 \ - \ - run_length=medium \ - metrics=all \ - \ - dataset=X--xysquares \ - \ - framework=ae \ - settings.framework.beta=0.0001 \ - settings.model.z_size=25 \ - settings.framework.recon_loss=mse,'mse_box_r${VAR.kernel_radius}_l${VAR.recon_loss_weight}_k${VAR.kernel_loss_weight}_norm_sum' \ - \ - sampling=default__bb - - -# HPARAM SWEEP -- TODO: update -# -- old, unused -# 1 * (2 * 8 * 2 * 2) = 160 -#submit_sweep \ -# +DUMMY.repeat=1 \ -# +EXTRA.tags='sweep_beta' \ -# hydra.job.name="vae_hparams" \ -# \ -# run_length=long \ -# metrics=all \ -# \ -# settings.framework.beta=0.000316,0.001,0.00316,0.01,0.0316,0.1,0.316,1.0 \ -# framework=betavae,adavae_os \ -# schedule=none \ -# settings.model.z_size=9,25 \ -# \ -# dataset=X--xysquares \ -# sampling=default__bb diff --git a/research/part01_data_overlap/plot01_data_traversal/animations/.gitignore b/research/part01_data_overlap/plot01_data_traversal/animations/.gitignore deleted file mode 100644 index 8b3117cc..00000000 --- a/research/part01_data_overlap/plot01_data_traversal/animations/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.png -*.jpg -*.gif diff --git a/research/part01_data_overlap/plot01_data_traversal/plots/.gitignore b/research/part01_data_overlap/plot01_data_traversal/plots/.gitignore deleted file mode 100644 index 225c3815..00000000 --- a/research/part01_data_overlap/plot01_data_traversal/plots/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.png -*.jpg diff --git a/research/part01_data_overlap/plot01_data_traversal/run_01_all_shared_data_prepare.sh b/research/part01_data_overlap/plot01_data_traversal/run_01_all_shared_data_prepare.sh deleted file mode 100644 index cc2f69e3..00000000 --- a/research/part01_data_overlap/plot01_data_traversal/run_01_all_shared_data_prepare.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash - -# This script is intended to prepare all shared data on the wits cluster -# you can probably modify it for your own purposes -# - data is loaded and processed into ~/downloads/datasets which is a -# shared drive, instead of /tmp/, which is a local drive. - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="prepare-data" -export PARTITION="stampede" -export PARALLELISM=32 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -DATASETS=( - cars3d - dsprites - # monte_rollouts - # mpi3d_real - # mpi3d_realistic - # mpi3d_toy - shapes3d - smallnorb - #X--adv-cars3d--WARNING - #X--adv-dsprites--WARNING - #X--adv-shapes3d--WARNING - #X--adv-smallnorb--WARNING - #X--dsprites-imagenet - #X--dsprites-imagenet-bg-20 - #X--dsprites-imagenet-bg-40 - #X--dsprites-imagenet-bg-60 - #X--dsprites-imagenet-bg-80 - #X--dsprites-imagenet-bg-100 - #X--dsprites-imagenet-fg-20 - #X--dsprites-imagenet-fg-40 - #X--dsprites-imagenet-fg-60 - #X--dsprites-imagenet-fg-80 - #X--dsprites-imagenet-fg-100 - #X--mask-adv-f-cars3d - #X--mask-adv-f-dsprites - #X--mask-adv-f-shapes3d - #X--mask-adv-f-smallnorb - #X--mask-adv-r-cars3d - #X--mask-adv-r-dsprites - #X--mask-adv-r-shapes3d - #X--mask-adv-r-smallnorb - #X--mask-dthr-cars3d - #X--mask-dthr-dsprites - #X--mask-dthr-shapes3d - #X--mask-dthr-smallnorb - #X--mask-ran-cars3d - #X--mask-ran-dsprites - #X--mask-ran-shapes3d - #X--mask-ran-smallnorb - "X--xyblocks" - #X--xyblocks_grey - "X--xysquares" - #X--xysquares_grey - #X--xysquares_rgb - xyobject - #xyobject_grey - #xyobject_shaded - #xyobject_shaded_grey -) - -local_sweep \ - run_action=prepare_data \ - run_location=stampede_shr \ - run_launcher=local \ - dataset="$(IFS=, ; echo "${DATASETS[*]}")" diff --git a/research/part01_data_overlap/plot01_data_traversal/run_02_plot_data_overlap.py b/research/part01_data_overlap/plot01_data_traversal/run_02_plot_data_overlap.py deleted file mode 100644 index 38f9d1db..00000000 --- a/research/part01_data_overlap/plot01_data_traversal/run_02_plot_data_overlap.py +++ /dev/null @@ -1,185 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import os -from typing import Optional - -import numpy as np -from matplotlib import pyplot as plt - -import research.code.util as H -from disent.dataset.data import Cars3d64Data -from disent.dataset.data import DSpritesData -from disent.dataset.data import GroundTruthData -from disent.dataset.data import Shapes3dData -from disent.dataset.data import SmallNorb64Data -from disent.dataset.data import XYObjectData -from disent.dataset.data import XYObjectShadedData -from research.code.dataset.data import XYSquaresData -from disent.util.function import wrapped_partial -from disent.util.seeds import TempNumpySeed - - -# ========================================================================= # -# core # -# ========================================================================= # - - -def ensure_rgb(img: np.ndarray) -> np.ndarray: - if img.shape[-1] == 1: - img = np.concatenate([img, img, img], axis=-1) - assert img.shape[-1] == 3, f'last channel of array is not of size 3 for RGB, got shape: {tuple(img.shape)}' - return img - - -def plot_dataset_overlap( - gt_data: GroundTruthData, - f_idxs=None, - obs_max: Optional[int] = None, - obs_spacing: int = 1, - rel_path=None, - plot_base: bool = False, - plot_combined: bool = True, - plot_sidebar: bool = False, - save=True, - seed=777, - plt_scale=4.5, - offset=0.75, -): - with TempNumpySeed(seed): - # choose an f_idx - f_idx = np.random.choice(gt_data.normalise_factor_idxs(f_idxs)) - f_name = gt_data.factor_names[f_idx] - num_cols = gt_data.factor_sizes[f_idx] - # get a traversal - factors, indices, obs = gt_data.sample_random_obs_traversal(f_idx=f_idx) - # get subset - if obs_max is not None: - max_obs_spacing, i = obs_spacing, 1 - while max_obs_spacing*obs_max > len(obs): - max_obs_spacing = obs_spacing-i - i += 1 - i = max((len(obs) - obs_max*max_obs_spacing) // 2, 0) - obs = obs[i:i+obs_max*obs_spacing:max_obs_spacing][:obs_max] - # convert - obs = np.array([ensure_rgb(x) for x in obs], dtype='float32') / 255 - # compute the distances - grid = np.zeros([len(obs), len(obs), *obs[0].shape]) - for i, i_obs in enumerate(obs): - for j, j_obs in enumerate(obs): - grid[i, j] = np.abs(i_obs - j_obs) - # normalize - grid /= grid.max() - - # make figure - factors, frames, _, _, c = grid.shape - assert c == 3 - - if plot_base: - # plot - fig, axs = H.plt_subplots_imshow(grid, label_size=18, title_size=24, title=f'{gt_data.name}: {f_name}', subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) - # save figure - if save and (rel_path is not None): - path = H.make_rel_path_add_ext(rel_path, ext='.png') - plt.savefig(path) - print(f'saved: {repr(path)}') - plt.show() - - if plot_combined: - # add obs - if True: - factors += 1 - frames += 1 - # scaled_obs = obs - scaled_obs = obs * 0.5 + 0.25 - # grid = 1 - grid - # grid = grid * 0.5 + 0.25 - grid = np.concatenate([scaled_obs[None, :], grid], axis=0) - add_row = np.concatenate([np.ones_like(obs[0:1]), scaled_obs], axis=0) - grid = np.concatenate([grid, add_row[:, None]], axis=1) - # plot - fig, axs = H.plt_subplots_imshow(grid, label_size=18, title_size=24, row_labels=["traversal"] + (["diff."] * len(obs)), col_labels=(["diff."] * len(obs)) + ["traversal"], title=f'{gt_data.name}: {f_name}', subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) - # save figure - if save and (rel_path is not None): - path = H.make_rel_path_add_ext(rel_path + '__combined', ext='.png') - plt.savefig(path) - print(f'saved: {repr(path)}') - plt.show() - - # plot - if plot_sidebar: - fig, axs = H.plt_subplots_imshow(obs[:, None], subplot_padding=None, figsize=(offset + (1/2.54)*1*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) - if save and (rel_path is not None): - path = H.make_rel_path_add_ext(rel_path + '__v', ext='.png') - plt.savefig(path) - print(f'saved: {repr(path)}') - plt.show() - fig, axs = H.plt_subplots_imshow(obs[None, :], subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(1+0.45)*plt_scale)) - if save and (rel_path is not None): - path = H.make_rel_path_add_ext(rel_path + '__h', ext='.png') - plt.savefig(path) - print(f'saved: {repr(path)}') - plt.show() - - -# ========================================================================= # -# entrypoint # -# ========================================================================= # - - -if __name__ == '__main__': - - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../../code/util/gadfly.mplstyle')) - - # options - all_squares = True - add_random_traversal = True - num_cols = 7 - seed = 48 - - for gt_data_cls, name in [ - (wrapped_partial(XYSquaresData, grid_spacing=1, grid_size=8, no_warnings=True), f'xy-squares-spacing1'), - (wrapped_partial(XYSquaresData, grid_spacing=2, grid_size=8, no_warnings=True), f'xy-squares-spacing2'), - (wrapped_partial(XYSquaresData, grid_spacing=4, grid_size=8, no_warnings=True), f'xy-squares-spacing4'), - (wrapped_partial(XYSquaresData, grid_spacing=8, grid_size=8, no_warnings=True), f'xy-squares-spacing8'), - ]: - plot_dataset_overlap(gt_data_cls(), rel_path=f'plots/overlap__{name}', obs_max=3, obs_spacing=4, seed=seed-40) - - for gt_data_cls, name in [ - (XYObjectData, f'xyobject'), - (XYObjectShadedData, f'xyobject_shaded'), - (DSpritesData, f'dsprites'), - (Shapes3dData, f'shapes3d'), - (Cars3d64Data, f'cars3d'), - (SmallNorb64Data, f'smallnorb'), - ]: - gt_data = gt_data_cls() - for f_idx, f_name in enumerate(gt_data.factor_names): - plot_dataset_overlap(gt_data, rel_path=f'plots/overlap__{name}__f{f_idx}-{f_name}', obs_max=3, obs_spacing=4, f_idxs=f_idx, seed=seed) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/part01_data_overlap/plot01_data_traversal/run_02_plot_traversals.py b/research/part01_data_overlap/plot01_data_traversal/run_02_plot_traversals.py deleted file mode 100644 index 25e5a0f5..00000000 --- a/research/part01_data_overlap/plot01_data_traversal/run_02_plot_traversals.py +++ /dev/null @@ -1,284 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import os -from typing import Optional -from typing import Sequence -from typing import Union - -import numpy as np -from matplotlib import pyplot as plt - -import research.code.util as H -from disent.dataset import DisentDataset -from disent.dataset.data import Cars3d64Data -from disent.dataset.data import DSpritesData -from research.code.dataset.data import DSpritesImagenetData -from disent.dataset.data import GroundTruthData -from disent.dataset.data import SelfContainedHdf5GroundTruthData -from disent.dataset.data import Shapes3dData -from disent.dataset.data import SmallNorb64Data -from research.code.dataset.data import XYBlocksData -from disent.dataset.data import XYObjectData -from disent.dataset.data import XYObjectShadedData -from research.code.dataset.data import XYSingleSquareData -from research.code.dataset.data import XYSquaresData -from disent.util.seeds import TempNumpySeed - - -# ========================================================================= # -# core # -# ========================================================================= # - - -def ensure_rgb(img: np.ndarray) -> np.ndarray: - if img.shape[-1] == 1: - img = np.concatenate([img, img, img], axis=-1) - assert img.shape[-1] == 3, f'last channel of array is not of size 3 for RGB, got shape: {tuple(img.shape)}' - return img - - -def plot_dataset_traversals( - gt_data: GroundTruthData, - f_idxs=None, - num_cols: Optional[int] = 8, - take_cols: Optional[int] = None, - base_factors=None, - add_random_traversal: bool = True, - pad: int = 8, - bg_color: int = 127, - border: bool = False, - rel_path: str = None, - save: bool = True, - seed: int = 777, - plt_scale: float = 4.5, - offset: float = 0.75, - transpose: bool = False, - title: Union[bool, str] = True, - label_size: int = 22, - title_size: int = 26, - labels_at_top: bool = False, - img_ext: str = '.jpg', -): - if take_cols is not None: - assert take_cols >= num_cols - # convert - dataset = DisentDataset(gt_data) - f_idxs = gt_data.normalise_factor_idxs(f_idxs) - num_cols = num_cols if (num_cols is not None) else min(max(gt_data.factor_sizes), 32) - # get traversal grid - row_labels = [gt_data.factor_names[i] for i in f_idxs] - grid, _, _ = H.visualize_dataset_traversal( - dataset=dataset, - data_mode='raw', - factor_names=f_idxs, - num_frames=num_cols if (take_cols is None) else take_cols, - seed=seed, - base_factors=base_factors, - traverse_mode='interval', - pad=pad, - bg_color=bg_color, - border=border, - ) - if take_cols is not None: - grid = grid[:, :num_cols, ...] - # add random traversal - if add_random_traversal: - with TempNumpySeed(seed): - row_labels = ['random'] + row_labels - row = dataset.dataset_sample_batch(num_samples=num_cols, mode='raw')[None, ...] # torch.Tensor - grid = np.concatenate([ensure_rgb(row), grid]) - # make figure - factors, frames, _, _, c = grid.shape - assert c == 3 - - # get title - if isinstance(title, bool): - title = gt_data.name if title else None - - if transpose: - col_titles = None - if labels_at_top: - col_titles, row_labels = row_labels, None - fig, axs = H.plt_subplots_imshow(np.swapaxes(grid, 0, 1), label_size=label_size, title_size=title_size, title=title, titles=col_titles, titles_size=label_size, col_labels=row_labels, subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)[::-1]) - else: - fig, axs = H.plt_subplots_imshow(grid, label_size=label_size, title_size=title_size, title=title, row_labels=row_labels, subplot_padding=None, figsize=(offset + (1/2.54)*frames*plt_scale, (1/2.54)*(factors+0.45)*plt_scale)) - - # save figure - if save and (rel_path is not None): - path = H.make_rel_path_add_ext(rel_path, ext=img_ext) - plt.savefig(path) - print(f'saved: {repr(path)}') - plt.show() - # done! - return fig, axs - - -def plot_incr_overlap( - rel_path: Optional[str] = None, - spacings: Union[Sequence[int], bool] = False, - seed: int = 777, - fidx: int = 1, - traversal_size: int = 8, - traversal_lim: Optional[int] = None, - save: bool = True, - show: bool = True, - img_ext: str = '.jpg', -): - if isinstance(spacings, bool): - spacings = ([1, 2, 3, 4, 5, 6, 7, 8] if spacings else [1, 4, 8]) - - if traversal_lim is None: - traversal_lim = traversal_size - assert traversal_size >= traversal_lim - - grid = [] - for s in spacings: - data = XYSquaresData(grid_spacing=s, grid_size=8, no_warnings=True) - with TempNumpySeed(seed): - factors, indices, obs = data.sample_random_obs_traversal(f_idx=data.normalise_factor_idx(fidx), num=traversal_size, mode='interval') - grid.append(obs[:traversal_lim]) - - w, h = traversal_lim * 2.54, len(spacings) * 2.54 - fig, axs = H.plt_subplots_imshow(grid, row_labels=[f'Space: {s}px' for s in spacings], figsize=(w, h), label_size=24) - fig.tight_layout() - - H.plt_rel_path_savefig(rel_path=rel_path, save=save, ext=img_ext, show=show) - - -# ========================================================================= # -# entrypoint # -# ========================================================================= # - - -if __name__ == '__main__': - - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../../code/util/gadfly.mplstyle')) - - # options - all_squares = False - num_cols = 7 - mini_cols = 5 - transpose_cols = 3 - seed = 47 - - INCLUDE_RANDOM_TRAVERSAL = False - TITLE = False - TITLE_MINI = False - TITLE_TRANSPOSE = False - - # get name - prefix = 'traversal' if INCLUDE_RANDOM_TRAVERSAL else 'traversal-noran' - - # plot increasing levels of overlap - plot_incr_overlap(rel_path=f'plots/traversal-incr-overlap__xy-squares', save=True, show=True, traversal_lim=None) - - # mini versions - plot_dataset_traversals(XYSingleSquareData(), rel_path=f'plots/traversal-mini__xy-single-square__spacing8', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(XYSquaresData(), rel_path=f'plots/traversal-mini__xy-squares__spacing8', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(Shapes3dData(), rel_path=f'plots/traversal-mini__shapes3d', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(DSpritesData(), rel_path=f'plots/traversal-mini__dsprites', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(SmallNorb64Data(), rel_path=f'plots/traversal-mini__smallnorb', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(Cars3d64Data(), rel_path=f'plots/traversal-mini__cars3d', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols, take_cols=mini_cols+1) - - # transpose versions - plot_dataset_traversals(XYSingleSquareData(), rel_path=f'plots/traversal-transpose__xy-single-square__spacing8', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(XYSquaresData(), rel_path=f'plots/traversal-transpose__xy-squares__spacing8', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(Shapes3dData(), rel_path=f'plots/traversal-transpose__shapes3d', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(DSpritesData(), rel_path=f'plots/traversal-transpose__dsprites', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(SmallNorb64Data(), rel_path=f'plots/traversal-transpose__smallnorb', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(Cars3d64Data(), rel_path=f'plots/traversal-transpose__cars3d', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols, take_cols=mini_cols+1) - - # save images - for i in ([1, 2, 3, 4, 5, 6, 7, 8] if all_squares else [1, 2, 4, 8]): - data = XYSquaresData(grid_spacing=i, grid_size=8, no_warnings=True) - plot_dataset_traversals(data, rel_path=f'plots/{prefix}__xy-squares__spacing{i}', title=TITLE, seed=seed-40, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(data, rel_path=f'plots/{prefix}__xy-squares__spacing{i}__some', title=TITLE, seed=seed-40, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols, f_idxs=[0, 3]) - - plot_dataset_traversals(XYSingleSquareData(), rel_path=f'plots/{prefix}__xy-single-square', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(Shapes3dData(), rel_path=f'plots/{prefix}__shapes3d', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(DSpritesData(), rel_path=f'plots/{prefix}__dsprites', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(SmallNorb64Data(), rel_path=f'plots/{prefix}__smallnorb', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(Cars3d64Data(), rel_path=f'plots/{prefix}__cars3d', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - - exit(0) - - # mini versions - plot_dataset_traversals(XYObjectData(), rel_path=f'plots/traversal-mini__xy-object', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(XYObjectShadedData(), rel_path=f'plots/traversal-mini__xy-object-shaded', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(XYBlocksData(), rel_path=f'plots/traversal-mini__xy-blocks', title=TITLE_MINI, seed=seed, transpose=False, add_random_traversal=False, num_cols=mini_cols) - - plot_dataset_traversals(DSpritesImagenetData(100, 'bg'), rel_path=f'plots/traversal-mini__dsprites-imagenet-bg-100', title=TITLE_MINI, seed=seed-6, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(DSpritesImagenetData( 50, 'bg'), rel_path=f'plots/traversal-mini__dsprites-imagenet-bg-50', title=TITLE_MINI, seed=seed-6, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(DSpritesImagenetData(100, 'fg'), rel_path=f'plots/traversal-mini__dsprites-imagenet-fg-100', title=TITLE_MINI, seed=seed-6, transpose=False, add_random_traversal=False, num_cols=mini_cols) - plot_dataset_traversals(DSpritesImagenetData( 50, 'fg'), rel_path=f'plots/traversal-mini__dsprites-imagenet-fg-50', title=TITLE_MINI, seed=seed-6, transpose=False, add_random_traversal=False, num_cols=mini_cols) - - # transpose versions - plot_dataset_traversals(XYObjectData(), rel_path=f'plots/traversal-transpose__xy-object', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(XYObjectShadedData(), rel_path=f'plots/traversal-transpose__xy-object-shaded', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(XYBlocksData(), rel_path=f'plots/traversal-transpose__xy-blocks', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - - plot_dataset_traversals(DSpritesImagenetData(100, 'bg'), rel_path=f'plots/traversal-transpose__dsprites-imagenet-bg-100', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed-6, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(DSpritesImagenetData( 50, 'bg'), rel_path=f'plots/traversal-transpose__dsprites-imagenet-bg-50', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed-6, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(DSpritesImagenetData(100, 'fg'), rel_path=f'plots/traversal-transpose__dsprites-imagenet-fg-100', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed-6, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - plot_dataset_traversals(DSpritesImagenetData( 50, 'fg'), rel_path=f'plots/traversal-transpose__dsprites-imagenet-fg-50', title=TITLE_TRANSPOSE, offset=0.95, label_size=23, seed=seed-6, labels_at_top=True, transpose=True, add_random_traversal=False, num_cols=transpose_cols) - - # save images - plot_dataset_traversals(XYObjectData(), rel_path=f'plots/{prefix}__xy-object', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(XYObjectShadedData(), rel_path=f'plots/{prefix}__xy-object-shaded', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(XYBlocksData(), rel_path=f'plots/{prefix}__xy-blocks', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - - plot_dataset_traversals(DSpritesImagenetData(100, 'bg'), rel_path=f'plots/{prefix}__dsprites-imagenet-bg-100', title=TITLE, seed=seed-6, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(DSpritesImagenetData( 50, 'bg'), rel_path=f'plots/{prefix}__dsprites-imagenet-bg-50', title=TITLE, seed=seed-6, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(DSpritesImagenetData(100, 'fg'), rel_path=f'plots/{prefix}__dsprites-imagenet-fg-100', title=TITLE, seed=seed-6, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - plot_dataset_traversals(DSpritesImagenetData( 50, 'fg'), rel_path=f'plots/{prefix}__dsprites-imagenet-fg-50', title=TITLE, seed=seed-6, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - - exit(0) - - BASE = os.path.abspath(os.path.join(__file__, '../../../out/adversarial_data_approx')) - - for folder in [ - # 'const' datasets - ('2021-08-18--00-58-22_FINAL-dsprites_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('2021-08-18--01-33-47_FINAL-shapes3d_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('2021-08-18--02-20-13_FINAL-cars3d_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('2021-08-18--03-10-53_FINAL-smallnorb_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - # 'invert' datasets - ('2021-08-18--03-52-31_FINAL-dsprites_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('2021-08-18--04-29-25_FINAL-shapes3d_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('2021-08-18--05-13-15_FINAL-cars3d_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('2021-08-18--06-03-32_FINAL-smallnorb_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - # stronger 'invert' datasets - ('2021-09-06--00-29-23_INVERT-VSTRONG-shapes3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), - ('2021-09-06--03-17-28_INVERT-VSTRONG-dsprites_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), - ('2021-09-06--05-42-06_INVERT-VSTRONG-cars3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), - ('2021-09-06--09-10-59_INVERT-VSTRONG-smallnorb_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), - ]: - plot_dataset_traversals(SelfContainedHdf5GroundTruthData(f'{BASE}/{folder}/data.h5'), rel_path=f'plots/{prefix}__{folder}.png', title=TITLE, seed=seed, add_random_traversal=INCLUDE_RANDOM_TRAVERSAL, num_cols=num_cols) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/part01_data_overlap/plot01_data_traversal/run_03_plot_dataset_animation.py b/research/part01_data_overlap/plot01_data_traversal/run_03_plot_dataset_animation.py deleted file mode 100644 index e6c609d4..00000000 --- a/research/part01_data_overlap/plot01_data_traversal/run_03_plot_dataset_animation.py +++ /dev/null @@ -1,76 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2022 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import os - -import numpy as np - -from disent.util.inout.paths import ensure_parent_dir_exists -from disent.util.seeds import TempNumpySeed -import research.code.util as H -import imageio - - -if __name__ == '__main__': - - OUTPUT_DIR = os.path.abspath(os.path.join(__file__, '..', 'animations')) - FRAMES_PER_TRAVERSAL = 18 - FRAMES_PER_SECOND = 8 - - for data_name, data_seed in [ - ('xysquares_8x8', [5, 5, 1, 5, 5, 1]), # ('x_R', 'y_R', 'x_G', 'y_G', 'x_B', 'y_B') - ('dsprites', [2, 3, 4, 16, 20]), # ('shape', 'scale', 'orientation', 'position_x', 'position_y') - ('cars3d', [2, 2, 79]), # ('elevation', 'azimuth', 'object_type') - ('shapes3d', [1, 6, 9, 4, 3, 2]), # ('floor_hue', 'wall_hue', 'object_hue', 'scale', 'shape', 'orientation') - ('smallnorb', [2, 4, 1, 2, 3]), # ('category', 'instance', 'elevation', 'rotation', 'lighting') - ]: - data = H.make_data(data_name, transform_mode='none') - - for mode in [ - 'cycle_from_start', - # 'cycle_from_start_ends', - ]: - frames = [] - # get starting point - if isinstance(data_seed, int): - with TempNumpySeed(data_seed): - base_factors = data.sample_factors() - else: - base_factors = np.array(data_seed) - print(f'{data_name}:', base_factors.tolist(), data.factor_names, data.factor_sizes) - # append factor traversals - for f_idx in range(data.num_factors): - traversal = data.sample_random_factor_traversal( - f_idx=f_idx, - base_factors=base_factors, - num=FRAMES_PER_TRAVERSAL, - mode=mode, - start_index=base_factors[f_idx], - ) - # append the frames - frames.extend(data[i] for i in data.pos_to_idx(traversal)) - # save the animation - path = ensure_parent_dir_exists(OUTPUT_DIR, f'animation__{data_name}__{mode}.gif') - imageio.mimsave(path, frames, fps=FRAMES_PER_SECOND) - print(f'saved: {path}') diff --git a/research/part01_data_overlap/plot02_data_distances/cache/.gitignore b/research/part01_data_overlap/plot02_data_distances/cache/.gitignore deleted file mode 100644 index 13974321..00000000 --- a/research/part01_data_overlap/plot02_data_distances/cache/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.pkl diff --git a/research/part01_data_overlap/plot02_data_distances/deprecated/run_01_x_z_recon_dists.sh b/research/part01_data_overlap/plot02_data_distances/deprecated/run_01_x_z_recon_dists.sh deleted file mode 100644 index 54ca819b..00000000 --- a/research/part01_data_overlap/plot02_data_distances/deprecated/run_01_x_z_recon_dists.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-01__gt-vs-learnt-dists" -export PARTITION="stampede" -export PARALLELISM=28 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - - -# 1 * (3 * 6 * 4 * 2) = 144 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep' \ - \ - model=linear,vae_fc,vae_conv64 \ - \ - run_length=medium \ - metrics=all \ - \ - dataset=xyobject,xyobject_shaded,shapes3d,dsprites,cars3d,smallnorb \ - sampling=default__bb \ - framework=ae,X--adaae_os,betavae,adavae_os \ - \ - settings.framework.beta=0.0316 \ - settings.optimizer.lr=3e-4 \ - settings.model.z_size=9,25 diff --git a/research/part01_data_overlap/plot02_data_distances/plots/.gitignore b/research/part01_data_overlap/plot02_data_distances/plots/.gitignore deleted file mode 100644 index 225c3815..00000000 --- a/research/part01_data_overlap/plot02_data_distances/plots/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.png -*.jpg diff --git a/research/part01_data_overlap/plot02_data_distances/run_data_correlation.py b/research/part01_data_overlap/plot02_data_distances/run_data_correlation.py deleted file mode 100644 index 5c94b711..00000000 --- a/research/part01_data_overlap/plot02_data_distances/run_data_correlation.py +++ /dev/null @@ -1,815 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from collections import defaultdict -from pprint import pprint -from typing import Dict -from typing import List -from typing import Optional -from typing import Sequence -from typing import Tuple -from typing import Union - -import numpy as np -import torch - -from disent.dataset.data import Cars3d64Data -from disent.dataset.data import DSpritesData -from disent.dataset.data import Shapes3dData -from disent.dataset.data import SmallNorb64Data -from disent.frameworks.helper.reconstructions import ReconLossHandler -from tqdm import tqdm - -import disent.registry as R -from disent.dataset import DisentDataset -from disent.dataset.transform import ToImgTensorF32 -from disent.util.function import wrapped_partial - -from research.code import register_to_disent -from research.code.dataset.data import XYSquaresData -from research.code.metrics._factored_components import _compute_dists -from research.code.metrics._factored_components import _compute_scores_from_dists -from research.code.metrics._factored_components import _numpy_concat_all_dicts - - -_RENAME_KEYS = { - 'mse/rsame_ground_data': 'rsame_ratio (mse)', - 'mse/rcorr_ground_data': 'rank_corr (mse)', - 'mse/lcorr_ground_data': 'linear_corr (mse)', - - 'mse_gau_r31_l1.0_k3969.0_norm_sum/rsame_ground_data': 'rsame_ratio (gau)', - 'mse_gau_r31_l1.0_k3969.0_norm_sum/rcorr_ground_data': 'rank_corr (gau)', - 'mse_gau_r31_l1.0_k3969.0_norm_sum/lcorr_ground_data': 'linear_corr (gau)', - - 'mse_box_r31_l1.0_k3969.0_norm_sum/rsame_ground_data': 'rsame_ratio (box)', - 'mse_box_r31_l1.0_k3969.0_norm_sum/rcorr_ground_data': 'rank_corr (box)', - 'mse_box_r31_l1.0_k3969.0_norm_sum/lcorr_ground_data': 'linear_corr (box)', - - 'mse_xy8_abs31_l1.0_k1.0_norm_none/rsame_ground_data': 'rsame_ratio (xy8)', - 'mse_xy8_abs31_l1.0_k1.0_norm_none/rcorr_ground_data': 'rank_corr (xy8)', - 'mse_xy8_abs31_l1.0_k1.0_norm_none/lcorr_ground_data': 'linear_corr (xy8)', - - 'mse_xy8_abs63_l1.0_k1.0_norm_none/rsame_ground_data': 'rsame_ratio (xy8,r63)', - 'mse_xy8_abs63_l1.0_k1.0_norm_none/rcorr_ground_data': 'rank_corr (xy8,r63)', - 'mse_xy8_abs63_l1.0_k1.0_norm_none/lcorr_ground_data': 'linear_corr (xy8,r63)', - - 'mse_xy8_r47_l1.0_k3969.0_norm_sum/rsame_ground_data': 'rsame_ratio (xy8,r47,OLD)', - 'mse_xy8_r47_l1.0_k3969.0_norm_sum/rcorr_ground_data': 'rank_corr (xy8,r47,OLD)', - 'mse_xy8_r47_l1.0_k3969.0_norm_sum/lcorr_ground_data': 'linear_corr (xy8,r47,OLD)', -} - -# ========================================================================= # -# plot # -# ========================================================================= # - - -def _n_digits(num: int): - if num > 0: - return int(np.log10(num) + 1) - if num < 0: - return int(np.log10(-num) + 2) # add an extra 1 for the minus sign - else: - return 1 - - -def _normalise_f_name_and_idx(dataset: DisentDataset, f_idx: Optional[Union[str, int]]) -> Tuple[Optional[int], str]: - if f_idx in ('random', None): - f_idx = None - f_name = 'random' - elif isinstance(f_idx, str): - f_idx = dataset.gt_data.normalise_factor_idx(f_idx) - f_name = dataset.gt_data.factor_names[f_idx] - else: - assert isinstance(f_idx, int) - f_name = dataset.gt_data.factor_names[f_idx] - return f_idx, f_name - - -@torch.no_grad() -def _compute_mean_rcorr_ground_data( - dataset: DisentDataset, - f_idx: Optional[Union[str, int]], - num_samples: int, - repeats: int, - progress: bool = True, - random_batch_size: int = 16, - losses: Sequence[str] = ('mse', 'mse_box_r31_l1.0_k3969.0') -): - f_idx, f_name = _normalise_f_name_and_idx(dataset, f_idx) - # recon loss handlers - recon_handlers = {loss: R.RECON_LOSSES[loss](reduction='mean').cuda() for loss in losses} - # storage for each loss - distance_measures: Dict[str, List[Dict[str, np.ndarray]]] = defaultdict(list) - - # repeat! - for i in tqdm(range(repeats), desc=f'{dataset.gt_data.name}: {f_name}', disable=not progress): - # sample random factors - if f_idx is None: - factors = dataset.gt_data.sample_factors(size=random_batch_size) - else: - factors = dataset.gt_data.sample_random_factor_traversal(f_idx=f_idx) - # encode factors - xs = dataset.dataset_batch_from_factors(factors, 'input').cuda() - factors = torch.from_numpy(factors).to(torch.float32).cpu() - # [COMPUTE SAME RATIO & CORRELATION] - for loss in losses: - recon_loss = recon_handlers[loss] - computed_dists = _compute_dists(num_samples, zs_traversal=None, xs_traversal=xs, factors=factors, recon_loss_fn=recon_loss) - distance_measures[loss].append(computed_dists) - - # concatenate all into arrays: - # then aggregate over first dimension: - distance_measures: Dict[str, float] = { - f'{loss}/{k}': v - for loss in losses - for k, v in _compute_scores_from_dists(_numpy_concat_all_dicts(distance_measures[loss])).items() - } - - # done! - return {_RENAME_KEYS.get(k, k): v for k, v in distance_measures.items()} - - -# ========================================================================= # -# entrypoint # -# ========================================================================= # - - -if __name__ == '__main__': - - def main(mode: str = 'compare_kernels', repeats: int = 16384): - gt_data_classes = { - # 'XYObject': wrapped_partial(XYObjectData), - # 'XYBlocks': wrapped_partial(XYBlocksData), - 'XYSquares': wrapped_partial(XYSquaresData), - 'Cars3d': wrapped_partial(Cars3d64Data), - 'Shapes3d': wrapped_partial(Shapes3dData), - 'SmallNorb': wrapped_partial(SmallNorb64Data), - 'DSprites': wrapped_partial(DSpritesData), - # 'Mpi3d': wrapped_partial(Mpi3dData), - - 'XYSquares-1-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=1, grid_size=8, no_warnings=True), - 'XYSquares-2-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=2, grid_size=8, no_warnings=True), - 'XYSquares-3-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=3, grid_size=8, no_warnings=True), - 'XYSquares-4-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=4, grid_size=8, no_warnings=True), - 'XYSquares-5-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=5, grid_size=8, no_warnings=True), - 'XYSquares-6-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=6, grid_size=8, no_warnings=True), - 'XYSquares-7-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=7, grid_size=8, no_warnings=True), - 'XYSquares-8-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=8, grid_size=8, no_warnings=True), - } - - include_factors = { - 'XYSquares-1-8': ('x_R',), - 'XYSquares-2-8': ('x_R',), - 'XYSquares-3-8': ('x_R',), - 'XYSquares-4-8': ('x_R',), - 'XYSquares-5-8': ('x_R',), - 'XYSquares-6-8': ('x_R',), - 'XYSquares-7-8': ('x_R',), - 'XYSquares-8-8': ('x_R',), - } - - # 16384 * 64 = 1_048_576 - num_samples = 64 - random_batch_size = 32 - progress = True - digits = 4 - - if mode == 'all_datasets': - ORDER = [ - 'linear_corr (mse)', - 'rank_corr (mse)', - 'linear_corr (box)', - 'rank_corr (box)', - ] - losses = ('mse', 'mse_box_r31_l1.0_k3969.0') - elif mode == 'compare_kernels': - gt_data_classes = {'XYSquares-8-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=8, grid_size=8, no_warnings=True)} - ORDER = { - 'rank_corr (mse)', - 'linear_corr (mse)', - 'rank_corr (gau)', - 'linear_corr (gau)', - 'rank_corr (box)', - 'linear_corr (box)', - 'rank_corr (xy8)', - 'linear_corr (xy8)', - # 'rank_corr (xy8,r47,OLD)', - # 'linear_corr (xy8,r47,OLD)', - } - # best hparams: - losses = ( - 'mse', # orig! - 'mse_gau_r31_l1.0_k3969.0_norm_sum', # gau: ORIG - # 'mse_gau_r63_l1.0_k3969.0_norm_sum', # gau: NEW (much better than r31) - 'mse_box_r31_l1.0_k3969.0_norm_sum', # box: ORIG - # 'mse_xy8_r47_l1.0_k3969.0_norm_sum', # learnt: OLD -- (DO NOT USE) - 'mse_xy8_abs63_l1.0_k1.0_norm_none', # learnt: NEW (SAME AS: 'mse_xy8_abs63_l1.0_k1.0_norm_none','mse_xy8_abs63_norm_none','mse_xy8_abs63') - ) - elif mode == 'compare_kernels_small': - gt_data_classes = {'XYSquares-8-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=8, grid_size=8, no_warnings=True)} - ORDER = { - 'rank_corr (mse)', - 'linear_corr (mse)', - 'rank_corr (gau)', - 'linear_corr (gau)', - 'rank_corr (box)', - 'linear_corr (box)', - 'rank_corr (xy8)', - 'linear_corr (xy8)', - # 'rank_corr (xy8,r47,OLD)', - # 'linear_corr (xy8,r47,OLD)', - } - # best hparams: - losses = ( - 'mse', # orig! - 'mse_gau_r31_l1.0_k3969.0_norm_sum', # gau: ORIG - # 'mse_gau_r63_l1.0_k3969.0_norm_sum', # gau: NEW (much better than r31) - 'mse_box_r31_l1.0_k3969.0_norm_sum', # box: ORIG - # 'mse_xy8_r47_l1.0_k3969.0_norm_sum', # learnt: OLD -- (DO NOT USE) - 'mse_xy8_abs31_l1.0_k1.0_norm_none', # learnt: NEW (SAME AS: 'mse_xy8_abs31_l1.0_k1.0_norm_none','mse_xy8_abs31_norm_none','mse_xy8_abs31') - ) - elif mode == 'box_hparams': - ORDER, gt_data_classes = [], {'XYSquares-8-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=8, grid_size=8, no_warnings=True)} - losses = tuple(f'mse_box_r{r}_l1.0_k{k}_norm_sum' for k in ['100.0', '396.9', '1000.0', '3969.0', '10000.0'] for r in ['15', '31', '47', '63']) - elif mode == 'gau_hparams': - ORDER, gt_data_classes = [], {'XYSquares-8-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=8, grid_size=8, no_warnings=True)} - losses = tuple(f'mse_gau_r{r}_l1.0_k{k}_norm_sum' for k in ['100.0', '396.9', '1000.0', '3969.0', '10000.0'] for r in ['15', '31', '47', '63']) - elif mode == 'xy8_hparams': - ORDER, gt_data_classes = [], {'XYSquares-8-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=8, grid_size=8, no_warnings=True)} - losses = ( - 'mse_xy8_abs63', # this should be the same as mse_xy8_abs63_l1.0_k1.0_norm_none -- USE THIS ONE! - 'mse_xy8_abs63_norm_none', # this should be the same as mse_xy8_abs63_l1.0_k1.0_norm_none - 'mse_xy8_abs63_l1.0_k1.0_norm_none', # this should be the same as mse_xy8_abs63_l1.0_k1.0_norm_none - 'mse_xy8_abs63_l1.0_k10.0_norm_none', - 'mse_xy8_abs63_l1.0_k100.0_norm_none', - 'mse_xy8_abs63_l1.0_k1000.0_norm_none', - 'mse_xy8_abs63_l1.0_k10000.0_norm_none', - # make sure scaling works - 'mse_xy8_abs63_l1.0_k1.0_norm_sum', - 'mse_xy8_abs63_l1.0_k10.0_norm_sum', - 'mse_xy8_abs63_l1.0_k100.0_norm_sum', - 'mse_xy8_abs63_l1.0_k1000.0_norm_sum', - 'mse_xy8_abs63_l1.0_k10000.0_norm_sum', - ) - elif mode == 'xy8_hparams_small': - ORDER, gt_data_classes = [], {'XYSquares-8-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=8, grid_size=8, no_warnings=True)} - losses = ( - 'mse_xy8_abs31', # this should be the same as mse_xy8_abs31_l1.0_k1.0_norm_none -- USE THIS ONE! - 'mse_xy8_abs31_norm_none', # this should be the same as mse_xy8_abs31_l1.0_k1.0_norm_none - 'mse_xy8_abs31_l1.0_k1.0_norm_none', # this should be the same as mse_xy8_abs31_l1.0_k1.0_norm_none - 'mse_xy8_abs31_l1.0_k10.0_norm_none', - 'mse_xy8_abs31_l1.0_k100.0_norm_none', - 'mse_xy8_abs31_l1.0_k1000.0_norm_none', - 'mse_xy8_abs31_l1.0_k10000.0_norm_none', - ) - else: - raise KeyError('invalid mode') - - - for name, data_cls in gt_data_classes.items(): - dataset = DisentDataset(data_cls(), transform=ToImgTensorF32(size=64)) - # get factor names - factor_names = tuple(include_factors.get(name, dataset.gt_data.factor_names)) + ('random',) - # compute over each factor name - for i, f_name in enumerate(factor_names): - # print variables - f_size = dataset.gt_data.factor_sizes[dataset.gt_data.normalise_factor_idx(f_name)] if (f_name != 'random') else len(dataset) - size_len = _n_digits(len(dataset)) - name_len = max(len(s) for s in factor_names) - # compute scores - try: - scores = _compute_mean_rcorr_ground_data(dataset, f_idx=f_name, num_samples=num_samples, repeats=repeats, random_batch_size=random_batch_size, progress=progress, losses=losses) - scores = {k: v for k, v in scores.items() if ('rsame_' not in k)} - order = (ORDER if ORDER else scores.keys()) - pprint(scores) - # NORMAL - # print(f'[{name}] f_idx={f_name:{name_len}s} f_size={f_size:{size_len}d} {" ".join(f"{k}={v:7.5f}" for k, v in scores.items())}') - # LATEX HEADINGS: - if i == 0: - print(f'[{name}] Factor Name & Factor Size & {" & ".join(f"{k:{digits}s}" for k in order if k in scores)}') - # LATEX - print(f'[{name}] {f_name:{name_len}s} & {f_size:{size_len}d} & {" & ".join(f"{scores[k]:{digits}.{digits-2}f}" for k in order if k in scores)}') - except Exception as e: - # NORMAL - # print(f'[{name}] f_idx={f_name:{name_len}s} f_size={f_size:{size_len}d} SKIPPED!') - # LATEX - print(f'[{name}] {f_name:{name_len}s} & {f_size:{size_len}d} & {" & ".join(f"N/A" for k in ORDER)}') - raise e - print() - - # RUN - register_to_disent() - - # main(mode='compare_kernels', repeats=8192) - main(mode='compare_kernels_small', repeats=8192) - # main(mode='xy8_hparams_small', repeats=128) - # main(mode='all_datasets', repeats=1024) - # main(mode='box_hparams', repeats=128) - # main(mode='gau_hparams', repeats=128) - # main(mode='xy8_hparams', repeats=128) - - - -# ========================================================================= # -# Results # -# ========================================================================= # - -# [Cars3d] Factor Name & Factor Size & rsame_ratio (mse) & rank_corr (mse) & linear_corr (mse) & rsame_ratio (aug) & rank_corr (aug) & linear_corr (aug) -# [Cars3d] elevation & 4 & 0.94 & 0.93 & 0.90 & 0.93 & 0.88 & 0.69 -# [Cars3d] azimuth & 24 & 0.65 & 0.34 & 0.30 & 0.62 & 0.25 & 0.08 -# [Cars3d] object_type & 183 & 0.52 & 0.04 & 0.04 & 0.50 & 0.01 & 0.00 -# [Cars3d] random & 17568 & 0.56 & 0.13 & 0.15 & 0.54 & 0.10 & 0.04 -# -# [Shapes3d] Factor Name & Factor Size & rsame_ratio (mse) & rank_corr (mse) & linear_corr (mse) & rsame_ratio (aug) & rank_corr (aug) & linear_corr (aug) -# [Shapes3d] floor_hue & 10 & 0.82 & 0.76 & 0.62 & 0.82 & 0.74 & 0.60 -# [Shapes3d] wall_hue & 10 & 0.82 & 0.74 & 0.60 & 0.82 & 0.72 & 0.55 -# [Shapes3d] object_hue & 10 & 0.82 & 0.71 & 0.53 & 0.82 & 0.63 & 0.41 -# [Shapes3d] scale & 8 & 0.95 & 0.88 & 0.81 & 0.95 & 0.87 & 0.71 -# [Shapes3d] shape & 4 & 0.91 & 0.79 & 0.69 & 0.90 & 0.80 & 0.58 -# [Shapes3d] orientation & 15 & 0.94 & 0.92 & 0.84 & 0.89 & 0.87 & 0.74 -# [Shapes3d] random & 480000 & 0.66 & 0.45 & 0.53 & 0.60 & 0.29 & 0.29 -# -# [SmallNorb] Factor Name & Factor Size & rsame_ratio (mse) & rank_corr (mse) & linear_corr (mse) & rsame_ratio (aug) & rank_corr (aug) & linear_corr (aug) -# [SmallNorb] category & 5 & 0.75 & 0.53 & 0.44 & 0.73 & 0.47 & 0.15 -# [SmallNorb] instance & 5 & 0.73 & 0.52 & 0.37 & 0.73 & 0.51 & 0.10 -# [SmallNorb] elevation & 9 & 0.94 & 0.90 & 0.81 & 0.78 & 0.64 & 0.51 -# [SmallNorb] rotation & 18 & 0.61 & 0.19 & 0.12 & 0.60 & 0.21 & 0.07 -# [SmallNorb] lighting & 6 & 0.64 & 0.29 & 0.07 & 0.64 & 0.28 & 0.07 -# [SmallNorb] random & 24300 & 0.54 & 0.14 & 0.10 & 0.54 & 0.14 & 0.07 -# -# [DSprites] Factor Name & Factor Size & rsame_ratio (mse) & rank_corr (mse) & linear_corr (mse) & rsame_ratio (aug) & rank_corr (aug) & linear_corr (aug) -# [DSprites] shape & 3 & 0.83 & 0.72 & 0.66 & 0.93 & 0.87 & 0.66 -# [DSprites] scale & 6 & 0.95 & 0.95 & 0.93 & 0.94 & 0.96 & 0.84 -# [DSprites] orientation & 40 & 0.60 & 0.17 & 0.13 & 0.63 & 0.21 & 0.15 -# [DSprites] position_x & 32 & 0.90 & 0.75 & 0.66 & 0.99 & 0.83 & 0.63 -# [DSprites] position_y & 32 & 0.90 & 0.75 & 0.65 & 0.99 & 0.83 & 0.63 -# [DSprites] random & 737280 & 0.64 & 0.38 & 0.43 & 0.66 & 0.36 & 0.29 -# -# [XYSquares-1-8] Factor Name & Factor Size & rsame_ratio (mse) & rank_corr (mse) & linear_corr (mse) & rsame_ratio (aug) & rank_corr (aug) & linear_corr (aug) -# [XYSquares-1-8] x_R & 8 & 1.00 & 1.00 & 1.00 & 0.97 & 0.99 & 0.98 -# [XYSquares-1-8] random & 262144 & 0.90 & 0.97 & 0.98 & 0.91 & 0.98 & 0.98 -# -# [XYSquares-2-8] Factor Name & Factor Size & rsame_ratio (mse) & rank_corr (mse) & linear_corr (mse) & rsame_ratio (aug) & rank_corr (aug) & linear_corr (aug) -# [XYSquares-2-8] x_R & 8 & 0.92 & 0.99 & 0.94 & 0.96 & 0.99 & 0.99 -# [XYSquares-2-8] random & 262144 & 0.77 & 0.83 & 0.85 & 0.92 & 0.99 & 0.99 -# -# [XYSquares-3-8] Factor Name & Factor Size & rsame_ratio (mse) & rank_corr (mse) & linear_corr (mse) & rsame_ratio (aug) & rank_corr (aug) & linear_corr (aug) -# [XYSquares-3-8] x_R & 8 & 0.84 & 0.95 & 0.86 & 0.96 & 0.99 & 0.99 -# [XYSquares-3-8] random & 262144 & 0.68 & 0.73 & 0.75 & 0.92 & 0.99 & 0.99 -# -# [XYSquares-4-8] Factor Name & Factor Size & rsame_ratio (mse) & rank_corr (mse) & linear_corr (mse) & rsame_ratio (aug) & rank_corr (aug) & linear_corr (aug) -# [XYSquares-4-8] x_R & 8 & 0.67 & 0.85 & 0.75 & 0.96 & 0.99 & 0.99 -# [XYSquares-4-8] random & 262144 & 0.47 & 0.58 & 0.67 & 0.92 & 0.99 & 0.99 -# -# [XYSquares-5-8] Factor Name & Factor Size & rsame_ratio (mse) & rank_corr (mse) & linear_corr (mse) & rsame_ratio (aug) & rank_corr (aug) & linear_corr (aug) -# [XYSquares-5-8] x_R & 8 & 0.67 & 0.85 & 0.72 & 0.95 & 0.99 & 0.99 -# [XYSquares-5-8] random & 262144 & 0.47 & 0.58 & 0.64 & 0.92 & 0.98 & 0.99 -# -# [XYSquares-6-8] Factor Name & Factor Size & rsame_ratio (mse) & rank_corr (mse) & linear_corr (mse) & rsame_ratio (aug) & rank_corr (aug) & linear_corr (aug) -# [XYSquares-6-8] x_R & 8 & 0.67 & 0.85 & 0.67 & 0.96 & 0.98 & 0.98 -# [XYSquares-6-8] random & 262144 & 0.47 & 0.58 & 0.61 & 0.90 & 0.97 & 0.98 -# -# [XYSquares-7-8] Factor Name & Factor Size & rsame_ratio (mse) & rank_corr (mse) & linear_corr (mse) & rsame_ratio (aug) & rank_corr (aug) & linear_corr (aug) -# [XYSquares-7-8] x_R & 8 & 0.67 & 0.85 & 0.60 & 0.96 & 0.98 & 0.97 -# [XYSquares-7-8] random & 262144 & 0.47 & 0.58 & 0.59 & 0.89 & 0.96 & 0.96 -# -# [XYSquares-8-8] Factor Name & Factor Size & rsame_ratio (mse) & rank_corr (mse) & linear_corr (mse) & rsame_ratio (aug) & rank_corr (aug) & linear_corr (aug) -# [XYSquares-8-8] x_R & 8 & 0.39 & 0.58 & 0.52 & 0.95 & 0.97 & 0.96 -# [XYSquares-8-8] random & 262144 & 0.21 & 0.37 & 0.55 & 0.87 & 0.94 & 0.95 - -# ========================================================================= # -# Results - Reformatted # -# ========================================================================= # - -# [Cars3d] Factor Name & linear_corr (mse) & rank_corr (mse) & linear_corr (aug) & rank_corr (aug) -# [Cars3d] elevation & 0.90 & 0.93 & 0.69 & 0.88 -# [Cars3d] azimuth & 0.30 & 0.34 & 0.08 & 0.25 -# [Cars3d] object_type & 0.04 & 0.04 & 0.00 & 0.01 -# [Cars3d] random & 0.15 & 0.13 & 0.04 & 0.10 -# -# [Shapes3d] Factor Name & linear_corr (mse) & rank_corr (mse) & linear_corr (aug) & rank_corr (aug) -# [Shapes3d] floor_hue & 0.62 & 0.76 & 0.60 & 0.74 -# [Shapes3d] wall_hue & 0.60 & 0.74 & 0.55 & 0.72 -# [Shapes3d] object_hue & 0.53 & 0.71 & 0.41 & 0.63 -# [Shapes3d] scale & 0.81 & 0.88 & 0.71 & 0.87 -# [Shapes3d] shape & 0.69 & 0.79 & 0.58 & 0.80 -# [Shapes3d] orientation & 0.84 & 0.92 & 0.74 & 0.87 -# [Shapes3d] random & 0.53 & 0.45 & 0.29 & 0.29 -# -# [SmallNorb] Factor Name & linear_corr (mse) & rank_corr (mse) & linear_corr (aug) & rank_corr (aug) -# [SmallNorb] category & 0.44 & 0.53 & 0.15 & 0.47 -# [SmallNorb] instance & 0.37 & 0.52 & 0.10 & 0.51 -# [SmallNorb] elevation & 0.81 & 0.90 & 0.51 & 0.64 -# [SmallNorb] rotation & 0.12 & 0.19 & 0.07 & 0.21 -# [SmallNorb] lighting & 0.07 & 0.29 & 0.07 & 0.28 -# [SmallNorb] random & 0.10 & 0.14 & 0.07 & 0.14 -# -# [DSprites] Factor Name & linear_corr (mse) & rank_corr (mse) & linear_corr (aug) & rank_corr (aug) -# [DSprites] shape & 0.66 & 0.72 & 0.66 & 0.87 -# [DSprites] scale & 0.93 & 0.95 & 0.84 & 0.96 -# [DSprites] orientation & 0.13 & 0.17 & 0.15 & 0.21 -# [DSprites] position_x & 0.66 & 0.75 & 0.63 & 0.83 -# [DSprites] position_y & 0.65 & 0.75 & 0.63 & 0.83 -# [DSprites] random & 0.43 & 0.38 & 0.29 & 0.36 -# -# [XYSquares-1-8] Factor Name & linear_corr (mse) & rank_corr (mse) & linear_corr (aug) & rank_corr (aug) -# [XYSquares-1-8] x_R & 1.00 & 1.00 & 0.98 & 0.99 -# [XYSquares-1-8] random & 0.98 & 0.97 & 0.98 & 0.98 -# -# [XYSquares-2-8] Factor Name & linear_corr (mse) & rank_corr (mse) & linear_corr (aug) & rank_corr (aug) -# [XYSquares-2-8] x_R & 0.94 & 0.99 & 0.99 & 0.99 -# [XYSquares-2-8] random & 0.85 & 0.83 & 0.99 & 0.99 -# -# [XYSquares-3-8] Factor Name & linear_corr (mse) & rank_corr (mse) & linear_corr (aug) & rank_corr (aug) -# [XYSquares-3-8] x_R & 0.86 & 0.95 & 0.99 & 0.99 -# [XYSquares-3-8] random & 0.75 & 0.73 & 0.99 & 0.99 -# -# [XYSquares-4-8] Factor Name & linear_corr (mse) & rank_corr (mse) & linear_corr (aug) & rank_corr (aug) -# [XYSquares-4-8] x_R & 0.75 & 0.85 & 0.99 & 0.99 -# [XYSquares-4-8] random & 0.67 & 0.58 & 0.99 & 0.99 -# -# [XYSquares-5-8] Factor Name & linear_corr (mse) & rank_corr (mse) & linear_corr (aug) & rank_corr (aug) -# [XYSquares-5-8] x_R & 0.72 & 0.85 & 0.99 & 0.99 -# [XYSquares-5-8] random & 0.64 & 0.58 & 0.99 & 0.98 -# -# [XYSquares-6-8] Factor Name & linear_corr (mse) & rank_corr (mse) & linear_corr (aug) & rank_corr (aug) -# [XYSquares-6-8] x_R & 0.67 & 0.85 & 0.98 & 0.98 -# [XYSquares-6-8] random & 0.61 & 0.58 & 0.98 & 0.97 -# -# [XYSquares-7-8] Factor Name & linear_corr (mse) & rank_corr (mse) & linear_corr (aug) & rank_corr (aug) -# [XYSquares-7-8] x_R & 0.60 & 0.85 & 0.97 & 0.98 -# [XYSquares-7-8] random & 0.59 & 0.58 & 0.96 & 0.96 -# -# [XYSquares-8-8] Factor Name & linear_corr (mse) & rank_corr (mse) & linear_corr (aug) & rank_corr (aug) -# [XYSquares-8-8] x_R & 0.52 & 0.58 & 0.96 & 0.97 -# [XYSquares-8-8] random & 0.55 & 0.37 & 0.95 & 0.94 - -# ========================================================================= # -# COMPARE KERNELS -- NEW # -# ========================================================================= # - -# [XYSquares-8-8] Factor Name & Factor Size & linear_corr (mse) & rank_corr (box) & rank_corr (xy8) & rank_corr (mse) & rank_corr (gau) & linear_corr (box) & linear_corr (xy8) & linear_corr (gau) -# [XYSquares-8-8] x_R & 8 & 0.52 & 0.97 & 0.99 & 0.58 & 0.88 & 0.96 & 1.00 & 0.84 -# [XYSquares-8-8] random & 262144 & 0.55 & 0.94 & 1.00 & 0.36 & 0.61 & 0.95 & 1.00 & 0.73 - -# MANUAL EDIT - -# Loss Name & Linear Corr. (factor) & Rank Corr. (factor) & Linear Corr. (random) & Rank Corr. (random) -# MSE & 0.52 & 0.58 & 0.55 & 0.36 -# MSE (Gau-Kernel) & 0.84 & 0.88 & 0.73 & 0.61 -# MSE (Box-Kernel) & 0.96 & 0.97 & 0.95 & 0.94 -# MSE (XY8-Kernel) & 1.00 & 0.99 & 1.00 & 1.00 - -# x_r = { -# 'linear_corr (mse)': 0.522849733858152, -# 'linear_corr (gau)': 0.8425233444009568, -# 'linear_corr (box)': 0.9568990533994665, -# 'linear_corr (xy8)': 0.9985841007312456, -# 'rank_corr (mse)': 0.5806083003878514, -# 'rank_corr (gau)': 0.8796127509043598, -# 'rank_corr (box)': 0.9725331862367547, -# 'rank_corr (xy8)': 0.9879068264304146, -# } - -# random = { -# 'linear_corr (mse)': 0.5549578765578173, -# 'linear_corr (gau)': 0.7252470489579737, -# 'linear_corr (box)': 0.945283121956908, -# 'linear_corr (xy8)': 0.9993413110499386, -# 'rank_corr (mse)': 0.36458667433987935, -# 'rank_corr (gau)': 0.6148508535668269, -# 'rank_corr (box)': 0.935241155400177, -# 'rank_corr (xy8)': 0.9981047204425746, -# } - -# ========================================================================= # -# HPARAMS # -# ========================================================================= # - -# SUMMARY: -# mse_box_r31_l1.0_k3969.0_norm_sum -- rcorr (x): 0.97, rcorr (ran): 0.94 -# mse_gau_r31_l1.0_k3969.0_norm_sum -- rcorr (x): 0.88, rcorr (ran): 0.61 -# mse_gau_r63_l1.0_k3969.0_norm_sum -- rcorr (x): 0.97, rcorr (ran): 0.87 -# mse_xy8_abs63_l1.0_k1.0_norm_none -- rcorr (x): 0.99, rcorr (ran): 0.998 (same as: `mse_xy8_abs63`,`mse_xy8_abs63_norm_none`,`mse_xy8_abs63_l1.0_k1.0_norm_none`) - -# x_R: (BOX) - -# {'mse_box_r15_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.8465267326733271, -# 'mse_box_r15_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.9074030172085413, -# 'mse_box_r15_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.8421812408768276, -# 'mse_box_r15_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.899732706846721, -# 'mse_box_r15_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.8364461294011671, -# 'mse_box_r15_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.902459955644352, -# 'mse_box_r15_l1.0_k396.9_norm_sum/lcorr_ground_data': 0.8387559590889287, -# 'mse_box_r15_l1.0_k396.9_norm_sum/rcorr_ground_data': 0.8992828209108001, -# 'mse_box_r15_l1.0_k3969.0_norm_sum/lcorr_ground_data': 0.839586011817049, -# 'mse_box_r15_l1.0_k3969.0_norm_sum/rcorr_ground_data': 0.9017972035516646, -# 'mse_box_r31_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.7456610052478235, -# 'mse_box_r31_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.9711368561490288, -# 'mse_box_r31_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.952658055712391, -# 'mse_box_r31_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.9719123703169099, -# 'mse_box_r31_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.9509994505296753, -# 'mse_box_r31_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.9715980270707917, -# 'mse_box_r31_l1.0_k396.9_norm_sum/lcorr_ground_data': 0.9185281602387867, -# 'mse_box_r31_l1.0_k396.9_norm_sum/rcorr_ground_data': 0.9716384205664865, -# 'mse_box_r31_l1.0_k3969.0_norm_sum/lcorr_ground_data': 0.952864628493186, # ORIG -- probs use this! -# 'mse_box_r31_l1.0_k3969.0_norm_sum/rcorr_ground_data': 0.9710944800602881, # ORIG -- probs use this! -# 'mse_box_r47_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.5469040351447404, -# 'mse_box_r47_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.7586339061180715, -# 'mse_box_r47_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.683911153268716, -# 'mse_box_r47_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.7564522298329677, -# 'mse_box_r47_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.7984639858449594, -# 'mse_box_r47_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.764245721003377, -# 'mse_box_r47_l1.0_k396.9_norm_sum/lcorr_ground_data': 0.6015761809920372, -# 'mse_box_r47_l1.0_k396.9_norm_sum/rcorr_ground_data': 0.7495935051981993, -# 'mse_box_r47_l1.0_k3969.0_norm_sum/lcorr_ground_data': 0.7929270714898712, -# 'mse_box_r47_l1.0_k3969.0_norm_sum/rcorr_ground_data': 0.7542703815316855, -# 'mse_box_r63_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.5189231212192755, -# 'mse_box_r63_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.5792379669971695, -# 'mse_box_r63_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.5261779645463969, -# 'mse_box_r63_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.5860239628489328, -# 'mse_box_r63_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.524179961771539, -# 'mse_box_r63_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.5853527764946593, -# 'mse_box_r63_l1.0_k396.9_norm_sum/lcorr_ground_data': 0.5222290384283436, -# 'mse_box_r63_l1.0_k396.9_norm_sum/rcorr_ground_data': 0.5833024458678896, -# 'mse_box_r63_l1.0_k3969.0_norm_sum/lcorr_ground_data': 0.5165526108194486, -# 'mse_box_r63_l1.0_k3969.0_norm_sum/rcorr_ground_data': 0.5722994771405968} - -# random: (BOX) - -# {'mse_box_r15_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.7355645492374561, -# 'mse_box_r15_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.6461296788386194, -# 'mse_box_r15_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.7369420684985439, -# 'mse_box_r15_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.6414962868012578, -# 'mse_box_r15_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.7325644896797724, -# 'mse_box_r15_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.6399353290724973, -# 'mse_box_r15_l1.0_k396.9_norm_sum/lcorr_ground_data': 0.730415588750465, -# 'mse_box_r15_l1.0_k396.9_norm_sum/rcorr_ground_data': 0.6395086234771381, -# 'mse_box_r15_l1.0_k3969.0_norm_sum/lcorr_ground_data': 0.7334653235918743, -# 'mse_box_r15_l1.0_k3969.0_norm_sum/rcorr_ground_data': 0.6393942199843501, -# 'mse_box_r31_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.7743633840498417, -# 'mse_box_r31_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.9233773009051243, -# 'mse_box_r31_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.9382410582431485, -# 'mse_box_r31_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.9339837949342503, -# 'mse_box_r31_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.9471825129300452, # BEST? -# 'mse_box_r31_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.9374386514298645, # BEST? -# 'mse_box_r31_l1.0_k396.9_norm_sum/lcorr_ground_data': 0.9109982140953756, -# 'mse_box_r31_l1.0_k396.9_norm_sum/rcorr_ground_data': 0.9313172089479078, -# 'mse_box_r31_l1.0_k3969.0_norm_sum/lcorr_ground_data': 0.9450903194446578, # ORIG -- probs use this -# 'mse_box_r31_l1.0_k3969.0_norm_sum/rcorr_ground_data': 0.9356454634461142, # ORIG -- probs use this -# 'mse_box_r47_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.5869369771248776, -# 'mse_box_r47_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.7701305240518559, -# 'mse_box_r47_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.751677221885856, -# 'mse_box_r47_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.7715966871218836, -# 'mse_box_r47_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.8327945543924842, -# 'mse_box_r47_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.771589715309024, -# 'mse_box_r47_l1.0_k396.9_norm_sum/lcorr_ground_data': 0.6611439615720487, -# 'mse_box_r47_l1.0_k396.9_norm_sum/rcorr_ground_data': 0.7694208909826364, -# 'mse_box_r47_l1.0_k3969.0_norm_sum/lcorr_ground_data': 0.8455723029971326, -# 'mse_box_r47_l1.0_k3969.0_norm_sum/rcorr_ground_data': 0.7839774335207814, -# 'mse_box_r63_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.5542709153289006, -# 'mse_box_r63_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.3649117298131543, -# 'mse_box_r63_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.5527700068276091, -# 'mse_box_r63_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.36332730774450345, -# 'mse_box_r63_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.563230626775863, -# 'mse_box_r63_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.3711849918502138, -# 'mse_box_r63_l1.0_k396.9_norm_sum/lcorr_ground_data': 0.5510390375629297, -# 'mse_box_r63_l1.0_k396.9_norm_sum/rcorr_ground_data': 0.36126596021015955, -# 'mse_box_r63_l1.0_k3969.0_norm_sum/lcorr_ground_data': 0.5618312259023005, -# 'mse_box_r63_l1.0_k3969.0_norm_sum/rcorr_ground_data': 0.36803101695849627} - -# x_R: (GAU) - -# {'mse_gau_r15_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.7273482838641963, -# 'mse_gau_r15_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.7144203401218819, -# 'mse_gau_r15_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.7259637743598086, -# 'mse_gau_r15_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.7005330607398159, -# 'mse_gau_r15_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.7264476800975389, -# 'mse_gau_r15_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.7086928934712139, -# 'mse_gau_r15_l1.0_k396.9_norm_sum/lcorr_ground_data': 0.7231276674231087, -# 'mse_gau_r15_l1.0_k396.9_norm_sum/rcorr_ground_data': 0.7012189371608948, -# 'mse_gau_r15_l1.0_k3969.0_norm_sum/lcorr_ground_data': 0.7293317481072964, -# 'mse_gau_r15_l1.0_k3969.0_norm_sum/rcorr_ground_data': 0.7098309693263373, -# 'mse_gau_r31_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.8383434338101533, -# 'mse_gau_r31_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.880150834452838, -# 'mse_gau_r31_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.8401334274585273, -# 'mse_gau_r31_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.8792685133944187, -# 'mse_gau_r31_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.8462596990408467, -# 'mse_gau_r31_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.8823476556222172, -# 'mse_gau_r31_l1.0_k396.9_norm_sum/lcorr_ground_data': 0.8383734716168756, -# 'mse_gau_r31_l1.0_k396.9_norm_sum/rcorr_ground_data': 0.8765377171861533, -# 'mse_gau_r31_l1.0_k3969.0_norm_sum/lcorr_ground_data': 0.8403361636272052, # ORIG -- probs use this? -# 'mse_gau_r31_l1.0_k3969.0_norm_sum/rcorr_ground_data': 0.8770459135570791, # ORIG -- probs use this? -# 'mse_gau_r47_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.8965053635227883, -# 'mse_gau_r47_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.942814653185387, -# 'mse_gau_r47_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.9066975740516656, -# 'mse_gau_r47_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.9411909410031053, -# 'mse_gau_r47_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.9047843494918939, -# 'mse_gau_r47_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.9409217203670263, -# 'mse_gau_r47_l1.0_k396.9_norm_sum/lcorr_ground_data': 0.9091836688143318, -# 'mse_gau_r47_l1.0_k396.9_norm_sum/rcorr_ground_data': 0.9419431101438522, -# 'mse_gau_r47_l1.0_k3969.0_norm_sum/lcorr_ground_data': 0.9050831582460364, # GOOD -# 'mse_gau_r47_l1.0_k3969.0_norm_sum/rcorr_ground_data': 0.9404859781515228, # GOOD -# 'mse_gau_r63_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.8869088324812824, -# 'mse_gau_r63_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.9681351360110224, -# 'mse_gau_r63_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.9432046233907826, -# 'mse_gau_r63_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.9676927614060077, -# 'mse_gau_r63_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.9376714841351936, -# 'mse_gau_r63_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.9669249655522885, -# 'mse_gau_r63_l1.0_k396.9_norm_sum/lcorr_ground_data': 0.9443273966580698, -# 'mse_gau_r63_l1.0_k396.9_norm_sum/rcorr_ground_data': 0.967771782673114, -# 'mse_gau_r63_l1.0_k3969.0_norm_sum/lcorr_ground_data': 0.9389288363812284, # GOOD -# 'mse_gau_r63_l1.0_k3969.0_norm_sum/rcorr_ground_data': 0.9677280298785211} # GOOD - -# random: (GAU) - -# {'mse_gau_r15_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.6159990533616333, -# 'mse_gau_r15_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.3550159374761337, -# 'mse_gau_r15_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.6190840974821674, -# 'mse_gau_r15_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.3666723181778829, -# 'mse_gau_r15_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.622168309786848, -# 'mse_gau_r15_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.36697781941110424, -# 'mse_gau_r15_l1.0_k396.9_norm_sum/lcorr_ground_data': 0.6283368419798032, -# 'mse_gau_r15_l1.0_k396.9_norm_sum/rcorr_ground_data': 0.3819506598563719, -# 'mse_gau_r15_l1.0_k3969.0_norm_sum/lcorr_ground_data': 0.6137272718227919, -# 'mse_gau_r15_l1.0_k3969.0_norm_sum/rcorr_ground_data': 0.3670218847742282, -# 'mse_gau_r31_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.7257352691684524, -# 'mse_gau_r31_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.6202714962447266, -# 'mse_gau_r31_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.7252995840844141, -# 'mse_gau_r31_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.6254115079120555, -# 'mse_gau_r31_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.7274790716936281, -# 'mse_gau_r31_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.6198841473186878, -# 'mse_gau_r31_l1.0_k396.9_norm_sum/lcorr_ground_data': 0.7340600310701334, -# 'mse_gau_r31_l1.0_k396.9_norm_sum/rcorr_ground_data': 0.6167873984375287, -# 'mse_gau_r31_l1.0_k3969.0_norm_sum/lcorr_ground_data': 0.7274600006378323, # ORIG -# 'mse_gau_r31_l1.0_k3969.0_norm_sum/rcorr_ground_data': 0.6118366263285681, # ORIG -# 'mse_gau_r47_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.8158084932971688, -# 'mse_gau_r47_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.7739331988198449, -# 'mse_gau_r47_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.8271920992120669, -# 'mse_gau_r47_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.778873657013877, -# 'mse_gau_r47_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.8250240787862765, -# 'mse_gau_r47_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.7798942756913891, -# 'mse_gau_r47_l1.0_k396.9_norm_sum/lcorr_ground_data': 0.825525769727308, -# 'mse_gau_r47_l1.0_k396.9_norm_sum/rcorr_ground_data': 0.777835640836142, -# 'mse_gau_r47_l1.0_k3969.0_norm_sum/lcorr_ground_data': 0.8241864657986688, -# 'mse_gau_r47_l1.0_k3969.0_norm_sum/rcorr_ground_data': 0.7777636698190691, -# 'mse_gau_r63_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.8482895371617932, -# 'mse_gau_r63_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.8636270383153233, -# 'mse_gau_r63_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.8925220988850137, -# 'mse_gau_r63_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.8666087312519845, -# 'mse_gau_r63_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.8905601116453018, -# 'mse_gau_r63_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.8697913786570708, -# 'mse_gau_r63_l1.0_k396.9_norm_sum/lcorr_ground_data': 0.8982607936070446, -# 'mse_gau_r63_l1.0_k396.9_norm_sum/rcorr_ground_data': 0.8757921515353694, -# 'mse_gau_r63_l1.0_k3969.0_norm_sum/lcorr_ground_data': 0.891751398475015, # GOOD -- probs use this? -# 'mse_gau_r63_l1.0_k3969.0_norm_sum/rcorr_ground_data': 0.8694677920447761} # GOOD -- probs use this? - -# x_R: (XY8) - -# {'mse_xy8_abs63/lcorr_ground_data': 0.9985332226841734, -# 'mse_xy8_abs63/rcorr_ground_data': 0.987917449596426, -# 'mse_xy8_abs63_l1.0_k1.0_norm_none/lcorr_ground_data': 0.9985750634395287, -# 'mse_xy8_abs63_l1.0_k1.0_norm_none/rcorr_ground_data': 0.9877437598037485, -# 'mse_xy8_abs63_l1.0_k1.0_norm_sum/lcorr_ground_data': 0.5189493011290895, -# 'mse_xy8_abs63_l1.0_k1.0_norm_sum/rcorr_ground_data': 0.9910276792501457, -# 'mse_xy8_abs63_l1.0_k10.0_norm_none/lcorr_ground_data': 0.9999575954417714, -# 'mse_xy8_abs63_l1.0_k10.0_norm_none/rcorr_ground_data': 0.9879777880324067, -# 'mse_xy8_abs63_l1.0_k10.0_norm_sum/lcorr_ground_data': 0.5269050272967563, -# 'mse_xy8_abs63_l1.0_k10.0_norm_sum/rcorr_ground_data': 0.9884123616001718, -# 'mse_xy8_abs63_l1.0_k100.0_norm_none/lcorr_ground_data': 0.9999304232258539, -# 'mse_xy8_abs63_l1.0_k100.0_norm_none/rcorr_ground_data': 0.9881616858702648, -# 'mse_xy8_abs63_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.5288342188349279, -# 'mse_xy8_abs63_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.9883748032254788, -# 'mse_xy8_abs63_l1.0_k1000.0_norm_none/lcorr_ground_data': 0.999926314488479, -# 'mse_xy8_abs63_l1.0_k1000.0_norm_none/rcorr_ground_data': 0.9880581865880144, -# 'mse_xy8_abs63_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.5269727435607732, -# 'mse_xy8_abs63_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.9878274986638773, -# 'mse_xy8_abs63_l1.0_k10000.0_norm_none/lcorr_ground_data': 0.9999250116548125, -# 'mse_xy8_abs63_l1.0_k10000.0_norm_none/rcorr_ground_data': 0.9878312192520812, -# 'mse_xy8_abs63_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.6102514374734425, -# 'mse_xy8_abs63_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.9878245763300847, -# 'mse_xy8_abs63_norm_none/lcorr_ground_data': 0.9985781813175294, -# 'mse_xy8_abs63_norm_none/rcorr_ground_data': 0.9880679193468579} - -# random: (XY8) - -# {'mse_xy8_abs63/lcorr_ground_data': 0.9993431447352555, # SHOULD USE THIS! (SAME) -# 'mse_xy8_abs63/rcorr_ground_data': 0.9981063918424973, # SHOULD USE THIS! (SAME) -# 'mse_xy8_abs63_l1.0_k1.0_norm_none/lcorr_ground_data': 0.9993355789889566, # SHOULD USE THIS! -# 'mse_xy8_abs63_l1.0_k1.0_norm_none/rcorr_ground_data': 0.9981534934650234, # SHOULD USE THIS! -# 'mse_xy8_abs63_l1.0_k1.0_norm_sum/lcorr_ground_data': 0.5575269078895938, -# 'mse_xy8_abs63_l1.0_k1.0_norm_sum/rcorr_ground_data': 0.9802105251456182, -# 'mse_xy8_abs63_l1.0_k10.0_norm_none/lcorr_ground_data': 0.9999006262214326, -# 'mse_xy8_abs63_l1.0_k10.0_norm_none/rcorr_ground_data': 0.9981098636624177, -# 'mse_xy8_abs63_l1.0_k10.0_norm_sum/lcorr_ground_data': 0.5633754524830307, -# 'mse_xy8_abs63_l1.0_k10.0_norm_sum/rcorr_ground_data': 0.9790422482307086, -# 'mse_xy8_abs63_l1.0_k100.0_norm_none/lcorr_ground_data': 0.999896386933507, -# 'mse_xy8_abs63_l1.0_k100.0_norm_none/rcorr_ground_data': 0.9981110027949393, -# 'mse_xy8_abs63_l1.0_k100.0_norm_sum/lcorr_ground_data': 0.5638532721375747, -# 'mse_xy8_abs63_l1.0_k100.0_norm_sum/rcorr_ground_data': 0.9809908504654221, -# 'mse_xy8_abs63_l1.0_k1000.0_norm_none/lcorr_ground_data': 0.9998916501321604, -# 'mse_xy8_abs63_l1.0_k1000.0_norm_none/rcorr_ground_data': 0.9980985297573617, -# 'mse_xy8_abs63_l1.0_k1000.0_norm_sum/lcorr_ground_data': 0.5664912801305914, -# 'mse_xy8_abs63_l1.0_k1000.0_norm_sum/rcorr_ground_data': 0.9813511742194941, -# 'mse_xy8_abs63_l1.0_k10000.0_norm_none/lcorr_ground_data': 0.9998963655043154, -# 'mse_xy8_abs63_l1.0_k10000.0_norm_none/rcorr_ground_data': 0.9981026861256567, -# 'mse_xy8_abs63_l1.0_k10000.0_norm_sum/lcorr_ground_data': 0.6673302758366136, -# 'mse_xy8_abs63_l1.0_k10000.0_norm_sum/rcorr_ground_data': 0.9807396889725206, -# 'mse_xy8_abs63_norm_none/lcorr_ground_data': 0.9993412106533186, # SHOULD USE THIS! (SAME) -# 'mse_xy8_abs63_norm_none/rcorr_ground_data': 0.9981519882968966} # SHOULD USE THIS! (SAME) - -# ========================================================================= # -# xy8_abs31 # -# ========================================================================= # - -# xy_squares: x_R -# {'linear_corr (xy8)': 0.9678735771797711, -# 'mse_xy8_abs31/lcorr_ground_data': 0.9687208658219576, -# 'mse_xy8_abs31/rcorr_ground_data': 0.9783804430856882, -# 'mse_xy8_abs31_l1.0_k10.0_norm_none/lcorr_ground_data': 0.9682637635856579, -# 'mse_xy8_abs31_l1.0_k10.0_norm_none/rcorr_ground_data': 0.9783654952815923, -# 'mse_xy8_abs31_l1.0_k100.0_norm_none/lcorr_ground_data': 0.968611081840099, -# 'mse_xy8_abs31_l1.0_k100.0_norm_none/rcorr_ground_data': 0.9785962279455712, -# 'mse_xy8_abs31_l1.0_k1000.0_norm_none/lcorr_ground_data': 0.9692789956253065, -# 'mse_xy8_abs31_l1.0_k1000.0_norm_none/rcorr_ground_data': 0.978977519791299, -# 'mse_xy8_abs31_l1.0_k10000.0_norm_none/lcorr_ground_data': 0.9689240381154687, -# 'mse_xy8_abs31_l1.0_k10000.0_norm_none/rcorr_ground_data': 0.9788376129373869, -# 'mse_xy8_abs31_norm_none/lcorr_ground_data': 0.9685526867870904, -# 'mse_xy8_abs31_norm_none/rcorr_ground_data': 0.9783801251042628, -# 'rank_corr (xy8)': 0.9781558343577775} - -# xy_squares: random -# {'linear_corr (xy8)': 0.9642915959954084, -# 'mse_xy8_abs31/lcorr_ground_data': 0.9643131940713038, -# 'mse_xy8_abs31/rcorr_ground_data': 0.9589865502353624, -# 'mse_xy8_abs31_l1.0_k10.0_norm_none/lcorr_ground_data': 0.9632682760808084, -# 'mse_xy8_abs31_l1.0_k10.0_norm_none/rcorr_ground_data': 0.9569984619564749, -# 'mse_xy8_abs31_l1.0_k100.0_norm_none/lcorr_ground_data': 0.9656460246275431, -# 'mse_xy8_abs31_l1.0_k100.0_norm_none/rcorr_ground_data': 0.9596193126172062, -# 'mse_xy8_abs31_l1.0_k1000.0_norm_none/lcorr_ground_data': 0.9646238691347586, -# 'mse_xy8_abs31_l1.0_k1000.0_norm_none/rcorr_ground_data': 0.9590847767760894, -# 'mse_xy8_abs31_l1.0_k10000.0_norm_none/lcorr_ground_data': 0.9652572250944952, -# 'mse_xy8_abs31_l1.0_k10000.0_norm_none/rcorr_ground_data': 0.9591961345585253, -# 'mse_xy8_abs31_norm_none/lcorr_ground_data': 0.9641075187530113, -# 'mse_xy8_abs31_norm_none/rcorr_ground_data': 0.9590779695430883, -# 'rank_corr (xy8)': 0.9593545150637656} - - -# xy_squares: x_R -# {'linear_corr (box)': 0.9566902061341364, -# 'linear_corr (gau)': 0.8421729148631358, -# 'linear_corr (mse)': 0.5229777048796336, -# 'linear_corr (xy8)': 0.9668309669291733, -# 'rank_corr (box)': 0.9724203612164438, -# 'rank_corr (gau)': 0.8791057586183358, -# 'rank_corr (mse)': 0.5812948243581433, -# 'rank_corr (xy8)': 0.9776622360136354} -# xy_squares: random -# {'linear_corr (box)': 0.9450759442133024, -# 'linear_corr (gau)': 0.7248028528773823, -# 'linear_corr (mse)': 0.55513621759871, -# 'linear_corr (xy8)': 0.9648509660103067, -# 'rank_corr (box)': 0.9352893421589664, -# 'rank_corr (gau)': 0.6144250118933472, -# 'rank_corr (mse)': 0.36447057769668934, -# 'rank_corr (xy8)': 0.9595310090244852} - - -# [XYSquares-8-8] Factor Name & Factor Size & rank_corr (gau) & rank_corr (mse) & linear_corr (mse) & linear_corr (box) & linear_corr (gau) & rank_corr (xy8) & linear_corr (xy8) & rank_corr (box) -# [XYSquares-8-8] x_R & 8 & 0.88 & 0.58 & 0.52 & 0.96 & 0.84 & 0.98 & 0.97 & 0.97 -# [XYSquares-8-8] random & 262144 & 0.61 & 0.36 & 0.56 & 0.95 & 0.72 & 0.96 & 0.96 & 0.94 - - -# MANUAL EDIT -- COMBINED WITH "COMPARE KERNELS -- NEW" ABOVE! - -# Loss Name & Linear Corr. (factor) & Rank Corr. (factor) & Linear Corr. (random) & Rank Corr. (random) -# MSE & 0.52 & 0.58 & 0.55 & 0.36 -# MSE (Gau-Kernel) & 0.84 & 0.88 & 0.73 & 0.61 -# MSE (Box-Kernel) & 0.96 & 0.97 & 0.95 & 0.94 -# MSE (XY8-Kernel) & 0.97 & 0.98 & 0.96 & 0.96 - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/part01_data_overlap/plot02_data_distances/run_plot_global_dists.py b/research/part01_data_overlap/plot02_data_distances/run_plot_global_dists.py deleted file mode 100644 index 8782bf5b..00000000 --- a/research/part01_data_overlap/plot02_data_distances/run_plot_global_dists.py +++ /dev/null @@ -1,479 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - - -import os -from collections import defaultdict -from typing import Dict - -import seaborn as sns -import numpy as np -import pandas as pd -import torch -from matplotlib import pyplot as plt -from matplotlib.ticker import MultipleLocator -from tqdm import tqdm - -import research.code.util as H -from disent.dataset import DisentDataset -from disent.dataset.data import Cars3d64Data -from disent.dataset.data import DSpritesData -from disent.dataset.data import Shapes3dData -from disent.dataset.data import SmallNorb64Data -from disent.dataset.data import XYObjectData -from disent.dataset.data import XYObjectShadedData -from research.code.dataset.data import XYSquaresData -from disent.dataset.transform import ToImgTensorF32 -from disent.util import to_numpy -from disent.util.function import wrapped_partial - - -# ========================================================================= # -# plot # -# ========================================================================= # - - -def plot_overlap(a, b, mode='abs'): - a, b = np.transpose(to_numpy(a), (1, 2, 0)), np.transpose(to_numpy(b), (1, 2, 0)) - if mode == 'binary': - d = np.float32(a != b) - elif mode == 'abs': - d = np.abs(a - b) - elif mode == 'diff': - d = a - b - else: - raise KeyError - d = (d - d.min()) / (d.max() - d.min()) - a, b, d = np.uint8(a * 255), np.uint8(b * 255), np.uint8(d * 255) - fig, (ax_a, ax_b, ax_d) = plt.subplots(1, 3) - ax_a.imshow(a) - ax_b.imshow(b) - ax_d.imshow(d) - plt.show() - - -# ========================================================================= # -# CORE # -# ========================================================================= # - - -def generate_data(dataset: DisentDataset, data_name: str, batch_size=64, samples=100_000, plot_diffs=False, load_cache=True, save_cache=True, overlap_loss: str = 'mse'): - # cache - file_path = os.path.join(os.path.dirname(__file__), f'cache/{data_name}_{samples}.pkl') - if load_cache: - if os.path.exists(file_path): - print(f'loaded: {file_path}') - return pd.read_pickle(file_path, compression='gzip') - - # generate - with torch.no_grad(): - # dataframe - df = defaultdict(lambda: defaultdict(list)) - - # randomly overlapped data - name = 'random' - for i in tqdm(range((samples + (batch_size-1) - 1) // (batch_size-1)), desc=f'{data_name}: {name}'): - # get random batch of unique elements - idxs = H.sample_unique_batch_indices(num_obs=len(dataset), num_samples=batch_size) - batch = dataset.dataset_batch_from_indices(idxs, mode='input') - # plot - if plot_diffs and (i == 0): - plot_overlap(batch[0], batch[1]) - # store overlap results - o = to_numpy(H.pairwise_overlap(batch[:-1], batch[1:], mode=overlap_loss)) - df[True][name].extend(o) - df[False][name].extend(o) - - # traversal overlaps - for f_idx in range(dataset.gt_data.num_factors): - name = f'f_{dataset.gt_data.factor_names[f_idx]}' - for i in tqdm(range((samples + (dataset.gt_data.factor_sizes[f_idx] - 1) - 1) // (dataset.gt_data.factor_sizes[f_idx] - 1)), desc=f'{data_name}: {name}'): - # get random batch that is a factor traversal - factors = dataset.gt_data.sample_random_factor_traversal(f_idx) - batch = dataset.dataset_batch_from_factors(factors, mode='input') - # shuffle indices - idxs = np.arange(len(factors)) - np.random.shuffle(idxs) - # plot - if plot_diffs and (i == 0): plot_overlap(batch[0], batch[1]) - # store overlap results - df[True][name].extend(to_numpy(H.pairwise_overlap(batch[:-1], batch[1:], mode=overlap_loss))) - df[False][name].extend(to_numpy(H.pairwise_overlap(batch[idxs[:-1]], batch[idxs[1:]], mode=overlap_loss))) - - # make dataframe! - df = pd.DataFrame({ - 'overlap': [d for ordered, data in df.items() for name, dat in data.items() for d in dat], - 'samples': [name for ordered, data in df.items() for name, dat in data.items() for d in dat], - 'ordered': [ordered for ordered, data in df.items() for name, dat in data.items() for d in dat], - 'data': [data_name for ordered, data in df.items() for name, dat in data.items() for d in dat], - }) - - # save into cache - if save_cache: - os.makedirs(os.path.dirname(file_path), exist_ok=True) - df.to_pickle(file_path, compression='gzip') - print(f'cached: {file_path}') - - return df - - -# ========================================================================= # -# plotting # -# ========================================================================= # - - -def dual_plot_from_generated_data(df: pd.DataFrame, data_name: str = None, save_name: str = None, tick_size: float = None, fig_l_pad=1, fig_w=7, fig_h=13): - # make subplots - cm = 1 / 2.54 - fig, (ax0, ax1) = plt.subplots(1, 2, figsize=((fig_l_pad+2*fig_w)*cm, fig_h*cm)) - if data_name is not None: - fig.suptitle(data_name, fontsize=20) - ax0.set_ylim(-0.025, 1.025) - ax1.set_ylim(-0.025, 1.025) - # plot - ax0.set_title('Ordered Traversals') - sns.ecdfplot(ax=ax0, data=df[df['ordered']==True], x="distance", hue="samples") - ax1.set_title('Shuffled Traversals') - sns.ecdfplot(ax=ax1, data=df[df['ordered']==False], x="distance", hue="samples") - # edit plots - ax0.set_xlabel('Visual Distance') - ax1.set_xlabel('Visual Distance') - if tick_size is not None: - ax0.xaxis.set_major_locator(MultipleLocator(base=tick_size)) - ax1.xaxis.set_major_locator(MultipleLocator(base=tick_size)) - # ax0.xaxis.set_major_formatter(FormatStrFormatter('%.2f')) - # ax1.xaxis.set_major_formatter(FormatStrFormatter('%.2f')) - ax0.set_ylabel('Cumulative Proportion') - ax1.set_ylabel(None) - ax1.set_yticklabels([]) - ax1.get_legend().remove() - plt.tight_layout() - # save - if save_name is not None: - path = os.path.join(os.path.dirname(__file__), 'plots', save_name) - os.makedirs(os.path.dirname(path), exist_ok=True) - plt.savefig(path) - print(f'saved: {path}') - # show - return fig - - -def all_plot_from_all_generated_data(dfs: dict, ordered=True, save_name: str = None, tick_sizes: Dict[str, float] = None, hide_extra_legends=False, fig_l_pad=1, fig_w=7, fig_h=13): - if not dfs: - return None - # make subplots - cm = 1 / 2.54 - fig, axs = plt.subplots(1, len(dfs), figsize=((fig_l_pad+len(dfs)*fig_w)*cm, fig_h * cm)) - axs = np.array(axs, dtype=object).reshape((-1,)) - # plot all - for i, (ax, (data_name, df)) in enumerate(zip(axs, dfs.items())): - # plot - ax.set_title(data_name) - sns.ecdfplot(ax=ax, data=df[df['ordered']==ordered], x="distance", hue="samples") - # edit plots - ax.set_ylim(-0.025, 1.025) - ax.set_xlabel('Visual Distance') - if (tick_sizes is not None) and (data_name in tick_sizes): - ax.xaxis.set_major_locator(MultipleLocator(base=tick_sizes[data_name])) - if i == 0: - ax.set_ylabel('Cumulative Proportion') - else: - if hide_extra_legends: - ax.get_legend().remove() - ax.set_ylabel(None) - ax.set_yticklabels([]) - plt.tight_layout() - # save - if save_name is not None: - path = os.path.join(os.path.dirname(__file__), 'plots', save_name) - os.makedirs(os.path.dirname(path), exist_ok=True) - plt.savefig(path) - print(f'saved: {path}') - # show - return fig - - -def plot_all(exp_name: str, gt_data_classes, tick_sizes: dict, samples: int, load=True, save=True, show_plt=True, show_dual_plt=False, save_plt=True, hide_extra_legends=False, fig_l_pad=1, fig_w=7, fig_h=13): - # generate data and plot! - dfs = {} - for data_name, data_cls in gt_data_classes.items(): - df = generate_data( - DisentDataset(data_cls(), transform=ToImgTensorF32()), - data_name, - batch_size=64, - samples=samples, - plot_diffs=False, - load_cache=load, - save_cache=save, - ) - dfs[data_name] = df - # flip overlap - df['distance'] = - df['overlap'] - del df['overlap'] - # plot ordered + shuffled - fig = dual_plot_from_generated_data( - df, - data_name=data_name, - save_name=f'{exp_name}/{data_name}_{samples}.png' if save_plt else None, - tick_size=tick_sizes.get(data_name, None), - fig_l_pad=fig_l_pad, - fig_w=fig_w, - fig_h=fig_h, - ) - - if show_dual_plt: - plt.show() - else: - plt.close(fig) - - def _all_plot_generated(dfs, ordered: bool, suffix: str): - fig = all_plot_from_all_generated_data( - dfs, - ordered=ordered, - save_name=f'{exp_name}/{exp_name}-{"ordered" if ordered else "shuffled"}{suffix}.png' if save_plt else None, - tick_sizes=tick_sizes, - hide_extra_legends=hide_extra_legends, - fig_l_pad=fig_l_pad, - fig_w=fig_w, - fig_h=fig_h, - ) - if show_plt: - plt.show() - else: - plt.close(fig) - - # all ordered plots - _all_plot_generated(dfs, ordered=True, suffix='') - _all_plot_generated({k: v for k, v in dfs.items() if k.lower().startswith('xy')}, ordered=True, suffix='-xy') - _all_plot_generated({k: v for k, v in dfs.items() if not k.lower().startswith('xy')}, ordered=True, suffix='-normal') - # all shuffled plots - _all_plot_generated(dfs, ordered=False, suffix='') - _all_plot_generated({k: v for k, v in dfs.items() if k.lower().startswith('xy')}, ordered=False, suffix='-xy') - _all_plot_generated({k: v for k, v in dfs.items() if not k.lower().startswith('xy')}, ordered=False, suffix='-normal') - # done! - return dfs - - -def plot_dfs_stacked(dfs, title: str, save_name: str = None, show_plt=True, tick_size: float = None, fig_l_pad=1, fig_w=7, fig_h=13, **kwargs): - # make new dataframe - df = pd.concat((df[df['samples']=='random'] for df in dfs.values())) - # make plot - cm = 1 / 2.54 - fig, ax = plt.subplots(1, 1, figsize=((fig_l_pad+1*fig_w)*cm, fig_h*cm)) - ax.set_title(title) - # plot - # sns.kdeplot(ax=ax, data=df, x="overlap", hue="data", bw_adjust=2) - sns.ecdfplot(ax=ax, data=df, x="overlap", hue="data") - # edit settins - # ax.set_ylim(-0.025, 1.025) - ax.set_xlabel('Overlap') - if tick_size is not None: - ax.xaxis.set_major_locator(MultipleLocator(base=tick_size)) - ax.set_ylabel('Cumulative Proportion') - plt.tight_layout() - # save - if save_name is not None: - path = os.path.join(os.path.dirname(__file__), 'plots', save_name) - os.makedirs(os.path.dirname(path), exist_ok=True) - plt.savefig(path) - print(f'saved: {path}') - # show - if show_plt: - plt.show() - else: - plt.close(fig) - - -def plot_unique_count(dfs, save_name: str = None, show_plt: bool = True, fig_l_pad=1, fig_w=1.5*7, fig_h=13): - df_uniques = pd.DataFrame({ - 'Grid Spacing': ['/'.join(data_name.split('-')[1:]) for data_name, df in dfs.items()], - 'Unique Overlap Values': [len(np.unique(df['overlap'].values, return_counts=True)[1]) for data_name, df in dfs.items()] - }) - # make plot - cm = 1 / 2.54 - fig, ax = plt.subplots(1, 1, figsize=((fig_l_pad+fig_w)*cm, fig_h*cm)) - ax.set_title('Increasing Overlap') - sns.barplot(data=df_uniques, x='Grid Spacing', y='Unique Overlap Values') - plt.gca().invert_xaxis() - plt.tight_layout() - # save - if save_name is not None: - path = os.path.join(os.path.dirname(__file__), 'plots', save_name) - os.makedirs(os.path.dirname(path), exist_ok=True) - plt.savefig(path) - print(f'saved: {path}') - # show - if show_plt: - plt.show() - else: - plt.close(fig) - - -# ========================================================================= # -# entrypoint # -# ========================================================================= # - - -if __name__ == '__main__': - - # TODO: update to new classes - # TODO: update to use registry - - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../../code/util/gadfly.mplstyle')) - - # common settings - SHARED_SETTINGS = dict( - samples=50_000, - load=True, - save=True, - show_plt=True, - save_plt=True, - show_dual_plt=False, - fig_l_pad=1, - fig_w=5.5, - fig_h=13, - tick_sizes={ - 'DSprites': 0.05, - 'Shapes3d': 0.2, - 'Cars3d': 0.05, - 'XYSquares': 0.01, - # increasing levels of overlap - 'XYSquares-1': 0.01, - 'XYSquares-2': 0.01, - 'XYSquares-3': 0.01, - 'XYSquares-4': 0.01, - 'XYSquares-5': 0.01, - 'XYSquares-6': 0.01, - 'XYSquares-7': 0.01, - 'XYSquares-8': 0.01, - # increasing levels of overlap 2 - 'XYSquares-1-8': 0.01, - 'XYSquares-2-8': 0.01, - 'XYSquares-3-8': 0.01, - 'XYSquares-4-8': 0.01, - 'XYSquares-5-8': 0.01, - 'XYSquares-6-8': 0.01, - 'XYSquares-7-8': 0.01, - 'XYSquares-8-8': 0.01, - }, - ) - - # EXPERIMENT 0 -- visual overlap on existing datasets - - dfs = plot_all( - exp_name='dataset-overlap', - gt_data_classes={ - # 'XYBlocks': wrapped_partial(XYBlocksData), - 'XYSquares': wrapped_partial(XYSquaresData), - 'DSprites': wrapped_partial(DSpritesData), - 'Shapes3d': wrapped_partial(Shapes3dData), - 'Cars3d': wrapped_partial(Cars3d64Data), - 'SmallNorb': wrapped_partial(SmallNorb64Data), - # 'Mpi3d': wrapped_partial(Mpi3dData), - }, - hide_extra_legends=False, - **{**SHARED_SETTINGS, 'fig_h': 11} - ) - - # EXPERIMENT -- p03e03 - - dfs = plot_all( - exp_name='differing-gt-factors', - gt_data_classes={ - 'XYObject': wrapped_partial(XYObjectData), - 'XYObjectShaded': wrapped_partial(XYObjectShadedData), - }, - hide_extra_legends=False, - **SHARED_SETTINGS - ) - - # EXPERIMENT 1 -- increasing visual overlap - - dfs = plot_all( - exp_name='increasing-overlap', - gt_data_classes={ - 'XYSquares-1': wrapped_partial(XYSquaresData, grid_spacing=1), - 'XYSquares-2': wrapped_partial(XYSquaresData, grid_spacing=2), - 'XYSquares-3': wrapped_partial(XYSquaresData, grid_spacing=3), - 'XYSquares-4': wrapped_partial(XYSquaresData, grid_spacing=4), - 'XYSquares-5': wrapped_partial(XYSquaresData, grid_spacing=5), - 'XYSquares-6': wrapped_partial(XYSquaresData, grid_spacing=6), - 'XYSquares-7': wrapped_partial(XYSquaresData, grid_spacing=7), - 'XYSquares-8': wrapped_partial(XYSquaresData, grid_spacing=8), - }, - hide_extra_legends=True, - **SHARED_SETTINGS - ) - - plot_unique_count( - dfs=dfs, - save_name='increasing-overlap/xysquares-increasing-overlap-counts.png', - ) - - plot_dfs_stacked( - dfs=dfs, - title='Increasing Overlap', - exp_name='increasing-overlap', - save_name='increasing-overlap/xysquares-increasing-overlap.png', - tick_size=0.01, - fig_w=13 - ) - - # EXPERIMENT 2 -- increasing visual overlap fixed dim size - - dfs = plot_all( - exp_name='increasing-overlap-fixed', - gt_data_classes={ - 'XYSquares-1-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=1, grid_size=8), - 'XYSquares-2-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=2, grid_size=8), - 'XYSquares-3-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=3, grid_size=8), - 'XYSquares-4-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=4, grid_size=8), - 'XYSquares-5-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=5, grid_size=8), - 'XYSquares-6-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=6, grid_size=8), - 'XYSquares-7-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=7, grid_size=8), - 'XYSquares-8-8': wrapped_partial(XYSquaresData, square_size=8, grid_spacing=8, grid_size=8), - }, - hide_extra_legends=True, - **SHARED_SETTINGS - ) - - plot_unique_count( - dfs=dfs, - save_name='increasing-overlap-fixed/xysquares-increasing-overlap-fixed-counts.png', - ) - - plot_dfs_stacked( - dfs=dfs, - title='Increasing Overlap', - exp_name='increasing-overlap-fixed', - save_name='increasing-overlap-fixed/xysquares-increasing-overlap-fixed.png', - tick_size=0.01, - fig_w=13 - ) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/part01_data_overlap/plot02_data_distances/run_plot_traversal_dists.py b/research/part01_data_overlap/plot02_data_distances/run_plot_traversal_dists.py deleted file mode 100644 index f2313217..00000000 --- a/research/part01_data_overlap/plot02_data_distances/run_plot_traversal_dists.py +++ /dev/null @@ -1,683 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import os -from collections import defaultdict -from typing import Any -from typing import Callable -from typing import Dict -from typing import List -from typing import Literal -from typing import Optional -from typing import Sequence -from typing import Tuple -from typing import Union - -import imageio -import matplotlib.pyplot as plt -import numpy as np -import torch -import torch.nn.functional as F -from tqdm import tqdm - -import research.code.util as H -from disent.dataset.data import GroundTruthData -from disent.dataset.data import SelfContainedHdf5GroundTruthData -from disent.dataset.util.state_space import NonNormalisedFactors -from disent.dataset.transform import ToImgTensorF32 -from disent.dataset.util.stats import compute_data_mean_std -from disent.util.inout.paths import ensure_parent_dir_exists -from disent.util.seeds import TempNumpySeed - - -# ========================================================================= # -# Factor Traversal Stats # -# ========================================================================= # -from disent.util.visualize.vis_util import make_image_grid - - -SampleModeHint = Union[Literal['random'], Literal['near'], Literal['combinations']] - - -@torch.no_grad() -def sample_factor_traversal_info( - gt_data: GroundTruthData, - f_idx: Optional[int] = None, - circular_distance: bool = False, - sample_mode: SampleModeHint = 'random', -) -> dict: - # load traversal -- TODO: this is the bottleneck! not threaded - factors, indices, obs = gt_data.sample_random_obs_traversal(f_idx=f_idx, obs_collect_fn=torch.stack) - # get pairs - idxs_a, idxs_b = H.pair_indices(max_idx=len(indices), mode=sample_mode) - # compute deltas - deltas = F.mse_loss(obs[idxs_a], obs[idxs_b], reduction='none').mean(dim=[-3, -2, -1]).numpy() - fdists = H.np_factor_dists(factors[idxs_a], factors[idxs_b], factor_sizes=gt_data.factor_sizes, circular_if_factor_sizes=circular_distance, p=1) - # done! - return dict( - # traversals - factors=factors, # np.ndarray - indices=indices, # np.ndarray - obs=obs, # torch.Tensor - # pairs - idxs_a=idxs_a, # np.ndarray - idxs_b=idxs_b, # np.ndarray - deltas=deltas, # np.ndarray - fdists=fdists, # np.ndarray - ) - - -def sample_factor_traversal_info_and_distmat( - gt_data: GroundTruthData, - f_idx: Optional[int] = None, - circular_distance: bool = False, -) -> dict: - dat = sample_factor_traversal_info(gt_data=gt_data, f_idx=f_idx, sample_mode='combinations', circular_distance=circular_distance) - # extract - factors, idxs_a, idxs_b, deltas, fdists = dat['factors'], dat['idxs_a'], dat['idxs_b'], dat['deltas'], dat['fdists'] - # generate deltas matrix - deltas_matrix = np.zeros([factors.shape[0], factors.shape[0]]) - deltas_matrix[idxs_a, idxs_b] = deltas - deltas_matrix[idxs_b, idxs_a] = deltas - # generate distance matrix - fdists_matrix = np.zeros([factors.shape[0], factors.shape[0]]) - fdists_matrix[idxs_a, idxs_b] = fdists - fdists_matrix[idxs_b, idxs_a] = fdists - # done! - return dict(**dat, deltas_matrix=deltas_matrix, fdists_matrix=fdists_matrix) - - -# ========================================================================= # -# Factor Traversal Collector # -# ========================================================================= # - - -def _collect_stats_for_factors( - gt_data: GroundTruthData, - f_idxs: Sequence[int], - stats_fn: Callable[[GroundTruthData, int, int], Dict[str, Any]], - keep_keys: Sequence[str], - stats_callback: Optional[Callable[[Dict[str, List[Any]], int, int], None]] = None, - return_stats: bool = True, - num_traversal_sample: int = 100, -) -> List[Dict[str, List[Any]]]: - # prepare - f_idxs = gt_data.normalise_factor_idxs(f_idxs) - # generate data per factor - f_stats = [] - for i, f_idx in enumerate(f_idxs): - factor_name = gt_data.factor_names[f_idx] - factor_size = gt_data.factor_sizes[f_idx] - # repeatedly generate stats per factor - stats = defaultdict(list) - for _ in tqdm(range(num_traversal_sample), desc=f'{gt_data.name}: {factor_name}'): - data = stats_fn(gt_data, i, f_idx) - for key in keep_keys: - stats[key].append(data[key]) - # save factor stats - if return_stats: - f_stats.append(stats) - if stats_callback: - stats_callback(stats, i, f_idx) - # done! - if return_stats: - return f_stats - - -# ========================================================================= # -# Plot Traversal Stats # -# ========================================================================= # - - -_COLORS = { - 'blue': (None, 'Blues', 'Blues'), - 'red': (None, 'Reds', 'Reds'), - 'purple': (None, 'Purples', 'Purples'), - 'green': (None, 'Greens', 'Greens'), - 'orange': (None, 'Oranges', 'Oranges'), -} - - -def plot_traversal_stats_OLD( - dataset_or_name: Union[str, GroundTruthData], - num_repeats: int = 256, - f_idxs: Optional[NonNormalisedFactors] = None, - circular_distance: bool = False, - color='blue', - color_gt_dist='blue', - color_im_dist='purple', - suffix: Optional[str] = None, - save_path: Optional[str] = None, - plot_freq: bool = True, - plot_title: Union[bool, str] = False, - fig_block_size: float = 4.0, - col_titles: Union[bool, List[str]] = True, - hide_axis: bool = True, - hide_labels: bool = True, - y_size_offset: float = 0.0, - x_size_offset: float = 0.0, -): - # TODO: this function is actually terrible... - # - - - - - - - - - - - - - - - - - # - - def stats_fn(gt_data, i, f_idx): - return sample_factor_traversal_info_and_distmat(gt_data=gt_data, f_idx=f_idx, circular_distance=circular_distance) - - def plot_ax(stats: dict, i: int, f_idx: int): - deltas = np.concatenate(stats['deltas']) - fdists = np.concatenate(stats['fdists']) - fdists_matrix = np.mean(stats['fdists_matrix'], axis=0) - deltas_matrix = np.mean(stats['deltas_matrix'], axis=0) - - # ensure that if we limit the number of points, that we get good values - with TempNumpySeed(777): np.random.shuffle(deltas) - with TempNumpySeed(777): np.random.shuffle(fdists) - - # subplot! - if plot_freq: - ax0, ax1, ax2, ax3 = axs[:, i] - else: - (ax0, ax1), (ax2, ax3) = (None, None), axs[:, i] - - # get title - curr_title = None - if isinstance(col_titles, bool): - if col_titles: - curr_title = gt_data.factor_names[f_idx] - else: - curr_title = col_titles[i] - - # set column titles - if curr_title is not None: - (ax0 if plot_freq else ax2).set_title(f'{curr_title}\n', fontsize=24) - - # plot the frequency stuffs - if plot_freq: - ax0.violinplot([deltas], vert=False) - ax0.set_xlabel('deltas') - ax0.set_ylabel('proportion') - - ax1.set_title('deltas vs. fdists') - ax1.scatter(x=deltas[:15_000], y=fdists[:15_000], s=20, alpha=0.1, c=c_points) - H.plt_2d_density( - x=deltas[:10_000], xmin=deltas.min(), xmax=deltas.max(), - y=fdists[:10_000], ymin=fdists.min() - 0.5, ymax=fdists.max() + 0.5, - n_bins=100, - ax=ax1, pcolormesh_kwargs=dict(cmap=cmap_density, alpha=0.5), - ) - ax1.set_xlabel('deltas') - ax1.set_ylabel('fdists') - - # ax2.set_title('fdists') - ax2.imshow(fdists_matrix, cmap=gt_cmap_img) - if not hide_labels: ax2.set_xlabel('f_idx') - if not hide_labels: ax2.set_ylabel('f_idx') - if hide_axis: H.plt_hide_axis(ax2) - - # ax3.set_title('divergence') - ax3.imshow(deltas_matrix, cmap=im_cmap_img) - if not hide_labels: ax3.set_xlabel('f_idx') - if not hide_labels: ax3.set_ylabel('f_idx') - if hide_axis: H.plt_hide_axis(ax3) - - - # - - - - - - - - - - - - - - - - - # - - # initialize - gt_data: GroundTruthData = H.make_data(dataset_or_name) if isinstance(dataset_or_name, str) else dataset_or_name - f_idxs = gt_data.normalise_factor_idxs(f_idxs) - - c_points, cmap_density, cmap_img = _COLORS[color] - im_c_points, im_cmap_density, im_cmap_img = _COLORS[color if (color_im_dist is None) else color_im_dist] - gt_c_points, gt_cmap_density, gt_cmap_img = _COLORS[color if (color_gt_dist is None) else color_gt_dist] - - n = 4 if plot_freq else 2 - - # get additional spacing - title_offset = 0 if (isinstance(col_titles, bool) and not col_titles) else 0.15 - - # settings - r, c = [n, len(f_idxs)] - h, w = [(n+title_offset)*fig_block_size + y_size_offset, len(f_idxs)*fig_block_size + x_size_offset] - - # initialize plot - fig, axs = plt.subplots(r, c, figsize=(w, h), squeeze=False) - - if isinstance(plot_title, str): - fig.suptitle(f'{plot_title}\n', fontsize=25) - elif plot_title: - fig.suptitle(f'{gt_data.name} [circular={circular_distance}]{f" {suffix}" if suffix else ""}\n', fontsize=25) - - # generate plot - _collect_stats_for_factors( - gt_data=gt_data, - f_idxs=f_idxs, - stats_fn=stats_fn, - keep_keys=['deltas', 'fdists', 'deltas_matrix', 'fdists_matrix'], - stats_callback=plot_ax, - num_traversal_sample=num_repeats, - ) - - # finalize plot - fig.tight_layout() # (pad=1.4 if hide_labels else 1.08) - - # save the path - if save_path is not None: - assert save_path.endswith('.png') - ensure_parent_dir_exists(save_path) - plt.savefig(save_path) - print(f'saved {gt_data.name} to: {save_path}') - - # show it! - plt.show() - - # - - - - - - - - - - - - - - - - - # - return fig - - -# TODO: fix -def plot_traversal_stats( - dataset_or_name: Union[str, GroundTruthData], - num_repeats: int = 256, - f_idxs: Optional[NonNormalisedFactors] = None, - circular_distance: bool = False, - color='blue', - color_gt_dist='blue', - color_im_dist='purple', - suffix: Optional[str] = None, - save_path: Optional[str] = None, - plot_freq: bool = True, - plot_title: Union[bool, str] = False, - plt_scale: float = 6, - col_titles: Union[bool, List[str]] = True, - hide_axis: bool = True, - hide_labels: bool = True, - y_size_offset: float = 0.45, - x_size_offset: float = 0.75, - disable_labels: bool = False, - bottom_labels: bool = False, - label_size: int = 23, - return_dists: bool = True -): - # - - - - - - - - - - - - - - - - - # - - def stats_fn(gt_data, i, f_idx): - return sample_factor_traversal_info_and_distmat( - gt_data=gt_data, f_idx=f_idx, circular_distance=circular_distance - ) - - grid_t = [] - grid_titles = [] - - def plot_ax(stats: dict, i: int, f_idx: int): - fdists_matrix = np.mean(stats['fdists_matrix'], axis=0) - deltas_matrix = np.mean(stats['deltas_matrix'], axis=0) - grid_t.append([fdists_matrix, deltas_matrix]) - # get the title - if isinstance(col_titles, bool): - if col_titles: - grid_titles.append(gt_data.factor_names[f_idx]) - else: - grid_titles.append(col_titles[i]) - - # initialize - gt_data: GroundTruthData = H.make_data(dataset_or_name) if isinstance(dataset_or_name, str) else dataset_or_name - f_idxs = gt_data.normalise_factor_idxs(f_idxs) - - # get title - if isinstance(plot_title, str): - suptitle = f'{plot_title}' - elif plot_title: - suptitle = f'{gt_data.name} [circular={circular_distance}]{f" {suffix}" if suffix else ""}' - else: - suptitle = None - - # generate plot - _collect_stats_for_factors( - gt_data=gt_data, - f_idxs=f_idxs, - stats_fn=stats_fn, - keep_keys=['deltas', 'fdists', 'deltas_matrix', 'fdists_matrix'], - stats_callback=plot_ax, - num_traversal_sample=num_repeats, - ) - - labels = None - if (not disable_labels) and grid_titles: - labels = grid_titles - - # settings - fig, axs = H.plt_subplots_imshow( - grid=list(zip(*grid_t)), - title=suptitle, - titles=None if bottom_labels else labels, - titles_size=label_size, - col_labels=labels if bottom_labels else None, - label_size=label_size, - subplot_padding=None, - figsize=((1/2.54) * len(f_idxs) * plt_scale + x_size_offset, (1/2.54) * (2) * plt_scale + y_size_offset) - ) - - # recolor axes - for (ax0, ax1) in axs.T: - ax0.images[0].set_cmap('Blues') - ax1.images[0].set_cmap('Purples') - - fig.tight_layout() - - # save the path - if save_path is not None: - assert save_path.endswith('.png') - ensure_parent_dir_exists(save_path) - plt.savefig(save_path) - print(f'saved {gt_data.name} to: {save_path}') - - # show it! - plt.show() - - # - - - - - - - - - - - - - - - - - # - if return_dists: - return fig, grid_t, grid_titles - return fig - - -# ========================================================================= # -# MAIN - DISTS # -# ========================================================================= # - - -@torch.no_grad() -def factor_stats(gt_data: GroundTruthData, f_idxs=None, min_samples: int = 100_000, min_repeats: int = 5000, recon_loss: str = 'mse', sample_mode: str = 'random') -> Tuple[Sequence[int], List[np.ndarray]]: - from disent.registry import RECON_LOSSES - from disent.frameworks.helper.reconstructions import ReconLossHandler - recon_loss: ReconLossHandler = RECON_LOSSES[recon_loss](reduction='mean') - - f_dists = [] - f_idxs = gt_data.normalise_factor_idxs(f_idxs) - # for each factor - for f_idx in f_idxs: - dists = [] - with tqdm(desc=gt_data.factor_names[f_idx], total=min_samples) as p: - # for multiple random factor traversals along the factor - while len(dists) < min_samples or p.n < min_repeats: - # based on: sample_factor_traversal_info(...) # TODO: should add recon loss to that function instead - factors, indices, obs = gt_data.sample_random_obs_traversal(f_idx=f_idx, obs_collect_fn=torch.stack) - # random pairs -- we use this because it does not include [i == i] - idxs_a, idxs_b = H.pair_indices(max_idx=len(indices), mode=sample_mode) - # get distances - d = recon_loss.compute_pairwise_loss(obs[idxs_a], obs[idxs_b]) - d = d.numpy().tolist() - # H.plt_subplots_imshow([[np.moveaxis(o.numpy(), 0, -1) for o in obs]]) - # plt.show() - dists.extend(d) - p.update(len(d)) - # aggregate the average distances - f_dists.append(np.array(dists)[:min_samples]) - - return f_idxs, f_dists - - -def get_random_dists(gt_data: GroundTruthData, num_samples: int = 100_000, recon_loss: str = 'mse'): - from disent.registry import RECON_LOSSES - from disent.frameworks.helper.reconstructions import ReconLossHandler - recon_loss: ReconLossHandler = RECON_LOSSES[recon_loss](reduction='mean') - - dists = [] - with tqdm(desc=gt_data.name, total=num_samples) as p: - # for multiple random factor traversals along the factor - while len(dists) < num_samples: - # random pair - i, j = np.random.randint(0, len(gt_data), size=2) - # get distance - d = recon_loss.compute_pairwise_loss(gt_data[i][None, ...], gt_data[j][None, ...]) - # plt.show() - dists.append(float(d.flatten())) - p.update() - # done! - return np.array(dists) - - -def print_ave_dists(gt_data: GroundTruthData, num_samples: int = 100_000, recon_loss: str = 'mse'): - dists = get_random_dists(gt_data=gt_data, num_samples=num_samples, recon_loss=recon_loss) - f_mean = np.mean(dists) - f_std = np.std(dists) - print(f'[{gt_data.name}] RANDOM ({len(gt_data)}, {len(dists)}) - mean: {f_mean:7.4f} std: {f_std:7.4f}') - - -def print_ave_factor_stats(gt_data: GroundTruthData, f_idxs=None, min_samples: int = 100_000, min_repeats: int = 5000, recon_loss: str = 'mse', sample_mode: str = 'random'): - # compute average distances - f_idxs, f_dists = factor_stats(gt_data=gt_data, f_idxs=f_idxs, min_repeats=min_repeats, min_samples=min_samples, recon_loss=recon_loss, sample_mode=sample_mode) - # compute dists - f_means = [np.mean(d) for d in f_dists] - f_stds = [np.std(d) for d in f_dists] - # sort in order of importance - order = np.argsort(f_means)[::-1] - # print information - for i in order: - f_idx, f_mean, f_std = f_idxs[i], f_means[i], f_stds[i] - print(f'[{gt_data.name}] {gt_data.factor_names[f_idx]} ({gt_data.factor_sizes[f_idx]}, {len(f_dists[f_idx])}) - mean: {f_mean:7.4f} std: {f_std:7.4f}') - - -def main_compute_dists(factor_samples: int = 50_000, min_repeats: int = 5000, random_samples: int = 50_000, recon_loss: str = 'mse', sample_mode: str = 'random', seed: int = 777): - # plot standard datasets - for name in ['dsprites', 'shapes3d', 'cars3d', 'smallnorb', 'xysquares_8x8_s8', 'xyobject', 'xyobject_shaded']: - gt_data = H.make_data(name) - if factor_samples is not None: - with TempNumpySeed(seed): - print_ave_factor_stats(gt_data, min_samples=factor_samples, min_repeats=min_repeats, recon_loss=recon_loss, sample_mode=sample_mode) - if random_samples is not None: - with TempNumpySeed(seed): - print_ave_dists(gt_data, num_samples=random_samples, recon_loss=recon_loss) - -# [dsprites] position_y (32, 50000) - mean: 0.0584 std: 0.0378 -# [dsprites] position_x (32, 50000) - mean: 0.0559 std: 0.0363 -# [dsprites] scale (6, 50000) - mean: 0.0250 std: 0.0148 -# [dsprites] shape (3, 50000) - mean: 0.0214 std: 0.0095 -# [dsprites] orientation (40, 50000) - mean: 0.0172 std: 0.0106 -# [dsprites] RANDOM (737280, 50000) - mean: 0.0754 std: 0.0289 - -# [3dshapes] wall_hue (10, 50000) - mean: 0.1122 std: 0.0661 -# [3dshapes] floor_hue (10, 50000) - mean: 0.1086 std: 0.0623 -# [3dshapes] object_hue (10, 50000) - mean: 0.0416 std: 0.0292 -# [3dshapes] shape (4, 50000) - mean: 0.0207 std: 0.0161 -# [3dshapes] scale (8, 50000) - mean: 0.0182 std: 0.0153 -# [3dshapes] orientation (15, 50000) - mean: 0.0116 std: 0.0079 -# [3dshapes] RANDOM (480000, 50000) - mean: 0.2432 std: 0.0918 - -# [cars3d] azimuth (24, 50000) - mean: 0.0355 std: 0.0185 -# [cars3d] object_type (183, 50000) - mean: 0.0349 std: 0.0176 -# [cars3d] elevation (4, 50000) - mean: 0.0174 std: 0.0100 -# [cars3d] RANDOM (17568, 50000) - mean: 0.0519 std: 0.0188 - -# [smallnorb] lighting (6, 50000) - mean: 0.0531 std: 0.0563 -# [smallnorb] category (5, 50000) - mean: 0.0113 std: 0.0066 -# [smallnorb] rotation (18, 50000) - mean: 0.0090 std: 0.0071 -# [smallnorb] instance (5, 50000) - mean: 0.0068 std: 0.0048 -# [smallnorb] elevation (9, 50000) - mean: 0.0034 std: 0.0030 -# [smallnorb] RANDOM (24300, 50000) - mean: 0.0535 std: 0.0529 - -# [xy_squares] y_B (8, 50000) - mean: 0.0104 std: 0.0000 -# [xy_squares] x_B (8, 50000) - mean: 0.0104 std: 0.0000 -# [xy_squares] y_G (8, 50000) - mean: 0.0104 std: 0.0000 -# [xy_squares] x_G (8, 50000) - mean: 0.0104 std: 0.0000 -# [xy_squares] y_R (8, 50000) - mean: 0.0104 std: 0.0000 -# [xy_squares] x_R (8, 50000) - mean: 0.0104 std: 0.0000 -# [xy_squares] RANDOM (262144, 50000) - mean: 0.0308 std: 0.0022 - -# [xy_object] y (25, 50000) - mean: 0.0116 std: 0.0139 -# [xy_object] x (25, 50000) - mean: 0.0115 std: 0.0140 -# [xy_object] color (24, 50000) - mean: 0.0088 std: 0.0084 -# [xy_object] scale (5, 50000) - mean: 0.0051 std: 0.0056 -# [xy_object] RANDOM (75000, 50000) - mean: 0.0145 std: 0.0111 - -# [xy_object] y (25, 50000) - mean: 0.0124 std: 0.0140 -# [xy_object] x (25, 50000) - mean: 0.0122 std: 0.0144 -# [xy_object] color (6, 50000) - mean: 0.0090 std: 0.0101 -# [xy_object] scale (5, 50000) - mean: 0.0050 std: 0.0055 -# [xy_object] intensity (4, 50000) - mean: 0.0033 std: 0.0039 -# [xy_object] RANDOM (75000, 50000) - mean: 0.0145 std: 0.0111 - - -def _grid_plot_save(path: str, imgs: Sequence[np.ndarray], show: bool = True): - img = make_image_grid(imgs, pad=0, border=False, num_cols=-1) - H.plt_imshow(img, show=True) - imageio.imsave(path, img) - - -# ========================================================================= # -# MAIN - PLOTTING # -# ========================================================================= # - - -def _make_self_contained_dataset(h5_path): - return SelfContainedHdf5GroundTruthData(h5_path=h5_path, transform=ToImgTensorF32()) - - -def _print_data_mean_std(data_or_name, print_mean_std: bool = True): - if print_mean_std: - data = H.make_data(data_or_name) if isinstance(data_or_name, str) else data_or_name - name = data_or_name if isinstance(data_or_name, str) else data.name - mean, std = compute_data_mean_std(data) - print(f'{name}\n vis_mean: {mean.tolist()}\n vis_std: {std.tolist()}') - - -def main_plotting(plot_all=False, print_mean_std=False): - CIRCULAR = False - PLOT_FREQ = False - - def sp(name): - prefix = 'CIRCULAR_' if CIRCULAR else 'DIST_' - prefix = prefix + ('FREQ_' if PLOT_FREQ else 'NO-FREQ_') - return os.path.join(os.path.dirname(__file__), 'plots', f'{prefix}{name}.png') - - # plot xysquares with increasing overlap - for s in [1, 2, 3, 4, 5, 6, 7, 8]: - plot_traversal_stats(circular_distance=CIRCULAR, plt_scale=8, label_size=26, x_size_offset=0, y_size_offset=0.6, save_path=sp(f'xysquares_8x8_s{s}'), color='blue', dataset_or_name=f'xysquares_8x8_s{s}', f_idxs=[1], col_titles=[f'Space: {s}px'], plot_freq=PLOT_FREQ) - _print_data_mean_std(f'xysquares_8x8_s{s}', print_mean_std) - - # plot xysquares with increasing overlap -- combined into one image - _grid_plot_save(path=sp(f'xysquares_8x8_all'), imgs=[imageio.imread(sp(f'xysquares_8x8_s{s}'))[:, 2:-2, :3] for s in range(1, 9)]) - _grid_plot_save(path=sp(f'xysquares_8x8_some'), imgs=[imageio.imread(sp(f'xysquares_8x8_s{s}'))[:, 2:-2, :3] for s in [1, 2, 4, 8]]) - - # plot standard datasets - for name in ['dsprites', 'shapes3d', 'cars3d', 'smallnorb', 'xyobject', 'xyobject_shaded']: - plot_traversal_stats(circular_distance=CIRCULAR, x_size_offset=0, y_size_offset=0.6, num_repeats=256, disable_labels=False, save_path=sp(name), color='blue', dataset_or_name=name, plot_freq=PLOT_FREQ) - _print_data_mean_std(name, print_mean_std) - - if not plot_all: - return - - # plot adversarial dsprites datasets - for fg in [True, False]: - for vis in [100, 75, 50, 25, 0]: - name = f'dsprites_imagenet_{"fg" if fg else "bg"}_{vis}' - plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(name), color='orange', dataset_or_name=name, plot_freq=PLOT_FREQ, x_size_offset=0.4) - _print_data_mean_std(name, print_mean_std) - - BASE = os.path.abspath(os.path.join(__file__, '../../../out/adversarial_data_approx')) - - # plot adversarial datasets - for color, folder in [ - # 'const' datasets - ('purple', '2021-08-18--00-58-22_FINAL-dsprites_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('purple', '2021-08-18--01-33-47_FINAL-shapes3d_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('purple', '2021-08-18--02-20-13_FINAL-cars3d_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('purple', '2021-08-18--03-10-53_FINAL-smallnorb_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - # 'invert' datasets - ('orange', '2021-08-18--03-52-31_FINAL-dsprites_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('orange', '2021-08-18--04-29-25_FINAL-shapes3d_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('orange', '2021-08-18--05-13-15_FINAL-cars3d_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - ('orange', '2021-08-18--06-03-32_FINAL-smallnorb_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06'), - # stronger 'invert' datasets - ('red', '2021-09-06--00-29-23_INVERT-VSTRONG-shapes3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), - ('red', '2021-09-06--03-17-28_INVERT-VSTRONG-dsprites_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), - ('red', '2021-09-06--05-42-06_INVERT-VSTRONG-cars3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), - ('red', '2021-09-06--09-10-59_INVERT-VSTRONG-smallnorb_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06'), - ]: - data = _make_self_contained_dataset(f'{BASE}/{folder}/data.h5') - plot_traversal_stats(circular_distance=CIRCULAR, save_path=sp(folder), color=color, dataset_or_name=data, plot_freq=PLOT_FREQ, x_size_offset=0.4) - _print_data_mean_std(data, print_mean_std) - - -# ========================================================================= # -# STATS # -# ========================================================================= # - - -if __name__ == '__main__': - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../../code/util/gadfly.mplstyle')) - # run! - main_plotting() - main_compute_dists() - - -# ========================================================================= # -# STATS # -# ========================================================================= # - - -# 2021-08-18--00-58-22_FINAL-dsprites_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.04375297] -# vis_std: [0.06837677] -# 2021-08-18--01-33-47_FINAL-shapes3d_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.48852729, 0.5872147 , 0.59863929] -# vis_std: [0.08931785, 0.18920148, 0.23331079] -# 2021-08-18--02-20-13_FINAL-cars3d_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.88888636, 0.88274618, 0.87782785] -# vis_std: [0.18967542, 0.20009377, 0.20805905] -# 2021-08-18--03-10-53_FINAL-smallnorb_self_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.74029344] -# vis_std: [0.06706581] -# -# 2021-08-18--03-52-31_FINAL-dsprites_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.0493243] -# vis_std: [0.09729655] -# 2021-08-18--04-29-25_FINAL-shapes3d_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.49514523, 0.58791172, 0.59616399] -# vis_std: [0.08637031, 0.1895267 , 0.23397072] -# 2021-08-18--05-13-15_FINAL-cars3d_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.88851889, 0.88029857, 0.87666017] -# vis_std: [0.200735 , 0.2151134, 0.2217553] -# 2021-08-18--06-03-32_FINAL-smallnorb_invert_margin_0.005_aw10.0_close_p_random_n_s50001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.73232105] -# vis_std: [0.08755041] -# -# 2021-09-06--00-29-23_INVERT-VSTRONG-shapes3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.47992192, 0.51311111, 0.54627272] -# vis_std: [0.28653814, 0.29201543, 0.27395435] -# 2021-09-06--03-17-28_INVERT-VSTRONG-dsprites_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.20482841] -# vis_std: [0.33634909] -# 2021-09-06--05-42-06_INVERT-VSTRONG-cars3d_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.76418207, 0.75554032, 0.75075393] -# vis_std: [0.31892905, 0.32751031, 0.33319886] -# 2021-09-06--09-10-59_INVERT-VSTRONG-smallnorb_invert_margin_0.05_aw10.0_same_k1_close_s200001_Adam_lr0.0005_wd1e-06 -# vis_mean: [0.69691603] -# vis_std: [0.21310608] - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/part01_data_overlap/plot02_data_distances/util_compute_traversal_dist_pairs.py b/research/part01_data_overlap/plot02_data_distances/util_compute_traversal_dist_pairs.py deleted file mode 100644 index 5980ec2a..00000000 --- a/research/part01_data_overlap/plot02_data_distances/util_compute_traversal_dist_pairs.py +++ /dev/null @@ -1,274 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -from pathlib import Path -from typing import Optional - -import numpy as np -import psutil -import ray -import torch -from ray.util.queue import Queue -from tqdm import tqdm - -import research.code.util as H -from disent.dataset.data import GroundTruthData -from disent.util.inout.files import AtomicSaveFile -from disent.util.profiling import Timer -from disent.util.seeds import TempNumpySeed - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Dataset Distances # -# ========================================================================= # - - -@ray.remote -def _compute_given_dists(gt_data, idxs, obs_pair_idxs, progress_queue=None): - # checks - assert idxs.ndim == 1 - assert obs_pair_idxs.ndim == 2 - assert len(obs_pair_idxs) == len(idxs) - # storage - with torch.no_grad(), Timer() as timer: - obs_pair_dists = torch.zeros(*obs_pair_idxs.shape, dtype=torch.float32) - # progress - done = 0 - # for each observation - for i, (obs_idx, pair_idxs) in enumerate(zip(idxs, obs_pair_idxs)): - # load data - obs = gt_data[obs_idx].flatten() - batch = torch.stack([gt_data[i].flatten() for i in pair_idxs], dim=0) - # compute distances - obs_pair_dists[i, :] = torch.mean((batch - obs[None, :])**2, dim=-1, dtype=torch.float32) - # add progress - done += 1 - if progress_queue is not None: - if timer.elapsed > 0.2: - timer.restart() - progress_queue.put(done) - done = 0 - # final update - if progress_queue is not None: - if done > 0: - progress_queue.put(done) - # done! - return obs_pair_dists.numpy() - - -def compute_dists(gt_data: GroundTruthData, obs_pair_idxs: np.ndarray, jobs_per_cpu: int = 1): - """ - Compute all the distances for ground truth data. - - obs_pair_idxs is a 2D array (len(gt_dat), N) that is a list - of paired indices to each element in the dataset. - """ - # checks - assert obs_pair_idxs.ndim == 2 - assert obs_pair_idxs.shape[0] == len(gt_data) - assert jobs_per_cpu > 0 - # get workers - num_cpus = int(ray.available_resources().get('CPU', 1)) - num_workers = int(num_cpus * jobs_per_cpu) - # get chunks - pair_idxs_chunks = np.array_split(obs_pair_idxs, num_workers) - start_idxs = [0] + np.cumsum([len(c) for c in pair_idxs_chunks]).tolist() - # progress queue - progress_queue = Queue() - ref_gt_data = ray.put(gt_data) - # make workers - futures = [ - _compute_given_dists.remote(ref_gt_data, np.arange(i, i+len(chunk)), chunk, progress_queue) - for i, chunk in zip(start_idxs, pair_idxs_chunks) - ] - # check progress - with tqdm(desc=gt_data.name, total=len(gt_data)) as progress: - completed = 0 - while completed < len(gt_data): - done = progress_queue.get() - completed += done - progress.update(done) - # done - obs_pair_dists = np.concatenate(ray.get(futures), axis=0) - return obs_pair_dists - -# ========================================================================= # -# Distance Types # -# ========================================================================= # - - -def dataset_pair_idxs__random(gt_data: GroundTruthData, num_pairs: int = 25) -> np.ndarray: - # purely random pairs... - return np.random.randint(0, len(gt_data), size=[len(gt_data), num_pairs]) - - -def dataset_pair_idxs__nearby(gt_data: GroundTruthData, num_pairs: int = 10, radius: int = 5) -> np.ndarray: - radius = np.array(radius) - assert radius.ndim in (0, 1) - if radius.ndim == 1: - assert radius.shape == (gt_data.num_factors,) - # get all positions - pos = gt_data.idx_to_pos(np.arange(len(gt_data))) - # generate random offsets - offsets = np.random.randint(-radius, radius + 1, size=[len(gt_data), num_pairs, gt_data.num_factors]) - # broadcast random offsets & wrap around - nearby_pos = (pos[:, None, :] + offsets) % gt_data.factor_sizes - # convert back to indices - nearby_idxs = gt_data.pos_to_idx(nearby_pos) - # done! - return nearby_idxs - - -def dataset_pair_idxs__nearby_scaled(gt_data: GroundTruthData, num_pairs: int = 10, min_radius: int = 2, radius_ratio: float = 0.2) -> np.ndarray: - return dataset_pair_idxs__nearby( - gt_data=gt_data, - num_pairs=num_pairs, - radius=np.maximum((np.array(gt_data.factor_sizes) * radius_ratio).astype('int'), min_radius), - ) - - -_PAIR_IDXS_FNS = { - 'random': dataset_pair_idxs__random, - 'nearby': dataset_pair_idxs__nearby, - 'nearby_scaled': dataset_pair_idxs__nearby_scaled, -} - - -def dataset_pair_idxs(mode: str, gt_data: GroundTruthData, num_pairs: int = 10, **kwargs): - if mode not in _PAIR_IDXS_FNS: - raise KeyError(f'invalid mode: {repr(mode)}, must be one of: {sorted(_PAIR_IDXS_FNS.keys())}') - return _PAIR_IDXS_FNS[mode](gt_data, num_pairs=num_pairs, **kwargs) - - -# ========================================================================= # -# Cache Distances # -# ========================================================================= # - -def _get_default_seed( - pairs_per_obs: int, - pair_mode: str, - dataset_name: str, -): - import hashlib - seed_key = (pairs_per_obs, pair_mode, dataset_name) - seed_hash = hashlib.md5(str(seed_key).encode()) - seed = int(seed_hash.hexdigest()[:8], base=16) % (2**32) # [0, 2**32-1] - return seed - - -def cached_compute_dataset_pair_dists( - dataset_name: str = 'smallnorb', - pair_mode: str = 'nearby_scaled', # random, nearby, nearby_scaled - pairs_per_obs: int = 64, - seed: Optional[int] = None, - # cache settings - cache_dir: str = 'data/cache', - force: bool = False, - # normalize - scaled: bool = True, -): - # checks - assert (seed is None) or isinstance(seed, int), f'seed must be an int or None, got: {type(seed)}' - assert isinstance(pairs_per_obs, int), f'pairs_per_obs must be an int, got: {type(pairs_per_obs)}' - assert pair_mode in _PAIR_IDXS_FNS, f'pair_mode is invalid, got: {repr(pair_mode)}, must be one of: {sorted(_PAIR_IDXS_FNS.keys())}' - # get default seed - if seed is None: - seed = _get_default_seed(pairs_per_obs=pairs_per_obs, pair_mode=pair_mode, dataset_name=dataset_name) - # cache path - cache_path = Path(cache_dir, f'dist-pairs_{dataset_name}_{pairs_per_obs}_{pair_mode}_{seed}.npz') - # generate if it does not exist - if force or not cache_path.exists(): - log.info(f'generating cached distances for: {dataset_name} to: {cache_path}') - # load data - gt_data = H.make_data(dataset_name, transform_mode='float32') - # generate idxs - with TempNumpySeed(seed=seed): - obs_pair_idxs = dataset_pair_idxs(pair_mode, gt_data, num_pairs=pairs_per_obs) - obs_pair_dists = compute_dists(gt_data, obs_pair_idxs) - # generate & save - with AtomicSaveFile(file=cache_path, overwrite=force) as path: - np.savez_compressed(path, **{ # TODO: this used to be `savez` might cause problems with change? this is untested? - 'dataset_name': dataset_name, - 'seed': seed, - 'obs_pair_idxs': obs_pair_idxs, - 'obs_pair_dists': obs_pair_dists, - }) - # load cached data - else: - log.info(f'loading cached distances for: {dataset_name} from: {cache_path}') - data = np.load(cache_path) - obs_pair_idxs = data['obs_pair_idxs'] - obs_pair_dists = data['obs_pair_dists'] - # normalize the max distance to 1.0 - if scaled: - obs_pair_dists /= np.max(obs_pair_dists) - # done! - return obs_pair_idxs, obs_pair_dists - - -# ========================================================================= # -# TEST! # -# ========================================================================= # - - -def generate_common_cache(force=False, force_seed=None): - import itertools - # settings - sweep_pairs_per_obs = [128, 32, 256, 64, 16] - sweep_pair_modes = ['nearby_scaled', 'random', 'nearby'] - sweep_dataset_names = ['cars3d', 'smallnorb', 'shapes3d', 'dsprites', 'xysquares'] - # info - log.info(f'Computing distances for sweep of size: {len(sweep_pairs_per_obs)*len(sweep_pair_modes)*len(sweep_dataset_names)}') - # sweep - for i, (pairs_per_obs, pair_mode, dataset_name) in enumerate(itertools.product(sweep_pairs_per_obs, sweep_pair_modes, sweep_dataset_names)): - # deterministic seed based on settings - if force_seed is None: - seed = _get_default_seed(pairs_per_obs=pairs_per_obs, pair_mode=pair_mode, dataset_name=dataset_name) - else: - seed = force_seed - # info - log.info(f'[{i}] Computing distances for: {repr(dataset_name)} {repr(pair_mode)} {repr(pairs_per_obs)} {repr(seed)}') - # get the dataset and delete the transform - cached_compute_dataset_pair_dists( - dataset_name=dataset_name, - pair_mode=pair_mode, - pairs_per_obs=pairs_per_obs, - seed=seed, - force=force, - scaled=True - ) - - -if __name__ == '__main__': - logging.basicConfig(level=logging.INFO) - ray.init(num_cpus=psutil.cpu_count(logical=False)) - generate_common_cache() - - -# ========================================================================= # -# DONE # -# ========================================================================= # diff --git a/research/part01_data_overlap/plot02_data_distances/util_compute_traversal_dists.py b/research/part01_data_overlap/plot02_data_distances/util_compute_traversal_dists.py deleted file mode 100644 index 9c3a1eda..00000000 --- a/research/part01_data_overlap/plot02_data_distances/util_compute_traversal_dists.py +++ /dev/null @@ -1,302 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -import warnings -from typing import Sequence - -import ray - -import logging -import os -from typing import Tuple - -import numpy as np -import torch -from matplotlib import pyplot as plt -from tqdm import tqdm - -import research.code.util as H -from disent.dataset.data import GroundTruthData -from disent.dataset.util.state_space import StateSpace -from disent.util.strings.fmt import bytes_to_human - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Dataset Stats # -# ========================================================================= # - - -def factor_dist_matrix_shapes(gt_data: GroundTruthData) -> np.ndarray: - # shape: (f_idx, num_factors + 1) - return np.array([factor_dist_matrix_shape(gt_data=gt_data, f_idx=f_idx) for f_idx in range(gt_data.num_factors)]) - - -def factor_dist_matrix_shape(gt_data: GroundTruthData, f_idx: int) -> Tuple[int, ...]: - # using triangular matrices complicates everything - # (np.prod(self._gt_data.factor_sizes) * self._gt_data.factor_sizes[i]) # symmetric, including diagonal in distance matrix - # (np.prod(self._gt_data.factor_sizes) * (self._gt_data.factor_sizes[i] - 1)) // 2 # upper triangular matrix excluding diagonal - # (np.prod(self._gt_data.factor_sizes) * (self._gt_data.factor_sizes[i] + 1)) // 2 # upper triangular matrix including diagonal - return (*np.delete(gt_data.factor_sizes, f_idx), gt_data.factor_sizes[f_idx], gt_data.factor_sizes[f_idx]) - - -def print_dist_matrix_stats(gt_data: GroundTruthData): - # assuming storage as f32 - num_pairs = factor_dist_matrix_shapes(gt_data).prod(axis=1).sum(axis=0) - pre_compute_bytes = num_pairs * (32 // 8) - pairwise_compute_bytes = num_pairs * (32 // 8) * np.prod(gt_data.x_shape) * 2 - traversal_compute_bytes = np.prod(gt_data.x_shape) * np.prod(gt_data.factor_sizes) * gt_data.num_factors - # string - print( - f'{f"{gt_data.name}:":12s} ' - f'{num_pairs:10d} (pairs) ' - f'{bytes_to_human(pre_compute_bytes):>22s} (pre-comp. f32) ' - f'{"x".join(str(i) for i in gt_data.img_shape):>11s} (obs. size)' - f'{bytes_to_human(pairwise_compute_bytes):>22s} (comp. f32) ' - f'{bytes_to_human(traversal_compute_bytes):>22s} (opt. f32)' - ) - - -# ========================================================================= # -# Dataset Compute # -# ========================================================================= # - - -def _iter_batch_ranges(total, batch_size): - assert total >= 0 - assert batch_size > 0 - for i in range(0, total, batch_size): - yield range(i, min(i + batch_size, total)) - - -def _check_gt_data(gt_data: GroundTruthData): - obs = gt_data[0] - # checks - assert isinstance(obs, torch.Tensor) - assert obs.dtype == torch.float32 - - -@ray.remote -def _compute_dists( - idxs: Sequence[int], - # thread data - f_states: StateSpace, - f_idx: int, - gt_data: GroundTruthData, - masked: bool, - a_idxs: np.ndarray, - b_idxs: np.ndarray, -): - results = [] - for idx in idxs: - # translate traversal position to dataset position - base_pos = f_states.idx_to_pos(int(idx)) - base_factors = np.insert(base_pos, f_idx, 0) - # load traversal: (f_size, H*W*C) - traversal = [gt_data[i].flatten().numpy() for i in gt_data.iter_traversal_indices(f_idx=f_idx, base_factors=base_factors)] - traversal = np.stack(traversal, axis=0) - # compute distances - if masked: - B, NUM = traversal.shape - # compute mask - mask = (traversal[0] != traversal[1]) - for item in traversal[2:]: - mask |= (traversal[0] != item) - traversal = traversal[:, mask] - # compute distances - dists = np.sum((traversal[a_idxs] - traversal[b_idxs]) ** 2, axis=1, dtype='float32') / NUM # might need to be float64 - else: - dists = np.mean((traversal[a_idxs] - traversal[b_idxs]) ** 2, axis=1, dtype='float32') - # return data - results.append((base_pos, dists)) - return results - - -def get_as_completed(obj_ids): - # https://github.com/ray-project/ray/issues/5554 - while obj_ids: - done, obj_ids = ray.wait(obj_ids) - yield ray.get(done[0]) - - -@torch.no_grad() -def compute_factor_dist_matrices( - gt_data: GroundTruthData, - f_idx: int, - masked: bool = True, - traversals_per_batch: int = 64, -): - if not ray.is_initialized(): - warnings.warn(f'Ray has not yet been initialized, consider calling `ray.init(...)` and specifying the CPU requirements.') - _check_gt_data(gt_data) - # load data - f_states = StateSpace(factor_sizes=np.delete(gt_data.factor_sizes, f_idx)) - a_idxs, b_idxs = H.pair_indices_combinations(gt_data.factor_sizes[f_idx]) - total = len(f_states) - # move to shared memory - ID_f_states = ray.put(f_states) - ID_gt_data = ray.put(gt_data) - ID_a_idxs = ray.put(a_idxs) - ID_b_idxs = ray.put(b_idxs) - # results - f_dist_matrices = np.zeros(factor_dist_matrix_shape(gt_data=gt_data, f_idx=f_idx), dtype='float32') - # generate futures - futures = [ - _compute_dists.remote( - idxs=sub_range, - f_idx=f_idx, - masked=masked, - f_states=ID_f_states, - gt_data=ID_gt_data, - a_idxs=ID_a_idxs, - b_idxs=ID_b_idxs, - ) - for sub_range in _iter_batch_ranges(total, batch_size=traversals_per_batch) - ] - # apply multithreading to compute traversal distances - with tqdm(total=total, desc=f'{gt_data.name}: {f_idx+1} of {gt_data.num_factors}') as p: - # compute distance matrices - for results in get_as_completed(futures): - for base_pos, dists in results: - f_dist_matrices[(*base_pos, a_idxs, b_idxs)] = dists - f_dist_matrices[(*base_pos, b_idxs, a_idxs)] = dists - p.update(len(results)) - # return distances - return f_dist_matrices - - -def compute_all_factor_dist_matrices( - gt_data: GroundTruthData, - masked: bool = True, - traversals_per_batch: int = 64, -): - """ - ALGORITHM: - for each factor: O(num_factors) - for each traversal: O(prod()) - for element in traversal: O(n) - -- compute overlapping mask - -- we use this mask to only transfer and compute over the needed data - -- we transfer the MASKED traversal to the GPU not the pairs - for each pair in the traversal: O(n*(n-1)/2) | O(n**2) - -- compute each unique pairs distance - -- return distances - """ - # for each factor, compute pairwise overlap - all_dist_matrices = [] - for f_idx in range(gt_data.num_factors): - f_dist_matrices = compute_factor_dist_matrices( - gt_data=gt_data, - f_idx=f_idx, - masked=masked, - traversals_per_batch=traversals_per_batch, - ) - all_dist_matrices.append(f_dist_matrices) - return all_dist_matrices - - -# TODO: replace this with cachier maybe? -def cached_compute_all_factor_dist_matrices( - dataset_name: str = 'smallnorb', - masked: bool = False, - traversals_per_batch: int = 64, - # cache settings - cache_dir: str = 'data/cache', - force: bool = False, - # normalize - normalize_mode: str = 'all', -): - import os - from disent.util.inout.files import AtomicSaveFile - # load data - gt_data = H.make_data(dataset_name, transform_mode='float32') - # check cache - name = f'dist-matrices_{dataset_name}_masked.npz' if masked else f'dist-matrices_{dataset_name}_full.npz' - cache_path = os.path.abspath(os.path.join(cache_dir, name)) - # generate if it does not exist - if force or not os.path.exists(cache_path): - log.info(f'generating cached distances for: {dataset_name} to: {cache_path}') - # generate & save - with AtomicSaveFile(file=cache_path, overwrite=force) as path: - all_dist_matrices = compute_all_factor_dist_matrices(gt_data, masked=masked, traversals_per_batch=traversals_per_batch) - np.savez_compressed(path, **{f_name: f_dists for f_name, f_dists in zip(gt_data.factor_names, all_dist_matrices)}) # TODO: change to savez_compressed is untested! - # load data - log.info(f'loading cached distances for: {dataset_name} from: {cache_path}') - data = np.load(cache_path) - dist_mats = [data[f_name] for f_name in gt_data.factor_names] - # normalize the max distance to 1.0 - if (normalize_mode == 'none') or (normalize_mode is None): - pass - elif normalize_mode == 'all': - M = np.max([np.max(v) for v in dist_mats]) - dist_mats = [v / M for v in dist_mats] - log.info(f'normalized max over all distances: {M} to 1.0') - elif normalize_mode == 'each': - Ms = [v.max() for v in dist_mats] - dist_mats = [v / M for v, M in zip(dist_mats, Ms)] - log.info(f'normalized max over each factor distance: {Ms} to 1.0') - else: - raise KeyError(f'invalid normalize mode: {repr(normalize_mode)}') - - # done! - return dist_mats - - -# ========================================================================= # -# TEST! # -# ========================================================================= # - - -def generate_common_cache(): - for name in ['cars3d', 'smallnorb', 'shapes3d', 'dsprites', 'xysquares']: - # get the dataset and delete the transform - gt_data = H.make_data(name, transform_mode='float32') - print_dist_matrix_stats(gt_data) - f_dist_matrices = cached_compute_all_factor_dist_matrices( - dataset_name=name, - force=True, - masked=True, - traversals_per_batch=32, - ) - # plot distance matrices - H.plt_subplots_imshow( - grid=[[d.reshape([-1, *d.shape[-2:]]).mean(axis=0) for d in f_dist_matrices]], - subplot_padding=0.5, - figsize=(20, 10), - ) - plt.show() - - -def _test_masked_equals_unmasked(): - for name in ['cars3d', 'smallnorb', 'shapes3d', 'dsprites', 'xysquares']: - dists_a = compute_all_factor_dist_matrices(gt_data=H.make_data(name, transform_mode='float32'), masked=True, traversals_per_batch=32) - dists_b = compute_all_factor_dist_matrices(gt_data=H.make_data(name, transform_mode='float32'), masked=False, traversals_per_batch=32) - for a, b in zip(dists_a, dists_b): - assert np.allclose(a, b) - - -if __name__ == '__main__': - ray.init(num_cpus=min(os.cpu_count(), 32)) - generate_common_cache() diff --git a/research/part01_data_overlap/plot03_wandb/plot_p01_all_experiments.py b/research/part01_data_overlap/plot03_wandb/plot_p01_all_experiments.py deleted file mode 100644 index 65236e96..00000000 --- a/research/part01_data_overlap/plot03_wandb/plot_p01_all_experiments.py +++ /dev/null @@ -1,374 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2022 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import os -from typing import Optional -from typing import Tuple - -import pandas as pd -import seaborn as sns -from cachier import cachier as _cachier -from matplotlib import pyplot as plt -import matplotlib.lines as mlines - -import research.code.util as H -from disent.util.function import wrapped_partial - -from research.code.util._wandb_plots import drop_non_unique_cols -from research.code.util._wandb_plots import drop_unhashable_cols -from research.code.util._wandb_plots import load_runs as _load_runs - - -# ========================================================================= # -# Helper # -# ========================================================================= # - - -DF = pd.DataFrame - -# cachier instance -CACHIER: _cachier = wrapped_partial(_cachier, cache_dir=os.path.join(os.path.dirname(__file__), 'plots/.cache')) - - -@CACHIER() -def load_runs(project: str, include_history: bool = False): - return _load_runs(project=project, include_history=include_history) - - -def clear_cache(clear_data=True, clear_wandb=False): - from research.code.util._wandb_plots import clear_runs_cache - if clear_wandb: - clear_runs_cache() - if clear_data: - load_runs.clear_cache() - - -# ========================================================================= # -# Prepare Data # -# ========================================================================= # - - -# common keys -K_GROUP = 'Run Group' -K_DATASET = 'Dataset' -K_FRAMEWORK = 'Framework' -K_SPACING = 'Grid Spacing' -K_BETA = 'Beta' -K_LOSS = 'Recon. Loss' -K_Z_SIZE = 'Latent Dims.' -K_REPEAT = 'Repeat' -K_STATE = 'State' -# K_MIG_END = 'MIG Score\n(End)' -# K_DCI_END = 'DCI Score\n(End)' -K_MIG_MAX = 'MIG Score' -K_DCI_MAX = 'DCI Score' - - -def load_general_data(project: str): - # load data - df = load_runs(project) - # filter out unneeded columns - df, dropped_hash = drop_unhashable_cols(df) - df, dropped_diverse = drop_non_unique_cols(df) - # rename columns - return df.rename(columns={ - 'EXTRA/tags': K_GROUP, - 'dataset/name': K_DATASET, - 'framework/name': K_FRAMEWORK, - 'dataset/data/grid_spacing': K_SPACING, - 'settings/framework/beta': K_BETA, - 'settings/framework/recon_loss': K_LOSS, - 'settings/model/z_size': K_Z_SIZE, - 'DUMMY/repeat': K_REPEAT, - 'state': K_STATE, - # 'final_metric/mig.discrete_score.max': K_MIG_END, - # 'final_metric/dci.disentanglement.max': K_DCI_END, - 'epoch_metric/mig.discrete_score.max': K_MIG_MAX, - 'epoch_metric/dci.disentanglement.max': K_DCI_MAX, - }) - - -# ========================================================================= # -# Plot Experiments # -# ========================================================================= # - - -PINK = '#FE375F' # usually: Beta-VAE -PURPLE = '#5E5BE5' # maybe: Ada-TVAE -BLUE = '#1A93FE' # maybe: TVAE -LBLUE = '#63D2FE' -ORANGE = '#FE9F0A' # usually: Ada-VAE -GREEN = '#2FD157' - -LGREEN = '#9FD911' # usually: MSE -LBLUE2 = '#36CFC8' # usually: MSE-Overlap - - -# ========================================================================= # -# Experiment 2 # -# ========================================================================= # - - -def plot_e02_incr_overlap_xysquares( - rel_path: Optional[str] = None, - save: bool = True, - show: bool = True, - reg_order: int = 4, - color_betavae: str = PINK, - color_adavae: str = ORANGE, - titles: bool = False, - figsize: Tuple[float, float] = (10, 5), - include: Tuple[str] = ('mig', 'dci'), -): - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - df = load_general_data(f'{os.environ["WANDB_USER"]}/CVPR-01__incr_overlap') - # select run groups - df = df[df[K_GROUP].isin(['sweep_xy_squares_overlap', 'sweep_xy_squares_overlap_small_beta'])] - # print common key values - print('K_GROUP: ', list(df[K_GROUP].unique())) - print('K_FRAMEWORK:', list(df[K_FRAMEWORK].unique())) - print('K_SPACING: ', list(df[K_SPACING].unique())) - print('K_BETA: ', list(df[K_BETA].unique())) - print('K_REPEAT: ', list(df[K_REPEAT].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - BETA = 0.00316 # if grid_spacing < 6 - BETA = 0.001 # if grid_spacing >= 6 - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - orig = df - # select runs - # df = df[df[K_STATE] == 'finished'] - # df = df[df[K_REPEAT].isin([1, 2, 3])] - # select adavae - adavae_selector = (df[K_FRAMEWORK] == 'adavae_os') & (df[K_BETA] == 0.001) # 0.001, 0.0001 - data_adavae = df[adavae_selector] - # select - betavae_selector_a = (df[K_FRAMEWORK] == 'betavae') & (df[K_BETA] == 0.001) & (df[K_SPACING] >= 3) - betavae_selector_b = (df[K_FRAMEWORK] == 'betavae') & (df[K_BETA] == 0.00316) & (df[K_SPACING] < 3) - data_betavae = df[betavae_selector_a | betavae_selector_b] - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - print('ADAGVAE', len(orig), '->', len(data_adavae)) - print('BETAVAE', len(orig), '->', len(data_betavae)) - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - fig, axs = plt.subplots(1, len(include), figsize=figsize, squeeze=False) - axs = axs.reshape(-1) - # Legend entries - marker_ada = mlines.Line2D([], [], color=color_adavae, marker='o', markersize=12, label='Ada-GVAE') - marker_beta = mlines.Line2D([], [], color=color_betavae, marker='X', markersize=12, label='Beta-VAE') # why does 'x' not work? only 'X'? - # PLOT: MIG - if 'mig' in include: - sns.regplot(ax=axs[0], x=K_SPACING, y=K_MIG_MAX, data=data_adavae, seed=777, order=reg_order, robust=False, color=color_adavae, marker='o') - sns.regplot(ax=axs[0], x=K_SPACING, y=K_MIG_MAX, data=data_betavae, seed=777, order=reg_order, robust=False, color=color_betavae, marker='x', line_kws=dict(linestyle='dashed')) - axs[0].legend(handles=[marker_beta, marker_ada], fontsize=14) - axs[0].set_ylim([-0.1, 1.1]) - axs[0].set_xlim([0.8, 8.2]) - if titles: axs[0].set_title('Framework Mig Scores') - # PLOT: DCI - if 'dci' in include: - sns.regplot(ax=axs[-1], x=K_SPACING, y=K_DCI_MAX, data=data_adavae, seed=777, order=reg_order, robust=False, color=color_adavae, marker='o') - sns.regplot(ax=axs[-1], x=K_SPACING, y=K_DCI_MAX, data=data_betavae, seed=777, order=reg_order, robust=False, color=color_betavae, marker='x', line_kws=dict(linestyle='dashed')) - axs[-1].legend(handles=[marker_beta, marker_ada], fontsize=14) - axs[-1].set_ylim([-0.1, 1.1]) - axs[-1].set_xlim([0.8, 8.2]) - if titles: axs[-1].set_title('Framework DCI Scores') - # PLOT: - fig.tight_layout() - H.plt_rel_path_savefig(rel_path, save=save, show=show) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - return fig, axs - - -# ========================================================================= # -# Experiment 1 # -# ========================================================================= # - - -def plot_e01_hparam_tuning( - rel_path: Optional[str] = None, - save: bool = True, - show: bool = True, - color_betavae: str = PINK, - color_adavae: str = ORANGE, -): - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - df = load_general_data(f'{os.environ["WANDB_USER"]}/CVPR-00__basic-hparam-tuning') - # select run groups - df = df[df[K_GROUP].isin(['sweep_beta'])] - # print common key values - print('K_GROUP: ', list(df[K_GROUP].unique())) - print('K_DATASET: ', list(df[K_DATASET].unique())) - print('K_FRAMEWORK:', list(df[K_FRAMEWORK].unique())) - print('K_BETA: ', list(df[K_BETA].unique())) - print('K_Z_SIZE: ', list(df[K_Z_SIZE].unique())) - print('K_REPEAT: ', list(df[K_REPEAT].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - orig = df - # select runs - df = df[df[K_STATE] == 'finished'] - # [1.0, 0.316, 0.1, 0.0316, 0.01, 0.00316, 0.001, 0.000316] - # df = df[(0.000316 < df[K_BETA]) & (df[K_BETA] < 1.0)] - print('NUM', len(orig), '->', len(df)) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - df = df[[K_DATASET, K_FRAMEWORK, K_MIG_MAX, K_DCI_MAX]] - df[K_DATASET].replace('xysquares_minimal', 'XYSquares', inplace=True) - df[K_DATASET].replace('smallnorb', 'NORB', inplace=True) - df[K_DATASET].replace('cars3d', 'Cars3D', inplace=True) - df[K_DATASET].replace('3dshapes', 'Shapes3D', inplace=True) - df[K_DATASET].replace('dsprites', 'dSprites', inplace=True) - df[K_FRAMEWORK].replace('adavae_os', 'Ada-GVAE', inplace=True) - df[K_FRAMEWORK].replace('betavae', 'Beta-VAE', inplace=True) - PALLETTE = {'Ada-GVAE': color_adavae, 'Beta-VAE': color_betavae} - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - fig, axs = plt.subplots(1, 2, figsize=(10, 4)) - (ax0, ax1) = axs - # PLOT: MIG - sns.violinplot(x=K_DATASET, y=K_MIG_MAX, hue=K_FRAMEWORK, palette=PALLETTE, split=True, cut=0, width=0.75, data=df, ax=ax0, scale='width', inner='quartile') - ax0.set_ylim([-0.1, 1.1]) - ax0.legend(bbox_to_anchor=(0.425, 0.9), fontsize=13) - sns.violinplot(x=K_DATASET, y=K_DCI_MAX, hue=K_FRAMEWORK, palette=PALLETTE, split=True, cut=0, width=0.75, data=df, ax=ax1, scale='width', inner='quartile') - ax1.set_ylim([-0.1, 1.1]) - ax1.get_legend().remove() - # PLOT: - fig.tight_layout() - H.plt_rel_path_savefig(rel_path, save=save, show=show, dpi=300) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - return fig, axs - - -# ========================================================================= # -# Experiment 3 # -# ========================================================================= # - - -def plot_e03_modified_loss_xysquares( - rel_path: Optional[str] = None, - save: bool = True, - show: bool = True, - color_betavae: str = PINK, - color_adavae: str = ORANGE, - color_mse: str = LGREEN, - color_mse_overlap: str = LBLUE2, -): - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - df = load_general_data(f'{os.environ["WANDB_USER"]}/CVPR-09__vae_overlap_loss') - # select run groups - df = df[df[K_GROUP].isin(['sweep_overlap_boxblur_specific', 'sweep_overlap_boxblur'])] - # print common key values - print('K_GROUP: ', list(df[K_GROUP].unique())) - print() - print('K_DATASET: ', list(df[K_DATASET].unique())) - print('K_FRAMEWORK:', list(df[K_FRAMEWORK].unique())) - print('K_Z_SIZE: ', list(df[K_Z_SIZE].unique())) - print('K_LOSS: ', list(df[K_LOSS].unique())) - print('K_BETA: ', list(df[K_BETA].unique())) - print() - print('K_REPEAT: ', list(df[K_REPEAT].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - orig = df - # select runs - df = df[df[K_STATE] == 'finished'] # TODO: update - df = df[df[K_DATASET] == 'xysquares_minimal'] - df = df[df[K_BETA].isin([0.0001, 0.0316])] - df = df[df[K_Z_SIZE] == 25] - # df = df[df[K_FRAMEWORK] == 'betavae'] # 18 - # df = df[df[K_FRAMEWORK] == 'adavae_os'] # 21 - # df = df[df[K_LOSS] == 'mse'] # 20 - # df = df[df[K_LOSS] != 'mse'] # 19 - # # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # TEMP - # df[K_MIG] = df['final_metric/mig.discrete_score.max'] - # df[K_DCI] = df['final_metric/dci.disentanglement.max'] - - print('NUM', len(orig), '->', len(df)) - - df = df[[K_DATASET, K_FRAMEWORK, K_LOSS, K_BETA, K_MIG_MAX, K_DCI_MAX]] - df[K_DATASET].replace('xysquares_minimal', 'XYSquares', inplace=True) - df[K_FRAMEWORK].replace('adavae_os', 'Ada-GVAE', inplace=True) - df[K_FRAMEWORK].replace('betavae', 'Beta-VAE', inplace=True) - df[K_LOSS].replace('mse_box_r31_l1.0_k3969.0', 'MSE-boxblur', inplace=True) - df[K_LOSS].replace('mse', 'MSE', inplace=True) - PALLETTE = {'Ada-GVAE': color_adavae, 'Beta-VAE': color_betavae, 'MSE': color_mse, 'MSE-boxblur': color_mse_overlap} - - print(df) - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - fig, axs = plt.subplots(1, 2, figsize=(10, 4)) - (ax0, ax1) = axs - # PLOT: MIG - sns.violinplot(x=K_FRAMEWORK, y=K_MIG_MAX, hue=K_LOSS, palette=PALLETTE, split=True, cut=0, width=0.5, data=df, ax=ax0, scale='width', inner='quartile') - ax0.set_ylim([-0.1, 1.1]) - ax0.legend(fontsize=13) - # ax0.legend(bbox_to_anchor=(0.425, 0.9), fontsize=13) - sns.violinplot(x=K_FRAMEWORK, y=K_DCI_MAX, hue=K_LOSS, palette=PALLETTE, split=True, cut=0, width=0.5, data=df, ax=ax1, scale='width', inner='quartile') - ax1.set_ylim([-0.1, 1.1]) - ax1.get_legend().remove() - # PLOT: - fig.tight_layout() - H.plt_rel_path_savefig(rel_path, save=save, show=show, dpi=300) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - -# ========================================================================= # -# Entrypoint # -# ========================================================================= # - - -if __name__ == '__main__': - - assert 'WANDB_USER' in os.environ, 'specify "WANDB_USER" environment variable' - - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../../code/util/gadfly.mplstyle')) - - # clear_cache(clear_data=True, clear_wandb=True) - # clear_cache(clear_data=True, clear_wandb=False) - - def main(): - plot_e01_hparam_tuning(rel_path='plots/p01e01_hparam-tuning', show=True) # was: exp_hparams-exp - plot_e02_incr_overlap_xysquares(rel_path='plots/p01e02_incr-overlap-xysquares', show=True) # was: exp_incr-overlap - # plot_e02_incr_overlap_xysquares(rel_path='plots/p01e02_incr-overlap-xysquares_mig', show=True, include=('mig',), figsize=(6.5, 4)) # was: exp_incr-overlap - # plot_e02_incr_overlap_xysquares(rel_path='plots/p01e02_incr-overlap-xysquares_dci', show=True, include=('dci',), figsize=(6.5, 4)) # was: exp_incr-overlap - plot_e03_modified_loss_xysquares(rel_path='plots/p01e03_modified-loss-xysquares', show=True) # was: exp_overlap-loss - - main() - - -# ========================================================================= # -# DONE # -# ========================================================================= # diff --git a/research/part01_data_overlap/plot03_wandb/plots/.gitignore b/research/part01_data_overlap/plot03_wandb/plots/.gitignore deleted file mode 100644 index 225c3815..00000000 --- a/research/part01_data_overlap/plot03_wandb/plots/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.png -*.jpg diff --git a/research/part02_metric_learning/e00_metrics/submit_beta_data_latent_correlation.sh b/research/part02_metric_learning/e00_metrics/submit_beta_data_latent_correlation.sh deleted file mode 100644 index bc802ae2..00000000 --- a/research/part02_metric_learning/e00_metrics/submit_beta_data_latent_correlation.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - - -# OVERVIEW: -# - this experiment is designed to find the correlation between our proposed -# "factored components" metrics and beta values of the frameworks. - - -# OUTCOMES: - - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="MSC-p02e00_beta-data-latent-corr" -export PARTITION="stampede" -export PARALLELISM=28 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - - -# -- metrics in the original experiment were updated after the runs -# we could have reused those results, but we needed the updated metric values... -# 1 * (2 * 9 * 2 * 5) = 180 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_beta_corr' \ - hydra.job.name="vae_beta" \ - \ - run_length=medium \ - metrics=all \ - \ - settings.optimizer.lr=1e-3,1e-4 \ - settings.framework.beta=0.0001,0.000316,0.001,0.00316,0.01,0.0316,0.1,0.316,1.0 \ - framework=betavae,adavae_os \ - schedule=none \ - settings.model.z_size=25 \ - \ - dataset=dsprites,shapes3d,cars3d,smallnorb,X--xysquares \ - sampling=default__bb diff --git a/research/part02_metric_learning/e01_naive_triplet/OLD/submit_01_triplet_hparam_sweep.sh b/research/part02_metric_learning/e01_naive_triplet/OLD/submit_01_triplet_hparam_sweep.sh deleted file mode 100644 index 27144bdf..00000000 --- a/research/part02_metric_learning/e01_naive_triplet/OLD/submit_01_triplet_hparam_sweep.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash - -# -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2022 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-02__naive-triplet-hparams" -export PARTITION="batch" -export PARALLELISM=24 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -# general sweep of hyper parameters for triplet -# 1 * (3*3*3*2*3 = 162) = 162 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_tvae_params' \ - \ - run_length=long \ - metrics=all \ - \ - framework=tvae \ - settings.framework.beta=0.0316,0.01,0.1 \ - \ - framework.cfg.triplet_margin_max=0.1,1.0,10.0 \ - framework.cfg.triplet_scale=0.1,1.0,0.01 \ - framework.cfg.triplet_p=1,2 \ - \ - dataset=xysquares,cars3d,smallnorb \ - sampling=gt_dist__manhat - -# check sampling strategy -# 2 * (4 * 5 = 20) = 40 -echo PARAMS NOT SET FROM PREVIOUS SWEEP -exit 1 - -# TODO: set the parameters -submit_sweep \ - +DUMMY.repeat=1,2 \ - +EXTRA.tags='sweep_tvae_sampling' \ - \ - run_length=long \ - metrics=all \ - \ - framework=tvae \ - settings.framework.beta=??? \ - \ - framework.cfg.triplet_margin_max=??? \ - framework.cfg.triplet_scale=??? \ - framework.cfg.triplet_p=??? \ - \ - dataset=xysquares,cars3d,shapes3d,dsprites,smallnorb \ - sampling=gt_dist__manhat_scaled,gt_dist__manhat,gt__dist_combined,gt_dist__factors diff --git a/research/part02_metric_learning/e01_naive_triplet/OLD/submit_02_check_vae_equivalence.sh b/research/part02_metric_learning/e01_naive_triplet/OLD/submit_02_check_vae_equivalence.sh deleted file mode 100644 index 8fac79c2..00000000 --- a/research/part02_metric_learning/e01_naive_triplet/OLD/submit_02_check_vae_equivalence.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash - -# -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2022 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-02__naive-triplet-equivalence" -export PARTITION="batch" -export PARALLELISM=24 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -# make sure the tvae is actually working -# like a vae when the triplet loss is disabled -# 1 * (4=4) = 4 -submit_sweep \ - +DUMMY.repeat=1,2 \ - +EXTRA.tags='check_equivalence' \ - \ - run_length=medium \ - metrics=all \ - \ - framework=tvae \ - framework.cfg.triplet_scale=0.0 \ - settings.framework.beta=0.0316 \ - \ - dataset=xysquares \ - sampling=gt_dist__manhat_scaled,gt_dist__manhat,gt__dist_combined,gt_dist__factors - -# check how sampling effects beta and adavae -# 2 * (2*3=6) = 12 -submit_sweep \ - +DUMMY.repeat=1,2 \ - +EXTRA.tags='check_vae_sampling' \ - \ - run_length=medium \ - metrics=all \ - \ - framework=betavae,adavae \ - settings.framework.beta=0.0316 \ - \ - dataset=xysquares \ - sampling=gt_dist__manhat_scaled,gt_dist__manhat,gt__dist_combined,gt_dist__factors diff --git a/research/part02_metric_learning/e01_naive_triplet/submit_01_triplet_param_tuning.sh b/research/part02_metric_learning/e01_naive_triplet/submit_01_triplet_param_tuning.sh deleted file mode 100644 index a912322d..00000000 --- a/research/part02_metric_learning/e01_naive_triplet/submit_01_triplet_param_tuning.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/bash - - -# OVERVIEW: -# - this experiment is designed to find the optimal triplet hyper-parameters - -# OUTCOMES: -# - ??? - - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="MSC-p02e01_triplet-param-tuning" -export PARTITION="stampede" -export PARALLELISM=32 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - - -# SWEEP FOR GOOD TVAE PARAMS -# 1 * (1*2*2*1*2*5*2) = 160 -# * triplet_scale: 0.01 is too low (from provisional experiments) -# * triplet_margin_max: 0.1 is too low (from provisional experiments) -# * beta: 0.01 (and 0.0316) chosen from previous experiment sweeps -# results: -# - z_size: generally worse disentanglement on larger z, so lets make the problem harder and use 25 later for ada methods. -# - triplet_margin_max: 10 seems good (maybe too strong), 1 is ok (maybe too weak), 0.1 is too weak -# - triplet_scale: 1.0 is good, 0.1 is ok but too weak often, 0.01 is definitely too weak. -# - triplet_p: 2 somtimes produces better results, but generally 1 is better for the linearity ratio and latent-ground correspondance. -# - dataset: actually struggles on cars3d, xysquares needs strong triplet to learn. -# - sampling: both `gt_dist__manhat` and `gt_dist__manhat_scaled` seem ok, need to test with ADA methods. Maybe `gt_dist__manhat` for simplicity? -# summary: -# - triplet_margin_max=10.0, triplet_scale=1.0, triplet_p=1, sampling=gt_dist__manhat,gt_dist__manhat_scaled, z_size=25 -# todo: -# - detach_decoder: see how much this affects things -# - triplet_loss: see if other versions are better -# MAKE PLOTS: -# -- use this to figure out if l1 or l2 is better -# -- use this to hparam tune for later experiments -# SWEEP FOR GOOD PARAMS - RERUN: -# changes: -# - no longer checking z_size=9,25 -# + added detach_decoder=FALSE,TRUE -# results: -# - pretty much the same as before, just needed the fixed metrics... -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_tvae_params_basic_RERUN' \ - hydra.job.name="tvae_params" \ - \ - run_length=medium \ - metrics=all \ - \ - settings.framework.beta=0.01 \ - framework=tvae \ - schedule=none \ - settings.model.z_size=25 \ - \ - framework.cfg.triplet_margin_max=1.0,10.0 \ - framework.cfg.triplet_scale=0.1,1.0 \ - framework.cfg.triplet_p=1,2 \ - framework.cfg.detach_decoder=FALSE,TRUE \ - framework.cfg.triplet_loss=triplet \ - \ - dataset=cars3d,smallnorb,shapes3d,dsprites,X--xysquares \ - sampling=gt_dist__manhat,gt_dist__manhat_scaled - -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_tvae_params_basic_RERUN_soft' \ - hydra.job.name="tvae_params" \ - \ - run_length=medium \ - metrics=all \ - \ - settings.framework.beta=0.01 \ - framework=tvae \ - schedule=none \ - settings.model.z_size=25 \ - \ - framework.cfg.triplet_margin_max=1.0,10.0 \ - framework.cfg.triplet_scale=0.1,1.0 \ - framework.cfg.triplet_p=1,2 \ - framework.cfg.detach_decoder=FALSE,TRUE \ - framework.cfg.triplet_loss=triplet_soft \ - \ - dataset=cars3d,smallnorb,shapes3d,dsprites,X--xysquares \ - sampling=gt_dist__manhat,gt_dist__manhat_scaled - -# TRIPLET TYPES: -# - N/A: [triplet_soft] -# - max: [triplet, triplet_sigmoid] -# - max+min: [min_clamped_triplet, split_clamped_triplet] - -# TRIPLET OPTIONS: -# framework.cfg.detach_decoder: FALSE -# framework.cfg.triplet_loss: triplet -# framework.cfg.triplet_margin_min: 0.001 -# framework.cfg.triplet_margin_max: 1 -# framework.cfg.triplet_scale: 0.1 -# framework.cfg.triplet_p: 1 - -# SWEEP TRIPLET MODES: -# RERUN NOTES: -# -- we no longer want this because triplet_soft doesnt seem to work -# well with axis-aligned triplet or detached decoders... -# 1 * (2*2*2*2*5) = 80 -#submit_sweep \ -# +DUMMY.repeat=1 \ -# +EXTRA.tags='sweep_tvae_modes_basic' \ -# hydra.job.name="tvae_modes" \ -# \ -# run_length=medium \ -# metrics=all \ -# \ -# settings.framework.beta=0.01 \ -# framework=tvae \ -# schedule=none \ -# settings.model.z_size=25 \ -# \ -# sampling=gt_dist__manhat,gt_dist__manhat_scaled -# \ -# framework.cfg.detach_decoder=FALSE,TRUE \ -# framework.cfg.triplet_margin_max=10.0 \ -# framework.cfg.triplet_scale=1.0 \ -# framework.cfg.triplet_p=1,2 \ -# framework.cfg.triplet_loss=triplet_soft,triplet \ -# \ -# dataset=cars3d,smallnorb,shapes3d,dsprites,X--xysquares diff --git a/research/part02_metric_learning/e02_axis_triplet/OLD/submit_01.sh b/research/part02_metric_learning/e02_axis_triplet/OLD/submit_01.sh deleted file mode 100644 index 7d1dd8d8..00000000 --- a/research/part02_metric_learning/e02_axis_triplet/OLD/submit_01.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-03__axis-triplet-3.0" -export PARTITION="batch" -export PARALLELISM=24 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 43200 "C-disent" # 12 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# SHORT RUNS: -# - test for best ada loss types -# 1 * (2*4*2*8=112) = 128 -submit_sweep \ - +DUMMY.repeat=1 \ - \ - framework=X--adatvae \ - dataset=xysquares \ - run_length=short \ - \ - framework.cfg.triplet_margin_max=1.0 \ - framework.cfg.triplet_scale=0.1 \ - framework.cfg.triplet_p=1 \ - sampling=gt_dist_manhat \ - \ - model.z_size=25,9 \ - \ - framework.cfg.thresh_ratio=0.5 \ - framework.cfg.ada_triplet_ratio=1.0 \ - schedule=adavae_thresh,adavae_all,adavae_ratio,none \ - framework.cfg.ada_triplet_sample=TRUE,FALSE \ - framework.cfg.ada_triplet_loss=framework.cfg.ada_triplet_loss=triplet,triplet_soft_ave,triplet_soft_neg_ave,triplet_all_soft_ave,triplet_hard_ave,triplet_hard_neg_ave,triplet_hard_neg_ave_pull,triplet_all_hard_ave - -# ADA TRIPLET LOSS MODES (short runs): -# - generally dont use sampling, except for: triplet_hard_neg_ave_pull -# - soft averages dont work if scheduling thresh or ratio separately, need to do both at the same time -# - hard averages perform well initially, but performance decays more toward the end of schedules -# ======================= -# [X] triplet -# -# [-] triplet_soft_ave [NOTE: OK, but just worse than, triplet_all_soft_ave] -# triplet_soft_neg_ave [NOTE: better disentanglement than triplet_all_soft_ave, but worse axis align] -# triplet_all_soft_ave -# -# triplet_hard_neg_ave -# triplet_hard_neg_ave_pull (weight = 0.1, triplet_hard_neg_ave_pull_soft) -# [X] triplet_hard_ave -# [X] triplet_hard_neg_ave_pull (weight = 1.0) -# [X] triplet_all_hard_ave diff --git a/research/part02_metric_learning/e02_axis_triplet/OLD/submit_02.sh b/research/part02_metric_learning/e02_axis_triplet/OLD/submit_02.sh deleted file mode 100644 index 55e47dba..00000000 --- a/research/part02_metric_learning/e02_axis_triplet/OLD/submit_02.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-03__axis-triplet-3.0" -export PARTITION="batch" -export PARALLELISM=30 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# MED RUNS: -# - test for best hparams for all soft ave loss -# 2 * (2*3*3*3=54) = 104 -submit_sweep \ - +DUMMY.repeat=1,2 \ - +EXTRA.tags='med-run+soft-hparams' \ - \ - framework=X--adatvae \ - run_length=medium \ - model.z_size=25 \ - \ - framework.cfg.triplet_margin_max=1.0,5.0 \ - framework.cfg.triplet_scale=0.1,0.02,0.5 \ - framework.cfg.triplet_p=1 \ - sampling=gt_dist_manhat \ - \ - framework.cfg.thresh_ratio=0.5 \ - framework.cfg.ada_triplet_ratio=1.0 \ - framework.cfg.ada_triplet_soft_scale=0.25,1.0,4.0 \ - framework.cfg.ada_triplet_sample=FALSE \ - \ - schedule=adavae_all,adavae_thresh,adavae_ratio \ - framework.cfg.ada_triplet_loss=triplet_all_soft_ave \ - dataset=xysquares diff --git a/research/part02_metric_learning/e02_axis_triplet/OLD/submit_03.sh b/research/part02_metric_learning/e02_axis_triplet/OLD/submit_03.sh deleted file mode 100644 index 43de6d3b..00000000 --- a/research/part02_metric_learning/e02_axis_triplet/OLD/submit_03.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-03__axis-triplet-3.0" -export PARTITION="stampede" -export PARALLELISM=32 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# LONG RUNS: -# - test best losses & best hparams from test1 on different datasets with long runs -# + [not tested] triplet_soft_neg_ave -# + triplet_all_soft_ave -# + triplet_hard_neg_ave -# + triplet_hard_neg_ave_pull - -# 1 * (2*3*4*4=96) = 96 -#submit_sweep \ -# +DUMMY.repeat=1 \ -# +EXTRA.tags='long-run' \ -# \ -# framework=X--adatvae \ -# run_length=long \ -# model.z_size=25 \ -# \ -# framework.cfg.triplet_margin_max=1.0 \ -# framework.cfg.triplet_scale=0.1 \ -# framework.cfg.triplet_p=1 \ -# sampling=gt_dist_manhat,gt_dist_manhat_scaled \ -# \ -# framework.cfg.thresh_ratio=0.5 \ -# framework.cfg.ada_triplet_ratio=1.0 \ -# framework.cfg.ada_triplet_soft_scale=1.0 \ -# framework.cfg.ada_triplet_sample=FALSE \ -# \ -# schedule=adavae_all,adavae_thresh,adavae_ratio \ -# framework.cfg.ada_triplet_loss=triplet,triplet_all_soft_ave,triplet_hard_neg_ave,triplet_hard_neg_ave_pull \ -# dataset=xysquares,shapes3d,cars3d,dsprites - -# 2*2*3*4*4 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='med-run+datasets+swap-chance+manhat-scaled' \ - \ - framework=X--adatvae \ - run_length=medium \ - model.z_size=25 \ - \ - sampling=gt_dist_manhat_scaled,gt_dist_manhat \ - schedule=adavae_all,adavae_thresh,adavae_ratio \ - sampling.triplet_swap_chance=0,0.1 \ - \ - framework.cfg.triplet_margin_max=1.0 \ - framework.cfg.triplet_scale=0.1 \ - framework.cfg.triplet_p=1 \ - \ - framework.cfg.thresh_ratio=0.5 \ - framework.cfg.ada_triplet_ratio=1.0 \ - framework.cfg.ada_triplet_soft_scale=1.0 \ - framework.cfg.ada_triplet_sample=FALSE \ - \ - framework.cfg.ada_triplet_loss=triplet,triplet_all_soft_ave,triplet_hard_neg_ave,triplet_hard_neg_ave_pull \ - dataset=xysquares,shapes3d,cars3d,dsprites diff --git a/research/part02_metric_learning/e02_axis_triplet/OLD/submit_04.sh b/research/part02_metric_learning/e02_axis_triplet/OLD/submit_04.sh deleted file mode 100644 index 9259e5ac..00000000 --- a/research/part02_metric_learning/e02_axis_triplet/OLD/submit_04.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-03__axis-triplet-4.0" -export PARTITION="stampede" -export PARALLELISM=24 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# RESULT: -# - BAD: ada_thresh_mode=symmetric_kl, rather use "dist" -# - BAD: framework.cfg.adaave_decode_orig=FALSE, rather use TRUE -# - adat_share_ave_mode depends on other settings, but usually doesnt matter -# - adaave_augment_orig depends on other settings, but usually doesnt matter -# - GOOD: adat_triplet_loss=triplet_hard_neg_ave -# - NOTE: schedule=adavae_up_ratio usually converges sooner -# - NOTE: schedule=adavae_up_all usually converges later (makes sense because its a doubling effect a ^ 2) -# - NOTE: schedule=adavae_up_thresh usually is worse at converging - - -# 3*2*4*2*2*2 == 192 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='short-run__ada-best-loss-combo' \ - \ - framework=X--adaavetvae \ - run_length=short \ - model.z_size=25 \ - \ - schedule=adavae_up_all,adavae_up_ratio,adavae_up_thresh \ - sampling=gt_dist_manhat \ - sampling.triplet_swap_chance=0 \ - dataset=xysquares \ - \ - framework.cfg.triplet_loss=triplet \ - framework.cfg.triplet_margin_min=0.001 \ - framework.cfg.triplet_margin_max=1 \ - framework.cfg.triplet_scale=0.1 \ - framework.cfg.triplet_p=1 \ - \ - framework.cfg.detach=FALSE \ - framework.cfg.detach_decoder=FALSE \ - framework.cfg.detach_no_kl=FALSE \ - framework.cfg.detach_std=NULL \ - \ - framework.module.ada_average_mode=gvae \ - framework.module.ada_thresh_mode=symmetric_kl,dist \ - framework.module.ada_thresh_ratio=0.5 \ - \ - framework.module.adat_triplet_loss=triplet,triplet_soft_ave_all,triplet_hard_neg_ave,triplet_hard_ave_all \ - framework.module.adat_triplet_ratio=1.0 \ - framework.module.adat_triplet_soft_scale=1.0 \ - framework.module.adat_triplet_pull_weight=0.1 \ - \ - framework.module.adat_share_mask_mode=posterior \ - framework.module.adat_share_ave_mode=all,neg \ - \ - framework.module.adaave_augment_orig=TRUE,FALSE \ - framework.module.adaave_decode_orig=TRUE,FALSE - -# TRY THESE TOO: -# framework.module.adat_share_ave_mode=all,neg,pos,pos_neg \ -# framework.module.adat_share_mask_mode=posterior,sample,sample_each \ -# framework.module.adat_triplet_loss=triplet,triplet_soft_ave_all,triplet_hard_neg_ave,triplet_hard_neg_ave_pull,triplet_hard_ave_all \ - -# # 3*2*8*2*3*2*2 -#submit_sweep \ -# +DUMMY.repeat=1 \ -# +EXTRA.tags='short-run__ada-best-loss-combo' \ -# \ -# framework=X--adaavetvae \ -# run_length=short \ -# model.z_size=25 \ -# \ -# schedule=adavae_all,adavae_thresh,adavae_ratio \ -# sampling=gt_dist_manhat \ -# sampling.triplet_swap_chance=0 \ -# dataset=xysquares \ -# \ -# triplet_loss=triplet \ -# triplet_margin_min=0.001 \ -# triplet_margin_max=1 \ -# triplet_scale=0.1 \ -# triplet_p=1 \ -# \ -# detach=FALSE \ -# detach_decoder=FALSE \ -# detach_no_kl=FALSE \ -# detach_std=NULL \ -# \ -# ada_average_mode=gvae \ -# ada_thresh_mode=symmetric_kl,dist \ -# ada_thresh_ratio=0.5 \ -# \ -# adat_triplet_loss=triplet,triplet_soft_ave_neg,triplet_soft_ave_p_n,triplet_soft_ave_all,triplet_hard_ave,triplet_hard_neg_ave,triplet_hard_neg_ave_pull,triplet_hard_ave_all \ -# adat_triplet_ratio=1.0 \ -# adat_triplet_soft_scale=1.0 \ -# adat_triplet_pull_weight=0.1 \ -# \ -# adat_share_mask_mode=posterior,dist \ -# adat_share_ave_mode=all,pos_neg,pos,neg \ -# \ -# adaave_augment_orig=TRUE,FALSE \ -# adaave_decode_orig=TRUE,FALSE diff --git a/research/part02_metric_learning/e02_axis_triplet/OLD/submit_05.sh b/research/part02_metric_learning/e02_axis_triplet/OLD/submit_05.sh deleted file mode 100644 index 4e10e3ee..00000000 --- a/research/part02_metric_learning/e02_axis_triplet/OLD/submit_05.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-03__axis-triplet-5.0" -export PARTITION="stampede" -export PARALLELISM=16 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# 1 * (3*6*5) == 90 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='ada-best-pull-weight' \ - \ - framework=X--adanegtvae \ - run_length=short,medium,long \ - model.z_size=25 \ - \ - schedule=adavae_down_all,adavae_up_all,adavae_down_ratio,adavae_up_ratio,adavae_down_thresh,adavae_up_thresh \ - sampling=gt_dist_manhat \ - sampling.triplet_swap_chance=0 \ - dataset=xysquares \ - \ - framework.cfg.triplet_loss=triplet \ - framework.cfg.triplet_margin_min=0.001 \ - framework.cfg.triplet_margin_max=1 \ - framework.cfg.triplet_scale=0.1 \ - framework.cfg.triplet_p=1 \ - \ - framework.cfg.detach=FALSE \ - framework.cfg.detach_decoder=FALSE \ - framework.cfg.detach_no_kl=FALSE \ - framework.cfg.detach_std=NULL \ - \ - framework.cfg.ada_average_mode=gvae \ - framework.cfg.ada_thresh_mode=dist \ - framework.cfg.ada_thresh_ratio=0.5 \ - \ - framework.cfg.adat_triplet_ratio=1.0 \ - framework.cfg.adat_triplet_pull_weight=-1.0,-0.1,0.0,0.1,1.0 \ - \ - framework.cfg.adat_share_mask_mode=posterior diff --git a/research/part02_metric_learning/e02_axis_triplet/submit_01_axis_aligned_triplet.sh b/research/part02_metric_learning/e02_axis_triplet/submit_01_axis_aligned_triplet.sh deleted file mode 100644 index cb3f5b65..00000000 --- a/research/part02_metric_learning/e02_axis_triplet/submit_01_axis_aligned_triplet.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/bin/bash - - -# OVERVIEW: -# - this experiment is designed to find the optimal ada-triplet function and hyper-parameters - -# OUTCOMES: -# - ??? - - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="MSC-p02e02_axis-aligned-triplet" -export PARTITION="stampede" -export PARALLELISM=24 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 172800 "C-disent" # 48 hours - -# SWEEP FOR GOOD ADANEG-TVAE PARAMS (OLD) -# 1 * (2*2*5*2*5) = 200 -# OUTCOMES: -# - this experiment was wrong! The schedules were not set up correctly! we did not define -# the correct initial values for `adat_triplet_share_scale` or `ada_thresh_ratio` -# - HOWEVER: The learning rate less than 1e-3 did not really work well, the batch size of -# 256 is large enough that we can use this high learning rate. -# - HOWEVER: On average: ada_thresh_mode="dist" performs FAR better than "symmetric_kl" -# - HOWEVER: On average, gt_dist__manhat_scaled sometimes performs better, but metrics and distance measurement don't line up properly. -#submit_sweep \ -# +DUMMY.repeat=1 \ -# +EXTRA.tags='sweep_adanegtvae_params_basic' \ -# hydra.job.name="adanegtvae_params" \ -# \ -# run_length=medium \ -# metrics=all \ -# \ -# settings.framework.beta=0.01 \ -# settings.model.z_size=25 \ -# settings.optimizer.lr=1e-3,3e-4 \ -# \ -# framework.cfg.detach_decoder=FALSE \ -# \ -# sampling=gt_dist__manhat,gt_dist__manhat_scaled \ -# schedule=OLD_adavae_up_all,OLD_adavae_up_all_full,OLD_adavae_up_ratio,OLD_adavae_up_ratio_full,OLD_adavae_up_thresh \ -# \ -# framework=X--adanegtvae \ -# framework.cfg.ada_thresh_mode=symmetric_kl,dist \ -# \ -# framework.cfg.triplet_margin_max=10.0 \ -# framework.cfg.triplet_scale=1.0 \ -# framework.cfg.triplet_p=1 \ -# framework.cfg.triplet_loss=triplet_soft \ -# \ -# dataset=cars3d,smallnorb,shapes3d,dsprites,X--xysquares - -# Try These: -# - adavae modes: framework.cfg.adat_triplet_loss=triplet,triplet_soft_ave_neg,triplet_soft_ave_p_n,triplet_soft_ave_all,triplet_hard_ave,triplet_hard_neg_ave,triplet_hard_neg_ave_pull,triplet_hard_ave_all,triplet_hard_neg_ave_scaled \ -# - try run length long -# - try run detach - - -# SWEEP FOR GOOD ADANEG-TVAE PARAMS -# 1 * (2*5*2*5) = 100 -# OUTCOMES: -# - `gt_dist__manhat_scaled` is generally better (problem is metrics sometimes need to be scaled to match this, so metrics might actaully be off?) -# - `adanegtvae_up_thresh` is generally bad, does not converge well. Might converge better for longer runs if givem more time? Usually quite stable though, recon loss is decent. -# - `*_full` version of schedules generally converge a bit quicker, but the recon loss decays badly towards the end. -# - `!*_full` versions (the not full versions) might get better scores if given more time, are generally more stable too in terms of recon loss. -# - `dist` is MUCH better than `symmetric_kl` --- always use the former! -# NOTE: -# -- distances don't always correlate well, especially if the dataset is more difficult to learn in terms of recon loss. -# we could try and increase the triplet_scale to compensate for this in future experiments? -# OR: we can detach the decoder? -# -- I am not sure if the soft-margin formulation hurts learning of -# distances? It might, try revert to normal triplet loss? -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_adanegtvae_params_longmed' \ - hydra.job.name="adanegtvae_hparams_longmed" \ - \ - run_length=longmed \ - metrics=all \ - \ - settings.framework.beta=0.01 \ - settings.model.z_size=25 \ - \ - sampling=gt_dist__manhat,gt_dist__manhat_scaled \ - framework.cfg.ada_thresh_mode=dist,symmetric_kl \ - \ - framework=X--adanegtvae \ - framework.cfg.detach_decoder=FALSE \ - \ - schedule=adanegtvae_up_all,adanegtvae_up_all_full,adanegtvae_up_ratio,adanegtvae_up_ratio_full,adanegtvae_up_thresh \ - framework.cfg.ada_thresh_ratio=0.5 \ - framework.cfg.adat_triplet_share_scale=0.5 \ - \ - framework.cfg.triplet_margin_max=10.0 \ - framework.cfg.triplet_scale=1.0 \ - framework.cfg.triplet_p=1 \ - framework.cfg.triplet_loss=triplet_soft \ - \ - dataset=cars3d,smallnorb,shapes3d,dsprites,X--xysquares - - -# SWEEP FOR ALTERNATIVE ADANEG-TVAE PARAMS -# 1 * (2*2*2*2*5) = 80 -# OUTCOMES: -# - ??? -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_adanegtvae_alt_params_longmed' \ - hydra.job.name="adanegtvae_hparams_alt" \ - \ - run_length=longmed \ - metrics=all \ - \ - settings.framework.beta=0.01 \ - settings.model.z_size=25 \ - \ - sampling=gt_dist__manhat_scaled \ - framework.cfg.ada_thresh_mode=dist \ - \ - schedule=adanegtvae_up_ratio,adanegtvae_up_all \ - framework.cfg.triplet_scale=10.0,1.0 \ - framework.cfg.detach_decoder=FALSE,TRUE \ - framework.cfg.triplet_loss=triplet,triplet_soft \ - dataset=cars3d,smallnorb,shapes3d,dsprites,X--xysquares \ - \ - framework=X--adanegtvae \ - \ - framework.cfg.ada_thresh_ratio=0.5 \ - framework.cfg.adat_triplet_share_scale=0.5 \ - \ - framework.cfg.triplet_margin_max=10.0 \ - framework.cfg.triplet_p=1 - - -# THIS REPLACES THE ABOVE SWEEPS: -# - we don't want soft-margin triplet anymore! -# 1 * (5*2*2*5*2) = 200 -#submit_sweep \ -# +DUMMY.repeat=1 \ -# +EXTRA.tags='sweep_adanegtvae_hard_triplet_params_longmed' \ -# hydra.job.name="adanegtvae_hparams_alt" \ -# \ -# run_length=longmed \ -# metrics=all \ -# \ -# settings.framework.beta=0.01 \ -# settings.model.z_size=25 \ -# \ -# sampling=gt_dist__manhat_scaled \ -# framework.cfg.ada_thresh_mode=dist \ -# \ -# schedule=adanegtvae_up_all,adanegtvae_up_all_full,adanegtvae_up_ratio,adanegtvae_up_ratio_full,adanegtvae_up_thresh \ -# framework.cfg.triplet_scale=10.0,1.0 \ -# framework.cfg.detach_decoder=FALSE,TRUE \ -# framework.cfg.triplet_loss=triplet \ -# dataset=cars3d,smallnorb,shapes3d,dsprites,X--xysquares \ -# \ -# framework=X--adanegtvae \ -# \ -# framework.cfg.ada_thresh_ratio=0.5 \ -# framework.cfg.adat_triplet_share_scale=0.5 \ -# \ -# framework.cfg.triplet_margin_max=1.0,10.0 \ -# framework.cfg.triplet_p=1 diff --git a/research/part02_metric_learning/e03_unsupervised_triplet/OLD/submit_01.sh b/research/part02_metric_learning/e03_unsupervised_triplet/OLD/submit_01.sh deleted file mode 100644 index 62d09a80..00000000 --- a/research/part02_metric_learning/e03_unsupervised_triplet/OLD/submit_01.sh +++ /dev/null @@ -1,62 +0,0 @@ -##!/bin/bash -# -## ========================================================================= # -## Settings # -## ========================================================================= # -# -#export USERNAME="n_michlo" -#export PROJECT="final-04__data-overlap-triplet" -#export PARTITION="stampede" -#export PARALLELISM=32 -# -## source the helper file -#source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" -# -## ========================================================================= # -## Experiment # -## ========================================================================= # -# -#clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours -# -## 1 * (3*2*2*5*2) == 120 -#submit_sweep \ -# +DUMMY.repeat=1 \ -# +EXTRA.tags='med-best' \ -# \ -# framework=X--dotvae_aug \ -# run_length=medium \ -# model.z_size=25 \ -# \ -# schedule=adavae_up_all,adavae_up_ratio,none \ -# sampling=gt_dist_manhat \ -# sampling.triplet_swap_chance=0 \ -# dataset=xysquares \ -# \ -# framework.cfg.triplet_loss=triplet \ -# framework.cfg.triplet_margin_min=0.001 \ -# framework.cfg.triplet_margin_max=1 \ -# framework.cfg.triplet_scale=0.1,0.01 \ -# framework.cfg.triplet_p=1 \ -# \ -# framework.cfg.detach=FALSE \ -# framework.cfg.detach_decoder=FALSE \ -# framework.cfg.detach_no_kl=FALSE \ -# framework.cfg.detach_std=NULL \ -# \ -# framework.cfg.ada_average_mode=gvae \ -# framework.cfg.ada_thresh_mode=dist \ -# framework.cfg.ada_thresh_ratio=0.5 \ -# \ -# framework.cfg.adat_triplet_share_scale=0.95 \ -# \ -# framework.cfg.adat_share_mask_mode=posterior \ -# \ -# framework.cfg.overlap_num=4096 \ -# framework.cfg.overlap_mine_ratio=0.05,0.1 \ -# framework.cfg.overlap_mine_triplet_mode=none,hard_neg,semi_hard_neg,hard_pos,easy_pos \ -# \ -# framework.cfg.overlap_augment_mode='augment' \ -# framework.cfg.overlap_augment.p=1.0 \ -# framework.cfg.overlap_augment.radius=[61,61],[0,61] \ -# framework.cfg.overlap_augment.random_mode='batch' \ -# framework.cfg.overlap_augment.random_same_xy=TRUE diff --git a/research/part02_metric_learning/e03_unsupervised_triplet/OLD/submit_02.sh b/research/part02_metric_learning/e03_unsupervised_triplet/OLD/submit_02.sh deleted file mode 100644 index 535ec11b..00000000 --- a/research/part02_metric_learning/e03_unsupervised_triplet/OLD/submit_02.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-04__data-overlap-triplet" -export PARTITION="batch" -export PARALLELISM=16 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# 1 * (2*8*4) == 64 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='best-augment-strength__alt' \ - \ - framework=X--dotvae_aug \ - run_length=short \ - model=conv64alt \ - model.z_size=25 \ - \ - schedule=adavae_up_ratio_full,adavae_up_all_full \ - sampling=gt_dist_manhat \ - sampling.triplet_swap_chance=0 \ - dataset=xysquares \ - \ - framework.cfg.triplet_loss=triplet \ - framework.cfg.triplet_margin_min=0.001 \ - framework.cfg.triplet_margin_max=1 \ - framework.cfg.triplet_scale=0.1 \ - framework.cfg.triplet_p=1 \ - \ - framework.cfg.detach=FALSE \ - framework.cfg.detach_decoder=FALSE \ - framework.cfg.detach_no_kl=FALSE \ - framework.cfg.detach_std=NULL \ - \ - framework.cfg.ada_average_mode=gvae \ - framework.cfg.ada_thresh_mode=dist \ - framework.cfg.ada_thresh_ratio=0.5 \ - \ - framework.cfg.adat_triplet_share_scale=1.0 \ - \ - framework.cfg.adat_share_mask_mode=posterior \ - \ - framework.cfg.overlap_augment_mode='augment' \ - framework.cfg.overlap_augment.kernel=xy1_r47,xy8_r47,box_r47,gau_r47 \ - \ - framework.cfg.overlap_num=4096 \ - framework.module.overlap_mine_ratio=0.1 \ - framework.module.overlap_mine_triplet_mode=none,hard_neg,semi_hard_neg,hard_pos,easy_pos,ran:hard_neg+hard_pos,ran:hard_neg+easy_pos,ran:hard_pos+easy_pos - - # framework.module.overlap_augment.kernel=xy1_r47,xy8_r47,box_r47,gau_r47,box_r15,box_r31,box_r63,gau_r15,gau_r31,gau_r63 diff --git a/research/part02_metric_learning/e03_unsupervised_triplet/OLD/submit_03_test_softada_vs_ada.sh b/research/part02_metric_learning/e03_unsupervised_triplet/OLD/submit_03_test_softada_vs_ada.sh deleted file mode 100644 index ea224030..00000000 --- a/research/part02_metric_learning/e03_unsupervised_triplet/OLD/submit_03_test_softada_vs_ada.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -# -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="test-hard-vs-soft-ada" -export PARTITION="stampede" -export PARALLELISM=16 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# 3 * (3 * 1) = 9 -submit_sweep \ - +DUMMY.repeat=1,2,3 \ - +EXTRA.tags='sweep_02' \ - \ - run_length=medium \ - metrics=all \ - \ - framework.beta=1 \ - framework=adavae_os,adagvae_minimal_os,X--softadagvae_minimal_os \ - model.z_size=25 \ - \ - dataset=shapes3d \ - \ - hydra.launcher.exclude='"mscluster93,mscluster94,mscluster97"' # we don't want to sweep over these diff --git a/research/part02_metric_learning/e03_unsupervised_triplet/array_01_MSC-p02e03_unsupervised-axis-triplet.txt b/research/part02_metric_learning/e03_unsupervised_triplet/array_01_MSC-p02e03_unsupervised-axis-triplet.txt deleted file mode 100644 index f0d99cb2..00000000 --- a/research/part02_metric_learning/e03_unsupervised_triplet/array_01_MSC-p02e03_unsupervised-axis-triplet.txt +++ /dev/null @@ -1,100 +0,0 @@ -+EXTRA.sweep_num=1 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=2 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=3 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=4 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=5 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=6 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=7 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=8 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=9 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=10 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=11 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=12 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=13 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=14 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=15 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=16 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=17 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=18 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=19 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=20 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=21 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=22 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=23 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=24 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=25 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=26 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=27 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=28 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=29 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=30 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=31 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=32 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=33 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=34 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=35 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=36 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=37 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=38 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=39 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=40 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=41 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=42 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=43 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=44 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=45 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=46 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=47 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=48 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=49 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=50 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=51 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=52 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=53 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=54 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=55 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=56 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=57 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=58 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=59 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=60 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=61 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=62 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=63 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=64 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=65 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=66 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=67 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=68 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=69 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=70 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=71 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=72 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=73 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=74 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=75 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=76 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=77 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=78 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=79 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=80 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed hydra.job.name=dotvae_hparams_longmed run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=81 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=82 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=83 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=84 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=85 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=86 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=87 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=88 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=89 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=90 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=512 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=91 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=92 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=93 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=94 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=95 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=96 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=97 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=98 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=99 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=100 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_longmed_xy hydra.job.name=dotvae_hparams_longmed_xy run_length=longmed metrics=all settings.framework.beta=0.01 settings.model.z_size=25 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.2 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null diff --git a/research/part02_metric_learning/e03_unsupervised_triplet/array_01_MSC-p02e03_unsupervised-axis-triplet_ALT.txt b/research/part02_metric_learning/e03_unsupervised_triplet/array_01_MSC-p02e03_unsupervised-axis-triplet_ALT.txt deleted file mode 100644 index b42c2ae7..00000000 --- a/research/part02_metric_learning/e03_unsupervised_triplet/array_01_MSC-p02e03_unsupervised-axis-triplet_ALT.txt +++ /dev/null @@ -1,25 +0,0 @@ -+EXTRA.sweep_num=1 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=2 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=3 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=4 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=5 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=cars3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=6 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=7 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=8 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=9 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=10 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=smallnorb settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=11 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=12 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=13 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=14 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=15 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=shapes3d settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=16 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=17 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=18 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=19 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=20 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long hydra.job.name=dotvae_hparams_9_long run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=dsprites settings.framework.recon_loss=mse framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=21 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long_xy hydra.job.name=dotvae_hparams_9_long_xy run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=none framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=22 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long_xy hydra.job.name=dotvae_hparams_9_long_xy run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=23 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long_xy hydra.job.name=dotvae_hparams_9_long_xy run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=semi_hard_neg framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=24 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long_xy hydra.job.name=dotvae_hparams_9_long_xy run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=hard_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null -+EXTRA.sweep_num=25 +DUMMY.repeat=1 +EXTRA.tags=sweep_dotvae_hard_params_9_long_xy hydra.job.name=dotvae_hparams_9_long_xy run_length=long metrics=all settings.framework.beta=0.01 settings.model.z_size=9 framework.cfg.triplet_loss=triplet framework.cfg.overlap_num=1024 sampling=gt_dist__manhat_scaled framework.cfg.ada_thresh_mode=dist framework=X--dotvae framework.cfg.detach_decoder=False schedule=adanegtvae_up_ratio framework.cfg.ada_thresh_ratio=0.5 framework.cfg.adat_triplet_share_scale=0.5 framework.cfg.triplet_margin_max=10.0 framework.cfg.triplet_scale=1.0 framework.cfg.triplet_p=1 dataset=X--xysquares +VAR.recon_loss_weight=1.0 +VAR.kernel_loss_weight=3969.0 +VAR.kernel_radius=31 settings.framework.recon_loss=mse_box_r$\{VAR.kernel_radius\}_l$\{VAR.recon_loss_weight\}_k$\{VAR.kernel_loss_weight\} framework.cfg.overlap_loss=null framework.cfg.overlap_mine_ratio=0.1 framework.cfg.overlap_mine_triplet_mode=easy_pos framework.cfg.overlap_augment_mode=augment framework.cfg.overlap_augment=null diff --git a/research/part02_metric_learning/e03_unsupervised_triplet/submit_01_unsupervised_axis_triplet.sh b/research/part02_metric_learning/e03_unsupervised_triplet/submit_01_unsupervised_axis_triplet.sh deleted file mode 100644 index 6b199a2f..00000000 --- a/research/part02_metric_learning/e03_unsupervised_triplet/submit_01_unsupervised_axis_triplet.sh +++ /dev/null @@ -1,217 +0,0 @@ -#!/bin/bash - - -# OVERVIEW: -# - this experiment is designed to find the optimal ada-triplet function and hyper-parameters - -# OUTCOMES: -# - ??? - - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="MSC-p02e03_unsupervised-axis-triplet" -export PARTITION="stampede" -export PARALLELISM=24 - -# the path to the generated arguments file -# - this needs to before we source the helper file -ARGS_FILE="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")/array_01_${PROJECT}.txt" -ARGS_FILE_ALT="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")/array_01_${PROJECT}_ALT.txt" - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Generate Experiment # -# ========================================================================= # - -# SWEEP FOR GOOD UNSUPERVISED DO-ADA-TVAE PARAMS -# 1 * (2*4*2*5) = 80 -ARGS_FILE="$ARGS_FILE" gen_sbatch_args_file \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_dotvae_hard_params_longmed' \ - hydra.job.name="dotvae_hparams_longmed" \ - \ - run_length=longmed \ - metrics=all \ - \ - settings.framework.beta=0.01 \ - settings.model.z_size=25 \ - \ - framework.cfg.triplet_loss=triplet \ - framework.cfg.overlap_num=512,1024 \ - sampling=gt_dist__manhat_scaled \ - framework.cfg.ada_thresh_mode=dist \ - \ - framework=X--dotvae \ - framework.cfg.detach_decoder=FALSE \ - \ - schedule=adanegtvae_up_ratio \ - framework.cfg.ada_thresh_ratio=0.5 \ - framework.cfg.adat_triplet_share_scale=0.5 \ - \ - framework.cfg.triplet_margin_max=10.0 \ - framework.cfg.triplet_scale=1.0 \ - framework.cfg.triplet_p=1 \ - \ - dataset=cars3d,smallnorb,shapes3d,dsprites \ - \ - \ - \ - \ - settings.framework.recon_loss=mse \ - \ - framework.cfg.overlap_loss=NULL \ - framework.cfg.overlap_mine_ratio=0.1,0.2 \ - framework.cfg.overlap_mine_triplet_mode=none,hard_neg,semi_hard_neg,hard_pos,easy_pos \ - \ - framework.cfg.overlap_augment_mode=augment \ - framework.cfg.overlap_augment=NULL - -# -- part of the above sweep! -# 1 * (2*1*2*5) = 20 -ARGS_FILE="$ARGS_FILE" APPEND_ARGS=1 ARGS_START_NUM=81 gen_sbatch_args_file \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_dotvae_hard_params_longmed_xy' \ - hydra.job.name="dotvae_hparams_longmed_xy" \ - \ - run_length=longmed \ - metrics=all \ - \ - settings.framework.beta=0.01 \ - settings.model.z_size=25 \ - \ - framework.cfg.triplet_loss=triplet \ - framework.cfg.overlap_num=512,1024 \ - sampling=gt_dist__manhat_scaled \ - framework.cfg.ada_thresh_mode=dist \ - \ - framework=X--dotvae \ - framework.cfg.detach_decoder=FALSE \ - \ - schedule=adanegtvae_up_ratio \ - framework.cfg.ada_thresh_ratio=0.5 \ - framework.cfg.adat_triplet_share_scale=0.5 \ - \ - framework.cfg.triplet_margin_max=10.0 \ - framework.cfg.triplet_scale=1.0 \ - framework.cfg.triplet_p=1 \ - \ - dataset=X--xysquares \ - \ - +VAR.recon_loss_weight=1.0 \ - +VAR.kernel_loss_weight=3969.0 \ - +VAR.kernel_radius=31 \ - settings.framework.recon_loss='mse_box_r${VAR.kernel_radius}_l${VAR.recon_loss_weight}_k${VAR.kernel_loss_weight}' \ - \ - framework.cfg.overlap_loss=NULL \ - framework.cfg.overlap_mine_ratio=0.1,0.2 \ - framework.cfg.overlap_mine_triplet_mode=none,hard_neg,semi_hard_neg,hard_pos,easy_pos \ - \ - framework.cfg.overlap_augment_mode=augment \ - framework.cfg.overlap_augment=NULL - -# RUN THE EXPERIMENT: -# clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours -ARGS_FILE="$ARGS_FILE" submit_sbatch_args_file - -# ========================================================================= # -# Generate Experiment 2 # -# ========================================================================= # - -# SWEEP UNSUPERVISED DO-ADA-TVAE WITH BEST SETTING FROM ABOVE, BUT WITH -# SMALLER Z_SIZE TO MATCH ORIG EXPERIMENTS AND IMPROVE DISENTANGLEMENT SCORES? -# 1 * (1*4*1*5) = 20 -ARGS_FILE="$ARGS_FILE_ALT" gen_sbatch_args_file \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_dotvae_hard_params_9_long' \ - hydra.job.name="dotvae_hparams_9_long" \ - \ - run_length=long \ - metrics=all \ - \ - settings.framework.beta=0.01 \ - settings.model.z_size=9 \ - \ - framework.cfg.triplet_loss=triplet \ - framework.cfg.overlap_num=1024 \ - sampling=gt_dist__manhat_scaled \ - framework.cfg.ada_thresh_mode=dist \ - \ - framework=X--dotvae \ - framework.cfg.detach_decoder=FALSE \ - \ - schedule=adanegtvae_up_ratio \ - framework.cfg.ada_thresh_ratio=0.5 \ - framework.cfg.adat_triplet_share_scale=0.5 \ - \ - framework.cfg.triplet_margin_max=10.0 \ - framework.cfg.triplet_scale=1.0 \ - framework.cfg.triplet_p=1 \ - \ - dataset=cars3d,smallnorb,shapes3d,dsprites \ - \ - settings.framework.recon_loss=mse \ - \ - framework.cfg.overlap_loss=NULL \ - framework.cfg.overlap_mine_ratio=0.1 \ - framework.cfg.overlap_mine_triplet_mode=none,hard_neg,semi_hard_neg,hard_pos,easy_pos \ - \ - framework.cfg.overlap_augment_mode=augment \ - framework.cfg.overlap_augment=NULL - -# -- part of the above sweep! -# 1 * (1*1*1*5) = 5 -ARGS_FILE="$ARGS_FILE_ALT" APPEND_ARGS=1 ARGS_START_NUM=21 gen_sbatch_args_file \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_dotvae_hard_params_9_long_xy' \ - hydra.job.name="dotvae_hparams_9_long_xy" \ - \ - run_length=long \ - metrics=all \ - \ - settings.framework.beta=0.01 \ - settings.model.z_size=9 \ - \ - framework.cfg.triplet_loss=triplet \ - framework.cfg.overlap_num=1024 \ - sampling=gt_dist__manhat_scaled \ - framework.cfg.ada_thresh_mode=dist \ - \ - framework=X--dotvae \ - framework.cfg.detach_decoder=FALSE \ - \ - schedule=adanegtvae_up_ratio \ - framework.cfg.ada_thresh_ratio=0.5 \ - framework.cfg.adat_triplet_share_scale=0.5 \ - \ - framework.cfg.triplet_margin_max=10.0 \ - framework.cfg.triplet_scale=1.0 \ - framework.cfg.triplet_p=1 \ - \ - dataset=X--xysquares \ - \ - +VAR.recon_loss_weight=1.0 \ - +VAR.kernel_loss_weight=3969.0 \ - +VAR.kernel_radius=31 \ - settings.framework.recon_loss='mse_box_r${VAR.kernel_radius}_l${VAR.recon_loss_weight}_k${VAR.kernel_loss_weight}' \ - \ - framework.cfg.overlap_loss=NULL \ - framework.cfg.overlap_mine_ratio=0.1 \ - framework.cfg.overlap_mine_triplet_mode=none,hard_neg,semi_hard_neg,hard_pos,easy_pos \ - \ - framework.cfg.overlap_augment_mode=augment \ - framework.cfg.overlap_augment=NULL - -# RUN THE EXPERIMENT: -# clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours -ARGS_FILE="$ARGS_FILE_ALT" submit_sbatch_args_file - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/part02_metric_learning/plot00_all/make_graphs.py b/research/part02_metric_learning/plot00_all/make_graphs.py deleted file mode 100644 index 1ecf8708..00000000 --- a/research/part02_metric_learning/plot00_all/make_graphs.py +++ /dev/null @@ -1,436 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import itertools -import os -from typing import Optional -from typing import Sequence -from typing import Tuple - -import numpy as np -import torch -from matplotlib import cm -from matplotlib import pyplot as plt -from tqdm import tqdm - -import research.code.util as H -from research.code.metrics._factored_components import compute_axis_score -from research.code.metrics._factored_components import compute_linear_score -from disent.util.seeds import seed - - -# ========================================================================= # -# distance function # -# ========================================================================= # - - -def _rotation_matrix(d, i, j, deg): - assert 0 <= i < j <= d - mat = torch.eye(d, dtype=torch.float32) - r = np.deg2rad(deg) - s, c = np.sin(r), np.cos(r) - mat[i, i] = c - mat[j, j] = c - mat[j, i] = -s - mat[i, j] = s - return mat - - -def rotation_matrix_2d(deg): - return _rotation_matrix(d=2, i=0, j=1, deg=deg) - - -def _random_rotation_matrix(d): - mat = torch.eye(d, dtype=torch.float32) - for i in range(d): - for j in range(i+1, d): - mat @= _rotation_matrix(d, i, j, np.random.randint(0, 360)) - return mat - - -def make_2d_line_points(n: int = 100, deg: float = 30, std_x: float = 1.0, std_y: float = 0.005): - points = torch.randn(n, 2, dtype=torch.float32) * torch.as_tensor([[std_x, std_y]], dtype=torch.float32) - points = points @ rotation_matrix_2d(deg) - return points - - -def make_nd_line_points(n: int = 100, dims: int = 4, std_x: float = 1.0, std_y: float = 0.005): - if not isinstance(dims, int): - m, M = dims - dims = np.randint(m, M) - # generate numbers - xs = torch.randn(n, dims, dtype=torch.float32) - # axis standard deviations - if isinstance(std_y, (float, int)): - std_y = torch.full((dims-1,), fill_value=std_y, dtype=torch.float32) - else: - m, M = std_y - std_y = torch.rand(dims-1, dtype=torch.float32) * (M - m) + m - # scale axes - std = torch.cat([torch.as_tensor([std_x]), std_y]) - xs = xs * std[None, :] - # rotate - return xs @ _random_rotation_matrix(dims) - - -def make_line_points(n: int = 100, deg: float = None, dims: int = 2, std_x: float = 1.0, std_y: float = 0.1): - if deg is None: - return make_nd_line_points(n=n, dims=dims, std_x=std_x, std_y=std_y) - else: - assert dims == 2, f'if "deg" is not None, then "dims" must equal 2, currently set to: {repr(dims)}' - return make_2d_line_points(n=n, deg=deg, std_x=std_x, std_y=std_y) - - -# def random_line(std, n=100): -# std = torch.as_tensor(std, dtype=torch.float32) -# (d,) = std.shape -# # generate numbers -# xs = torch.randn(n, d, dtype=torch.float32) -# # scale axes -# xs = xs * std[None, :] -# # rotate -# return xs @ _random_rotation_matrix(d) - - -# ========================================================================= # -# GAUSSIAN # -# ========================================================================= # - - -def gaussian_1d(x, s): return 1 / (np.sqrt(2 * np.pi) * s) * torch.exp(-(x**2)/(2*s**2)) -def gaussian_1d_dx(x, s): return gaussian_1d(x, s) * (-x/s**2) -def gaussian_1d_dx2(x, s): return gaussian_1d(x, s) * ((x**2 - s**2)/s**4) - - -def gaussian_2d(x, y, sx, sy): return gaussian_1d(x, sx) * gaussian_1d(y, sy) -def gaussian_2d_dy(x, y, sx, sy): return gaussian_1d(x, sx) * gaussian_1d_dx(y, sy) -def gaussian_2d_dy2(x, y, sx, sy): return gaussian_1d(x, sx) * gaussian_1d_dx2(y, sy) - - -def rotated_radius_meshgrid(radius: float, num_points: int, deg: float = 0, device=None, return_orig=False) -> Tuple[torch.Tensor, torch.Tensor]: - # x & y values centered around zero - # p = torch.arange(size, device=device) - (size-1)/2 - p = torch.linspace(-radius, radius, num_points, device=device) - x, y = torch.meshgrid(p, p) - # matrix multiplication along first axis | https://pytorch.org/docs/stable/generated/torch.einsum.html - rx, ry = torch.einsum('dxy,kd->kxy', torch.stack([x, y]), rotation_matrix_2d(deg)) - # result - if return_orig: - return (rx, ry), (x, y) - return rx, ry - - -def rotated_guassian2d(std_x: float, std_y: float, deg: float, trunc_sigma: Optional[float] = None, num_points: int = 511): - radius = (2.25*max(std_x, std_y)) if (trunc_sigma is None) else trunc_sigma - (xs_r, ys_r), (xs, ys) = rotated_radius_meshgrid(radius=radius, num_points=num_points, deg=deg, return_orig=True) - zs = gaussian_2d(xs_r, ys_r, sx=std_x, sy=std_y) - zs /= zs.sum() - return xs, ys, zs - - -def plot_gaussian( - deg: float = 0.0, - std_x: float = 1.0, - std_y: float = 0.1, - # contour - contour_resolution: int = 255, - contour_trunc_sigma: Optional[float] = None, - contour_kwargs: Optional[dict] = None, - # dots - dots_num: Optional[int] = None, - dots_kwargs: Optional[dict] = None, - # axis - ax=None, -): - if ax is None: - fig = plt.figure() - ax = fig.gca() - # set limits - trunc_sigma = (2.05 * max(std_x, std_y)) if (contour_trunc_sigma is None) else contour_trunc_sigma - ax.set_xlim([-trunc_sigma, trunc_sigma]) - ax.set_ylim([-trunc_sigma, trunc_sigma]) - # plot contour - xs, ys, zs = rotated_guassian2d(std_x=std_x, std_y=std_y, deg=deg, trunc_sigma=trunc_sigma, num_points=contour_resolution) - ax.contourf(xs, ys, zs, **({} if contour_kwargs is None else contour_kwargs)) - # plot dots - if dots_num is not None: - points = make_line_points(n=dots_num, dims=2, deg=deg, std_x=std_x, std_y=std_y) - ax.scatter(*points.T, **({} if dots_kwargs is None else dots_kwargs)) - # done - return ax - - -# ========================================================================= # -# Generate Average Plots # -# ========================================================================= # - - -def score_grid( - deg_rotations: Sequence[Optional[float]], - y_std_ratios: Sequence[float], - x_std: float = 1.0, - num_points: int = 1000, - num_dims: int = 2, - use_std: bool = True, - use_max: bool = False, - norm: bool = True, - return_points: bool = False, -): - h, w = len(y_std_ratios), len(deg_rotations) - # grids - axis_scores = torch.zeros([h, w], dtype=torch.float64) - linear_scores = torch.zeros([h, w], dtype=torch.float64) - if return_points: - all_points = torch.zeros([h, w, num_points, num_dims], dtype=torch.float64) - # compute scores - for i, y_std_ratio in enumerate(y_std_ratios): - for j, deg in enumerate(deg_rotations): - points = make_line_points(n=num_points, dims=num_dims, deg=deg, std_x=x_std, std_y=x_std * y_std_ratio) - axis_scores[i, j] = compute_axis_score(points, use_std=use_std, use_max=use_max, norm=norm) - linear_scores[i, j] = compute_linear_score(points, use_std=use_std, use_max=use_max, norm=norm) - if return_points: - all_points[i, j] = points - # results - if return_points: - return axis_scores, linear_scores, all_points - return axis_scores, linear_scores - - -def ave_score_grid( - deg_rotations: Sequence[Optional[float]], - y_std_ratios: Sequence[float], - x_std: float = 1.0, - num_points: int = 1000, - num_dims: int = 2, - use_std: bool = True, - use_max: bool = False, - norm: bool = True, - repeats: int = 10, -): - results = [] - # repeat - for i in tqdm(range(repeats)): - results.append(score_grid(deg_rotations=deg_rotations, y_std_ratios=y_std_ratios, x_std=x_std, num_points=num_points, num_dims=num_dims, use_std=use_std, use_max=use_max, norm=norm)) - # average results - all_axis_scores, all_linear_scores = zip(*results) - axis_scores = torch.mean(torch.stack(all_axis_scores, dim=0), dim=0) - linear_scores = torch.mean(torch.stack(all_linear_scores, dim=0), dim=0) - # results - return axis_scores, linear_scores - - -def make_ave_scores_plot( - std_num: int = 21, - deg_num: int = 21, - ndim: Optional[int] = None, - # extra - num_points: int = 1000, - repeats: int = 25, - x_std: float = 1.0, - use_std: bool = True, - use_max: bool = False, - norm: bool = True, - # cmap - cmap_axis: str = 'GnBu_r', # 'RdPu_r', 'GnBu_r', 'Blues_r', 'viridis', 'plasma', 'magma' - cmap_linear: str = 'RdPu_r', # 'RdPu_r', 'GnBu_r', 'Blues_r', 'viridis', 'plasma', 'magma' - vertical: bool = True, - # subplot settings - subplot_size: float = 4., - subplot_padding: float = 1.5, -): - # make sure to handle the random case - deg_num = std_num if (ndim is None) else deg_num - axis_scores, linear_scores = ave_score_grid( - deg_rotations=np.linspace(0., 180., num=deg_num) if (ndim is None) else [None], - y_std_ratios=np.linspace(0., 1., num=std_num), - x_std=x_std, - num_points=num_points, - num_dims=2 if (ndim is None) else ndim, - use_std=use_std, - use_max=use_max, - norm=norm, - repeats=repeats, - ) - # make plot - fig, axs = H.plt_subplots( - nrows=1+int(vertical), - ncols=1+int(not vertical), - titles=['Linear', 'Axis'], - row_labels=f'$σ_y$ - Standard Deviation', - col_labels=f'θ - Rotation Degrees', - figsize=(subplot_size + 0.5, subplot_size * 2 * (deg_num / std_num) + 0.75)[::1 if vertical else -1] - ) - (ax0, ax1) = axs.flatten() - # subplots - ax0.imshow(linear_scores, cmap=cmap_linear, extent=[0., 180., 1., 0.]) - ax1.imshow(axis_scores, cmap=cmap_axis, extent=[0., 180., 1., 0.]) - for ax in axs.flatten(): - ax.set_aspect(180 * (std_num / deg_num)) - if len(ax.get_xticks()): - ax.set_xticks(np.linspace(0., 180., 5)) - # layout - fig.tight_layout(pad=subplot_padding) - # done - return fig, axs - - -# ========================================================================= # -# HELPER # -# ========================================================================= # - - -def plot_scores(ax, axis_score, linear_score): - from matplotlib.lines import Line2D - assert 0 <= linear_score <= 1 - assert 0 <= axis_score <= 1 - linear_rgb = cm.get_cmap('RdPu_r')(np.clip(linear_score, 0., 1.)) - axis_rgb = cm.get_cmap('GnBu_r')(np.clip(axis_score, 0., 1.)) - ax.legend(handles=[ - Line2D([0], [0], label=f'Linear: {float(linear_score):.2f}', color=linear_rgb, marker='o', markersize=10, linestyle='None'), - Line2D([0], [0], label=f'Axis: {float(axis_score):.2f}', color=axis_rgb, marker='o', markersize=10, linestyle='None'), - ]) - return ax - - -# ========================================================================= # -# Generate Grid Plots # -# ========================================================================= # - - -def make_grid_gaussian_score_plot( - # grid - y_stds: Sequence[float] = (0.8, 0.2, 0.05)[::-1], # (0.8, 0.4, 0.2, 0.1, 0.05), - deg_rotations: Sequence[float] = (0, 22.5, 45, 67.5, 90, 112.5, 135, 157.5), # (0, 15, 30, 45, 60, 75, 90, 105, 120, 135, 150, 165), - # plot dot options - dots_num: Optional[int] = None, - # score options - num_points: int = 10000, - repeats: int = 100, - use_std: bool = True, - use_max: bool = False, - norm: bool = True, - # grid options - subplot_size: float = 2.125, - subplot_padding: float = 0.5, - subplot_contour_kwargs: Optional[dict] = None, - subplot_dots_kwargs: Optional[dict] = None, -): - # defaults - if subplot_contour_kwargs is None: subplot_contour_kwargs = dict(cmap='Blues') - if subplot_dots_kwargs is None: subplot_dots_kwargs = dict(cmap='Purples') - - # make figure - nrows, ncols = len(y_stds), len(deg_rotations) - fig, axs = H.plt_subplots( - nrows=nrows, ncols=ncols, - row_labels=[f'$σ_y$ = {std_y}' for std_y in y_stds], - col_labels=[f'θ = {deg}°' for deg in deg_rotations], - hide_axis='all', - figsize=(ncols*subplot_size, nrows*subplot_size), - ) - - # progress - p = tqdm(total=axs.size, desc='generating_plot') - # generate plot - for (y, std_y), (x, deg) in itertools.product(enumerate(y_stds), enumerate(deg_rotations)): - # compute scores - axis_score, linear_score = [], [] - for k in range(repeats): - points = make_2d_line_points(n=num_points, deg=deg, std_x=1.0, std_y=std_y) - axis_score.append(compute_axis_score(points, use_std=use_std, use_max=use_max, norm=norm)) - linear_score.append(compute_linear_score(points, use_std=use_std, use_max=use_max, norm=norm)) - axis_score, linear_score = np.mean(axis_score), np.mean(linear_score) - # generate subplots - plot_gaussian(ax=axs[y, x], deg=deg, std_x=1.0, std_y=std_y, dots_num=dots_num, contour_trunc_sigma=2.05, contour_kwargs=subplot_contour_kwargs, dots_kwargs=subplot_dots_kwargs) - plot_scores(ax=axs[y, x], axis_score=axis_score, linear_score=linear_score) - # update progress - p.update() - plt.tight_layout(pad=subplot_padding) - - return fig, axs - - -# ========================================================================= # -# MAIN # -# ========================================================================= # - - -if __name__ == '__main__': - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../../code/util/gadfly.mplstyle')) - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # - - # plot everything - seed(777) - make_grid_gaussian_score_plot( - repeats=250, - num_points=25000, - ) - plt.savefig(H.make_rel_path_add_ext('plots/metric_grid', ext='.png')) - plt.show() - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # - - # plot everything -- minimal - seed(777) - make_grid_gaussian_score_plot( - y_stds=(0.8, 0.4, 0.2, 0.1, 0.05)[::-1], # (0.8, 0.4, 0.2, 0.1, 0.05), - deg_rotations=(0, 22.5, 45, 67.5, 90), - repeats=250, - num_points=25000, - ) - plt.savefig(H.make_rel_path_add_ext('plots/metric_grid_minimal_5x5', ext='.png')) - plt.show() - - # plot everything -- minimal - seed(777) - make_grid_gaussian_score_plot( - y_stds=(0.8, 0.4, 0.2, 0.05)[::-1], # (0.8, 0.4, 0.2, 0.1, 0.05), - deg_rotations=(0, 22.5, 45, 67.5, 90), - repeats=250, - num_points=25000, - ) - plt.savefig(H.make_rel_path_add_ext('plots/metric_grid_minimal_4x5', ext='.png')) - plt.show() - - # plot everything -- minimal - seed(777) - fig, axs = make_grid_gaussian_score_plot( - y_stds=(0.8, 0.2, 0.05)[::-1], # (0.8, 0.4, 0.2, 0.1, 0.05), - deg_rotations=(0, 22.5, 45, 67.5, 90), - repeats=250, - num_points=25000, - ) - plt.savefig(H.make_rel_path_add_ext('plots/metric_grid_minimal_3x5', ext='.png')) - plt.show() - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # - - seed(777) - make_ave_scores_plot(repeats=250, num_points=10000, use_max=False) - plt.savefig(H.make_rel_path_add_ext('plots/metric_scores', ext='.png')) - plt.show() - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # diff --git a/research/part02_metric_learning/plot00_all/plot_p02_wandb_experiments.py b/research/part02_metric_learning/plot00_all/plot_p02_wandb_experiments.py deleted file mode 100644 index 5dfa20f6..00000000 --- a/research/part02_metric_learning/plot00_all/plot_p02_wandb_experiments.py +++ /dev/null @@ -1,1240 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2022 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import itertools -import logging -import os -from pprint import pprint -from typing import Optional -from typing import Sequence -from typing import Union - -import numpy as np -import pandas as pd -import seaborn as sns -from matplotlib import pyplot as plt -import matplotlib as mpl -import matplotlib.lines as mlines -import matplotlib.patches as mpatches -from cachier import cachier as _cachier - -import research.code.util as H -from disent.util.function import wrapped_partial -from disent.util.profiling import Timer -from research.code.util._wandb_plots import drop_non_unique_cols -from research.code.util._wandb_plots import drop_unhashable_cols -from research.code.util._wandb_plots import load_runs as _load_runs - - -# ========================================================================= # -# Helper # -# ========================================================================= # - - -# cachier instance -CACHIER: _cachier = wrapped_partial(_cachier, cache_dir=os.path.join(os.path.dirname(__file__), 'plots/.cache')) - - -@CACHIER() -def load_runs(project: str, include_history: bool = False): - return _load_runs(project=project, include_history=include_history) - - -def clear_cache(clear_data=True, clear_wandb=False): - from research.code.util._wandb_plots import clear_runs_cache - if clear_wandb: - clear_runs_cache() - if clear_data: - load_runs.clear_cache() - - -# ========================================================================= # -# Prepare Data # -# ========================================================================= # - - -# common keys -K_GROUP = 'Run Group' -K_DATASET = 'Dataset' -K_FRAMEWORK = 'Framework' -K_BETA = 'Beta' -K_LOSS = 'Recon. Loss' -K_Z_SIZE = 'Latent Dims.' -K_REPEAT = 'Repeat' -K_STATE = 'State' -K_LR = 'Learning Rate' -K_SCHEDULE = 'Schedule' -K_SAMPLER = 'Sampler' -K_ADA_MODE = 'Threshold Mode' - -K_MIG_END = 'MIG Score\n(End)' -K_DCI_END = 'DCI Score\n(End)' -K_MIG_MAX = 'MIG Score' -K_DCI_MAX = 'DCI Score' -K_LCORR_GT_F = 'Linear Corr.\n(factors)' -K_RCORR_GT_F = 'Rank Corr.\n(factors)' -K_LCORR_GT_G = 'Global Linear Corr.\n(factors)' -K_RCORR_GT_G = 'Global Rank Corr.\n(factors)' -K_LCORR_DATA_F = 'Linear Corr.\n(data)' -K_RCORR_DATA_F = 'Rank Corr.\n(data)' -K_LCORR_DATA_G = 'Global Linear Corr.\n(data)' -K_RCORR_DATA_G = 'Global Rank Corr.\n(data)' -K_AXIS = 'Axis Ratio' -K_LINE = 'Linear Ratio' - -K_TRIPLET_SCALE = 'Triplet Scale' # framework.cfg.triplet_margin_max -K_TRIPLET_MARGIN = 'Triplet Margin' # framework.cfg.triplet_scale -K_TRIPLET_P = 'Triplet P' # framework.cfg.triplet_p -K_DETACH = 'Detached' # framework.cfg.detach_decoder -K_TRIPLET_MODE = 'Triplet Mode' # framework.cfg.triplet_loss - -# ALL_K_SCORES = [ -# K_MIG, -# K_DCI, -# # K_LCORR_GT_F, -# K_RCORR_GT_F, -# # K_LCORR_GT_G, -# # K_RCORR_GT_G, -# # K_LCORR_DATA_F, -# K_RCORR_DATA_F, -# # K_LCORR_DATA_G, -# # K_RCORR_DATA_G, -# # K_LINE, -# # K_AXIS, -# ] - - -def load_general_data(project: str, include_history: bool = False, keep_cols: Sequence[str] = None): - # keep columns - if keep_cols is None: - keep_cols = [] - keep_cols = list(keep_cols) - # load data - with Timer('loading data'): - df = load_runs(project, include_history=include_history) - # process data - with Timer('processing data'): - # filter out unneeded columns - df, dropped_hash = drop_unhashable_cols(df, skip=['history'] + keep_cols) - df, dropped_diverse = drop_non_unique_cols(df, skip=['history'] + keep_cols) - # rename columns - df = df.rename(columns={ - 'settings/optimizer/lr': K_LR, - 'EXTRA/tags': K_GROUP, - 'dataset/name': K_DATASET, - 'framework/name': K_FRAMEWORK, - 'settings/framework/beta': K_BETA, - 'settings/framework/recon_loss': K_LOSS, - 'settings/model/z_size': K_Z_SIZE, - 'DUMMY/repeat': K_REPEAT, - 'state': K_STATE, - 'epoch_metric/mig.discrete_score.max': K_MIG_MAX, - 'epoch_metric/dci.disentanglement.max': K_DCI_MAX, - 'final_metric/mig.discrete_score.max': K_MIG_END, - 'final_metric/dci.disentanglement.max': K_DCI_END, - # scores - 'epoch_metric/distances.lcorr_ground_latent.l1.factor.max': K_LCORR_GT_F, - 'epoch_metric/distances.rcorr_ground_latent.l1.factor.max': K_RCORR_GT_F, - 'epoch_metric/distances.lcorr_ground_latent.l1.global.max': K_LCORR_GT_G, - 'epoch_metric/distances.rcorr_ground_latent.l1.global.max': K_RCORR_GT_G, - 'epoch_metric/distances.lcorr_latent_data.l2.factor.max': K_LCORR_DATA_F, - 'epoch_metric/distances.rcorr_latent_data.l2.factor.max': K_RCORR_DATA_F, - 'epoch_metric/distances.lcorr_latent_data.l2.global.max': K_LCORR_DATA_G, - 'epoch_metric/distances.rcorr_latent_data.l2.global.max': K_RCORR_DATA_G, - 'epoch_metric/linearity.axis_ratio.var.max': K_AXIS, - 'epoch_metric/linearity.linear_ratio.var.max': K_LINE, - # adaptive methods - 'schedule/name': K_SCHEDULE, - 'sampling/name': K_SAMPLER, - 'framework/cfg/ada_thresh_mode': K_ADA_MODE, - # triplet experiments - 'framework/cfg/triplet_margin_max': K_TRIPLET_MARGIN, - 'framework/cfg/triplet_scale': K_TRIPLET_SCALE, - 'framework/cfg/triplet_p': K_TRIPLET_P, - 'framework/cfg/detach_decoder': K_DETACH, - 'framework/cfg/triplet_loss': K_TRIPLET_MODE, - }) - return df - - -def rename_entries(df: pd.DataFrame): - df = df.copy() - # replace values in the df - for key, value, new_value in [ - (K_DATASET, 'xysquares_minimal', 'xysquares'), - (K_SCHEDULE, 'adanegtvae_up_all_full', 'Schedule: Both (strong)'), - (K_SCHEDULE, 'adanegtvae_up_all', 'Schedule: Both (weak)'), - (K_SCHEDULE, 'adanegtvae_up_ratio_full', 'Schedule: Weight (strong)'), - (K_SCHEDULE, 'adanegtvae_up_ratio', 'Schedule: Weight (weak)'), - (K_SCHEDULE, 'adanegtvae_up_thresh', 'Schedule: Threshold'), - (K_TRIPLET_MODE, 'triplet', 'Triplet Loss (Hard Margin)'), - (K_TRIPLET_MODE, 'triplet_soft', 'Triplet Loss (Soft Margin)'), - (K_TRIPLET_P, 1, 'L1 Distance'), - (K_TRIPLET_P, 2, 'L2 Distance'), - # (K_SAMPLER, 'gt_dist__manhat', 'Ground-Truth Dist Sampling'), - # (K_SAMPLER, 'gt_dist__manhat_scaled', 'Ground-Truth Dist Sampling (Scaled)'), - ]: - if key in df.columns: - df[key].replace(value, new_value, inplace=True) - # df.loc[df[key] == value, key] = new_value - return df - - -def drop_and_copy_old_invalid_xysquares(df: pd.DataFrame) -> pd.DataFrame: - # hack to replace the invalid sampling with the correct sampling! - df = df.copy() - # drop invalid items - sel_drop = (df[K_DATASET] == 'xysquares') & (df[K_SAMPLER] == 'gt_dist__manhat_scaled') - df = df.drop(df[sel_drop].index) - # copy invalid items & rename entries - sel_copy = (df[K_DATASET] == 'xysquares') & (df[K_SAMPLER] == 'gt_dist__manhat') - assert np.sum(sel_drop) == np.sum(sel_copy), f'drop: {np.sum(sel_drop)}, copy: {np.sum(sel_copy)}' - df_append = df[sel_copy].copy() - df_append.loc[df_append[K_SAMPLER] == 'gt_dist__manhat', K_SAMPLER] = 'gt_dist__manhat_scaled' - # append the rows! - df = df.append(df_append) - # done! - return df - - -# ========================================================================= # -# Plot Experiments # -# ========================================================================= # - - -PINK = '#FE375F' # usually: Beta-VAE -PURPLE = '#5E5BE5' # maybe: Ada-TVAE -LPURPLE = '#b0b6ff' # maybe: Ada-TVAE (alt) -BLUE = '#1A93FE' # maybe: TVAE -LBLUE = '#63D2FE' -ORANGE = '#FE9F0A' # usually: Ada-VAE -GREEN = '#2FD157' - -LGREEN = '#9FD911' # usually: MSE -LBLUE2 = '#36CFC8' # usually: MSE-Overlap - - -# ========================================================================= # -# Experiment p02e00 # -# ========================================================================= # - -def plot_e00_beta_metric_correlation( - rel_path: Optional[str] = None, - save: bool = True, - show: bool = True, - color_betavae: str = PINK, - color_adavae: str = ORANGE, - grid_size_v: float = 2.25, - grid_size_h: float = 2.75, - metrics: Sequence[str] = (K_MIG_MAX, K_RCORR_GT_F, K_RCORR_DATA_F), # (K_MIG, K_DCI, K_RCORR_GT_F, K_RCORR_DATA_F, K_AXIS, K_LINE) -): - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - df: pd.DataFrame = load_general_data(f'{os.environ["WANDB_USER"]}/MSC-p02e00_beta-data-latent-corr') - # select run groups - df = df[df[K_GROUP].isin(['sweep_beta_corr'])] - df = df[~df[K_DATASET].isin(['xyobject'])] - df = df[df[K_STATE].isin(['finished', 'running'])] - # df = df[df[K_LR].isin([0.0001])] # 0.0001, 0.001 - # sort everything - df = df.sort_values([K_FRAMEWORK, K_DATASET, K_BETA, K_LR]) - # print common key values - print('K_GROUP: ', list(df[K_GROUP].unique())) - print('K_FRAMEWORK:', list(df[K_FRAMEWORK].unique())) - print('K_BETA: ', list(df[K_BETA].unique())) - print('K_REPEAT: ', list(df[K_REPEAT].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - print('K_DATASET: ', list(df[K_DATASET].unique())) - print('K_LR: ', list(df[K_LR].unique())) - # number of runs - print(f'total={len(df)}') - df = rename_entries(df) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # plot vars - x_m = 10**-4 / (10**0.5) - x_M = 10**0 * (10**0.5) - y_m = -0.05 - y_M = 1.05 - ax_m = x_m * 1.05 - ax_M = x_M / 1.05 - ay_m = y_m + 0.01 - ay_M = y_M - 0.01 - - # rename entries - df.loc[df[K_DATASET] == 'xysquares_minimal', K_DATASET] = 'xysquares' - - # manual remove invalid values - df = df[df[K_LINE].notna()] - df = df[df[K_AXIS].notna()] - # make sure we do not have invalid values! - for k in metrics: - df = df[df[k].notna()] - # df = df[df[k] >= 0] - # df = df[df[k] <= 1] - # for k in metrics: - # df[k] = df[k].replace(np.nan, 0) - - ALL_DATASETS = list(df[K_DATASET].unique()) - num_datasets = len(df[K_DATASET].unique()) - num_scores = len(metrics) - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - # make plot - fig, axs = plt.subplots(num_scores, num_datasets, figsize=(grid_size_h*num_datasets, num_scores*grid_size_v)) - # fill plot - for i, score_key in enumerate(metrics): - for j, dataset in enumerate(ALL_DATASETS): - # filter data - ada_df = df[(df[K_FRAMEWORK] == 'adavae_os') & (df[K_DATASET] == dataset)] - beta_df = df[(df[K_FRAMEWORK] == 'betavae') & (df[K_DATASET] == dataset)] - # plot! - # sns.regplot(ax=axs[i, j], x=K_BETA, y=score_key, data=ada_df, seed=777, order=2, robust=False, color=color_adavae, marker='o') - # sns.regplot(ax=axs[i, j], x=K_BETA, y=score_key, data=beta_df, seed=777, order=2, robust=False, color=color_betavae, marker='x', line_kws=dict(linestyle='dashed')) - sns.lineplot(ax=axs[i, j], x=K_BETA, y=score_key, data=ada_df, ci=None, color=color_adavae) - sns.lineplot(ax=axs[i, j], x=K_BETA, y=score_key, data=beta_df, ci=None, color=color_betavae, linestyle='dashed') - sns.scatterplot(ax=axs[i, j], x=K_BETA, y=score_key, data=ada_df, color=color_adavae, marker='o') - sns.scatterplot(ax=axs[i, j], x=K_BETA, y=score_key, data=beta_df, color=color_betavae, marker='X') - # set the axis limits - axs[i, j].set(xscale='log', ylim=(y_m, y_M), xlim=(x_m, x_M)) - # hide labels - if i == 0: - axs[i, j].set_title(dataset) - if i < num_scores-1: - axs[i, j].set_xlabel(None) - axs[i, j].set_xticklabels([]) - if j > 0: - axs[i, j].set_ylabel(None) - axs[i, j].set_yticklabels([]) - # draw border - axs[i, j].plot([ax_m, ax_m], [ay_m, ay_M], color='#cccccc', linewidth=1) # left - axs[i, j].plot([ax_m, ax_M], [ay_m, ay_m], color='#cccccc', linewidth=1) # bottom - axs[i, j].plot([ax_M, ax_M], [ay_m, ay_M], color='#cccccc', linewidth=1) # right - axs[i, j].plot([ax_m, ax_M], [ay_M, ay_M], color='#cccccc', linewidth=1) # top - # add axis labels - axs[i, j].set_xticks([10**i for i in [-4, -3, -2, -1, 0]]) - # add the legend to the top right plot - marker_ada = mlines.Line2D([], [], color=color_adavae, marker='o', markersize=12, label='Ada-GVAE') - marker_beta = mlines.Line2D([], [], color=color_betavae, marker='X', markersize=12, label='Beta-VAE') # why does 'x' not work? only 'X'? - axs[0, -1].legend(handles=[marker_beta, marker_ada], fontsize=14) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - # PLOT - fig.tight_layout() - H.plt_rel_path_savefig(rel_path, save=save, show=show, dpi=150, ext='.png') - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - -# ========================================================================= # -# Experiment p02e02 # -# ========================================================================= # - - -# TODO: this should be replaced with hard triplet loss experiments! -def plot_e02_axis_triplet_schedules( - rel_path: Optional[str] = None, - save: bool = True, - show: bool = True, - grid_size_v: float = 1.4, - grid_size_h: float = 2.75, - metrics: Sequence[str] = (K_MIG_END, K_DCI_END, K_RCORR_GT_F, K_RCORR_DATA_F, K_AXIS, K_LINE), - adaptive_modes: Sequence[str] = ('dist',), - title: Optional[Union[str, bool]] = True, -): - if (title is True) and len(adaptive_modes) == 1: - title = adaptive_modes[0] - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - with Timer('getting data'): - df: pd.DataFrame = load_general_data(f'{os.environ["WANDB_USER"]}/MSC-p02e02_axis-aligned-triplet') - # select run groups - df = df[df[K_GROUP].isin(['sweep_adanegtvae_params_longmed'])] - df = df[df[K_ADA_MODE].isin(adaptive_modes)] - # sort everything - df = df.sort_values([K_FRAMEWORK, K_DATASET, K_SCHEDULE, K_SAMPLER, K_ADA_MODE]) - # print common key values - print('K_GROUP: ', list(df[K_GROUP].unique())) - print('K_FRAMEWORK: ', list(df[K_FRAMEWORK].unique())) - print('K_BETA: ', list(df[K_BETA].unique())) - print('K_REPEAT: ', list(df[K_REPEAT].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - print('K_DATASET: ', list(df[K_DATASET].unique())) - print('K_LR: ', list(df[K_LR].unique())) - print('K_TRIPLET_MODE: ', list(df[K_TRIPLET_MODE].unique())) - print('K_SCHEDULE: ', list(df[K_SCHEDULE].unique())) - print('K_SAMPLER: ', list(df[K_SAMPLER].unique())) - print('K_ADA_MODE: ', list(df[K_ADA_MODE].unique())) - # number of runs - print(f'total={len(df)}') - df = rename_entries(df) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - # hack to replace the invalid sampling with the correct sampling! - df = drop_and_copy_old_invalid_xysquares(df) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - # axis keys - col_x = K_DATASET - col_bars = K_SCHEDULE - col_hue = K_SAMPLER - # get rows and columns - all_bars = list(df[col_bars].unique()) - all_x = list(df[col_x].unique()) - all_y = metrics - all_hue = list(df[col_hue].unique()) - # num rows and cols - num_bars = len(all_bars) - num_x = len(all_x) - num_y = len(all_y) - num_hue = len(all_hue) - # palettes - colors = [PINK, ORANGE, BLUE, LBLUE, PURPLE] - hatches = [None, '..'] - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - # make plot - fig, axs = plt.subplots(num_y, num_x, figsize=(grid_size_h*num_x, num_y*grid_size_v)) - if isinstance(title, str): - fig.suptitle(title, fontsize=19) - # fill plot - for y, key_y in enumerate(all_y): # metric - for x, key_x in enumerate(all_x): # schedule - ax = axs[y, x] - # filter items - df_filtered = df[(df[col_x] == key_x)] - # plot! - sns.barplot(ax=ax, x=col_bars, y=key_y, hue=col_hue, data=df_filtered) - ax.set(ylim=(0, 1), xlim=(-0.5, num_bars - 0.5)) - # set colors - for i, (bar, color) in enumerate(zip(ax.patches, itertools.cycle(colors))): - bar.set_facecolor(color) - bar.set_edgecolor('white') - bar.set_linewidth(2) - if hatches[i // num_bars]: - bar.set_facecolor(color + '65') - bar.set_hatch(hatches[i // num_bars]) - # remove labels - if ax.get_legend(): - ax.get_legend().remove() - if y == 0: - ax.set_title(key_x) - # if y < num_y-1: - ax.set_xlabel(None) - ax.set_xticklabels([]) - if x > 0: - ax.set_ylabel(None) - ax.set_yticklabels([]) - # add the legend to the top right plot - handles_a = [mpl.patches.Patch(label=label, color=color) for label, color in zip(all_bars, colors)] - fig.legend(handles=handles_a, fontsize=12, bbox_to_anchor=(0.986, 0.03), loc='lower right', ncol=2, labelspacing=0.1) - # add the legend to the top right plot - assert np.all(df[col_hue].unique() == ['gt_dist__manhat', 'gt_dist__manhat_scaled']), f'{list(df[col_hue].unique())}' - handles_b = [mpl.patches.Patch(label='Ground-Truth Dist Sampling', facecolor='#505050ff', edgecolor='white', hatch=hatches[0]), # gt_dist__manhat - mpl.patches.Patch(label='Ground-Truth Dist Sampling (Scaled)', facecolor='#50505065', edgecolor='white', hatch=hatches[1])] # gt_dist__manhat_scaled - fig.legend(handles=handles_b, fontsize=12, bbox_to_anchor=(0.073, 0.03), loc='lower left', ncol=1, labelspacing=0.1) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - # plot! - fig.tight_layout() - H.plt_rel_path_savefig(rel_path, save=save, show=show, dpi=150, ext='.png') - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - -def lighten_color(color, amount=0.5): - """ - darkens the color if amount > 1 - otherwise lightens the color - - FROM: https://stackoverflow.com/questions/37765197/darken-or-lighten-a-color-in-matplotlib - """ - import matplotlib.colors as mc - import colorsys - try: - c = mc.cnames[color] - except: - c = color - c = colorsys.rgb_to_hls(*mc.to_rgb(c)) - return colorsys.hls_to_rgb(c[0], 1 - amount * (1 - c[1]), c[2]) - - -def _load_e01_hparam_tuning( - # show: bool = False, -) -> pd.DataFrame: - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - df = load_general_data(f'{os.environ["WANDB_USER"]}/CVPR-00__basic-hparam-tuning') - # select run groups - df = df[df[K_GROUP].isin(['sweep_beta'])] - # print common key values - print('K_GROUP: ', list(df[K_GROUP].unique())) - print('K_DATASET: ', list(df[K_DATASET].unique())) - print('K_FRAMEWORK:', list(df[K_FRAMEWORK].unique())) - print('K_BETA: ', list(df[K_BETA].unique())) - print('K_Z_SIZE: ', list(df[K_Z_SIZE].unique())) - print('K_REPEAT: ', list(df[K_REPEAT].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - orig = df - # select runs - df = df[df[K_STATE] == 'finished'] - # [1.0, 0.316, 0.1, 0.0316, 0.01, 0.00316, 0.001, 0.000316] - # df = df[(0.000316 < df[K_BETA]) & (df[K_BETA] < 1.0)] - print('NUM', len(orig), '->', len(df)) - df = rename_entries(df) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - df[K_FRAMEWORK].replace('adavae_os', 'Ada-GVAE', inplace=True) - df[K_FRAMEWORK].replace('betavae', 'Beta-VAE', inplace=True) - return df - -def _load_e00_beta_metric_correlation( - # metrics: Sequence[str] = (K_MIG_MAX, K_RCORR_GT_F, K_RCORR_DATA_F), -): - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - df: pd.DataFrame = load_general_data(f'{os.environ["WANDB_USER"]}/MSC-p02e00_beta-data-latent-corr') - # select run groups - df = df[df[K_GROUP].isin(['sweep_beta_corr'])] - df = df[~df[K_DATASET].isin(['xyobject'])] - df = df[df[K_STATE].isin(['finished', 'running'])] - # df = df[df[K_LR].isin([0.0001])] # 0.0001, 0.001 - # sort everything - df = df.sort_values([K_FRAMEWORK, K_DATASET, K_BETA, K_LR]) - # print common key values - print('K_GROUP: ', list(df[K_GROUP].unique())) - print('K_FRAMEWORK:', list(df[K_FRAMEWORK].unique())) - print('K_BETA: ', list(df[K_BETA].unique())) - print('K_REPEAT: ', list(df[K_REPEAT].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - print('K_DATASET: ', list(df[K_DATASET].unique())) - print('K_LR: ', list(df[K_LR].unique())) - # number of runs - print(f'total={len(df)}') - df = rename_entries(df) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - # rename entries - df[K_FRAMEWORK].replace('adavae_os', 'Ada-GVAE', inplace=True) - df[K_FRAMEWORK].replace('betavae', 'Beta-VAE', inplace=True) - # manual remove invalid values - df = df[df[K_LINE].notna()] - df = df[df[K_AXIS].notna()] - - return df - - -def plot_e02_axis_triplet_kl_vs_dist( - rel_path: Optional[str] = None, - save: bool = True, - show: bool = True, - grid_size_v: float = 1.4, - grid_size_h: float = 2.75, - metrics: Sequence[str] = (K_MIG_MAX, K_DCI_MAX, K_RCORR_GT_F, K_RCORR_DATA_F, K_AXIS, K_LINE), - sampling_modes: Sequence[str] = ('gt_dist__manhat_scaled',), - vals_schedules: Optional[Sequence[str]] = ('adanegtvae_up_all', 'adanegtvae_up_ratio', 'adanegtvae_up_thresh',), # ('adanegtvae_up_all', 'adanegtvae_up_all_full', 'adanegtvae_up_ratio', 'adanegtvae_up_ratio_full', 'adanegtvae_up_thresh') - title: Optional[Union[str, bool]] = True, - plot_vae_results: bool = True, - color_betavae: str = lighten_color(PINK, amount=1.33), # darken the color - color_adavae: str = ORANGE, -): - all_schedules = ('adanegtvae_up_all', 'adanegtvae_up_all_full', 'adanegtvae_up_ratio', 'adanegtvae_up_ratio_full', 'adanegtvae_up_thresh') - if not vals_schedules: - vals_schedules = all_schedules - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - if (title is True) and len(sampling_modes) == 1: - title = sampling_modes[0] - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - with Timer('getting data'): - df: pd.DataFrame = load_general_data(f'{os.environ["WANDB_USER"]}/MSC-p02e02_axis-aligned-triplet') - # select run groups - df = df[df[K_GROUP].isin(['sweep_adanegtvae_params_longmed'])] - df = df[df[K_SCHEDULE].isin(vals_schedules)] - # sort everything - df = df.sort_values([K_FRAMEWORK, K_DATASET, K_SCHEDULE, K_SAMPLER, K_ADA_MODE]) - # print common key values - print('K_GROUP: ', list(df[K_GROUP].unique())) - print('K_FRAMEWORK: ', list(df[K_FRAMEWORK].unique())) - print('K_BETA: ', list(df[K_BETA].unique())) - print('K_REPEAT: ', list(df[K_REPEAT].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - print('K_DATASET: ', list(df[K_DATASET].unique())) - print('K_LR: ', list(df[K_LR].unique())) - print('K_TRIPLET_MODE: ', list(df[K_TRIPLET_MODE].unique())) - print('K_SCHEDULE: ', list(df[K_SCHEDULE].unique())) - print('K_ADA_MODE: ', list(df[K_ADA_MODE].unique())) - # number of runs - df = rename_entries(df) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - # hack to replace the invalid sampling with the correct sampling! - df = drop_and_copy_old_invalid_xysquares(df) - df = df[df[K_SAMPLER].isin(sampling_modes)] # we can only filter this after the above! - print('K_SAMPLER: ', list(df[K_SAMPLER].unique())) - print(f'total={len(df)}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ # - if plot_vae_results: - vae_keys = [K_DATASET, K_FRAMEWORK, *metrics] - - # load the hparams from the original vae hparam tuning -- we don't have the new metrics in these runs! - # vae_df_tuning = _load_e01_hparam_tuning() - # vae_df_tuning = vae_df_tuning[[k for k in vae_keys if k in vae_df_tuning.columns]] - # vae_scores_tuning = vae_df_tuning.groupby([K_DATASET, K_FRAMEWORK]).mean().reset_index() - # print(vae_scores_tuning) - - # load the hparams from the section on metrics, comparing disentanglement to different beta values - vae_df_metrics = _load_e00_beta_metric_correlation() - vae_df_metrics = vae_df_metrics[[k for k in vae_keys if k in vae_df_metrics.columns]] - vae_scores_metrics = vae_df_metrics.groupby([K_DATASET, K_FRAMEWORK]).mean().reset_index() - print(vae_scores_metrics) - # /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ # - - # axis keys - col_x = K_DATASET - col_bars = K_SCHEDULE - col_hue = K_ADA_MODE - # get rows and columns - all_bars = list(df[col_bars].unique()) - all_x = list(df[col_x].unique()) - all_y = metrics - all_hue = list(df[col_hue].unique()) - # num rows and cols - num_bars = len(all_bars) - num_x = len(all_x) - num_y = len(all_y) - num_hue = len(all_hue) - # palettes - colors = [PINK, ORANGE, BLUE, LBLUE, PURPLE] - hatches = [None, '..'] - idxs = [all_schedules.index(v) for v in vals_schedules] - colors = [colors[i] for i in idxs] - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - # make plot - fig, axs = plt.subplots(num_y, num_x, figsize=(grid_size_h * num_x, num_y * grid_size_v)) - if isinstance(title, str): - fig.suptitle(title, fontsize=19) - # fill plot - for y, key_y in enumerate(all_y): # metric - for x, key_x in enumerate(all_x): # dataset - ax = axs[y, x] - # filter items - df_filtered = df[(df[col_x] == key_x)] - # plot! - sns.barplot(ax=ax, x=col_bars, y=key_y, hue=col_hue, data=df_filtered) - ax.set(ylim=(0, 1), xlim=(-0.5, num_bars - 0.5)) - # set colors - for i, (bar, color) in enumerate(zip(ax.patches, itertools.cycle(colors))): - bar.set_facecolor(color) - bar.set_edgecolor('white') - bar.set_linewidth(2) - if hatches[i // num_bars]: - bar.set_facecolor(color + '65') - bar.set_hatch(hatches[i // num_bars]) - # \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ # - if plot_vae_results: - # draw VAE results! - # -- extract single results - scores_betavae = vae_scores_metrics[(vae_scores_metrics[K_DATASET] == key_x) & (vae_scores_metrics[K_FRAMEWORK] == 'Beta-VAE')] - scores_adavae = vae_scores_metrics[(vae_scores_metrics[K_DATASET] == key_x) & (vae_scores_metrics[K_FRAMEWORK] == 'Ada-GVAE')] - assert len(scores_betavae) == 1 - assert len(scores_adavae) == 1 - scores_betavae = scores_betavae.to_dict('records')[0] - scores_adavae = scores_adavae.to_dict('records')[0] - # -- draw single results - ax.plot([-0.5, 2.5], [scores_betavae[key_y], scores_betavae[key_y]], color=color_betavae, linewidth=2.25, linestyle='--') - ax.plot([-0.5, 2.5], [scores_adavae[key_y], scores_adavae[key_y]], color=color_adavae, linewidth=2.25, linestyle='-') - # /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ # - # remove labels - if ax.get_legend(): - ax.get_legend().remove() - if y == 0: - ax.set_title(key_x) - # if y < num_y-1: - ax.set_xlabel(None) - ax.set_xticklabels([]) - if x > 0: - ax.set_ylabel(None) - ax.set_yticklabels([]) - # add the legend - handles_a = [mpl.patches.Patch(label=label, color=color) for label, color in zip(all_bars, colors)] - fig.legend(handles=handles_a, fontsize=12, bbox_to_anchor=(0.975, 0.03), loc='lower right', ncol=2, labelspacing=0.1) - # add the legend - assert np.all(df[col_hue].unique() == ['dist', 'symmetric_kl']), f'{list(df[col_hue].unique())}' - handles_b = [mpl.patches.Patch(label='Absolute Difference', facecolor='#505050ff', edgecolor='white', hatch=hatches[0]), # gt_dist__manhat - mpl.patches.Patch(label='KL Divergence', facecolor='#50505065', edgecolor='white', hatch=hatches[1])] # gt_dist__manhat_scaled - fig.legend(handles=handles_b, fontsize=12, bbox_to_anchor=(0.073, 0.03), loc='lower left', ncol=1, labelspacing=0.1) - # add the legend - if plot_vae_results: - handles_vae = [ - mpl.lines.Line2D([0], [0], color=color_betavae, label='Beta-GVAE', linewidth=2.75, linestyle='--'), - mpl.lines.Line2D([0], [0], color=color_adavae, label='Ada-GVAE', linewidth=2.75, linestyle='-'), - ] - fig.legend(handles=handles_vae, fontsize=12, bbox_to_anchor=(0.44, 0.03), loc='lower left', ncol=1, labelspacing=0.1) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - # plot! - fig.tight_layout() - H.plt_rel_path_savefig(rel_path, save=save, show=show, dpi=150, ext='.png') - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - -def plot_e02_axis_triplet_schedule_recon_loss( - rel_path: Optional[str] = None, - save: bool = True, - show: bool = True, - # filtering - vals_schedules: Optional[Sequence[str]] = ('adanegtvae_up_all', 'adanegtvae_up_ratio', 'adanegtvae_up_thresh',), # ('adanegtvae_up_all', 'adanegtvae_up_all_full', 'adanegtvae_up_ratio', 'adanegtvae_up_ratio_full', 'adanegtvae_up_thresh') - vals_detach: Sequence[bool] = (True, False), - vals_triplet_scale: Sequence[int] = (1.0, 10.0), - # vals_triplet_mode: Sequence[str] = ('triplet_soft',),] - color_triplet_soft: str = BLUE, - color_triplet_hard: str = PURPLE, -): - all_schedules = ('adanegtvae_up_all', 'adanegtvae_up_all_full', 'adanegtvae_up_ratio', 'adanegtvae_up_ratio_full', 'adanegtvae_up_thresh') - if not vals_schedules: - vals_schedules = all_schedules - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - df: pd.DataFrame = load_general_data(f'{os.environ["WANDB_USER"]}/MSC-p02e02_axis-aligned-triplet', include_history=True) - # select run groups - # TODO: add sweep_adanegtvae_params_longmed: - # sampling=gt_dist__manhat,gt_dist__manhat_scaled \ - # framework.cfg.ada_thresh_mode=dist,symmetric_kl \ - # framework.cfg.detach_decoder=FALSE \ - # schedule=adanegtvae_up_all,adanegtvae_up_all_full,adanegtvae_up_ratio,adanegtvae_up_ratio_full,adanegtvae_up_thresh \ - # framework.cfg.triplet_loss=triplet_soft \ - # dataset=cars3d,smallnorb,shapes3d,dsprites,X--xysquares - # CURRENT: - # schedule=adanegtvae_up_ratio,adanegtvae_up_all \ - # framework.cfg.triplet_scale=10.0,1.0 \ - # framework.cfg.detach_decoder=FALSE,TRUE \ - # framework.cfg.triplet_loss=triplet,triplet_soft \ - # dataset=cars3d,smallnorb,shapes3d,dsprites,X--xysquares \ - df = df[df[K_GROUP].isin(['sweep_adanegtvae_alt_params_longmed'])] - df = df[df[K_SCHEDULE].isin(vals_schedules)] - df = df[df[K_TRIPLET_SCALE].isin(vals_triplet_scale)] - df = df[df[K_DETACH].isin(vals_detach)] - df = df[df[K_TRIPLET_MODE].isin(['triplet'])] - # sort everything - df = df.sort_values([K_FRAMEWORK, K_DATASET, K_SCHEDULE, K_TRIPLET_SCALE, K_TRIPLET_MODE, K_DETACH]) - # print common key values - print('K_GROUP: ', list(df[K_GROUP].unique())) - print('K_FRAMEWORK: ', list(df[K_FRAMEWORK].unique())) - print('K_BETA: ', list(df[K_BETA].unique())) - print('K_REPEAT: ', list(df[K_REPEAT].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - print('K_DATASET: ', list(df[K_DATASET].unique())) - print('K_LR: ', list(df[K_LR].unique())) - print('K_TRIPLET_SCALE: ', list(df[K_TRIPLET_SCALE].unique())) - print('K_DETACH: ', list(df[K_DETACH].unique())) - print('K_TRIPLET_MODE: ', list(df[K_TRIPLET_MODE].unique())) - print('K_SCHEDULE: ', list(df[K_SCHEDULE].unique())) - print('K_SAMPLER: ', list(df[K_SAMPLER].unique())) - print('K_ADA_MODE: ', list(df[K_ADA_MODE].unique())) - # number of runs - df = rename_entries(df) - print(f'total={len(df)}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - # extract entries from run histories - # - we need to get the minimum reconstruction loss over the course of the runs - # - or we need to get the point with the reconstruction loss corresponding to the maximum metric? - df['recon_loss.min'] = [hist['recon_loss'].min() for hist in df['history']] - metric_key = 'recon_loss.min' - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - replace = [ - ('Schedule: Both (weak)', 'Schedule:\nBoth (weak)'), - ('Schedule: Both (strong)', 'Schedule:\nBoth (strong)'), - ('Schedule: Weight (weak)', 'Schedule:\nWeight (weak)'), - ('Schedule: Weight (strong)', 'Schedule:\nWeight (strong)'), - ('Schedule: Threshold', 'Schedule:\nThreshold'), - ] - for k, v in replace: - df[K_SCHEDULE].replace(k, v, inplace=True) - - PALLETTE = { - 'Triplet Loss (Soft Margin)': color_triplet_soft, - 'Triplet Loss (Hard Margin)': color_triplet_hard, - 'Schedule:\nBoth (weak)': PINK, - 'Schedule:\nBoth (strong)': ORANGE, - 'Schedule:\nWeight (weak)': BLUE, - 'Schedule:\nWeight (strong)': LBLUE, - 'Schedule:\nThreshold': PURPLE, - } - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - dataset_vals = df[K_DATASET].unique() - - fig, axs = plt.subplots(1, len(dataset_vals), figsize=(len(dataset_vals)*3.5, 3.33), squeeze=False) - axs = axs.reshape(-1) - - # PLOT: MIG - for i, (ax, dataset_val) in enumerate(zip(axs, dataset_vals)): - # filter items - df_filtered = df[df[K_DATASET] == dataset_val] - # plot everything - sns.violinplot(data=df_filtered, ax=ax, x=K_SCHEDULE, y=metric_key, palette=PALLETTE, split=True, cut=0, width=0.75, scale='width', inner='quartile') - ax.set_ylim([0, None]) - if i == 0: - if ax.get_legend(): - ax.get_legend().remove() - ax.set_xlabel(None) - ax.set_ylabel('Minimum Recon. Loss') - elif i == len(axs) - 1: - ax.legend(bbox_to_anchor=(0, 0.1), fontsize=11.5, loc='lower left', labelspacing=0.1) - ax.set_xlabel(None) - ax.set_ylabel(None) - else: - if ax.get_legend(): - ax.get_legend().remove() - ax.set_xlabel(None) - ax.set_ylabel(None) - ax.set_title(dataset_val) - - # PLOT: - fig.tight_layout() - H.plt_rel_path_savefig(rel_path, save=save, show=show, dpi=300) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - return fig, axs - - -# ========================================================================= # -# Experiment p02e01 # -# ========================================================================= # - - -def plot_e01_normal_triplet( - rel_path: Optional[str] = None, - save: bool = True, - show: bool = True, - grid_size_v: float = 1.4, - grid_size_h: float = 2.75, - metrics: Sequence[str] = (K_MIG_MAX, K_DCI_MAX, K_RCORR_GT_F, K_RCORR_DATA_F, K_AXIS, K_LINE), - title: str = None, - plot_vae_results: bool = True, - color_betavae: str = lighten_color(PINK, amount=1.33), # darken the color - color_adavae: str = ORANGE, -): - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - df: pd.DataFrame = load_general_data(f'{os.environ["WANDB_USER"]}/MSC-p02e01_triplet-param-tuning') - # select run groups - df = df[df[K_GROUP].isin(['sweep_tvae_params_basic_RERUN', 'sweep_tvae_params_basic_RERUN_soft'])] - df = df[df[K_DETACH].isin([False])] # True, False - df = df[df[K_TRIPLET_MARGIN].isin([1.0, 10.0])] # 1.0, 10.0 - df = df[df[K_TRIPLET_SCALE].isin([0.1, 1.0])] # 0.1, 1.0 - df = df[df[K_TRIPLET_P].isin([1, 2])] # 1, 2 - df = df[df[K_TRIPLET_MODE].isin(['triplet', 'triplet_soft'])] # 'triplet', 'triplet_soft' - df = df[df[K_SAMPLER].isin(['gt_dist__manhat', 'gt_dist__manhat_scaled'])] # 'gt_dist__manhat', 'gt_dist__manhat_scaled' - # sort everything - df = df.sort_values([K_DATASET, K_SAMPLER, K_TRIPLET_MODE, K_TRIPLET_P, K_TRIPLET_SCALE, K_TRIPLET_MARGIN, K_DETACH]) - # print common key values - with Timer('values'): - print('K_GROUP: ', list(df[K_GROUP].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - print('K_DATASET: ', list(df[K_DATASET].unique())) - print('K_SAMPLER: ', list(df[K_SAMPLER].unique())) - print('K_TRIPLET_MODE: ', list(df[K_TRIPLET_MODE].unique())) - print('K_TRIPLET_P: ', list(df[K_TRIPLET_P].unique())) - print('K_TRIPLET_SCALE: ', list(df[K_TRIPLET_SCALE].unique())) - print('K_TRIPLET_MARGIN: ', list(df[K_TRIPLET_MARGIN].unique())) - print('K_DETACH: ', list(df[K_DETACH].unique())) - # number of runs - print(f'total={len(df)}') - df = rename_entries(df) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ # - if plot_vae_results: - vae_keys = [K_DATASET, K_FRAMEWORK, *metrics] - # load the hparams from the section on metrics, comparing disentanglement to different beta values - vae_df_metrics = _load_e00_beta_metric_correlation() - vae_df_metrics = vae_df_metrics[[k for k in vae_keys if k in vae_df_metrics.columns]] - vae_scores_metrics = vae_df_metrics.groupby([K_DATASET, K_FRAMEWORK]).mean().reset_index() - print(vae_scores_metrics) - # /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ # - - hatches = [None, '..'] - - col_x = K_DATASET - col_bars = K_TRIPLET_MODE - col_hue = K_TRIPLET_P - - # get rows and columns - all_bars = list(df[col_bars].unique()) - all_x = list(df[col_x].unique()) - all_y = metrics - all_hue = list(df[col_hue].unique()) - - num_bars = len(all_bars) - num_x = len(all_x) - num_y = len(all_y) - num_hue = len(all_hue) - - colors = [ - # PINK, - # ORANGE, - PURPLE, - BLUE, - # LBLUE, - # GREEN, - # LGREEN, - ] - - # hack to replace the invalid sampling with the correct sampling! - df = drop_and_copy_old_invalid_xysquares(df) - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - # make plot - fig, axs = plt.subplots(num_y, num_x, figsize=(grid_size_h*num_x, num_y*grid_size_v)) - if title: - fig.suptitle(title, fontsize=19) - # fill plot - for y, key_y in enumerate(all_y): # metric - for x, key_x in enumerate(all_x): # dataset - ax = axs[y, x] - # filter items - df_filtered = df[(df[col_x] == key_x)] - # plot! - sns.barplot(ax=ax, x=col_bars, y=key_y, hue=col_hue, data=df_filtered) - ax.set(ylim=(0, 1), xlim=(-0.5, num_bars - 0.5)) - # set colors - for i, (bar, color) in enumerate(zip(ax.patches, itertools.cycle(colors))): - bar.set_facecolor(color) - bar.set_edgecolor('white') - bar.set_linewidth(2) - if hatches[i // num_bars]: - bar.set_facecolor(color + '65') - bar.set_hatch(hatches[i // num_bars]) - # \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ # - if plot_vae_results: - # draw VAE results! - # -- extract single results - scores_betavae = vae_scores_metrics[(vae_scores_metrics[K_DATASET] == key_x) & (vae_scores_metrics[K_FRAMEWORK] == 'Beta-VAE')] - scores_adavae = vae_scores_metrics[(vae_scores_metrics[K_DATASET] == key_x) & (vae_scores_metrics[K_FRAMEWORK] == 'Ada-GVAE')] - assert len(scores_betavae) == 1 - assert len(scores_adavae) == 1 - scores_betavae = scores_betavae.to_dict('records')[0] - scores_adavae = scores_adavae.to_dict('records')[0] - # -- draw single results - ax.plot([-0.5, 2.5], [scores_betavae[key_y], scores_betavae[key_y]], color=color_betavae, linewidth=2.25, linestyle='--') - ax.plot([-0.5, 2.5], [scores_adavae[key_y], scores_adavae[key_y]], color=color_adavae, linewidth=2.25, linestyle='-') - # /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ # - # remove labels - if ax.get_legend(): - ax.get_legend().remove() - if y == 0: - ax.set_title(key_x) - # if y < num_y-1: - ax.set_xlabel(None) - ax.set_xticklabels([]) - if x > 0: - ax.set_ylabel(None) - ax.set_yticklabels([]) - # add the legend to the top right plot - handles_a = [mpl.patches.Patch(label=label, color=color) for label, color in zip(all_bars, colors)] - fig.legend(handles=handles_a, fontsize=12, bbox_to_anchor=(0.983, 0.03), loc='lower right', ncol=1, labelspacing=0.1) - assert len(all_hue) == len(hatches) == 2, f'{all_hue}, {hatches}' - handles_b = [mpl.patches.Patch(label=all_hue[0], facecolor='#505050ff', edgecolor='white', hatch=hatches[0]), - mpl.patches.Patch(label=all_hue[1], facecolor='#50505065', edgecolor='white', hatch=hatches[1])] - fig.legend(handles=handles_b, fontsize=12, bbox_to_anchor=(0.077, 0.03), loc='lower left', ncol=1, labelspacing=0.1) - # add the legend - if plot_vae_results: - handles_vae = [ - mpl.lines.Line2D([0], [0], color=color_betavae, label='Beta-GVAE', linewidth=2.75, linestyle='--'), - mpl.lines.Line2D([0], [0], color=color_adavae, label='Ada-GVAE', linewidth=2.75, linestyle='-'), - ] - fig.legend(handles=handles_vae, fontsize=12, bbox_to_anchor=(0.47, 0.03), loc='lower left', ncol=1, labelspacing=0.1) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - # PLOT: - fig.tight_layout() - H.plt_rel_path_savefig(rel_path, save=save, show=show, dpi=150, ext='.png') - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - -def plot_e01_normal_triplet_recon_loss( - rel_path: Optional[str] = None, - save: bool = True, - show: bool = True, - color_triplet_soft: str = BLUE, - color_triplet_hard: str = PURPLE, - # filtering - vals_detach: Sequence[bool] = (True,), - vals_p: Sequence[int] = (1,), - vals_scale: Sequence[int] = (0.1, 1.0), - vals_margin: Sequence[int] = (1.0, 10.0), -): - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - df: pd.DataFrame = load_general_data(f'{os.environ["WANDB_USER"]}/MSC-p02e01_triplet-param-tuning', include_history=True) - # select run groups - df = df[df[K_GROUP].isin(['sweep_tvae_params_basic_RERUN', 'sweep_tvae_params_basic_RERUN_soft'])] - df = df[df[K_DETACH].isin(vals_detach)] # True, False - df = df[df[K_TRIPLET_MARGIN].isin(vals_margin)] # 1.0, 10.0 - df = df[df[K_TRIPLET_SCALE].isin(vals_scale)] # 0.1, 1.0 - df = df[df[K_TRIPLET_P].isin(vals_p)] # 1, 2 - df = df[df[K_TRIPLET_MODE].isin(['triplet', 'triplet_soft'])] # 'triplet', 'triplet_soft' - df = df[df[K_SAMPLER].isin(['gt_dist__manhat', 'gt_dist__manhat_scaled'])] # 'gt_dist__manhat', 'gt_dist__manhat_scaled' - # sort everything - df = df.sort_values([K_SAMPLER, K_DATASET, K_TRIPLET_MODE, K_TRIPLET_P, K_TRIPLET_SCALE, K_TRIPLET_MARGIN, K_DETACH]) - # print common key values - with Timer('values'): - print('K_GROUP: ', list(df[K_GROUP].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - print('K_DATASET: ', list(df[K_DATASET].unique())) - print('K_SAMPLER: ', list(df[K_SAMPLER].unique())) - print('K_TRIPLET_MODE: ', list(df[K_TRIPLET_MODE].unique())) - print('K_TRIPLET_P: ', list(df[K_TRIPLET_P].unique())) - print('K_TRIPLET_SCALE: ', list(df[K_TRIPLET_SCALE].unique())) - print('K_TRIPLET_MARGIN: ', list(df[K_TRIPLET_MARGIN].unique())) - print('K_DETACH: ', list(df[K_DETACH].unique())) - # number of runs - print(f'total={len(df)}') - df = rename_entries(df) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # print a list of columns - # pprint(df.iloc[0]['history'].columns) - - # extract entries from run histories - # - we need to get the minimum reconstruction loss over the course of the runs - # - or we need to get the point with the reconstruction loss corresponding to the maximum metric? - df['recon_loss.min'] = [hist['recon_loss'].min() for hist in df['history']] - - metric_key = 'recon_loss.min' - - # hack to replace the invalid sampling with the correct sampling! - df = drop_and_copy_old_invalid_xysquares(df) - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # df = df[[K_DATASET, K_FRAMEWORK, K_MIG, K_DCI]] - df[K_SAMPLER].replace('gt_dist__manhat', 'Dist. Sampling\n', inplace=True) - df[K_SAMPLER].replace('gt_dist__manhat_scaled', 'Dist. Sampling\n(Scaled)', inplace=True) - - # rename columns - for key, value, new_value in [ - (K_TRIPLET_MODE, 'Triplet Loss (Hard Margin)', 'Hard-Margin'), - (K_TRIPLET_MODE, 'Triplet Loss (Soft Margin)', 'Soft-Margin'), - ]: - if key in df.columns: - df[key].replace(value, new_value, inplace=True) - - PALLETTE = { - 'Soft-Margin': color_triplet_soft, - 'Hard-Margin': color_triplet_hard, - } - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - dataset_vals = df[K_DATASET].unique() - - fig, axs = plt.subplots(1, len(dataset_vals), figsize=(len(dataset_vals)*3.33, 3.33), squeeze=False) - axs = axs.reshape(-1) - - # PLOT: MIG - for i, (ax, dataset_val) in enumerate(zip(axs, dataset_vals)): - # filter items - df_filtered = df[df[K_DATASET] == dataset_val] - # plot everything - sns.violinplot(data=df_filtered, ax=ax, x=K_SAMPLER, y=metric_key, hue=K_TRIPLET_MODE, palette=PALLETTE, split=True, cut=0, width=0.75, scale='width', inner='quartile') - ax.set_ylim([0, None]) - if i == 0: - ax.legend(bbox_to_anchor=(0, 0), fontsize=12, loc='lower left', labelspacing=0.1) - ax.set_xlabel(None) - ax.set_ylabel('Minimum Recon. Loss') - else: - ax.get_legend().remove() - ax.set_xlabel(None) - ax.set_ylabel(None) - ax.set_title(dataset_val) - - # PLOT: - fig.tight_layout() - H.plt_rel_path_savefig(rel_path, save=save, show=show, dpi=300) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - return fig, axs - - -# ========================================================================= # -# EXPERIMENT 03 -- unsupervised triplet! # -# ========================================================================= # - - -def plot_e03_unsupervised_triplet_scores( - rel_path: Optional[str] = None, - save: bool = True, - show: bool = True, - grid_size_v: float = 3.33, - grid_size_h: float = 3.33, - metrics: Sequence[str] = (K_MIG_MAX, K_DCI_MAX, K_RCORR_GT_F, K_RCORR_DATA_F, K_AXIS, K_LINE), - plot_vae_results: bool = False, - color_betavae: str = lighten_color(PINK, amount=1.33), # darken the color - color_adavae: str = ORANGE, -): - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - df: pd.DataFrame = load_general_data(f'{os.environ["WANDB_USER"]}/MSC-p02e03_unsupervised-axis-triplet', include_history=False) - - # SWEEP STANDARD DATASETS - # - framework.cfg.overlap_num=512,1024 \ - # - dataset=cars3d,smallnorb,shapes3d,dsprites \ - # X settings.framework.recon_loss=mse \ - # - framework.cfg.overlap_mine_ratio=0.1,0.2 \ - # - framework.cfg.overlap_mine_triplet_mode=none,hard_neg,semi_hard_neg,hard_pos,easy_pos \ - # SWEEP XYSQUARES DATASET - # - framework.cfg.overlap_num=512,1024 \ - # - dataset=X--xysquares \ - # X settings.framework.recon_loss='mse_box_r31_l1.0_k3969.0' \ - # - framework.cfg.overlap_mine_ratio=0.1,0.2 \ - # - framework.cfg.overlap_mine_triplet_mode=none,hard_neg,semi_hard_neg,hard_pos,easy_pos \ - - K_MINE_NUM = 'framework/cfg/overlap_num' - K_MINE_RATIO = 'framework/cfg/overlap_mine_ratio' - K_MINE_MODE = 'framework/cfg/overlap_mine_triplet_mode' - - # select run groups - df = df[df[K_GROUP].isin(['sweep_dotvae_hard_params_longmed', 'sweep_dotvae_hard_params_longmed_xy'])] - df = df[df[K_MINE_MODE].isin(['none'])] - df = df[df[K_Z_SIZE].isin([25])] - - # sort everything - df = df.sort_values([K_DATASET, K_MINE_MODE, K_MINE_RATIO, K_MINE_NUM]) - # print common key values - with Timer('values'): - print('K_GROUP: ', list(df[K_GROUP].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - print('K_DATASET: ', list(df[K_DATASET].unique())) - print('K_SAMPLER: ', list(df[K_SAMPLER].unique())) - print('K_TRIPLET_MODE: ', list(df[K_TRIPLET_MODE].unique())) - print('K_TRIPLET_P: ', list(df[K_TRIPLET_P].unique())) - print('K_TRIPLET_SCALE: ', list(df[K_TRIPLET_SCALE].unique())) - print('K_TRIPLET_MARGIN: ', list(df[K_TRIPLET_MARGIN].unique())) - print('K_DETACH: ', list(df[K_DETACH].unique())) - print('K_Z_SIZE: ', list(df[K_Z_SIZE].unique())) - # number of runs - print(f'total={len(df)}') - df = rename_entries(df) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - # axis keys - col_bars = K_DATASET - # get rows and columns - all_bars = list(df[col_bars].unique()) - all_x = metrics - # num rows and cols - num_bars = len(all_bars) - num_x = len(all_x) - # palettes - colors = [PURPLE, BLUE, LBLUE, LGREEN, '#889299'] - # hatches = [None, '..'] - # idxs = [all_schedules.index(v) for v in vals_schedules] - # colors = [colors[i] for i in idxs] - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ # - if plot_vae_results: - vae_keys = [K_DATASET, K_FRAMEWORK, *metrics] - # load the hparams from the section on metrics, comparing disentanglement to different beta values - vae_df_metrics = _load_e00_beta_metric_correlation() - vae_df_metrics = vae_df_metrics[[k for k in vae_keys if k in vae_df_metrics.columns]] - vae_scores_metrics = vae_df_metrics.groupby([K_DATASET, K_FRAMEWORK]).mean().reset_index() - print(vae_scores_metrics) - # /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ # - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - # make plot - fig, axs = plt.subplots(1, num_x, figsize=(grid_size_h * num_x, 1 * grid_size_v)) - # fill plot - for x, key_x in enumerate(all_x): # metric - ax = axs[x] - # plot! - sns.barplot(ax=ax, x=col_bars, y=key_x, data=df) - ax.set(ylim=(-0.05, 1.05), xlim=(-0.5, num_bars - 0.5)) - # set colors - for i, (bar, color) in enumerate(zip(ax.patches, itertools.cycle(colors))): - bar.set_facecolor(color) - bar.set_edgecolor('white') - bar.set_linewidth(2) - - # \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ # - if plot_vae_results: - for i, k_dataset in enumerate(all_bars): - # draw VAE results! - # -- extract single results - scores_betavae = vae_scores_metrics[(vae_scores_metrics[K_DATASET] == k_dataset) & (vae_scores_metrics[K_FRAMEWORK] == 'Beta-VAE')] - scores_adavae = vae_scores_metrics[(vae_scores_metrics[K_DATASET] == k_dataset) & (vae_scores_metrics[K_FRAMEWORK] == 'Ada-GVAE')] - assert len(scores_betavae) == 1 - assert len(scores_adavae) == 1 - scores_betavae = scores_betavae.to_dict('records')[0] - scores_adavae = scores_adavae.to_dict('records')[0] - # -- draw single results - ax.plot([i-0.4, i+0.4], [scores_betavae[key_x], scores_betavae[key_x]], color=color_betavae, linewidth=2.25, linestyle='--') - ax.plot([i-0.4, i+0.4], [scores_adavae[key_x], scores_adavae[key_x]], color=color_adavae, linewidth=2.25, linestyle='-') - # /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ # - - # remove labels - # ax.set_title(key_x) - if ax.get_legend(): - ax.get_legend().remove() - # if y < num_y-1: - ax.set_xlabel(None) - ax.set_xticklabels([]) - # if x > 0: - # ax.set_ylabel(None) - # ax.set_yticklabels([]) - # add the legend to the top right plot - handles_a = [mpl.patches.Patch(label=label, color=color) for label, color in zip(all_bars, colors)] - fig.legend(handles=handles_a, fontsize=12, bbox_to_anchor=(0.99, 0.1), loc='lower right', ncol=1, labelspacing=0.1) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - # plot! - fig.tight_layout() - H.plt_rel_path_savefig(rel_path, save=save, show=show, dpi=150, ext='.png') - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - -# ========================================================================= # -# Entrypoint # -# ========================================================================= # - - -if __name__ == '__main__': - - assert 'WANDB_USER' in os.environ, 'specify "WANDB_USER" environment variable' - - logging.basicConfig(level=logging.INFO) - - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../../code/util/gadfly.mplstyle')) - - # clear_cache(clear_data=True, clear_wandb=True) - # clear_cache(clear_data=True, clear_wandb=False) - - def main(): - plot_e00_beta_metric_correlation(rel_path='plots/p02e00_metrics_some', show=True, metrics=(K_MIG_MAX, K_RCORR_GT_F, K_RCORR_DATA_F)) - plot_e00_beta_metric_correlation(rel_path='plots/p02e00_metrics_all', show=True, metrics=(K_MIG_MAX, K_RCORR_GT_F, K_RCORR_DATA_F, K_AXIS, K_LINE)) - plot_e00_beta_metric_correlation(rel_path='plots/p02e00_metrics_alt', show=True, metrics=(K_MIG_MAX, K_RCORR_GT_F, K_RCORR_DATA_F, K_LINE)) - - plot_e01_normal_triplet(rel_path='plots/p02e01_normal-triplet', show=True) - plot_e01_normal_triplet_recon_loss(rel_path='plots/p02e01_normal-l1-triplet_recon-loss_detached', vals_detach=(True,), vals_p=(1,), show=True) - plot_e01_normal_triplet_recon_loss(rel_path='plots/p02e01_normal-l1-triplet_recon-loss_attached', vals_detach=(False,), vals_p=(1,), show=True) - - plot_e02_axis_triplet_schedules(rel_path='plots/p02e02_axis__soft-triplet__dist', show=True, adaptive_modes=('dist',), title=False) - plot_e02_axis_triplet_kl_vs_dist(rel_path='plots/p02e02_axis__soft-triplet__kl-vs-dist', show=True, title=False) - - plot_e02_axis_triplet_schedule_recon_loss(rel_path='plots/p02e02_ada-triplet_recon-loss_detached', vals_detach=(True,), show=True) - plot_e02_axis_triplet_schedule_recon_loss(rel_path='plots/p02e02_ada-triplet_recon-loss_attached', vals_detach=(False,), show=True) - - plot_e03_unsupervised_triplet_scores(rel_path='plots/p02e03_unsupervised-triplet') - - main() - - -# ========================================================================= # -# DONE # -# ========================================================================= # diff --git a/research/part02_metric_learning/plot00_all/plots/.gitignore b/research/part02_metric_learning/plot00_all/plots/.gitignore deleted file mode 100644 index 225c3815..00000000 --- a/research/part02_metric_learning/plot00_all/plots/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.png -*.jpg diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/config/config_adversarial_dataset.yaml b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/config/config_adversarial_dataset.yaml deleted file mode 100644 index f3f3ad23..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/config/config_adversarial_dataset.yaml +++ /dev/null @@ -1,60 +0,0 @@ - -# ========================================================================= # -# CONFIG # -# ========================================================================= # - - -defaults: - - run_logging: wandb_fast - - run_location: griffin - - run_launcher: local - # entries in this file override entries from default lists - - _self_ - -settings: - job: - user: 'n_michlo' - project: 'DELETE' # exp-disentangle-dataset - name: 'no-name' # TEST-${framework.dataset_name}_${framework.adversarial_mode}_${framework.sampler_name}_s${trainer.max_steps}_${framework.optimizer_name}_lr${framework.optimizer_lr} # _wd${framework.optimizer_kwargs.weight_decay} - seed: 777 - exp: - show_every_n_steps: 500 - # saving - rel_save_dir: 'out/adversarial_data/' - save_prefix: 'PREFIX' - save_data: TRUE - dataset: - batch_size: 32 - -trainer: - # same as defaults: - run_length: ... - max_steps: 30001 - max_epochs: 30001 - -adv_system: - ### IMPORTANT SETTINGS ### - dataset_name: 'dsprites' # [cars3d, smallnorb, dsprites, shapes3d, xysquares_8x8_mini] - adversarial_mode: 'self' # [self, invert_margin_0.005] invert, invert_unbounded - sampler_name: 'close_p_random_n' # [close_p_random_n, same_k1_close] - - ### OTHER SETTINGS ### - # optimizer options - optimizer_name: 'Adam' - optimizer_lr: 1e-1 - optimizer_kwargs: NULL - # dataset config options - # | dataset_name: 'cars3d' # cars3d, smallnorb, xysquares_8x8_mini - dataset_batch_size: 2048 # x3 - dataset_num_workers: ${dataloader.num_workers} - data_root: ${dsettings.storage.data_root} - # adversarial loss options - # | adversarial_mode: 'invert_margin_0.005' # [self, invert_margin_0.005] invert, invert_unbounded - adversarial_swapped: FALSE - adversarial_masking: FALSE # can produce weird artefacts that look like they might go against the training process, eg. improve disentanglement on dsprites, not actually checked by trianing model on this. - adversarial_top_k: NULL # NULL or range(1, batch_size) - pixel_loss_mode: 'mse' - # sampling config - # | sampler_name: 'close_p_random_n' # [close_p_random_n] (see notes above) -- close_p_random_n, close_p_random_n_bb, same_k, same_k_close, same_k1_close, same_k (might be wrong!), same_k_close, same_k1_close, close_far, close_factor_far_random, close_far_same_factor, same_factor, random_bb, random_swap_manhat, random_swap_manhat_norm - # train options - train_batch_optimizer: TRUE - train_dataset_fp16: TRUE diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/config/config_adversarial_dataset_approx.yaml b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/config/config_adversarial_dataset_approx.yaml deleted file mode 100644 index e984f7e6..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/config/config_adversarial_dataset_approx.yaml +++ /dev/null @@ -1,121 +0,0 @@ - -# ========================================================================= # -# CONFIG # -# ========================================================================= # - -defaults: - - run_logging: wandb_fast - - run_location: griffin - - run_launcher: local - # entries in this file override entries from default lists - - _self_ - -settings: - job: - user: 'n_michlo' - project: 'DELETE' - name_prefix: 'B32' - name: '${settings.job.name_prefix}-${adv_system.dataset_name}_${adv_system.adversarial_mode}_${adv_system.samples_sort_mode}_aw${adv_system.loss_adversarial_weight}_${adv_system.sampler_name}_s${trainer.max_steps}_${adv_system.optimizer_name}_lr${adv_system.optimizer_lr}_wd${adv_system.optimizer_kwargs.weight_decay}_b${settings.dataset.batch_size}_${settings.exp.save_dtype}' - seed: 424242 - exp: - show_every_n_steps: 1000 - # saving - rel_save_dir: 'out/adversarial_data_approx/' - save_prefix: 'PREFIX' - save_model: FALSE - save_data: FALSE - save_dtype: float16 - dataset: - batch_size: 32 - -trainer: - # same as defaults: - run_length: ... - # - 15000 takes 40 mins with batch size 512 (heartofgold, 12 workers) - # - 50000 takes 33 mins with batch size 256 (griffin, 16 workers) - max_steps: 15000 - max_epochs: 15000 - -adv_system: - ### IMPORTANT SETTINGS ### - # best: - # - close_p_random_n - # note: sampler_name (adversarial_mode=invert_margin_0.005) - # - random_swap_manhattan: worst [no inversion before 5k] (probability of encountering close is too low, don't use! ++easiest to implement) - # - close_p_random_n: good [inversion before 5k] (easier to implement) - # - close_p_random_n_bb: good [inversion before 5k] (hard to implement, but pretty much the same as close_p_random_n) - # - same_k: bad [no inversion before 5k] (probability of encountering close is too low, don't use! --harder to implement, better guarantees than random_swap_manhattan) - # - same_k_close: ok [almost inversion before 5k] (harder to implement) - # - same_k1_close: best [inversion well before 5 k] (easier to implement) - # note: sampler_name (adversarial_mode=self) - # - close_p_random_n: seems better based on plot of fdists vs overlap (converges better, but loss is higher which makes sense) - # - same_k1_close: seems worse based on plot of fdists vs overlap (seems to maintain original shape more, might hinder disentanglement? not actually tested) - sampler_name: 'close_p_random_n' # [random_swap_manhattan, close_p_random_n, same_k1_close] - samples_sort_mode: 'swap' # [none, swap, sort_inorder, sort_reverse] - dataset_name: 'smallnorb' # [cars3d, smallnorb, dsprites, shapes3d, xysquares_8x8_mini] - adversarial_mode: 'triplet_margin_0.1' # [self, invert_margin_0.05, invert_margin_0.005] invert, invert_unbounded - - ### OTHER SETTINGS ### - # optimizer options - optimizer_name: 'adam' - optimizer_lr: 2e-3 - optimizer_kwargs: - weight_decay: 1e-5 - # dataset config options - dataset_batch_size: ${dataloader.batch_size} # x3 - dataset_num_workers: ${dataloader.num_workers} - data_root: ${dsettings.storage.data_root} - data_load_into_memory: FALSE # I don't think this is truly multi-threaded, possible lock on array access? - # adversarial loss options - adversarial_swapped: FALSE - adversarial_masking: FALSE # can produce weird artefacts that look like they might go against the training process, eg. improve disentanglement on dsprites, not actually checked by trianing model on this. - adversarial_top_k: NULL # NULL or range(1, batch_size) - pixel_loss_mode: 'mse' - # loss extras - loss_adversarial_weight: 10.0 - loss_out_of_bounds_weight: 1.0 # not really needed -- if this is too high it struggles to "invert" - loss_same_stats_weight: 0.0 # not really needed - loss_similarity_weight: 1.0 # important - # model settings - model_type: 'ae_conv64' # ae_conv64, ae_linear, ae_conv64norm - model_mask_mode: 'none' # std, diff, none - model_weight_init: 'xavier_normal' # [xavier_normal, default] - # logging settings - logging_scale_imgs: FALSE - - -# ========================================================================= # -# OLD EXPERIMENTS # -# ========================================================================= # - - -# EXPERIMENT SWEEP: -# -m framework.sampler_name=close_p_random_n framework.adversarial_mode=self,invert_margin_0.005 framework.dataset_name=dsprites,shapes3d,cars3d,smallnorb -# -m framework.loss_adversarial_weight=100.0 framework.sampler_name=same_k1_close framework.adversarial_mode=self2,self framework.dataset_name=dsprites,shapes3d,cars3d,smallnorb - -# EXPERIMENT INDIVIDUALS: -# framework.sampler_name=close_p_random_n framework.adversarial_mode=self framework.dataset_name=dsprites -# framework.sampler_name=close_p_random_n framework.adversarial_mode=self framework.dataset_name=shapes3d -# framework.sampler_name=close_p_random_n framework.adversarial_mode=self framework.dataset_name=cars3d -# framework.sampler_name=close_p_random_n framework.adversarial_mode=self framework.dataset_name=smallnorb - -# framework.sampler_name=close_p_random_n framework.adversarial_mode=invert_margin_0.005 framework.dataset_name=dsprites -# framework.sampler_name=close_p_random_n framework.adversarial_mode=invert_margin_0.005 framework.dataset_name=shapes3d -# framework.sampler_name=close_p_random_n framework.adversarial_mode=invert_margin_0.005 framework.dataset_name=cars3d -# framework.sampler_name=close_p_random_n framework.adversarial_mode=invert_margin_0.005 framework.dataset_name=smallnorb -# -# # 3dshapes does not seem to want to invert... -# framework.sampler_name=close_p_random_n framework.adversarial_mode=invert_margin_0.01 framework.dataset_name=shapes3d -# framework.sampler_name=close_p_random_n framework.adversarial_mode=invert_margin_0.10 framework.dataset_name=shapes3d - -# NEW EXPERIMENT: -# -m framework.sampler_name=same_k1_close,close_p_random_n framework.adversarial_mode=invert_margin_0.05 framework.dataset_name=dsprites,shapes3d,smallnorb,cars3d -# - continue -# DONE: -m framework.sampler_name=same_k1_close,close_p_random_n framework.adversarial_mode=invert_margin_0.05 framework.dataset_name=smallnorb,cars3d -# DOING: -m framework.sampler_name=close_p_random_n framework.adversarial_mode=invert_margin_0.05 framework.dataset_name=smallnorb,cars3d -# TODO: -m framework.sampler_name=close_p_random_n framework.adversarial_mode=invert_margin_0.05 framework.dataset_name=cars3d,smallnorb - -# NEW EXPERIMENT 2: -# -m framework.sampler_name=same_k1_close,close_p_random_n framework.adversarial_mode=invert_margin_0.05 framework.loss_out_of_bounds_weight=1000.0 framework.dataset_name=dsprites,shapes3d,smallnorb,cars3d - -# NEW EXPERIMENT 3: -# -m framework.sampler_name=same_k1_close framework.adversarial_mode=invert_margin_0.05 framework.loss_out_of_bounds_weight=10000.0 framework.dataset_name=shapes3d,dsprites,cars3d,smallnorb diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_01_gen_adversarial_disk.py b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_01_gen_adversarial_disk.py deleted file mode 100644 index 1383be48..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_01_gen_adversarial_disk.py +++ /dev/null @@ -1,497 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - - -""" -Generate an adversarial dataset -- Stores the mutating dataset on disk -- Loads minibatches from disk that are optimized and the saved back to the disk -- No model is used, images are directly optimized against eachother, could decay in some cases? - -This is quite memory efficient, but it is quite old! -- Should probably be re-written using ray -""" - - -import logging -import multiprocessing.synchronize -import os -from concurrent.futures import Executor -from concurrent.futures import Future -from concurrent.futures import ProcessPoolExecutor -from typing import Optional -from typing import Sequence - -import h5py -import numpy as np -import psutil -import torch -from tqdm import tqdm - -import research.code.util as H -from disent.util.deprecate import deprecated -from disent.util.inout.paths import ensure_parent_dir_exists -from disent.util.profiling import Timer -from disent.util.seeds import seed - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# losses # -# ========================================================================= # - - -def stochastic_const_loss(pred: torch.Tensor, mask: torch.Tensor, num_pairs: int, num_samples: int, loss='mse', reg_out_of_bounds=True, top_k: int = None, constant_targ: float = None) -> torch.Tensor: - ia, ib = torch.randint(0, len(pred), size=(2, num_samples), device=pred.device) - # constant dist loss - x_ds = (H.unreduced_loss(pred[ia], pred[ib], mode=loss) * mask[None, ...]).mean(dim=(-3, -2, -1)) - # compute constant loss - if constant_targ is None: - iA, iB = torch.randint(0, len(x_ds), size=(2, num_pairs), device=pred.device) - lcst = H.unreduced_loss(x_ds[iA], x_ds[iB], mode=loss) - else: - lcst = H.unreduced_loss(x_ds, torch.full_like(x_ds, constant_targ), mode=loss) - # aggregate constant loss - if top_k is None: - lcst = lcst.mean() - else: - lcst = torch.topk(lcst, k=top_k, largest=True).values.mean() - # values over the required range - if reg_out_of_bounds: - m = torch.nan_to_num((0 - pred[pred < 0]) ** 2, nan=0).mean() - M = torch.nan_to_num((pred[pred > 1] - 1) ** 2, nan=0).mean() - mM = m + M - else: - mM = 0. - # done! - return mM + lcst - - -# ========================================================================= # -# h5py dataset helper # -# ========================================================================= # - - -NAME_DATA = 'data' -NAME_VISITS = 'visits' -NAME_OBS = 'x_targ' - -_SAVE_TYPE_LOOKUP = { - 'uint8': torch.uint8, - 'float16': torch.float16, - 'float32': torch.float32, -} - -SAVE_TYPE = 'float16' -assert SAVE_TYPE in _SAVE_TYPE_LOOKUP - - -def _make_hdf5_dataset(path, dataset, overwrite_mode: str = 'continue') -> str: - path = ensure_parent_dir_exists(path) - # get read/write mode - if overwrite_mode == 'overwrite': - rw_mode = 'w' # create new file, overwrite if exists - elif overwrite_mode == 'fail': - rw_mode = 'x' # create new file, fail if exists - elif overwrite_mode == 'continue': - rw_mode = 'a' # create if missing, append if exists - # clear file consistency flags - # if clear_consistency_flags: - # if os.path.isfile(path): - # cmd = ["h5clear", "-s", "'{path}'"] - # print(f'clearing file consistency flags: {" ".join(cmd)}') - # try: - # subprocess.check_output(cmd) - # except FileNotFoundError: - # raise FileNotFoundError('h5clear utility is not installed!') - else: - raise KeyError(f'invalid overwrite_mode={repr(overwrite_mode)}') - # open in read write mode - log.info(f'Opening hdf5 dataset: overwrite_mode={repr(overwrite_mode)} exists={repr(os.path.exists(path))} path={repr(path)}') - with h5py.File(path, rw_mode, libver='earliest') as f: - # get data - num_obs = len(dataset) - obs_shape = dataset[0][NAME_OBS][0].shape - # make dset - if NAME_DATA not in f: - f.create_dataset( - NAME_DATA, - shape=(num_obs, *obs_shape), - dtype=SAVE_TYPE, - chunks=(1, *obs_shape), - track_times=False, - ) - # make set_dset - if NAME_VISITS not in f: - f.create_dataset( - NAME_VISITS, - shape=(num_obs,), - dtype='int64', - chunks=(1,), - track_times=False, - ) - return path - - -# def _read_hdf5_batch(h5py_path: str, idxs, return_visits=False): -# batch, visits = [], [] -# with h5py.File(h5py_path, 'r', swmr=True) as f: -# for i in idxs: -# visits.append(f[NAME_VISITS][i]) -# obs = torch.as_tensor(f[NAME_DATA][i], dtype=torch.float32) -# if SAVE_TYPE == 'uint8': -# obs /= 255 -# batch.append(obs) -# # return values -# if return_visits: -# return torch.stack(batch, dim=0), np.array(visits, dtype=np.int64) -# else: -# return torch.stack(batch, dim=0) - - -def _load_hdf5_batch(dataset, h5py_path: str, idxs, initial_noise: Optional[float] = None, return_visits=True): - """ - Load a batch from the disk -- always return float32 - - Can be used by multiple threads at a time. - - returns an item from the original dataset if an - observation has not been saved into the hdf5 dataset yet. - """ - batch, visits = [], [] - with h5py.File(h5py_path, 'r', swmr=True) as f: - for i in idxs: - v = f[NAME_VISITS][i] - if v > 0: - obs = torch.as_tensor(f[NAME_DATA][i], dtype=torch.float32) - if SAVE_TYPE == 'uint8': - obs /= 255 - else: - (obs,) = dataset[i][NAME_OBS] - obs = obs.to(torch.float32) - if initial_noise is not None: - obs += (torch.randn_like(obs) * initial_noise) - batch.append(obs) - visits.append(v) - # stack and check values - batch = torch.stack(batch, dim=0) - assert batch.dtype == torch.float32 - # return values - if return_visits: - return batch, np.array(visits, dtype=np.int64) - else: - return batch - - -def _save_hdf5_batch(h5py_path: str, batch, idxs): - """ - Save a float32 batch to disk. - - Can only be used by one thread at a time! - """ - assert batch.dtype == torch.float32 - with h5py.File(h5py_path, 'r+', libver='earliest') as f: - for obs, idx in zip(batch, idxs): - if SAVE_TYPE == 'uint8': - f[NAME_DATA][idx] = torch.clamp(torch.round(obs * 255), 0, 255).to(torch.uint8) - else: - f[NAME_DATA][idx] = obs.to(_SAVE_TYPE_LOOKUP[SAVE_TYPE]) - f[NAME_VISITS][idx] += 1 - - -# ========================================================================= # -# multiproc h5py dataset helper # -# ========================================================================= # - - -class FutureList(object): - def __init__(self, futures: Sequence[Future]): - self._futures = futures - - def result(self): - return [future.result() for future in self._futures] - - -# ========================================================================= # -# multiproc h5py dataset helper # -# ========================================================================= # - - -# SUBMIT: - - -def _submit_load_batch_futures(executor: Executor, num_splits: int, dataset, h5py_path: str, idxs, initial_noise: Optional[float] = None) -> FutureList: - return FutureList([ - executor.submit(__inner__load_batch, dataset=dataset, h5py_path=h5py_path, idxs=idxs, initial_noise=initial_noise) - for idxs in np.array_split(idxs, num_splits) - ]) - - -def _submit_save_batch(executor: Executor, h5py_path: str, batch, idxs) -> Future: - return executor.submit(__inner__save_batch, h5py_path=h5py_path, batch=batch, idxs=idxs) - - -NUM_WORKERS = psutil.cpu_count() -_BARRIER = None - - -def __inner__load_batch(dataset, h5py_path: str, idxs, initial_noise: Optional[float] = None): - _BARRIER.wait() - result = _load_hdf5_batch(dataset=dataset, h5py_path=h5py_path, idxs=idxs, initial_noise=initial_noise) - _BARRIER.wait() - return result - - -def __inner__save_batch(h5py_path, batch, idxs): - _save_hdf5_batch(h5py_path=h5py_path, batch=batch, idxs=idxs) - - -# WAIT: - - -def _wait_for_load_future(future: FutureList): - with Timer() as t: - xs, visits = zip(*future.result()) - xs = torch.cat(xs, dim=0) - visits = np.concatenate(visits, axis=0).mean(dtype=np.float32) - return (xs, visits), t - - -def _wait_for_save_future(future: Future): - with Timer() as t: - future.result() - return t - - -# ========================================================================= # -# adversarial dataset generator # -# ========================================================================= # - - -def run_generate_and_save_adversarial_dataset_mp( - dataset_name: str = 'shapes3d', - dataset_load_into_memory: bool = False, - optimizer: str = 'adam', - lr: float = 1e-2, - obs_masked: bool = True, - obs_initial_noise: Optional[float] = None, - loss_fn: str = 'mse', - batch_size: int = 1024*12, # approx - batch_sample_mode: str = 'shuffle', # range, shuffle, random - loss_num_pairs: int = 1024*4, - loss_num_samples: int = 1024*4*2, # only applies if loss_const_targ=None - loss_top_k: Optional[int] = None, - loss_const_targ: Optional[float] = 0.1, # replace stochastic pairwise constant loss with deterministic loss target - loss_reg_out_of_bounds: bool = False, - train_epochs: int = 8, - train_optim_steps: int = 125, - # skipped params - save_folder: str = 'out/overlap', - save_prefix: str = '', - overwrite_mode: str = 'fail', # continue, overwrite, fail - seed_: Optional[int] = 777, -) -> str: - # checks - if obs_initial_noise is not None: - assert not obs_masked, '`obs_masked` cannot be `True`, if using initial noise, ie. `obs_initial_noise is not None`' - - # deterministic! - seed(seed_) - - # ↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓ # - # make dataset - dataset = H.make_dataset(dataset_name, load_into_memory=dataset_load_into_memory, load_memory_dtype=torch.float16) - # get save path - assert not ('/' in save_prefix or '\\' in save_prefix) - name = H.params_as_string(H.get_caller_params(exclude=["save_folder", "save_prefix", "overwrite_mode", "seed_"])) - path = _make_hdf5_dataset(os.path.join(save_folder, f'{save_prefix}{name}.hdf5'), dataset=dataset, overwrite_mode=overwrite_mode) - # ↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑ # - - train_batches = (len(dataset) + batch_size - 1) // batch_size - # loop vars & progress bar - save_time = Timer() - prog = tqdm(total=train_epochs * train_batches * train_optim_steps, postfix={'loss': 0.0, '💯': 0.0, '🔍': 'N/A', '💾': 'N/A'}, ncols=100) - # multiprocessing pool - global _BARRIER # TODO: this is a hack and should be unique to each run - _BARRIER = multiprocessing.Barrier(NUM_WORKERS) - executor = ProcessPoolExecutor(NUM_WORKERS) - - # EPOCHS: - for e in range(train_epochs): - # generate batches - batch_idxs = H.generate_epoch_batch_idxs(num_obs=len(dataset), num_batches=train_batches, mode=batch_sample_mode) - # first data load - load_future = _submit_load_batch_futures(executor, num_splits=NUM_WORKERS, dataset=dataset, h5py_path=path, idxs=batch_idxs[0], initial_noise=obs_initial_noise) - - # TODO: log to WANDB - # TODO: SAMPLING STRATEGY MIGHT NEED TO CHANGE! - # - currently random pairs are generated, but the pairs that matter are the nearby ones. - # - sample pairs that increase and decrease along an axis - # - sample pairs that are nearby according to the factor distance metric - - # BATCHES: - for n in range(len(batch_idxs)): - # ↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓ # - # get batch -- transfer to gpu is the bottleneck - (x, visits), load_time = _wait_for_load_future(load_future) - x = x.cuda().requires_grad_(True) - # ↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑ # - - # queue loading an extra batch - if (n+1) < len(batch_idxs): - load_future = _submit_load_batch_futures(executor, num_splits=NUM_WORKERS, dataset=dataset, h5py_path=path, idxs=batch_idxs[n + 1], initial_noise=obs_initial_noise) - - # ↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓ # - # make optimizers - mask = H.make_changed_mask(x, masked=obs_masked) - optim = H.make_optimizer(x, name=optimizer, lr=lr) - # ↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑ # - - # OPTIMIZE: - for _ in range(train_optim_steps): - # final loss & update - # ↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓=↓ # - loss = stochastic_const_loss(x, mask, num_pairs=loss_num_pairs, num_samples=loss_num_samples, loss=loss_fn, reg_out_of_bounds=loss_reg_out_of_bounds, top_k=loss_top_k, constant_targ=loss_const_targ) - H.step_optimizer(optim, loss) - # ↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑=↑ # - - # update progress bar - logs = {'loss': float(loss), '💯': visits, '🔍': load_time.pretty, '💾': save_time.pretty} - prog.update() - prog.set_postfix(logs) - - # save optimized minibatch - if n > 0: - save_time = _wait_for_save_future(save_future) - save_future = _submit_save_batch(executor, h5py_path=path, batch=x.detach().cpu(), idxs=batch_idxs[n]) - - # final save - save_time = _wait_for_save_future(save_future) - - # cleanup all - executor.shutdown() - # return the path to the dataset - return path - - -# ========================================================================= # -# test adversarial dataset generator # -# ========================================================================= # - - -@deprecated('Replaced with run_02_gen_adversarial_dataset_approx') -def run_generate_adversarial_data( - dataset: str ='shapes3d', - factor: str ='wall_hue', - factor_mode: str = 'sample_random', - optimizer: str ='radam', - lr: float = 1e-2, - obs_num: int = 1024 * 10, - obs_noise_weight: float = 0, - obs_masked: bool = True, - loss_fn: str = 'mse', - loss_num_pairs: int = 4096, - loss_num_samples: int = 4096*2, # only applies if loss_const_targ=None - loss_top_k: int = None, - loss_const_targ: float = None, # replace stochastic pairwise constant loss with deterministic loss target - loss_reg_out_of_bounds: bool = False, - train_steps: int = 2000, - display_period: int = 500, -): - seed(777) - # make dataset - dataset = H.make_dataset(dataset) - # make batches - factors = H.sample_factors(dataset, num_obs=obs_num, factor_mode=factor_mode, factor=factor) - x = dataset.dataset_batch_from_factors(factors, 'target') - # make tensors to optimize - if torch.cuda.is_available(): - x = x.cuda() - x = torch.tensor(x + torch.randn_like(x) * obs_noise_weight, requires_grad=True) - # generate mask - mask = H.make_changed_mask(x, masked=obs_masked) - H.plt_imshow(H.to_img(mask.to(torch.float32)), show=True) - # make optimizer - optimizer = H.make_optimizer(x, name=optimizer, lr=lr) - - # optimize differences according to loss - prog = tqdm(range(train_steps+1), postfix={'loss': 0.0}) - for i in prog: - # final loss - loss = stochastic_const_loss(x, mask, num_pairs=loss_num_pairs, num_samples=loss_num_samples, loss=loss_fn, reg_out_of_bounds=loss_reg_out_of_bounds, top_k=loss_top_k, constant_targ=loss_const_targ) - # update variables - H.step_optimizer(optimizer, loss) - if i % display_period == 0: - log.warning(f'visualisation of `x[:9]` was disabled') - prog.set_postfix({'loss': float(loss)}) - - -# ========================================================================= # -# entrypoint # -# ========================================================================= # - -# TODO: add WANDB support for visualisation of dataset -# TODO: add graphing of visual overlap like exp 01 - -def main(): - logging.basicConfig(level=logging.INFO, format='(%(asctime)s) %(name)s:%(lineno)d [%(levelname)s]: %(message)s') - - paths = [] - for i, kwargs in enumerate([ - # dict(save_prefix='e128__fixed_unmask_const_', obs_masked=False, loss_const_targ=0.1, obs_initial_noise=None, optimizer='adam', dataset_name='cars3d'), - # dict(save_prefix='e128__fixed_unmask_const_', obs_masked=False, loss_const_targ=0.1, obs_initial_noise=None, optimizer='adam', dataset_name='smallnorb'), - # dict(save_prefix='e128__fixed_unmask_randm_', obs_masked=False, loss_const_targ=None, obs_initial_noise=None, optimizer='adam', dataset_name='cars3d'), - # dict(save_prefix='e128__fixed_unmask_randm_', obs_masked=False, loss_const_targ=None, obs_initial_noise=None, optimizer='adam', dataset_name='smallnorb'), - ]): - # generate dataset - try: - path = run_generate_and_save_adversarial_dataset_mp( - train_epochs=128, - train_optim_steps=175, - seed_=777, - overwrite_mode='overwrite', - dataset_load_into_memory=True, - lr=5e-3, - # batch_sample_mode='range', - **kwargs - ) - paths.append(path) - except Exception as e: - log.error(f'[{i}] FAILED RUN: {e} -- {repr(kwargs)}', exc_info=True) - # load some samples and display them - try: - log.warning(f'visualisation of `_read_hdf5_batch(paths[-1], display_idxs)` was disabled') - except Exception as e: - log.warning(f'[{i}] FAILED SHOW: {e} -- {repr(kwargs)}') - - for path in paths: - print(path) - - -# ========================================================================= # -# main # -# ========================================================================= # - - -if __name__ == '__main__': - main() diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_02_adv_dataset.sh b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_02_adv_dataset.sh deleted file mode 100644 index e864de99..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_02_adv_dataset.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# get the path to the script -PARENT_DIR="$(dirname "$(realpath -s "$0")")" -ROOT_DIR="$(dirname "$(dirname "$(dirname "$PARENT_DIR")")")" - -# TODO: fix this! -# TODO: this is out of date -PYTHONPATH="$ROOT_DIR" python3 "$PARENT_DIR/run_02_gen_adversarial_dataset.py" \ - -m \ - framework.sampler_name=same_k,close_far,same_factor,random_bb \ - framework.loss_mode=self,const,invert \ - framework.dataset_name=cars3d,smallnorb diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_02_gen_adversarial_dataset.py b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_02_gen_adversarial_dataset.py deleted file mode 100644 index d2640b84..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_02_gen_adversarial_dataset.py +++ /dev/null @@ -1,425 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -""" -Generate an adversarial dataset -- images are directly optimized against each other, could decay in some cases? -- All data is stored in memory, with minibatches taken and optimized. -""" - -import logging -import os -import warnings -from datetime import datetime -from typing import Iterator -from typing import List -from typing import Optional -from typing import Sequence - -import numpy as np -import pytorch_lightning as pl -import torch -import wandb -from omegaconf import OmegaConf -from torch.utils.data import DataLoader -from torch.utils.data import IterableDataset -from torch.utils.data.dataset import T_co - -import research.code.util as H -from disent.dataset import DisentDataset -from disent.dataset.sampling import BaseDisentSampler -from disent.dataset.util.hdf5 import H5Builder -from disent.util import to_numpy -from disent.util.deprecate import deprecated -from disent.util.inout.paths import ensure_parent_dir_exists -from disent.util.lightning.callbacks import BaseCallbackPeriodic -from disent.util.lightning.callbacks import LoggerProgressCallback -from disent.util.lightning.logger_util import wb_log_metrics -from disent.util.math.random import random_choice_prng -from disent.util.seeds import seed -from disent.util.seeds import TempNumpySeed -from disent.util.strings.fmt import bytes_to_human -from disent.util.strings.fmt import make_box_str -from disent.util.visualize.vis_util import make_image_grid -from experiment.run import hydra_get_callbacks -from experiment.run import hydra_get_gpus -from experiment.run import hydra_make_logger -from experiment.util.hydra_main import hydra_main -from experiment.util.hydra_utils import make_non_strict -from research.part03_learnt_overlap.__OLD__e02_learn_adversarial_data.util_gen_adversarial_dataset import adversarial_loss -from research.part03_learnt_overlap.__OLD__e02_learn_adversarial_data.util_gen_adversarial_dataset import make_adversarial_sampler - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# adversarial dataset generator # -# ========================================================================= # - - -class AdversarialModel(pl.LightningModule): - - def __init__( - self, - # optimizer options - optimizer_name: str = 'sgd', - optimizer_lr: float = 5e-2, - optimizer_kwargs: Optional[dict] = None, - # dataset config options - dataset_name: str = 'cars3d', - dataset_num_workers: int = min(os.cpu_count(), 16), - dataset_batch_size: int = 1024, # approx - data_root: str = 'data/dataset', - # data_load_into_memory: bool = False, - # adversarial loss options - adversarial_mode: str = 'self', - adversarial_swapped: bool = False, - adversarial_masking: bool = False, - adversarial_top_k: Optional[int] = None, - pixel_loss_mode: str = 'mse', - # loss extras - # loss_adversarial_weight: Optional[float] = 1.0, - # loss_same_stats_weight: Optional[float] = 0.0, - # loss_similarity_weight: Optional[float] = 0.0, - # loss_out_of_bounds_weight: Optional[float] = 0.0, - # sampling config - sampler_name: str = 'close_far', - # train options - train_batch_optimizer: bool = True, - train_dataset_fp16: bool = True, - train_is_gpu: bool = False, - # logging settings - # logging_scale_imgs: bool = False, - ): - super().__init__() - # check values - if train_dataset_fp16 and (not train_is_gpu): - warnings.warn('`train_dataset_fp16=True` is not supported on CPU, overriding setting to `False`') - train_dataset_fp16 = False - self._dtype_dst = torch.float32 - self._dtype_src = torch.float16 if train_dataset_fp16 else torch.float32 - # modify hparams - if optimizer_kwargs is None: - optimizer_kwargs = {} - # save hparams - self.save_hyperparameters() - # variables - self.dataset: DisentDataset = None - self.array: torch.Tensor = None - self.sampler: BaseDisentSampler = None - - # ================================== # - # setup # - # ================================== # - - def prepare_data(self) -> None: - # create dataset - self.dataset = H.make_dataset(self.hparams.dataset_name, load_into_memory=True, load_memory_dtype=self._dtype_src, data_root=self.hparams.data_root) - # load dataset into memory as fp16 - if self.hparams.train_batch_optimizer: - self.array = self.dataset.gt_data.array - else: - self.array = torch.nn.Parameter(self.dataset.gt_data.array, requires_grad=True) # move with model to correct device - # create sampler - self.sampler = make_adversarial_sampler(self.hparams.sampler_name) - self.sampler.init(self.dataset.gt_data) - - def _make_optimizer(self, params): - return H.make_optimizer( - params, - name=self.hparams.optimizer_name, - lr=self.hparams.optimizer_lr, - **self.hparams.optimizer_kwargs, - ) - - def configure_optimizers(self): - if self.hparams.train_batch_optimizer: - return None - else: - return self._make_optimizer(self.array) - - # ================================== # - # train step # - # ================================== # - - def training_step(self, batch, batch_idx): - # get indices - (a_idx, p_idx, n_idx) = batch['idx'] - # generate batches & transfer to correct device - if self.hparams.train_batch_optimizer: - (a_x, p_x, n_x), (params, param_idxs, optimizer) = self._load_batch(a_idx, p_idx, n_idx) - else: - a_x = self.array[a_idx] - p_x = self.array[p_idx] - n_x = self.array[n_idx] - # compute loss - loss = adversarial_loss( - ys=(a_x, p_x, n_x), - xs=None, - adversarial_mode=self.hparams.adversarial_mode, - adversarial_swapped=self.hparams.adversarial_swapped, - adversarial_masking=self.hparams.adversarial_masking, - adversarial_top_k=self.hparams.adversarial_top_k, - pixel_loss_mode=self.hparams.pixel_loss_mode, - ) - # log results - self.log_dict({ - 'loss': loss, - 'adv_loss': loss, - }, prog_bar=True) - # done! - if self.hparams.train_batch_optimizer: - self._update_with_batch(loss, params, param_idxs, optimizer) - return None - else: - return loss - - # ================================== # - # optimizer for each batch mode # - # ================================== # - - def _load_batch(self, a_idx, p_idx, n_idx): - with torch.no_grad(): - # get all indices - all_indices = np.stack([ - a_idx.detach().cpu().numpy(), - p_idx.detach().cpu().numpy(), - n_idx.detach().cpu().numpy(), - ], axis=0) - # find unique values - param_idxs, inverse_indices = np.unique(all_indices.flatten(), return_inverse=True) - inverse_indices = inverse_indices.reshape(all_indices.shape) - # load data with values & move to gpu - # - for batch size (256*3, 3, 64, 64) with num_workers=0, this is 5% faster - # than .to(device=self.device, dtype=DST_DTYPE) in one call, as we reduce - # the memory overhead in the transfer. This does slightly increase the - # memory usage on the target device. - # - for batch size (1024*3, 3, 64, 64) with num_workers=12, this is 15% faster - # but consumes slightly more memory: 2492MiB vs. 2510MiB - params = self.array[param_idxs].to(device=self.device).to(dtype=self._dtype_dst) - # make params and optimizer - params = torch.nn.Parameter(params, requires_grad=True) - optimizer = self._make_optimizer(params) - # get batches -- it is ok to index by a numpy array without conversion - a_x = params[inverse_indices[0, :]] - p_x = params[inverse_indices[1, :]] - n_x = params[inverse_indices[2, :]] - # return values - return (a_x, p_x, n_x), (params, param_idxs, optimizer) - - def _update_with_batch(self, loss, params, param_idxs, optimizer): - with TempNumpySeed(777): - std, mean = torch.std_mean(self.array[np.random.randint(0, len(self.array), size=128)]) - std, mean = std.cpu().numpy().tolist(), mean.cpu().numpy().tolist() - self.log_dict({'approx_mean': mean, 'approx_std': std}, prog_bar=True) - # backprop - H.step_optimizer(optimizer, loss) - # save values to dataset - with torch.no_grad(): - self.array[param_idxs] = params.detach().cpu().to(self._dtype_src) - - # ================================== # - # dataset # - # ================================== # - - def train_dataloader(self): - # sampling in dataloader - sampler = self.sampler - data_len = len(self.dataset.gt_data) - # generate the indices in a multi-threaded environment -- this is not deterministic if num_workers > 0 - class SamplerIndicesDataset(IterableDataset): - def __getitem__(self, index) -> T_co: - raise RuntimeError('this should never be called on an iterable dataset') - def __iter__(self) -> Iterator[T_co]: - while True: - yield {'idx': sampler(np.random.randint(0, data_len))} - # create data loader! - return DataLoader( - SamplerIndicesDataset(), - batch_size=self.hparams.dataset_batch_size, - num_workers=self.hparams.dataset_num_workers, - shuffle=False, - ) - - def make_train_periodic_callbacks(self, cfg) -> Sequence[pl.Callback]: - class ImShowCallback(BaseCallbackPeriodic): - def do_step(this, trainer: pl.Trainer, pl_module: pl.LightningModule): - if self.dataset is None: - log.warning('dataset not initialized, skipping visualisation') - # get dataset images - with TempNumpySeed(777): - # get scaling values - samples = self.dataset.dataset_sample_batch(num_samples=128, mode='raw').to(torch.float32) - m, M = float(torch.amin(samples)), float(torch.amax(samples)) - # add transform to dataset - self.dataset._transform = lambda x: H.to_img((x.to(torch.float32) - m) / (M - m)) # this is hacky, scale values to [0, 1] then to [0, 255] - # get images - image = make_image_grid(self.dataset.dataset_sample_batch(num_samples=16, mode='input')) - # get augmented traversals - with torch.no_grad(): - wandb_image, wandb_animation = H.visualize_dataset_traversal(self.dataset, data_mode='input', output_wandb=True) - # log images to WANDB - wb_log_metrics(trainer.logger, { - 'random_images': wandb.Image(image), - 'traversal_image': wandb_image, 'traversal_animation': wandb_animation, - }) - return [ImShowCallback(every_n_steps=cfg.exp.show_every_n_steps, begin_first_step=True)] - - -# ========================================================================= # -# Run Hydra # -# ========================================================================= # - - -ROOT_DIR = os.path.abspath(__file__ + '/../../../..') - - -@deprecated('Replaced with run_02_gen_adversarial_dataset_approx') -def run_gen_adversarial_dataset(cfg): - time_string = datetime.today().strftime('%Y-%m-%d--%H-%M-%S') - log.info(f'Starting run at time: {time_string}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # cleanup from old runs: - try: - wandb.finish() - except: - pass - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - cfg = make_non_strict(cfg) - # - - - - - - - - - - - - - - - # - # check CUDA setting - gpus = hydra_get_gpus(cfg) - # create logger - logger = hydra_make_logger(cfg) - # create callbacks - callbacks: List[pl.Callback] = [c for c in hydra_get_callbacks(cfg) if isinstance(c, LoggerProgressCallback)] - # - - - - - - - - - - - - - - - # - # check save dirs - assert not os.path.isabs(cfg.settings.exp.rel_save_dir), f'rel_save_dir must be relative: {repr(cfg.settings.exp.rel_save_dir)}' - save_dir = os.path.join(ROOT_DIR, cfg.settings.exp.rel_save_dir) - assert os.path.isabs(save_dir), f'save_dir must be absolute: {repr(save_dir)}' - # - - - - - - - - - - - - - - - # - # get the logger and initialize - if logger is not None: - logger.log_hyperparams(cfg) - # print the final config! - log.info('Final Config' + make_box_str(OmegaConf.to_yaml(cfg))) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # | | | | | | | | | | | | | | | # - seed(cfg.settings.job.seed) - # | | | | | | | | | | | | | | | # - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # make framework - framework = AdversarialModel(train_is_gpu=cfg.trainer.cuda, **cfg.adv_system) - callbacks.extend(framework.make_train_periodic_callbacks(cfg)) - # train - trainer = pl.Trainer( - logger=logger, - callbacks=callbacks, - # cfg.dsettings.trainer - gpus=gpus, - # cfg.trainer - max_epochs=cfg.trainer.max_epochs, - max_steps=cfg.trainer.max_steps, - log_every_n_steps=cfg.trainer.log_every_n_steps, - enable_progress_bar=cfg.trainer.enable_progress_bar, - # prepare_data_per_node=cfg.trainer.prepare_data_per_node, # TODO: moved into data module / framework ! - # we do this here so we don't run the final metrics - detect_anomaly=False, # this should only be enabled for debugging torch and finding NaN values, slows down execution, not by much though? - enable_checkpointing=False, - ) - trainer.fit(framework) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # get save paths - save_prefix = f'{cfg.settings.exp.save_prefix}_' if cfg.settings.exp.save_prefix else '' - save_path_data = os.path.join(save_dir, f'{save_prefix}{time_string}_{cfg.settings.job.name}', f'data.h5') - # create directories - if cfg.settings.exp.save_data: ensure_parent_dir_exists(save_path_data) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # compute standard deviation when saving and scale so - # that we have mean=0 and std=1 of the saved data! - with TempNumpySeed(777): - std, mean = torch.std_mean(framework.array[random_choice_prng(len(framework.array), size=2048, replace=False)]) - std, mean = float(std), float(mean) - log.info(f'normalizing saved dataset of shape: {tuple(framework.array.shape)} and dtype: {framework.array.dtype} with mean: {repr(mean)} and std: {repr(std)}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # save adversarial dataset - if cfg.settings.exp.save_data: - log.info(f'saving data to path: {repr(save_path_data)}') - # transfer to GPU - if torch.cuda.is_available(): - framework = framework.cuda() - # create new h5py file -- TODO: use this in other places! - with H5Builder(path=save_path_data, mode='atomic_w') as builder: - # this dataset is self-contained and can be loaded by SelfContainedHdf5GroundTruthData - # we normalize the values to have approx mean of 0 and std of 1 - builder.add_dataset_from_gt_data( - data=framework.dataset, # produces tensors - mutator=lambda x: np.moveaxis((to_numpy(x).astype('float32') - mean) / std, -3, -1).astype('float16'), # consumes tensors -> np.ndarrays - img_shape=(64, 64, None), - compression_lvl=9, - batch_size=32, - dtype='float16', - attrs=dict( - norm_mean=mean, - norm_std=std, - ) - ) - log.info(f'saved data size: {bytes_to_human(os.path.getsize(save_path_data))}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - -# ========================================================================= # -# Entry Point # -# ========================================================================= # - - -if __name__ == '__main__': - - # BENCHMARK (batch_size=256, optimizer=sgd, lr=1e-2, dataset_num_workers=0): - # - batch_optimizer=False, gpu=True, fp16=True : [3168MiB/5932MiB, 3.32/11.7G, 5.52it/s] - # - batch_optimizer=False, gpu=True, fp16=False : [5248MiB/5932MiB, 3.72/11.7G, 4.84it/s] - # - batch_optimizer=False, gpu=False, fp16=True : [same as fp16=False] - # - batch_optimizer=False, gpu=False, fp16=False : [0003MiB/5932MiB, 4.60/11.7G, 1.05it/s] - # --------- - # - batch_optimizer=True, gpu=True, fp16=True : [1284MiB/5932MiB, 3.45/11.7G, 4.31it/s] - # - batch_optimizer=True, gpu=True, fp16=False : [1284MiB/5932MiB, 3.72/11.7G, 4.31it/s] - # - batch_optimizer=True, gpu=False, fp16=True : [same as fp16=False] - # - batch_optimizer=True, gpu=False, fp16=False : [0003MiB/5932MiB, 1.80/11.7G, 4.18it/s] - - # BENCHMARK (batch_size=1024, optimizer=sgd, lr=1e-2, dataset_num_workers=12): - # - batch_optimizer=True, gpu=True, fp16=True : [2510MiB/5932MiB, 4.10/11.7G, 4.75it/s, 20% gpu util] (to(device).to(dtype)) - # - batch_optimizer=True, gpu=True, fp16=True : [2492MiB/5932MiB, 4.10/11.7G, 4.12it/s, 19% gpu util] (to(device, dtype)) - - CONFIGS_THIS_EXP = os.path.abspath(os.path.join(__file__, '../..', 'config')) - CONFIGS_RESEARCH = os.path.abspath(os.path.join(__file__, '../../../..', 'config')) - - # launch the action - hydra_main( - callback=run_gen_adversarial_dataset, - config_name='config_adversarial_dataset', - search_dirs_prepend=[CONFIGS_THIS_EXP, CONFIGS_RESEARCH], - log_level=logging.INFO, - ) diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_03_check.py b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_03_check.py deleted file mode 100644 index af480f60..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_03_check.py +++ /dev/null @@ -1,88 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - - -""" -Check the adversarial data generated in previous exerpiments -- This is old and outdated... -- Should use `plot02_data_distances/run_plot_traversal_dists.py` instead! -""" - - -import numpy as np -import torch -import torch.nn.functional as F -import torchvision -import matplotlib.pyplot as plt - -from disent.dataset.data import Shapes3dData - -# TODO: I think this can be replaced with the self contained hdf5 dataset classes? -from research.code.util._data import AdversarialOptimizedData - - -if __name__ == '__main__': - - def ave_pairwise_dist(data, n_samples=1000): - """ - Get the average distance between observations in the dataset - """ - # get stats - diff = [] - for i in range(n_samples): - a, b = np.random.randint(0, len(data), size=2) - a, b = data[a], data[b] - diff.append(F.mse_loss(a, b, reduction='mean').item()) - return np.mean(diff) - - def plot_samples(data, name=None): - """ - Display random observations from the dataset - """ - # get image - img = torchvision.utils.make_grid([data[i*1000] for i in range(9)], nrow=3) - img = torch.moveaxis(img, 0, -1).numpy() - # plot - if name is not None: - plt.title(name) - plt.imshow(img) - plt.show() - - - def main(): - base_data = Shapes3dData(in_memory=False, prepare=True, transform=torchvision.transforms.ToTensor()) - plot_samples(base_data) - print(ave_pairwise_dist(base_data)) - - for path in [ - 'out/overlap/fixed_masked_const_shapes3d_adam_0.01_True_None_mse_12288_shuffle_5120_10240_None_0.1_False_8_125.hdf5', - 'out/overlap/fixed_masked_randm_shapes3d_adam_0.01_True_None_mse_12288_shuffle_5120_10240_None_None_False_8_125.hdf5', - 'out/overlap/noise_unmask_randm_shapes3d_adam_0.01_False_0.001_mse_12288_shuffle_5120_10240_None_None_False_8_125.hdf5', - 'out/overlap/noise_unmask_randm_shapes3d_adam_0.01_False_0.1_mse_12288_shuffle_5120_10240_None_None_False_8_125.hdf5', - ]: - data = AdversarialOptimizedData(path, base_data, transform=torchvision.transforms.ToTensor()) - plot_samples(data) - print(ave_pairwise_dist(data)) - - main() diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_04_gen_adversarial_ruck.py b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_04_gen_adversarial_ruck.py deleted file mode 100644 index 4c12e5d4..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_04_gen_adversarial_ruck.py +++ /dev/null @@ -1,585 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -""" -This file generates pareto-optimal solutions to the multi-objective -optimisation problem of masking a dataset as to minimize some metric -for overlap, while maximizing the amount of data kept. - -- We solve this problem using the NSGA2 algorithm and save all the results - to disk to be loaded with `get_closest_mask` from `util_load_adversarial_mask.py` -""" - -import gzip -import logging -import os -import pickle -import random -import warnings -from datetime import datetime -from typing import Any -from typing import Dict -from typing import List -from typing import Optional -from typing import Tuple - -import numpy as np -import ray -import ruck -from matplotlib import pyplot as plt -from ruck import R -from ruck.external.ray import ray_map -from ruck.external.ray import ray_remote_put -from ruck.external.ray import ray_remote_puts -from ruck.external.deap import select_nsga2 - -import research.code.util as H -from disent.dataset.wrapper import MaskedDataset -from disent.util.function import wrapped_partial -from disent.util.inout.paths import ensure_parent_dir_exists -from disent.util.profiling import Timer -from disent.util.seeds import seed -from disent.util.visualize.vis_util import get_idx_traversal -from research.part01_data_overlap.plot02_data_distances.util_compute_traversal_dists import cached_compute_all_factor_dist_matrices -from research.part03_learnt_overlap.__OLD__e02_learn_adversarial_data.util_eval_adversarial import eval_individual - - -log = logging.getLogger(__name__) - - -''' -NOTES ON MULTI-OBJECTIVE OPTIMIZATION: - https://en.wikipedia.org/wiki/Pareto_efficiency - https://en.wikipedia.org/wiki/Multi-objective_optimization - https://www.youtube.com/watch?v=SL-u_7hIqjA - - IDEAL MULTI-OBJECTIVE OPTIMIZATION - 1. generate set of pareto-optimal solutions (solutions lying along optimal boundary) -- (A posteriori methods) - - converge to pareto optimal front - - maintain as diverse a population as possible along the front (nsga-ii?) - 2. choose one from set using higher level information - - NOTE: - most multi-objective problems are just - converted into single objective functions. - - WEIGHTED SUMS - -- need to know weights - -- non-uniform in pareto-optimal solutions - -- cannot find some pareto-optimal solutions in non-convex regions - `return w0 * score0 + w1 * score1 + ...` - - ε-CONSTRAINT: constrain all but one objective - -- need to know ε vectors - -- non-uniform in pareto-optimal solutions - -- any pareto-optimal solution can be found - * EMO is a generalisation? -''' - - -# ========================================================================= # -# Ruck Helper # -# ========================================================================= # - - -def mutate_oneof(*mutate_fns): - # TODO: move this into ruck - def mutate_fn(value): - fn = random.choice(mutate_fns) - return fn(value) - return mutate_fn - - -def plt_pareto_solutions( - population, - label_fitness_0: str, - label_fitness_1: str, - title: str = None, - plot: bool = True, - chosen_idxs_f0=None, - chosen_idxs_f1=None, - random_points=None, - **fig_kw, -): - # fitness values must be of type Tuple[float, float] for this function to work! - fig, axs = H.plt_subplots(1, 1, title=title if title else 'Pareto-Optimal Solutions', **fig_kw) - # plot fitness values - xs, ys = zip(*(m.fitness for m in population)) - axs[0, 0].set_xlabel(label_fitness_0) - axs[0, 0].set_ylabel(label_fitness_1) - # plot random - if random_points is not None: - axs[0, 0].scatter(*np.array(random_points).T, c='orange') - # plot normal - axs[0, 0].scatter(xs, ys) - # plot chosen - if chosen_idxs_f0 is not None: - axs[0, 0].scatter(*np.array([population[i].fitness for i in chosen_idxs_f0]).T, c='purple') - if chosen_idxs_f1 is not None: - axs[0, 0].scatter(*np.array([population[i].fitness for i in chosen_idxs_f1]).T, c='green') - # label axes - # layout - fig.tight_layout() - # plot - if plot: - plt.show() - # done! - return fig, axs - - -def individual_ave(dataset, individual, print_=False): - if isinstance(dataset, str): - dataset = H.make_data(dataset, transform_mode='none') - # masked - sub_data = MaskedDataset(data=dataset, mask=individual.flatten()) - if print_: - print(', '.join(f'{individual.reshape(sub_data._data.factor_sizes).sum(axis=f_idx).mean():2f}' for f_idx in range(sub_data._data.num_factors))) - # make obs - ave_obs = np.zeros_like(sub_data[0], dtype='float64') - for obs in sub_data: - ave_obs += obs - return ave_obs / ave_obs.max() - - -def plot_averages(dataset_name: str, values: list, subtitle: str, title_prefix: str = None, titles=None, show: bool = False): - data = H.make_data(dataset_name, transform_mode='none') - # average individuals - ave_imgs = [individual_ave(data, v) for v in values] - col_lbls = [f'{np.sum(v)} / {np.prod(v.shape)}' for v in values] - # make plots - fig_ave_imgs, _ = H.plt_subplots_imshow( - [ave_imgs], - col_labels=col_lbls, - titles=titles, - show=show, - vmin=0.0, - vmax=1.0, - figsize=(10, 3), - title=f'{f"{title_prefix} " if title_prefix else ""}Average Datasets\n{subtitle}', - ) - return fig_ave_imgs - - -def get_spaced(array, num: int): - return [array[i] for i in get_idx_traversal(len(array), num)] - - -# ========================================================================= # -# Evaluation # -# ========================================================================= # - - -@ray.remote -def evaluate_member( - value: np.ndarray, - gt_dist_matrices: np.ndarray, - factor_sizes: Tuple[int, ...], - fitness_overlap_mode: str, - fitness_overlap_aggregate: str, - fitness_overlap_include_singles: bool, -) -> Tuple[float, float]: - overlap_score, usage_ratio = eval_individual( - individual=value, - gt_dist_matrices=gt_dist_matrices, - factor_sizes=factor_sizes, - fitness_overlap_mode=fitness_overlap_mode, - fitness_overlap_aggregate=fitness_overlap_aggregate, - exclude_diag=True, - increment_single=fitness_overlap_include_singles, - backend='numba', - ) - - # weight components - # assert fitness_overlap_weight >= 0 - # assert fitness_usage_weight >= 0 - # w_ovrlp = fitness_overlap_weight * overlap_score - # w_usage = fitness_usage_weight * usage_ratio - - # GOALS: minimize overlap, maximize usage - # [min, max] objective -> target - # [ 0, 1] factor_score -> 0 - # [ 0, 1] kept_ratio -> 1 - - # linear scalarization - # loss = w_ovrlp - w_usage - - # No-preference method - # -- norm(f(x) - z_ideal) - # -- preferably scale variables - # z_ovrlp = fitness_overlap_weight * (overlap_score - 0.0) - # z_usage = fitness_usage_weight * (usage_ratio - 1.0) - # loss = np.linalg.norm([z_ovrlp, z_usage], ord=2) - - # convert minimization problem into maximization - # return - loss - - if overlap_score < 0: - log.warning(f'member has invalid overlap_score: {repr(overlap_score)}') - overlap_score = 1000 # minimizing target to 0 in range [0, 1] so this is bad - if usage_ratio < 0: - log.warning(f'member has invalid usage_ratio: {repr(usage_ratio)}') - usage_ratio = -1000 # maximizing target to 1 in range [0, 1] so this is bad - - return (-overlap_score, usage_ratio) - - -# ========================================================================= # -# Type Hints # -# ========================================================================= # - - -Values = List[ray.ObjectRef] -Population = List[ruck.Member[ray.ObjectRef]] - - -# ========================================================================= # -# Evolutionary System # -# ========================================================================= # - - -class DatasetMaskModule(ruck.EaModule): - - # STATISTICS - - def get_stats_groups(self): - remote_sum = ray.remote(np.mean).remote - return { - **super().get_stats_groups(), - 'mask': ruck.StatsGroup(lambda pop: ray.get([remote_sum(m.value) for m in pop]), min=np.min, max=np.max, mean=np.mean), - } - - def get_progress_stats(self): - return ('evals', 'fit:mean', 'mask:mean') - - # POPULATION - - def gen_starting_values(self) -> Values: - return [ - ray.put(np.random.random(np.prod(self.hparams.factor_sizes)) < (0.1 + np.random.random() * 0.8)) - for _ in range(self.hparams.population_size) - ] - - def select_population(self, population: Population, offspring: Population) -> Population: - return select_nsga2(population + offspring, len(population), weights=(1.0, 1.0)) - - def evaluate_values(self, values: Values) -> List[float]: - return ray.get([self._evaluate_value_fn(v) for v in values]) - - # INITIALISE - - def __init__( - self, - dataset_name: str = 'cars3d', - dist_normalize_mode: str = 'all', - population_size: int = 128, - # fitness settings - fitness_overlap_aggregate: str = 'mean', - fitness_overlap_mode: str = 'std', - fitness_overlap_include_singles: bool = True, - # ea settings - p_mate: float = 0.5, - p_mutate: float = 0.5, - p_mutate_flip: float = 0.05, - ): - # load the dataset - gt_data = H.make_data(dataset_name) - factor_sizes = gt_data.factor_sizes - # save hyper parameters to .hparams - self.save_hyperparameters(include=['factor_sizes']) - # compute all distances - gt_dist_matrices = cached_compute_all_factor_dist_matrices(dataset_name, normalize_mode=dist_normalize_mode) - gt_dist_matrices = ray.put(gt_dist_matrices) - # get offspring function - self.generate_offspring = wrapped_partial( - R.apply_mate_and_mutate, - mate_fn=ray_remote_puts(R.mate_crossover_nd).remote, - mutate_fn=ray_remote_put(mutate_oneof( - wrapped_partial(R.mutate_flip_bits, p=p_mutate_flip), - wrapped_partial(R.mutate_flip_bit_groups, p=p_mutate_flip), - )).remote, - p_mate=p_mate, - p_mutate=p_mutate, - map_fn=ray_map # parallelize - ) - # get evaluation function - self._evaluate_value_fn = wrapped_partial( - evaluate_member.remote, - gt_dist_matrices=gt_dist_matrices, - factor_sizes=factor_sizes, - fitness_overlap_mode=fitness_overlap_mode, - fitness_overlap_aggregate=fitness_overlap_aggregate, - fitness_overlap_include_singles=fitness_overlap_include_singles, - ) - - -# ========================================================================= # -# RUNNER # -# ========================================================================= # - - -def run( - dataset_name: str = 'shapes3d', # xysquares_8x8_toy_s4, xcolumns_8x_toy_s1 - dist_normalize_mode: str = 'all', # all, each, none - # population - generations: int = 250, - population_size: int = 128, - # fitness settings - fitness_overlap_mode: str = 'std', - fitness_overlap_aggregate: str = 'mean', - fitness_overlap_include_singles: bool = True, - # save settings - save: bool = False, - save_prefix: str = '', - seed_: Optional[int] = None, - # plot settings - plot: bool = False, - # wandb_settings - wandb_enabled: bool = True, - wandb_init: bool = True, - wandb_project: str = 'exp-adversarial-mask', - wandb_user: str = 'n_michlo', - wandb_job_name: str = None, - wandb_tags: Optional[List[str]] = None, - wandb_finish: bool = True, -) -> Dict[str, Any]: - # save the starting time for the save path - time_string = datetime.today().strftime('%Y-%m-%d--%H-%M-%S') - log.info(f'Starting run at time: {time_string}') - - # get hparams - hparams = dict(dataset_name=dataset_name, dist_normalize_mode=dist_normalize_mode, generations=generations, population_size=population_size, fitness_overlap_mode=fitness_overlap_mode, fitness_overlap_aggregate=fitness_overlap_aggregate, fitness_overlap_include_singles=fitness_overlap_include_singles, save=save, save_prefix=save_prefix, seed_=seed_, plot=plot, wandb_enabled=wandb_enabled, wandb_init=wandb_init, wandb_project=wandb_project, wandb_user=wandb_user, wandb_job_name=wandb_job_name) - # name - name = f'{(save_prefix + "_" if save_prefix else "")}{dataset_name}_{generations}x{population_size}_{dist_normalize_mode}_{fitness_overlap_mode}_{fitness_overlap_aggregate}_{fitness_overlap_include_singles}' - log.info(f'- Run name is: {name}') - - # enable wandb - wandb = None - if wandb_enabled: - import wandb - # cleanup from old runs: - if wandb_init: - if wandb_finish: - try: - wandb.finish() - except: - pass - # initialize - wandb.init( - entity=wandb_user, - project=wandb_project, - name=wandb_job_name if (wandb_job_name is not None) else name, - group=None, - tags=wandb_tags, - ) - # track hparams - wandb.config.update({f'adv/{k}': v for k, v in hparams.items()}) - - # This is not completely deterministic with ray - # although the starting population will always be the same! - seed_ = seed_ if (seed_ is not None) else int(np.random.randint(1, 2**31-1)) - seed(seed_) - - # run! - with Timer('ruck:onemax'): - problem = DatasetMaskModule( - dataset_name=dataset_name, - dist_normalize_mode=dist_normalize_mode, - population_size=population_size, - fitness_overlap_mode=fitness_overlap_mode, - fitness_overlap_aggregate=fitness_overlap_aggregate, - fitness_overlap_include_singles=fitness_overlap_include_singles, - ) - # train - population, logbook, halloffame = ruck.Trainer(generations=generations, progress=True).fit(problem) - # retrieve stats - log.info(f'start population: {logbook[0]}') - log.info(f'end population: {logbook[-1]}') - values = [ray.get(m.value) for m in halloffame] - - # log to wandb as steps - if wandb_enabled: - for i, stats in enumerate(logbook): - stats = {f'stats/{k}': v for k, v in stats.items()} - stats['current_step'] = i - wandb.log(stats, step=i) - - # generate average images - if plot or wandb_enabled: - # plot average - fig_ave_imgs_hof = plot_averages(dataset_name, values, title_prefix='HOF', subtitle=name, show=plot) - # get individuals -- this is not ideal because not evenly spaced - idxs_chosen_f0 = get_spaced(np.argsort([m.fitness[0] for m in population])[::-1], 5) # overlap - idxs_chosen_f1 = get_spaced(np.argsort([m.fitness[1] for m in population]), 5) # usage - chosen_values_f0 = [ray.get(population[i].value) for i in idxs_chosen_f0] - chosen_values_f1 = [ray.get(population[i].value) for i in idxs_chosen_f1] - random_fitnesses = problem.evaluate_values([ray.put(np.random.random(values[0].shape) < p) for p in np.linspace(0.025, 1, num=population_size+2)[1:-1]]) - # plot averages - fig_ave_imgs_f0 = plot_averages(dataset_name, chosen_values_f0, subtitle=name, titles=[f'{population[i].fitness[0]:2f}' for i in idxs_chosen_f0], title_prefix='Overlap -', show=plot) - fig_ave_imgs_f1 = plot_averages(dataset_name, chosen_values_f1, subtitle=name, titles=[f'{population[i].fitness[1]:2f}' for i in idxs_chosen_f1], title_prefix='Usage -', show=plot) - # plot parento optimal solutions - fig_pareto_sol, axs = plt_pareto_solutions( - population, - label_fitness_0='Overlap Score', - label_fitness_1='Usage Score', - title=f'Pareto-Optimal Solutions\n{name}', - plot=plot, - chosen_idxs_f0=idxs_chosen_f0, - chosen_idxs_f1=idxs_chosen_f1, - random_points=random_fitnesses, - figsize=(7, 7), - ) - # log average - if wandb_enabled: - wandb.log({ - 'ave_images_hof': wandb.Image(fig_ave_imgs_hof), - 'ave_images_overlap': wandb.Image(fig_ave_imgs_f0), - 'ave_images_usage': wandb.Image(fig_ave_imgs_f1), - 'pareto_solutions': wandb.Image(fig_pareto_sol), - }) - - # get summary - use_elems = np.sum(values[0]) - num_elems = np.prod(values[0].shape) - use_ratio = (use_elems / num_elems) - - # log summary - if wandb_enabled: - wandb.summary['num_elements'] = num_elems - wandb.summary['used_elements'] = use_elems - wandb.summary['used_elements_ratio'] = use_ratio - for k, v in logbook[0].items(): wandb.summary[f'log:start:{k}'] = v - for k, v in logbook[-1].items(): wandb.summary[f'log:end:{k}'] = v - - # generate paths - job_name = f'{time_string}_{name}' - - # collect results - results = { - 'hparams': hparams, - 'job_name': job_name, - 'save_path': None, - 'time_string': time_string, - 'values': [ray.get(m.value) for m in population], - 'scores': [m.fitness for m in population], - # score components - 'scores_overlap': [m.fitness[0] for m in population], - 'scores_usage': [m.fitness[1] for m in population], - # history data - 'logbook_history': logbook.history, - # we don't want these because they store object refs, and - # it means we need ray to unpickle them. - # 'population': population, - # 'halloffame_members': halloffame.members, - } - - if save: - # get save path, make parent dir & save! - results['save_path'] = ensure_parent_dir_exists(ROOT_DIR, 'out/adversarial_mask', job_name, 'data.pkl.gz') - # NONE : 122943493 ~= 118M (100.%) : 103.420ms - # lvl=1 : 23566691 ~= 23M (19.1%) : 1.223s - # lvl=2 : 21913595 ~= 21M (17.8%) : 1.463s - # lvl=3 : 20688319 ~= 20M (16.8%) : 2.504s - # lvl=4 : 18325859 ~= 18M (14.9%) : 1.856s # good - # lvl=5 : 17467772 ~= 17M (14.2%) : 3.332s # good - # lvl=6 : 16594660 ~= 16M (13.5%) : 7.163s # starting to slow - # lvl=7 : 16242279 ~= 16M (13.2%) : 12.407s - # lvl=8 : 15586416 ~= 15M (12.7%) : 1m:4s # far too slow - # lvl=9 : 15023324 ~= 15M (12.2%) : 3m:11s # far too slow - log.info(f'saving data to: {results["save_path"]}') - with gzip.open(results["save_path"], 'wb', compresslevel=5) as fp: - pickle.dump(results, fp) - log.info(f'saved data to: {results["save_path"]}') - - # cleanup wandb - if wandb_enabled: - if wandb_finish: - try: - wandb.finish() - except: - pass - - # done - return results - - -# ========================================================================= # -# ENTRYPOINT # -# ========================================================================= # - - -ROOT_DIR = os.path.abspath(__file__ + '/../../..') - - -def main(): - from itertools import product - - # (3 * 2 * 2 * 5) - for (fitness_overlap_include_singles, dist_normalize_mode, fitness_overlap_aggregate, fitness_overlap_mode, dataset_name) in product( - [True, False], - ['all', 'each', 'none'], - ['gmean', 'mean'], - ['std', 'range'], - ['xysquares_8x8_toy_s2', 'cars3d', 'smallnorb', 'shapes3d', 'dsprites'], - ): - print('='*100) - print(f'[STARTING]: dataset_name={repr(dataset_name)} dist_normalize_mode={repr(dist_normalize_mode)} fitness_overlap_mode={repr(fitness_overlap_mode)} fitness_overlap_aggregate={repr(fitness_overlap_aggregate)} fitness_overlap_include_singles={repr(fitness_overlap_include_singles)}') - try: - run( - dataset_name=dataset_name, - dist_normalize_mode=dist_normalize_mode, - # fitness - fitness_overlap_aggregate=fitness_overlap_aggregate, - fitness_overlap_mode=fitness_overlap_mode, - fitness_overlap_include_singles=fitness_overlap_include_singles, - # population - generations=1000, - population_size=256, - seed_=42, - save=True, - save_prefix='EXP', - plot=True, - wandb_enabled=True, - wandb_project='exp-adversarial-mask', - wandb_tags=['exp_factor_dists'] - ) - except KeyboardInterrupt: - warnings.warn('Exiting early') - exit(1) - # except: - # warnings.warn(f'[FAILED]: dataset_name={repr(dataset_name)} dist_normalize_mode={repr(dist_normalize_mode)} fitness_overlap_mode={repr(fitness_overlap_mode)} fitness_overlap_aggregate={repr(fitness_overlap_aggregate)}') - print('='*100) - - -if __name__ == '__main__': - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../../../code/util/gadfly.mplstyle')) - - # run - logging.basicConfig(level=logging.INFO) - ray.init(num_cpus=64) - main() - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_04_gen_adversarial_ruck_dist_pairs.py b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_04_gen_adversarial_ruck_dist_pairs.py deleted file mode 100644 index b09dbd8c..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/run_04_gen_adversarial_ruck_dist_pairs.py +++ /dev/null @@ -1,601 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -""" -This file generates pareto-optimal solutions to the multi-objective -optimisation problem of masking a dataset as to minimize some metric -for overlap, while maximizing the amount of data kept. - -- We solve this problem using the NSGA2 algorithm and save all the results - to disk to be loaded with `get_closest_mask` from `util_load_adversarial_mask.py` -""" - -import gzip -import logging -import os -import pickle -import random -import warnings -from datetime import datetime -from typing import Any -from typing import Dict -from typing import List -from typing import Optional -from typing import Tuple - -import numpy as np -import psutil -import ray -import ruck -from matplotlib import pyplot as plt -from ruck import R -from ruck.external.ray import ray_map -from ruck.external.ray import ray_remote_put -from ruck.external.ray import ray_remote_puts - -from ruck.external.deap import select_nsga2 as select_nsga2_deap -# from ruck.functional import select_nsga2 as select_nsga2_ruck # should rather use this! - -import research.code.util as H -from disent.dataset.wrapper import MaskedDataset -from disent.util.function import wrapped_partial -from disent.util.inout.paths import ensure_parent_dir_exists -from disent.util.profiling import Timer -from disent.util.seeds import seed -from disent.util.visualize.vis_util import get_idx_traversal -from research.part01_data_overlap.plot02_data_distances.util_compute_traversal_dist_pairs import cached_compute_dataset_pair_dists -from research.part03_learnt_overlap.__OLD__e02_learn_adversarial_data.util_eval_adversarial_dist_pairs import eval_masked_dist_pairs - - -log = logging.getLogger(__name__) - - -''' -NOTES ON MULTI-OBJECTIVE OPTIMIZATION: - https://en.wikipedia.org/wiki/Pareto_efficiency - https://en.wikipedia.org/wiki/Multi-objective_optimization - https://www.youtube.com/watch?v=SL-u_7hIqjA - - IDEAL MULTI-OBJECTIVE OPTIMIZATION - 1. generate set of pareto-optimal solutions (solutions lying along optimal boundary) -- (A posteriori methods) - - converge to pareto optimal front - - maintain as diverse a population as possible along the front (nsga-ii?) - 2. choose one from set using higher level information - - NOTE: - most multi-objective problems are just - converted into single objective functions. - - WEIGHTED SUMS - -- need to know weights - -- non-uniform in pareto-optimal solutions - -- cannot find some pareto-optimal solutions in non-convex regions - `return w0 * score0 + w1 * score1 + ...` - - ε-CONSTRAINT: constrain all but one objective - -- need to know ε vectors - -- non-uniform in pareto-optimal solutions - -- any pareto-optimal solution can be found - * EMO is a generalisation? -''' - - -# ========================================================================= # -# Ruck Helper # -# ========================================================================= # - - -def mutate_oneof(*mutate_fns): - # TODO: move this into ruck - def mutate_fn(value): - fn = random.choice(mutate_fns) - return fn(value) - return mutate_fn - - -def plt_pareto_solutions( - population, - label_fitness_0: str, - label_fitness_1: str, - title: str = None, - plot: bool = True, - chosen_idxs_f0=None, - chosen_idxs_f1=None, - random_points=None, - **fig_kw, -): - # fitness values must be of type Tuple[float, float] for this function to work! - fig, axs = H.plt_subplots(1, 1, title=title if title else 'Pareto-Optimal Solutions', **fig_kw) - # plot fitness values - xs, ys = zip(*(m.fitness for m in population)) - axs[0, 0].set_xlabel(label_fitness_0) - axs[0, 0].set_ylabel(label_fitness_1) - # plot random - if random_points is not None: - axs[0, 0].scatter(*np.array(random_points).T, c='orange') - # plot normal - axs[0, 0].scatter(xs, ys) - # plot chosen - if chosen_idxs_f0 is not None: - axs[0, 0].scatter(*np.array([population[i].fitness for i in chosen_idxs_f0]).T, c='purple') - if chosen_idxs_f1 is not None: - axs[0, 0].scatter(*np.array([population[i].fitness for i in chosen_idxs_f1]).T, c='green') - # label axes - # layout - fig.tight_layout() - # plot - if plot: - plt.show() - # done! - return fig, axs - - -def individual_ave(dataset, individual, print_=False): - if isinstance(dataset, str): - dataset = H.make_data(dataset, transform_mode='none') - # masked - sub_data = MaskedDataset(data=dataset, mask=individual.flatten()) - if print_: - print(', '.join(f'{individual.reshape(sub_data._data.factor_sizes).sum(axis=f_idx).mean():2f}' for f_idx in range(sub_data._data.num_factors))) - # make obs - ave_obs = np.zeros_like(sub_data[0], dtype='float64') - for obs in sub_data: - ave_obs += obs - return ave_obs / ave_obs.max() - - -def plot_averages(dataset_name: str, values: list, subtitle: str, title_prefix: str = None, titles=None, show: bool = False): - data = H.make_data(dataset_name, transform_mode='none') - # average individuals - ave_imgs = [individual_ave(data, v) for v in values] - col_lbls = [f'{np.sum(v)} / {np.prod(v.shape)}' for v in values] - # make plots - fig_ave_imgs, _ = H.plt_subplots_imshow( - [ave_imgs], - col_labels=col_lbls, - titles=titles, - show=show, - vmin=0.0, - vmax=1.0, - figsize=(10, 3), - title=f'{f"{title_prefix} " if title_prefix else ""}Average Datasets\n{subtitle}', - ) - return fig_ave_imgs - - -def get_spaced(array, num: int): - return [array[i] for i in get_idx_traversal(len(array), num)] - - -# ========================================================================= # -# Evaluation # -# ========================================================================= # - - -@ray.remote -def evaluate_member( - value: np.ndarray, - pair_obs_dists: np.ndarray, - pair_obs_idxs: np.ndarray, - fitness_overlap_mode: str, - fitness_overlap_include_singles: bool = True, -) -> Tuple[float, float]: - overlap_score, usage_ratio = eval_masked_dist_pairs( - mask=value, - pair_obs_dists=pair_obs_dists, - pair_obs_idxs=pair_obs_idxs, - fitness_mode=fitness_overlap_mode, - increment_single=fitness_overlap_include_singles, - backend='numba', - ) - - # weight components - # assert fitness_overlap_weight >= 0 - # assert fitness_usage_weight >= 0 - # w_ovrlp = fitness_overlap_weight * overlap_score - # w_usage = fitness_usage_weight * usage_ratio - - # GOALS: minimize overlap, maximize usage - # [min, max] objective -> target - # [ 0, 1] factor_score -> 0 - # [ 0, 1] kept_ratio -> 1 - - # linear scalarization - # loss = w_ovrlp - w_usage - - # No-preference method - # -- norm(f(x) - z_ideal) - # -- preferably scale variables - # z_ovrlp = fitness_overlap_weight * (overlap_score - 0.0) - # z_usage = fitness_usage_weight * (usage_ratio - 1.0) - # loss = np.linalg.norm([z_ovrlp, z_usage], ord=2) - - # convert minimization problem into maximization - # return - loss - - if overlap_score < 0: - log.warning(f'member has invalid overlap_score: {repr(overlap_score)}') - overlap_score = 1000 # minimizing target to 0 in range [0, 1] so this is bad - if usage_ratio < 0: - log.warning(f'member has invalid usage_ratio: {repr(usage_ratio)}') - usage_ratio = -1000 # maximizing target to 1 in range [0, 1] so this is bad - - return (-overlap_score, usage_ratio) - - -# ========================================================================= # -# Type Hints # -# ========================================================================= # - - -Values = List[ray.ObjectRef] -Population = List[ruck.Member[ray.ObjectRef]] - - -# ========================================================================= # -# Evolutionary System # -# ========================================================================= # - - -class DatasetDistPairMaskModule(ruck.EaModule): - - # STATISTICS - - def get_stats_groups(self): - remote_sum = ray.remote(np.mean).remote - return { - **super().get_stats_groups(), - 'mask': ruck.StatsGroup(lambda pop: ray.get([remote_sum(m.value) for m in pop]), min=np.min, max=np.max, mean=np.mean), - } - - def get_progress_stats(self): - return ('evals', 'fit:mean', 'mask:mean') - - # POPULATION - - def gen_starting_values(self) -> Values: - return [ - ray.put(np.random.random(np.prod(self.hparams.factor_sizes)) < (0.1 + np.random.random() * 0.8)) - for _ in range(self.hparams.population_size) - ] - - def select_population(self, population: Population, offspring: Population) -> Population: - return select_nsga2_deap(population + offspring, len(population)) - - def evaluate_values(self, values: Values) -> List[float]: - return ray.get([self._evaluate_value_fn(v) for v in values]) - - # INITIALISE - - def __init__( - self, - dataset_name: str = 'smallnorb', - pair_mode: str = 'nearby_scaled', # random, nearby, nearby_scaled - pairs_per_obs: int = 100, - pairs_seed: Optional[int] = None, - dists_scaled: bool = True, - # population - population_size: int = 128, - # fitness settings - fitness_overlap_mode: str = 'std', - fitness_overlap_include_singles: bool = True, - # ea settings - p_mate: float = 0.5, - p_mutate: float = 0.5, - p_mutate_flip: float = 0.05, - ): - # load the dataset - gt_data = H.make_data(dataset_name) - factor_sizes = gt_data.factor_sizes - # save hyper parameters to .hparams - self.save_hyperparameters(include=['factor_sizes']) - # compute all distances - obs_pair_idxs, obs_pair_dists = cached_compute_dataset_pair_dists(dataset_name, pair_mode=pair_mode, pairs_per_obs=pairs_per_obs, seed=pairs_seed, scaled=dists_scaled) - obs_pair_idxs = ray.put(obs_pair_idxs) - obs_pair_dists = ray.put(obs_pair_dists) - # get offspring function - self.generate_offspring = wrapped_partial( - R.apply_mate_and_mutate, - mate_fn=ray_remote_puts(R.mate_crossover_nd).remote, - mutate_fn=ray_remote_put(mutate_oneof( - wrapped_partial(R.mutate_flip_bits, p=p_mutate_flip), - wrapped_partial(R.mutate_flip_bit_groups, p=p_mutate_flip), - )).remote, - p_mate=p_mate, - p_mutate=p_mutate, - map_fn=ray_map # parallelize - ) - # get evaluation function - self._evaluate_value_fn = wrapped_partial( - evaluate_member.remote, - pair_obs_dists=obs_pair_dists, - pair_obs_idxs=obs_pair_idxs, - fitness_overlap_mode=fitness_overlap_mode, - fitness_overlap_include_singles=fitness_overlap_include_singles, - ) - - -# ========================================================================= # -# RUNNER # -# ========================================================================= # - - -def run( - dataset_name: str = 'shapes3d', # xysquares_8x8_toy_s4, xcolumns_8x_toy_s1 - pair_mode: str = 'nearby_scaled', - pairs_per_obs: int = 64, - dists_scaled: bool = True, - # population - generations: int = 250, - population_size: int = 128, - # fitness settings - fitness_overlap_mode: str = 'std', - fitness_overlap_include_singles: bool = True, - # save settings - save: bool = False, - save_prefix: str = '', - seed_: Optional[int] = None, - # plot settings - plot: bool = False, - # wandb_settings - wandb_enabled: bool = True, - wandb_init: bool = True, - wandb_project: str = 'exp-adversarial-mask', - wandb_user: str = 'n_michlo', - wandb_job_name: str = None, - wandb_tags: Optional[List[str]] = None, - wandb_finish: bool = True, -) -> Dict[str, Any]: - # save the starting time for the save path - time_string = datetime.today().strftime('%Y-%m-%d--%H-%M-%S') - log.info(f'Starting run at time: {time_string}') - - # get hparams - hparams = dict(dataset_name=dataset_name, pair_mode=pair_mode, pairs_per_obs=pairs_per_obs, dists_scaled=dists_scaled, generations=generations, population_size=population_size, fitness_overlap_mode=fitness_overlap_mode, fitness_overlap_include_singles=fitness_overlap_include_singles, save=save, save_prefix=save_prefix, seed_=seed_, plot=plot, wandb_enabled=wandb_enabled, wandb_init=wandb_init, wandb_project=wandb_project, wandb_user=wandb_user, wandb_job_name=wandb_job_name, wandb_tags=wandb_tags, wandb_finish=wandb_finish) - # name - name = f'{(save_prefix + "_" if save_prefix else "")}{dataset_name}_{generations}x{population_size}_{pair_mode}_{pairs_per_obs}_{dists_scaled}_{fitness_overlap_mode}_{fitness_overlap_include_singles}' - log.info(f'- Run name is: {name}') - - # enable wandb - wandb = None - if wandb_enabled: - import wandb - # cleanup from old runs: - if wandb_init: - if wandb_finish: - try: - wandb.finish() - except: - pass - # initialize - wandb.init( - entity=wandb_user, - project=wandb_project, - name=wandb_job_name if (wandb_job_name is not None) else name, - group=None, - tags=wandb_tags, - ) - # track hparams - wandb.config.update({f'adv/{k}': v for k, v in hparams.items()}) - - # This is not completely deterministic with ray - # although the starting population will always be the same! - seed_ = seed_ if (seed_ is not None) else int(np.random.randint(1, 2**31-1)) - seed(seed_) - - # run! - with Timer('ruck:onemax'): - problem = DatasetDistPairMaskModule( - dataset_name=dataset_name, - pair_mode=pair_mode, - pairs_per_obs=pairs_per_obs, - # pairs_seed=pairs_seed, - dists_scaled=dists_scaled, - # population - population_size=population_size, - # fitness settings - fitness_overlap_mode=fitness_overlap_mode, - fitness_overlap_include_singles=fitness_overlap_include_singles, - # ea settings - # p_mate=p_mate, - # p_mutate=p_mutate, - # p_mutate_flip=p_mutate_flip, - ) - # train - population, logbook, halloffame = ruck.Trainer(generations=generations, progress=True).fit(problem) - # retrieve stats - log.info(f'start population: {logbook[0]}') - log.info(f'end population: {logbook[-1]}') - values = [ray.get(m.value) for m in halloffame] - - # log to wandb as steps - if wandb_enabled: - for i, stats in enumerate(logbook): - stats = {f'stats/{k}': v for k, v in stats.items()} - stats['current_step'] = i - wandb.log(stats, step=i) - - # generate average images - if plot or wandb_enabled: - # plot average - fig_ave_imgs_hof = plot_averages(dataset_name, values, title_prefix='HOF', subtitle=name, show=plot) - # get individuals -- this is not ideal because not evenly spaced - idxs_chosen_f0 = get_spaced(np.argsort([m.fitness[0] for m in population])[::-1], 5) # overlap - idxs_chosen_f1 = get_spaced(np.argsort([m.fitness[1] for m in population]), 5) # usage - chosen_values_f0 = [ray.get(population[i].value) for i in idxs_chosen_f0] - chosen_values_f1 = [ray.get(population[i].value) for i in idxs_chosen_f1] - random_fitnesses = problem.evaluate_values([ray.put(np.random.random(values[0].shape) < p) for p in np.linspace(0.025, 1, num=population_size+2)[1:-1]]) - # plot averages - fig_ave_imgs_f0 = plot_averages(dataset_name, chosen_values_f0, subtitle=name, titles=[f'{population[i].fitness[0]:2f}' for i in idxs_chosen_f0], title_prefix='Overlap -', show=plot) - fig_ave_imgs_f1 = plot_averages(dataset_name, chosen_values_f1, subtitle=name, titles=[f'{population[i].fitness[1]:2f}' for i in idxs_chosen_f1], title_prefix='Usage -', show=plot) - # plot parento optimal solutions - fig_pareto_sol, axs = plt_pareto_solutions( - population, - label_fitness_0='Overlap Score', - label_fitness_1='Usage Score', - title=f'Pareto-Optimal Solutions\n{name}', - plot=plot, - chosen_idxs_f0=idxs_chosen_f0, - chosen_idxs_f1=idxs_chosen_f1, - random_points=random_fitnesses, - figsize=(7, 7), - ) - # plot factor usage ratios - # TODO: PLOT 2D matrix of all permutations of factors aggregated - # log average - if wandb_enabled: - wandb.log({ - 'ave_images_hof': wandb.Image(fig_ave_imgs_hof), - 'ave_images_overlap': wandb.Image(fig_ave_imgs_f0), - 'ave_images_usage': wandb.Image(fig_ave_imgs_f1), - 'pareto_solutions': wandb.Image(fig_pareto_sol), - }) - - # get summary - use_elems = np.sum(values[0]) - num_elems = np.prod(values[0].shape) - use_ratio = (use_elems / num_elems) - - # log summary - if wandb_enabled: - wandb.summary['num_elements'] = num_elems - wandb.summary['used_elements'] = use_elems - wandb.summary['used_elements_ratio'] = use_ratio - for k, v in logbook[0].items(): wandb.summary[f'log:start:{k}'] = v - for k, v in logbook[-1].items(): wandb.summary[f'log:end:{k}'] = v - - # generate paths - job_name = f'{time_string}_{name}' - - # collect results - results = { - 'hparams': hparams, - 'job_name': job_name, - 'save_path': None, - 'time_string': time_string, - 'values': [ray.get(m.value) for m in population], - 'scores': [m.fitness for m in population], - # score components - 'scores_overlap': [m.fitness[0] for m in population], - 'scores_usage': [m.fitness[1] for m in population], - # history data - 'logbook_history': logbook.history, - # we don't want these because they store object refs, and - # it means we need ray to unpickle them. - # 'population': population, - # 'halloffame_members': halloffame.members, - } - - if save: - # get save path, make parent dir & save! - results['save_path'] = ensure_parent_dir_exists(ROOT_DIR, 'out/adversarial_mask', job_name, 'data.pkl.gz') - # NONE : 122943493 ~= 118M (100.%) : 103.420ms - # lvl=1 : 23566691 ~= 23M (19.1%) : 1.223s - # lvl=2 : 21913595 ~= 21M (17.8%) : 1.463s - # lvl=3 : 20688319 ~= 20M (16.8%) : 2.504s - # lvl=4 : 18325859 ~= 18M (14.9%) : 1.856s # good - # lvl=5 : 17467772 ~= 17M (14.2%) : 3.332s # good - # lvl=6 : 16594660 ~= 16M (13.5%) : 7.163s # starting to slow - # lvl=7 : 16242279 ~= 16M (13.2%) : 12.407s - # lvl=8 : 15586416 ~= 15M (12.7%) : 1m:4s # far too slow - # lvl=9 : 15023324 ~= 15M (12.2%) : 3m:11s # far too slow - log.info(f'saving data to: {results["save_path"]}') - with gzip.open(results["save_path"], 'wb', compresslevel=5) as fp: - pickle.dump(results, fp) - log.info(f'saved data to: {results["save_path"]}') - - # cleanup wandb - if wandb_enabled: - if wandb_finish: - try: - wandb.finish() - except: - pass - - # done - return results - - -# ========================================================================= # -# ENTRYPOINT # -# ========================================================================= # - - -ROOT_DIR = os.path.abspath(__file__ + '/../../..') - - -def main(): - from itertools import product - - # (2*1 * 3*1*2 * 5) = 60 - for i, (fitness_overlap_include_singles, dists_scaled, pair_mode, pairs_per_obs, fitness_overlap_mode, dataset_name) in enumerate(product( - [True, False], - [True], # [True, False] - ['nearby_scaled', 'nearby', 'random'], - [256], # [64, 16, 256] - ['std', 'range'], - ['xysquares_8x8_toy_s2', 'cars3d', 'smallnorb', 'shapes3d', 'dsprites'], # ['xysquares_8x8_toy_s2'] - )): - print('='*100) - print(f'[STARTING]: i={i} dataset_name={repr(dataset_name)} pair_mode={repr(pair_mode)} pairs_per_obs={repr(pairs_per_obs)} dists_scaled={repr(dists_scaled)} fitness_overlap_mode={repr(fitness_overlap_mode)} fitness_overlap_include_singles={repr(fitness_overlap_include_singles)}') - try: - run( - dataset_name=dataset_name, - pair_mode=pair_mode, - pairs_per_obs=pairs_per_obs, - dists_scaled=dists_scaled, - fitness_overlap_mode=fitness_overlap_mode, - fitness_overlap_include_singles=fitness_overlap_include_singles, - # population - generations=1000, # 1000 - population_size=384, - seed_=42, - save=True, - save_prefix='DISTS-SCALED', - plot=True, - wandb_enabled=True, - wandb_project='exp-adversarial-mask', - wandb_tags=['exp_pair_dists'] - ) - except KeyboardInterrupt: - warnings.warn('Exiting early') - exit(1) - except: - warnings.warn(f'[FAILED] i={i}') - print('='*100) - - -if __name__ == '__main__': - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../../../code/util/gadfly.mplstyle')) - - # run - logging.basicConfig(level=logging.INFO) - ray.init(num_cpus=psutil.cpu_count(logical=False)) - main() - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/submit_02_train_adversarial_data.sh b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/submit_02_train_adversarial_data.sh deleted file mode 100644 index c626a6a1..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/submit_02_train_adversarial_data.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-06__adversarial-modified-data" -export PARTITION="stampede" -export PARALLELISM=28 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# 1 * (4 * 2 * 2) = 16 -local_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_griffin' \ - run_location='griffin' \ - \ - run_length=short \ - metrics=fast \ - \ - framework.beta=0.001,0.00316,0.01,0.000316 \ - framework=betavae,adavae_os \ - model.z_size=25 \ - \ - dataset=X--adv-dsprites--WARNING,X--adv-shapes3d--WARNING \ - sampling=default__bb # \ - # \ - # hydra.launcher.exclude='"mscluster93,mscluster94,mscluster97"' # we don't want to sweep over these - -# 2 * (8 * 2 * 4) = 128 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_beta' \ - \ - run_length=short \ - metrics=fast \ - \ - framework.beta=0.000316,0.001,0.00316,0.01,0.0316,0.1,0.316,1.0 \ - framework=betavae,adavae_os \ - model.z_size=25 \ - \ - dataset=dsprites,shapes3d,cars3d,smallnorb \ - sampling=default__bb \ - \ - hydra.launcher.exclude='"mscluster93,mscluster94,mscluster97,mscluster99"' # we don't want to sweep over these diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/submit_04_train_dsprites_imagenet.sh b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/submit_04_train_dsprites_imagenet.sh deleted file mode 100644 index 9919a19d..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/submit_04_train_dsprites_imagenet.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-06__dsprites-imagenet" -export PARTITION="stampede" -export PARALLELISM=36 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# (3*2*2*11) = 132 -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_dsprites_imagenet' \ - \ - run_callbacks=vis \ - run_length=medium \ - metrics=fast \ - \ - model.z_size=9,16 \ - framework.beta=0.0316,0.01,0.1 \ - framework=adavae_os,betavae \ - \ - dataset=dsprites,X--dsprites-imagenet-bg-20,X--dsprites-imagenet-bg-40,X--dsprites-imagenet-bg-60,X--dsprites-imagenet-bg-80,X--dsprites-imagenet-bg-100,X--dsprites-imagenet-fg-20,X--dsprites-imagenet-fg-40,X--dsprites-imagenet-fg-60,X--dsprites-imagenet-fg-80,X--dsprites-imagenet-fg-100 \ - sampling=default__bb \ - \ - hydra.launcher.exclude='"mscluster93,mscluster94,mscluster97,mscluster99"' # we don't want to sweep over these diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/submit_04_train_masked_data.sh b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/submit_04_train_masked_data.sh deleted file mode 100644 index f673155a..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/submit_04_train_masked_data.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-06__masked-datasets" -export PARTITION="stampede" -export PARALLELISM=28 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# 3 * (12 * 2 * 2) = 144 -#submit_sweep \ -# +DUMMY.repeat=1,2,3 \ -# +EXTRA.tags='sweep_01' \ -# \ -# run_length=medium \ -# \ -# framework.beta=0.001 \ -# framework=betavae,adavae_os \ -# model.z_size=9 \ -# \ -# dataset=X--mask-adv-f-dsprites,X--mask-ran-dsprites,dsprites,X--mask-adv-f-shapes3d,X--mask-ran-shapes3d,shapes3d,X--mask-adv-f-smallnorb,X--mask-ran-smallnorb,smallnorb,X--mask-adv-f-cars3d,X--mask-ran-cars3d,cars3d \ -# sampling=random \ -# \ -# hydra.launcher.exclude='"mscluster93,mscluster94,mscluster97"' # we don't want to sweep over these - -# TODO: beta needs to be tuned! -# 3 * (12*3*2 = 72) = 216 -submit_sweep \ - +DUMMY.repeat=1,2,3 \ - +EXTRA.tags='sweep_usage_ratio' \ - \ - run_callbacks=vis \ - run_length=short \ - metrics=all \ - \ - framework.beta=0.001 \ - framework=betavae,adavae_os \ - model.z_size=25 \ - framework.optional.usage_ratio=0.5,0.2,0.05 \ - \ - dataset=X--mask-adv-f-dsprites,X--mask-ran-dsprites,dsprites,X--mask-adv-f-shapes3d,X--mask-ran-shapes3d,shapes3d,X--mask-adv-f-smallnorb,X--mask-ran-smallnorb,smallnorb,X--mask-adv-f-cars3d,X--mask-ran-cars3d,cars3d \ - sampling=random \ - \ - hydra.launcher.exclude='"mscluster93,mscluster94,mscluster97,mscluster99"' # we don't want to sweep over these diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/submit_04_train_masked_data_dist_pairs.sh b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/submit_04_train_masked_data_dist_pairs.sh deleted file mode 100644 index c0780527..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/deprecated/submit_04_train_masked_data_dist_pairs.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="final-06__masked-datasets-dist-pairs" -export PARTITION="stampede" -export PARALLELISM=36 - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours - -# TODO: update this script -echo UPDATE THIS SCRIPT -exit 1 - -# (3*2*3*12 = 72) = 216 -# TODO: z_size needs tuning -submit_sweep \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_dist_pairs_usage_ratio' \ - \ - run_callbacks=vis \ - run_length=short \ - metrics=all \ - \ - framework.beta=0.0316,0.01,0.1 \ - framework=betavae,adavae_os \ - model.z_size=16 \ - framework.optional.usage_ratio=0.5,0.2,0.05 \ - \ - dataset=X--mask-adv-r-dsprites,X--mask-ran-dsprites,dsprites,X--mask-adv-r-shapes3d,X--mask-ran-shapes3d,shapes3d,X--mask-adv-r-smallnorb,X--mask-ran-smallnorb,smallnorb,X--mask-adv-r-cars3d,X--mask-ran-cars3d,cars3d \ - sampling=random \ - \ - hydra.launcher.exclude='"mscluster93,mscluster94,mscluster97,mscluster99"' # we don't want to sweep over these diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/run_02_adv_dataset_approx.sh b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/run_02_adv_dataset_approx.sh deleted file mode 100644 index 75dd8b44..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/run_02_adv_dataset_approx.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# get the path to the script -PARENT_DIR="$(dirname "$(realpath -s "$0")")" -ROOT_DIR="$(dirname "$(dirname "$PARENT_DIR")")" - -# maybe lower lr or increase batch size? -#PYTHONPATH="$ROOT_DIR" python3 "$PARENT_DIR/run_02_gen_adversarial_dataset_approx.py" \ -# -m \ -# adv_system.sampler_name=close_p_random_n,same_k1_close \ -# adv_system.adversarial_mode=self,invert_margin_0.005 \ -# adv_system.dataset_name=dsprites,shapes3d,cars3d,smallnorb - -#PYTHONPATH="$ROOT_DIR" python3 "$PARENT_DIR/run_02_gen_adversarial_dataset_approx.py" \ -# -m \ -# settings.dataset.batch_size=32,256 \ -# adv_system.loss_out_of_bounds_weight=0.0,1.0 \ -# \ -# adv_system.sampler_name=close_p_random_n \ -# adv_system.adversarial_mode=invert_margin_0.05,invert_margin_0.005,invert_margin_0.0005 \ -# adv_system.dataset_name=smallnorb - -PYTHONPATH="$ROOT_DIR" python3 "$PARENT_DIR/run_02_gen_adversarial_dataset_approx.py" \ - -m "$@" \ - \ - +meta.tag='unbounded_manhat' \ - settings.job.name_prefix=MANHAT \ - \ - adv_system.sampler_name=same_k_close,random_swap_manhattan,close_p_random_n \ - adv_system.samples_sort_mode=swap,sort_reverse,none,sort_inorder \ - \ - adv_system.adversarial_mode=triplet_unbounded \ - adv_system.dataset_name=smallnorb \ - \ - trainer.max_steps=7500 \ - trainer.max_epochs=7500 \ - \ - adv_system.optimizer_lr=5e-3 \ - settings.exp.show_every_n_steps=500 \ - \ - settings.dataset.batch_size=128 diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/run_02_gen_adversarial_dataset_approx.py b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/run_02_gen_adversarial_dataset_approx.py deleted file mode 100644 index 25eb46c5..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/run_02_gen_adversarial_dataset_approx.py +++ /dev/null @@ -1,616 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -""" -Generate an adversarial dataset by approximating the difference between -the dataset and the target adversarial images using a model. - adv = obs + diff(obs) -""" - -import logging -import os -from datetime import datetime -from typing import List -from typing import Optional -from typing import Sequence -from typing import Tuple - -import numpy as np -import pytorch_lightning as pl -import torch -import torch.nn.functional as F -import wandb -from omegaconf import OmegaConf -from torch.utils.data import DataLoader - -import research.code.util as H -from disent import registry -from disent.dataset import DisentDataset -from disent.dataset.sampling import BaseDisentSampler -from disent.dataset.util.hdf5 import H5Builder -from disent.model import AutoEncoder -from disent.nn.activations import Swish -from disent.nn.modules import DisentModule -from disent.nn.weights import init_model_weights -from disent.util import to_numpy -from disent.util.function import wrapped_partial -from disent.util.inout.paths import ensure_parent_dir_exists -from disent.util.lightning.callbacks import BaseCallbackPeriodic -from disent.util.lightning.callbacks import LoggerProgressCallback -from disent.util.lightning.logger_util import wb_has_logger -from disent.util.lightning.logger_util import wb_log_metrics -from disent.util.seeds import seed -from disent.util.seeds import TempNumpySeed -from disent.util.strings.fmt import bytes_to_human -from disent.util.strings.fmt import make_box_str -from disent.util.visualize.vis_util import make_image_grid -from experiment.run import hydra_get_gpus -from experiment.run import hydra_get_callbacks -from experiment.run import hydra_make_logger -from experiment.util.hydra_main import hydra_main -from experiment.util.hydra_utils import make_non_strict -from research.part03_learnt_overlap.__OLD__e02_learn_adversarial_data.util_gen_adversarial_dataset import adversarial_loss -from research.part03_learnt_overlap.__OLD__e02_learn_adversarial_data.util_gen_adversarial_dataset import make_adversarial_sampler -from research.part03_learnt_overlap.__OLD__e02_learn_adversarial_data.util_gen_adversarial_dataset import sort_samples - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Dataset Mask # -# ========================================================================= # - -@torch.no_grad() -def _sample_stacked_batch(dataset: DisentDataset) -> torch.Tensor: - batch = next(iter(DataLoader(dataset, batch_size=1024, num_workers=0, shuffle=True))) - batch = torch.cat(batch['x_targ'], dim=0) - return batch - -@torch.no_grad() -def gen_approx_dataset_mask(dataset: DisentDataset, model_mask_mode: Optional[str]) -> Optional[torch.Tensor]: - if model_mask_mode in ('none', None): - mask = None - elif model_mask_mode == 'diff': - batch = _sample_stacked_batch(dataset) - mask = ~torch.all(batch[1:] == batch[0:1], dim=0) - elif model_mask_mode == 'std': - batch = _sample_stacked_batch(dataset) - mask = torch.std(batch, dim=0) - m, M = torch.amin(mask), torch.amax(mask) - mask = (mask - m) / (M - m) - else: - raise KeyError(f'invalid `model_mask_mode`: {repr(model_mask_mode)}') - # done - return mask - - -# ========================================================================= # -# adversarial dataset generator # -# ========================================================================= # - - -class AeModel(AutoEncoder): - def forward(self, x): - return self.decode(self.encode(x)) - - -def make_delta_model(model_type: str, x_shape: Tuple[int, ...]): - C, H, W = x_shape - # get model - if model_type.startswith('ae_'): - return AeModel( - encoder=registry.MODELS[f'encoder_{model_type[len("ae_"):]}'](x_shape=x_shape, z_size=64, z_multiplier=1), - decoder=registry.MODELS[f'decoder_{model_type[len("ae_"):]}'](x_shape=x_shape, z_size=64, z_multiplier=1), - ) - elif model_type == 'fcn_small': - return torch.nn.Sequential( - torch.nn.ReflectionPad2d(1), torch.nn.Conv2d(in_channels=C, out_channels=5, kernel_size=3), Swish(), - torch.nn.ReflectionPad2d(1), torch.nn.Conv2d(in_channels=5, out_channels=7, kernel_size=3), Swish(), - torch.nn.ReflectionPad2d(1), torch.nn.Conv2d(in_channels=7, out_channels=9, kernel_size=3), Swish(), - torch.nn.ReflectionPad2d(1), torch.nn.Conv2d(in_channels=9, out_channels=7, kernel_size=3), Swish(), - torch.nn.ReflectionPad2d(1), torch.nn.Conv2d(in_channels=7, out_channels=5, kernel_size=3), Swish(), - torch.nn.ReflectionPad2d(1), torch.nn.Conv2d(in_channels=5, out_channels=C, kernel_size=3), - ) - else: - raise KeyError(f'invalid model type: {repr(model_type)}') - - -class AdversarialAugmentModel(DisentModule): - - def __init__(self, model_type: str, x_shape=(3, 64, 64), mask=None, meta: dict = None): - super().__init__() - # make layers - self.delta_model = make_delta_model(model_type=model_type, x_shape=x_shape) - self.meta = meta if meta else {} - # mask - if mask is not None: - self.register_buffer('mask', mask[None, ...]) - assert self.mask.ndim == 4 # (1, C, H, W) - - def forward(self, x): - assert x.ndim == 4 - # compute - if hasattr(self, 'mask'): - return x + self.delta_model(x) * self.mask - else: - return x + self.delta_model(x) - - -# ========================================================================= # -# adversarial dataset generator # -# ========================================================================= # - - -class AdversarialModel(pl.LightningModule): - - def __init__( - self, - # optimizer options - optimizer_name: str = 'sgd', - optimizer_lr: float = 5e-2, - optimizer_kwargs: Optional[dict] = None, - # dataset config options - dataset_name: str = 'cars3d', - dataset_num_workers: int = min(os.cpu_count(), 16), - dataset_batch_size: int = 1024, # approx - data_root: str = 'data/dataset', - data_load_into_memory: bool = False, - # adversarial loss options - adversarial_mode: str = 'self', - adversarial_swapped: bool = False, - adversarial_masking: bool = False, - adversarial_top_k: Optional[int] = None, - pixel_loss_mode: str = 'mse', - # loss extras - loss_adversarial_weight: Optional[float] = 1.0, - loss_same_stats_weight: Optional[float] = 0.0, - loss_similarity_weight: Optional[float] = 0.0, - loss_out_of_bounds_weight: Optional[float] = 0.0, - # sampling config - sampler_name: str = 'close_far', - samples_sort_mode: str = 'none', - # model settings - model_type: str = 'ae_linear', - model_mask_mode: Optional[str] = 'none', - model_weight_init: str = 'xavier_normal', - # logging settings - logging_scale_imgs: bool = False, - # log_wb_stats_table: bool = True, - ): - super().__init__() - # modify hparams - if optimizer_kwargs is None: - optimizer_kwargs = {} # value used by save_hyperparameters - # save hparams - self.save_hyperparameters() - # variables - self.dataset: DisentDataset = None - self.sampler: BaseDisentSampler = None - self.model: DisentModule = None - - # ================================== # - # setup # - # ================================== # - - def prepare_data(self) -> None: - # create dataset - self.dataset = H.make_dataset( - self.hparams.dataset_name, - load_into_memory=self.hparams.data_load_into_memory, - load_memory_dtype=torch.float32, - data_root=self.hparams.data_root, - sampler=make_adversarial_sampler(self.hparams.sampler_name), - ) - # make the model - self.model = AdversarialAugmentModel( - model_type=self.hparams.model_type, - x_shape=(self.dataset.gt_data.img_channels, 64, 64), - mask=gen_approx_dataset_mask(dataset=self.dataset, model_mask_mode=self.hparams.model_mask_mode), - # if we save the model we can restore things! - meta=dict( - dataset_name=self.hparams.dataset_name, - dataset_factor_sizes=self.dataset.gt_data.factor_sizes, - dataset_factor_names=self.dataset.gt_data.factor_names, - sampler_name=self.hparams.sampler_name, - hparams=dict(self.hparams) - ), - ) - # initialize model - self.model = init_model_weights(self.model, mode=self.hparams.model_weight_init) - - def train_dataloader(self): - return DataLoader( - self.dataset, - batch_size=self.hparams.dataset_batch_size, - num_workers=self.hparams.dataset_num_workers, - shuffle=True, - ) - - def configure_optimizers(self): - return H.make_optimizer( - self.model, - name=self.hparams.optimizer_name, - lr=self.hparams.optimizer_lr, - **self.hparams.optimizer_kwargs, - ) - - # ================================== # - # train step # - # ================================== # - - def forward(self, x): - return self.model(x) - - def training_step(self, batch, batch_idx): - (a_x, p_x, n_x) = batch['x_targ'] - # sort inputs - a_x, p_x, n_x = sort_samples(a_x, p_x, n_x, sort_mode=self.hparams.samples_sort_mode, pixel_loss_mode=self.hparams.pixel_loss_mode) - # feed forward - a_y = self.model(a_x) - p_y = self.model(p_x) - n_y = self.model(n_x) - # compute loss - loss_adv = 0 - if (self.hparams.loss_adversarial_weight is not None) and (self.hparams.loss_adversarial_weight > 0): - loss_adv, loss_adv_stats = adversarial_loss( - ys=(a_y, p_y, n_y), - xs=(a_x, p_x, n_x), - adversarial_mode=self.hparams.adversarial_mode, - adversarial_swapped=self.hparams.adversarial_swapped, - adversarial_masking=self.hparams.adversarial_masking, - adversarial_top_k=self.hparams.adversarial_top_k, - pixel_loss_mode=self.hparams.pixel_loss_mode, - return_stats=True, - ) - loss_adv *= self.hparams.loss_adversarial_weight - self.log_dict(loss_adv_stats) - # additional loss components - # - keep stats the same - loss_stats = 0 - if (self.hparams.loss_same_stats_weight is not None) and (self.hparams.loss_same_stats_weight > 0): - loss_stats += (self.hparams.loss_same_stats_weight/3) * (( - F.mse_loss(a_y.mean(dim=[-3, -2, -1]), a_x.mean(dim=[-3, -2, -1]), reduction='mean') + - F.mse_loss(p_y.mean(dim=[-3, -2, -1]), p_x.mean(dim=[-3, -2, -1]), reduction='mean') + - F.mse_loss(n_y.mean(dim=[-3, -2, -1]), n_x.mean(dim=[-3, -2, -1]), reduction='mean') - ) + ( - F.mse_loss(a_y.std(dim=[-3, -2, -1]), a_x.std(dim=[-3, -2, -1]), reduction='mean') + - F.mse_loss(p_y.std(dim=[-3, -2, -1]), p_x.std(dim=[-3, -2, -1]), reduction='mean') + - F.mse_loss(n_y.std(dim=[-3, -2, -1]), n_x.std(dim=[-3, -2, -1]), reduction='mean') - )) - # - try keep similar to inputs - loss_sim = 0 - if (self.hparams.loss_similarity_weight is not None) and (self.hparams.loss_similarity_weight > 0): - loss_sim = (self.hparams.loss_similarity_weight / 3) * ( - F.mse_loss(a_y, a_x, reduction='mean') + - F.mse_loss(p_y, p_x, reduction='mean') + - F.mse_loss(n_y, n_x, reduction='mean') - ) - # - regularize if out of bounds - loss_out = 0 - if (self.hparams.loss_out_of_bounds_weight is not None) and (self.hparams.loss_out_of_bounds_weight > 0): - zeros = torch.zeros_like(a_y) - loss_out = (self.hparams.loss_out_of_bounds_weight / 6) * ( - torch.where(a_y < 0, -a_y, zeros).mean() + torch.where(a_y > 1, a_y-1, zeros).mean() + - torch.where(p_y < 0, -p_y, zeros).mean() + torch.where(p_y > 1, p_y-1, zeros).mean() + - torch.where(n_y < 0, -n_y, zeros).mean() + torch.where(n_y > 1, n_y-1, zeros).mean() - ) - # final loss - loss = loss_adv + loss_sim + loss_out - # log everything - self.log_dict({ - 'loss': loss, - 'loss_stats': loss_stats, - 'loss_adv': loss_adv, - 'loss_out': loss_out, - 'loss_sim': loss_sim, - }, prog_bar=True) - # done! - return loss - - # ================================== # - # dataset # - # ================================== # - - @torch.no_grad() - def batch_to_adversarial_imgs(self, batch: torch.Tensor, m=0, M=1, mode='uint8') -> np.ndarray: - batch = batch.to(device=self.device, dtype=torch.float32) - batch = self.model(batch) - batch = (batch - m) / (M - m) - if mode == 'uint8': return H.to_imgs(batch).numpy() - elif mode == 'float32': return torch.moveaxis(batch, -3, -1).to(torch.float32).cpu().numpy() - elif mode == 'float16': return torch.moveaxis(batch, -3, -1).to(torch.float16).cpu().numpy() - else: raise KeyError(f'invalid output mode: {repr(mode)}') - - def make_train_periodic_callbacks(self, cfg) -> Sequence[BaseCallbackPeriodic]: - - # dataset transform helper - @TempNumpySeed(42) - @torch.no_grad() - def make_scale_uint8_transform(transform): - # get scaling values - if self.hparams.logging_scale_imgs: - samples = self.dataset.dataset_sample_batch(num_samples=128, mode='raw').to(torch.float32) - samples = self.model(samples.to(self.device)).cpu() - m, M = float(torch.amin(samples)), float(torch.amax(samples)) - else: - m, M = 0, 1 - # make transform - def _transform(x): - if transform is not None: - x = transform(x) - x = self.batch_to_adversarial_imgs(x[None, ...], m=m, M=M)[0] - return x - # done! - return _transform - - # show image callback - class _BaseDatasetCallback(BaseCallbackPeriodic): - @TempNumpySeed(777) - @torch.no_grad() - def do_step(this, trainer: pl.Trainer, system: AdversarialModel): - if not wb_has_logger(trainer.logger): - log.warning(f'no wandb logger found, skipping visualisation: {system.__class__.__name__}') - return - if system.dataset is None: - log.warning(f'dataset not initialized, skipping visualisation: {system.__class__.__name__}') - return - log.info(f'visualising: {this.__class__.__name__}') - try: - this._do_step(trainer, system) - except: - log.error('Failed to do visualise callback step!', exc_info=True) - - # override this - def _do_step(this, trainer: pl.Trainer, system: AdversarialModel): - raise NotImplementedError - - # show image callback - class ImShowCallback(_BaseDatasetCallback): - def _do_step(this, trainer: pl.Trainer, system: AdversarialModel): - # make dataset with required transform - # -- this is inefficient for multiple subclasses of this class, we need to recompute the transform each time - dataset = system.dataset.shallow_copy(transform=make_scale_uint8_transform(system.dataset.transform)) - # get images & traversal - image = make_image_grid(dataset.dataset_sample_batch(num_samples=16, mode='input'), pad=4) - wandb_image, wandb_animation = H.visualize_dataset_traversal(dataset, data_mode='input', output_wandb=True, pad=4) - # log images to WANDB - wb_log_metrics(trainer.logger, { - 'random_images': wandb.Image(image), - 'traversal_image': wandb_image, - 'traversal_animation': wandb_animation, - }) - - # factor distances callback - class DistsPlotCallback(_BaseDatasetCallback): - def _do_step(this, trainer: pl.Trainer, system: AdversarialModel): - from disent.util.lightning.callbacks._callback_vis_dists import compute_factor_distances, plt_factor_distances - - # make distances function - def dists_fn(xs_a, xs_b): - dists = H.pairwise_loss(xs_a, xs_b, mode=system.hparams.pixel_loss_mode, mean_dtype=torch.float32, mask=None) - return [dists] - - def transform_batch(batch): - return system.model(batch.to(device=system.device)) - - # compute various distances matrices for each factor - dists_names, f_grid = compute_factor_distances( - dataset=system.dataset, - dists_fn=dists_fn, - dists_names=['dists'], - traversal_repeats=100, - batch_size=system.hparams.dataset_batch_size, - include_gt_factor_dists=True, - transform_batch=transform_batch, - seed=777, - data_mode='input', - ) - # plot these results - fig, axs = plt_factor_distances( - gt_data=system.dataset.gt_data, - f_grid=f_grid, - dists_names=dists_names, - title=f'{system.hparams.model_type.capitalize()}: {system.hparams.dataset_name.capitalize()} Distances', - plt_block_size=1.25, - plt_transpose=True, - plt_cmap='Blues', - ) - # recolour dists axis - for ax in axs[-1, :]: - ax.images[0].set_cmap('Reds') - # generate image & close matplotlib instace - from matplotlib import pyplot as plt - img = wandb.Image(fig) - plt.close() - # log the plot to wandb - if True: - wb_log_metrics(trainer.logger, { - 'factor_distances': img - }) - - # show stats callback - class StatsShowCallback(_BaseDatasetCallback): - def _do_step(this, trainer: pl.Trainer, system: AdversarialModel): - # make dataset with required transform - # -- this is inefficient for multiple subclasses of this class, we need to recompute the transform each time - dataset = system.dataset.shallow_copy(transform=make_scale_uint8_transform(system.dataset.transform)) - # get batches - batch, factors = dataset.dataset_sample_batch_with_factors(num_samples=512, mode='input') - batch = batch.to(torch.float32) - a_idx = torch.randint(0, len(batch), size=[4*len(batch)]) - b_idx = torch.randint(0, len(batch), size=[4*len(batch)]) - mask = (a_idx != b_idx) - # TODO: check that this is deterministic - # compute distances - deltas = to_numpy(H.pairwise_overlap(batch[a_idx[mask]], batch[b_idx[mask]], mode='mse')) - fdists = to_numpy(torch.abs(factors[a_idx[mask]] - factors[b_idx[mask]]).sum(dim=-1)) - sdists = to_numpy((torch.abs(factors[a_idx[mask]] - factors[b_idx[mask]]) / to_numpy(dataset.gt_data.factor_sizes)[None, :]).sum(dim=-1)) - # log to wandb - from matplotlib import pyplot as plt - plt.scatter(fdists, deltas); img_fdists = wandb.Image(plt); plt.close() - plt.scatter(sdists, deltas); img_sdists = wandb.Image(plt); plt.close() - wb_log_metrics(trainer.logger, { - 'fdists_vs_overlap': img_fdists, - 'sdists_vs_overlap': img_sdists, - }) - - # done! - return [ - ImShowCallback(every_n_steps=cfg.settings.exp.show_every_n_steps, begin_first_step=True), - DistsPlotCallback(every_n_steps=cfg.settings.exp.show_every_n_steps, begin_first_step=True), - StatsShowCallback(every_n_steps=cfg.settings.exp.show_every_n_steps, begin_first_step=True), - ] - - -# ========================================================================= # -# Run Hydra # -# ========================================================================= # - - -ROOT_DIR = os.path.abspath(__file__ + '/../../..') - - -def run_gen_adversarial_dataset(cfg): - time_string = datetime.today().strftime('%Y-%m-%d--%H-%M-%S') - log.info(f'Starting run at time: {time_string}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # cleanup from old runs: - try: - wandb.finish() - except: - pass - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - cfg = make_non_strict(cfg) - # - - - - - - - - - - - - - - - # - # check CUDA setting - gpus = hydra_get_gpus(cfg) - # create logger - logger = hydra_make_logger(cfg) - # create callbacks - callbacks: List[pl.Callback] = [c for c in hydra_get_callbacks(cfg) if isinstance(c, LoggerProgressCallback)] - # - - - - - - - - - - - - - - - # - # check save dirs - assert not os.path.isabs(cfg.settings.exp.rel_save_dir), f'rel_save_dir must be relative: {repr(cfg.settings.exp.rel_save_dir)}' - save_dir = os.path.join(ROOT_DIR, cfg.settings.exp.rel_save_dir) - assert os.path.isabs(save_dir), f'save_dir must be absolute: {repr(save_dir)}' - # - - - - - - - - - - - - - - - # - # get the logger and initialize - if logger is not None: - logger.log_hyperparams(cfg) - # print the final config! - log.info('Final Config' + make_box_str(OmegaConf.to_yaml(cfg))) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # | | | | | | | | | | | | | | | # - seed(cfg.settings.job.seed) - # | | | | | | | | | | | | | | | # - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # make framework - framework = AdversarialModel(**cfg.adv_system) - callbacks.extend(framework.make_train_periodic_callbacks(cfg)) - # train - trainer = pl.Trainer( - logger=logger, - callbacks=callbacks, - # cfg.dsettings.trainer - gpus=gpus, - # cfg.trainer - max_epochs=cfg.trainer.max_epochs, - max_steps=cfg.trainer.max_steps, - log_every_n_steps=cfg.trainer.log_every_n_steps, - enable_progress_bar=cfg.trainer.enable_progress_bar, - # prepare_data_per_node=cfg.trainer.prepare_data_per_node, # TODO: moved into data module / framework ! - # we do this here so we don't run the final metrics - detect_anomaly=False, # this should only be enabled for debugging torch and finding NaN values, slows down execution, not by much though? - enable_checkpointing=False, - ) - trainer.fit(framework) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # get save paths - save_prefix = f'{cfg.settings.exp.save_prefix}_' if cfg.settings.exp.save_prefix else '' - save_path_model = os.path.join(save_dir, f'{save_prefix}{time_string}_{cfg.settings.job.name}', f'model.pt') - save_path_data = os.path.join(save_dir, f'{save_prefix}{time_string}_{cfg.settings.job.name}', f'data.h5') - # create directories - if cfg.settings.exp.save_model: ensure_parent_dir_exists(save_path_model) - if cfg.settings.exp.save_data: ensure_parent_dir_exists(save_path_data) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # save adversarial model - if cfg.settings.exp.save_model: - log.info(f'saving model to path: {repr(save_path_model)}') - torch.save(framework.model, save_path_model) - log.info(f'saved model size: {bytes_to_human(os.path.getsize(save_path_model))}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # save adversarial dataset - if cfg.settings.exp.save_data: - log.info(f'saving data to path: {repr(save_path_data)}') - # transfer to GPU - if torch.cuda.is_available(): - framework = framework.cuda() - # create new h5py file -- TODO: use this in other places! - with H5Builder(path=save_path_data, mode='atomic_w') as builder: - # this dataset is self-contained and can be loaded by SelfContainedHdf5GroundTruthData - builder.add_dataset_from_gt_data( - data=framework.dataset, # produces tensors - mutator=wrapped_partial(framework.batch_to_adversarial_imgs, mode=cfg.settings.exp.save_dtype), # consumes tensors -> np.ndarrays - img_shape=(64, 64, None), - compression_lvl=4, - dtype=cfg.settings.exp.save_dtype, - batch_size=32, - ) - log.info(f'saved data size: {bytes_to_human(os.path.getsize(save_path_data))}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - -# ========================================================================= # -# Entry Point # -# ========================================================================= # - - -if __name__ == '__main__': - - # BENCHMARK (batch_size=256, optimizer=sgd, lr=1e-2, dataset_num_workers=0): - # - batch_optimizer=False, gpu=True, fp16=True : [3168MiB/5932MiB, 3.32/11.7G, 5.52it/s] - # - batch_optimizer=False, gpu=True, fp16=False : [5248MiB/5932MiB, 3.72/11.7G, 4.84it/s] - # - batch_optimizer=False, gpu=False, fp16=True : [same as fp16=False] - # - batch_optimizer=False, gpu=False, fp16=False : [0003MiB/5932MiB, 4.60/11.7G, 1.05it/s] - # --------- - # - batch_optimizer=True, gpu=True, fp16=True : [1284MiB/5932MiB, 3.45/11.7G, 4.31it/s] - # - batch_optimizer=True, gpu=True, fp16=False : [1284MiB/5932MiB, 3.72/11.7G, 4.31it/s] - # - batch_optimizer=True, gpu=False, fp16=True : [same as fp16=False] - # - batch_optimizer=True, gpu=False, fp16=False : [0003MiB/5932MiB, 1.80/11.7G, 4.18it/s] - - # BENCHMARK (batch_size=1024, optimizer=sgd, lr=1e-2, dataset_num_workers=12): - # - batch_optimizer=True, gpu=True, fp16=True : [2510MiB/5932MiB, 4.10/11.7G, 4.75it/s, 20% gpu util] (to(device).to(dtype)) - # - batch_optimizer=True, gpu=True, fp16=True : [2492MiB/5932MiB, 4.10/11.7G, 4.12it/s, 19% gpu util] (to(device, dtype)) - - CONFIGS_THIS_EXP = os.path.abspath(os.path.join(__file__, '..', 'config')) - CONFIGS_RESEARCH = os.path.abspath(os.path.join(__file__, '../../..', 'config')) - - # launch the action - hydra_main( - callback=run_gen_adversarial_dataset, - config_name='config_adversarial_dataset_approx', - search_dirs_prepend=[CONFIGS_THIS_EXP, CONFIGS_RESEARCH], - log_level=logging.INFO, - ) diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/util_eval_adversarial.py b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/util_eval_adversarial.py deleted file mode 100644 index b32699be..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/util_eval_adversarial.py +++ /dev/null @@ -1,348 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -from typing import Tuple - -import numpy as np -from numba import njit -from scipy.stats import gmean - - -# ========================================================================= # -# Aggregate # -# ========================================================================= # - - -_NP_AGGREGATE_FNS = { - 'sum': np.sum, - 'mean': np.mean, - 'gmean': gmean, # no negatives - 'max': lambda a, axis, dtype: np.amax(a, axis=axis), # propagate NaNs - 'min': lambda a, axis, dtype: np.amin(a, axis=axis), # propagate NaNs - 'std': np.std, -} - - -def np_aggregate(array, mode: str, axis=0, dtype=None): - try: - fn = _NP_AGGREGATE_FNS[mode] - except KeyError: - raise KeyError(f'invalid aggregate mode: {repr(mode)}, must be one of: {sorted(_NP_AGGREGATE_FNS.keys())}') - result = fn(array, axis=axis, dtype=dtype) - if dtype is not None: - result = result.astype(dtype) - return result - - -# ========================================================================= # -# Factor Evaluation - SLOW # -# ========================================================================= # - - -def eval_factor_fitness_numpy( - individual: np.ndarray, - f_idx: int, - f_dist_matrices: np.ndarray, - factor_sizes: Tuple[int, ...], - fitness_mode: str, - exclude_diag: bool, - increment_single: bool = True, -) -> float: - assert increment_single, f'`increment_single=False` is not supported for numpy fitness evaluation' - # generate missing mask axis - mask = individual.reshape(factor_sizes) - mask = np.moveaxis(mask, f_idx, -1) - f_mask = mask[..., :, None] & mask[..., None, :] - # the diagonal can change statistics - if exclude_diag: - diag = np.arange(f_mask.shape[-1]) - f_mask[..., diag, diag] = False - # mask the distance array | we negate the mask so that TRUE means the item is disabled - f_dists = np.ma.masked_where(~f_mask, f_dist_matrices) - - # get distances - if fitness_mode == 'range': agg_vals = np.ma.max(f_dists, axis=-1) - np.ma.min(f_dists, axis=-1) - elif fitness_mode == 'max': agg_vals = np.ma.max(f_dists, axis=-1) - elif fitness_mode == 'std': agg_vals = np.ma.std(f_dists, axis=-1) - else: raise KeyError(f'invalid fitness_mode: {repr(fitness_mode)}') - - # mean -- there is still a slight difference between this version - # and the numba version, but this helps improve things... - # It might just be a precision error? - fitness_sparse = np.ma.masked_where(~mask, agg_vals).mean() - - # combined scores - return fitness_sparse - - -# ========================================================================= # -# Factor Evaluation - FAST # -# ========================================================================= # - - -@njit -def eval_factor_fitness_numba__std_nodiag( - mask: np.ndarray, - f_dists: np.ndarray, - increment_single: bool = True -): - """ - This is about 10x faster than the built in numpy version - """ - assert f_dists.shape == (*mask.shape, mask.shape[-1]) - # totals - total = 0.0 - count = 0 - # iterate over values -- np.ndindex is usually quite fast - for I in np.ndindex(mask.shape[:-1]): - # mask is broadcast to the distance matrix - m_row = mask[I] - d_mat = f_dists[I] - # handle each distance matrix -- enumerate is usually faster than range - for i, m in enumerate(m_row): - if not m: - continue - # get vars - dists = d_mat[i] - # init vars - n = 0 - s = 0.0 - s2 = 0.0 - # handle each row -- enumerate is usually faster than range - for j, d in enumerate(dists): - if i == j: - continue - if not m_row[j]: - continue - n += 1 - s += d - s2 += d*d - # ^^^ END j - # update total - if n > 1: - mean2 = (s * s) / (n * n) - m2 = (s2 / n) - # is this just needed because of precision errors? - if m2 > mean2: - total += np.sqrt(m2 - mean2) - count += 1 - elif increment_single and (n == 1): - total += 0. - count += 1 - # ^^^ END i - if count == 0: - return -1 - else: - return total / count - - -@njit -def eval_factor_fitness_numba__range_nodiag( - mask: np.ndarray, - f_dists: np.ndarray, - increment_single: bool = True, -): - """ - This is about 10x faster than the built in numpy version - """ - assert f_dists.shape == (*mask.shape, mask.shape[-1]) - # totals - total = 0.0 - count = 0 - # iterate over values -- np.ndindex is usually quite fast - for I in np.ndindex(mask.shape[:-1]): - # mask is broadcast to the distance matrix - m_row = mask[I] - d_mat = f_dists[I] - # handle each distance matrix -- enumerate is usually faster than range - for i, m in enumerate(m_row): - if not m: - continue - # get vars - dists = d_mat[i] - # init vars - num_checked = False - m = 0.0 - M = 0.0 - # handle each row -- enumerate is usually faster than range - for j, d in enumerate(dists): - if i == j: - continue - if not m_row[j]: - continue - # update range - if num_checked > 0: - if d < m: - m = d - if d > M: - M = d - else: - m = d - M = d - # update num checked - num_checked += 1 - # ^^^ END j - # update total - if (num_checked > 1) or (increment_single and num_checked == 1): - total += (M - m) - count += 1 - # ^^^ END i - if count == 0: - return -1 - else: - return total / count - - -def eval_factor_fitness_numba( - individual: np.ndarray, - f_idx: int, - f_dist_matrices: np.ndarray, - factor_sizes: Tuple[int, ...], - fitness_mode: str, - exclude_diag: bool, - increment_single: bool = True, -): - """ - We only keep this function as a compatibility layer between: - - eval_factor_fitness_numpy - - eval_factor_fitness_numba__range_nodiag - """ - assert exclude_diag, 'fast version of eval only supports `exclude_diag=True`' - # usually a view - mask = np.moveaxis(individual.reshape(factor_sizes), f_idx, -1) - # call - if fitness_mode == 'range': - return eval_factor_fitness_numba__range_nodiag(mask=mask, f_dists=f_dist_matrices, increment_single=increment_single) - elif fitness_mode == 'std': - return eval_factor_fitness_numba__std_nodiag(mask=mask, f_dists=f_dist_matrices, increment_single=increment_single) - else: - raise KeyError(f'fast version of eval only supports `fitness_mode in ("range", "std")`, got: {repr(fitness_mode)}') - - -# ========================================================================= # -# Individual Evaluation # -# ========================================================================= # - - -_EVAL_BACKENDS = { - 'numpy': eval_factor_fitness_numpy, - 'numba': eval_factor_fitness_numba, -} - - -def eval_individual( - individual: np.ndarray, - gt_dist_matrices: np.ndarray, - factor_sizes: Tuple[int, ...], - fitness_overlap_mode: str, - fitness_overlap_aggregate: str, - exclude_diag: bool, - increment_single: bool = True, - backend: str = 'numba', -) -> Tuple[float, float]: - # get function - if backend not in _EVAL_BACKENDS: - raise KeyError(f'invalid backend: {repr(backend)}, must be one of: {sorted(_EVAL_BACKENDS.keys())}') - eval_fn = _EVAL_BACKENDS[backend] - # evaluate all factors - factor_scores = np.array([ - [eval_fn(individual, f_idx, f_dist_matrices, factor_sizes=factor_sizes, fitness_mode=fitness_overlap_mode, exclude_diag=exclude_diag, increment_single=increment_single)] - for f_idx, f_dist_matrices in enumerate(gt_dist_matrices) - ]) - # aggregate - factor_score = np_aggregate(factor_scores[:, 0], mode=fitness_overlap_aggregate, dtype='float64') - kept_ratio = individual.mean() - # check values just in case something goes wrong! - factor_score = np.nan_to_num(factor_score, nan=float('-inf')) - kept_ratio = np.nan_to_num(kept_ratio, nan=float('-inf')) - # return values! - return float(factor_score), float(kept_ratio) - - -# ========================================================================= # -# Equality Checks # -# ========================================================================= # - - -def _check_equal( - dataset_name: str = 'dsprites', - fitness_mode: str = 'std', # range, std - n: int = 5, -): - from research.part01_data_overlap.plot02_data_distances.util_compute_traversal_dists import cached_compute_all_factor_dist_matrices - from timeit import timeit - import research.code.util as H - - # load data - gt_data = H.make_data(dataset_name) - print(f'{dataset_name} {gt_data.factor_sizes} : {fitness_mode}') - - # get distances & individual - all_dist_matrices = cached_compute_all_factor_dist_matrices(dataset_name) # SHAPE FOR: s=factor_sizes, i=f_idx | (*s[:i], *s[i+1:], s[i], s[i]) - mask = np.random.random(len(gt_data)) < 0.5 # SHAPE: (-1,) - - def eval_factor(backend: str, f_idx: int, increment_single=True): - return _EVAL_BACKENDS[backend]( - individual=mask, - f_idx=f_idx, - f_dist_matrices=all_dist_matrices[f_idx], - factor_sizes=gt_data.factor_sizes, - fitness_mode=fitness_mode, - exclude_diag=True, - increment_single=increment_single, - ) - - def eval_all(backend: str, increment_single=True): - return np.around([eval_factor(backend, i, increment_single=increment_single) for i in range(gt_data.num_factors)], decimals=15) - - new_vals = eval_all('numba', increment_single=False) - new_time = timeit(lambda: eval_all('numba', increment_single=False), number=n) / n - print(f'- NEW {new_time:.5f}s {new_vals} (increment_single=False)') - - new_vals = eval_all('numba') - new_time = timeit(lambda: eval_all('numba'), number=n) / n - print(f'- NEW {new_time:.5f}s {new_vals}') - - old_vals = eval_all('numpy') - old_time = timeit(lambda: eval_all('numpy'), number=n) / n - print(f'- OLD {old_time:.5f}s {old_vals}') - print(f'* speedup: {np.around(old_time/new_time, decimals=2)}x') - - if not np.allclose(new_vals, old_vals): - print('[WARNING]: values are not close!') - - -if __name__ == '__main__': - - for dataset_name in ['smallnorb', 'shapes3d', 'dsprites']: - print('='*100) - _check_equal(dataset_name, fitness_mode='std') - print() - _check_equal(dataset_name, fitness_mode='range') - print('='*100) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/util_eval_adversarial_dist_pairs.py b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/util_eval_adversarial_dist_pairs.py deleted file mode 100644 index 5cc7980a..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/util_eval_adversarial_dist_pairs.py +++ /dev/null @@ -1,290 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -from typing import Tuple - -import numpy as np -from numba import njit - - -# ========================================================================= # -# Factor Evaluation - SLOW # -# ========================================================================= # - - -def eval_dist_pairs_numpy( - mask: np.ndarray, - pair_obs_dists: np.ndarray, - pair_obs_idxs: np.ndarray, - fitness_mode: str, - increment_single: bool = True -) -> float: - assert increment_single, f'`increment_single=False` is not supported for numpy fitness evaluation' - # mask the distance array | we negate the mask so that TRUE means the item is disabled - dists = np.ma.masked_where(~mask[pair_obs_idxs], pair_obs_dists) - # get distances - if fitness_mode == 'range': agg_vals = np.ma.max(dists, axis=-1) - np.ma.min(dists, axis=-1) - elif fitness_mode == 'std': agg_vals = np.ma.std(dists, axis=-1) - else: raise KeyError(f'invalid fitness_mode: {repr(fitness_mode)}') - # mean -- there is still a slight difference between this version - # and the numba version, but this helps improve things... - # It might just be a precision error? - fitness_sparse = np.ma.masked_where(~mask, agg_vals).mean() - # combined scores - return fitness_sparse - - -# ========================================================================= # -# Factor Evaluation - FAST # -# ========================================================================= # - - -@njit -def eval_dist_pairs_numba__std( - mask: np.ndarray, - pair_obs_dists: np.ndarray, - pair_obs_idxs: np.ndarray, - increment_single: bool = True -): - """ - This is about 10x faster than the built in numpy version - -- something is wrong compared to the numpy version, maybe the - numpy version is wrong because of the mean taken after masking? - """ - assert len(mask) == len(pair_obs_dists) - assert len(mask) == len(pair_obs_idxs) - assert pair_obs_dists.shape == pair_obs_idxs.shape - # totals - total = 0.0 - count = 0 - # iterate over values -- np.ndindex is usually quite fast - for i, m in enumerate(mask): - # skip if invalid - if not m: - continue - # get pair info - dists = pair_obs_dists[i] - idxs = pair_obs_idxs[i] - # init vars - n = 0 - s = 0.0 - s2 = 0.0 - # handle each distance matrix -- enumerate is usually faster than range - for j, d in zip(idxs, dists): - # skip if invalid - if not mask[j]: - continue - # compute std - n += 1 - s += d - s2 += d*d - # update total -- TODO: numpy includes this, but we might not want to? - if n > 1: - mean2 = (s * s) / (n * n) - m2 = (s2 / n) - # is this just needed because of precision errors? - if m2 > mean2: - total += np.sqrt(m2 - mean2) - count += 1 - elif increment_single and (n == 1): - total += 0. - count += 1 - # ^^^ END i - if count == 0: - return -1 - else: - return total / count - - -@njit -def eval_dist_pairs_numba__range( - mask: np.ndarray, - pair_obs_dists: np.ndarray, - pair_obs_idxs: np.ndarray, - increment_single: bool = True -): - """ - This is about 10x faster than the built in numpy version - """ - assert len(mask) == len(pair_obs_dists) - assert len(mask) == len(pair_obs_idxs) - assert pair_obs_dists.shape == pair_obs_idxs.shape - # totals - total = 0.0 - count = 0 - # iterate over values -- np.ndindex is usually quite fast - for i, m in enumerate(mask): - # skip if invalid - if not m: - continue - # get pair info - dists = pair_obs_dists[i] - idxs = pair_obs_idxs[i] - # init vars - num_checked = 0 - m = 0.0 - M = 0.0 - # handle each distance matrix -- enumerate is usually faster than range - for j, d in zip(idxs, dists): - # skip if invalid - if not mask[j]: - continue - # update range - if num_checked > 0: - if d < m: m = d - if d > M: M = d - else: - m = d - M = d - # update num checked - num_checked += 1 - # update total - if (num_checked > 1) or (increment_single and num_checked == 1): - total += (M - m) - count += 1 - # ^^^ END i - if count == 0: - return -1 - else: - return total / count - - -def eval_dist_pairs_numba( - mask: np.ndarray, - pair_obs_dists: np.ndarray, - pair_obs_idxs: np.ndarray, - fitness_mode: str, - increment_single: bool = True -): - """ - We only keep this function as a compatibility layer between: - - eval_numpy - - eval_numba__range_nodiag - """ - # call - if fitness_mode == 'range': - return eval_dist_pairs_numba__range(mask=mask, pair_obs_dists=pair_obs_dists, pair_obs_idxs=pair_obs_idxs, increment_single=increment_single) - elif fitness_mode == 'std': - return eval_dist_pairs_numba__std(mask=mask, pair_obs_dists=pair_obs_dists, pair_obs_idxs=pair_obs_idxs, increment_single=increment_single) - else: - raise KeyError(f'fast version of eval only supports `fitness_mode in ("range", "std")`, got: {repr(fitness_mode)}') - - -# ========================================================================= # -# Individual Evaluation # -# ========================================================================= # - - -_EVAL_BACKENDS = { - 'numpy': eval_dist_pairs_numpy, - 'numba': eval_dist_pairs_numba, -} - - -def eval_masked_dist_pairs( - mask: np.ndarray, - pair_obs_dists: np.ndarray, - pair_obs_idxs: np.ndarray, - fitness_mode: str, - increment_single: bool = True, - backend: str = 'numba', -) -> Tuple[float, float]: - # get function - if backend not in _EVAL_BACKENDS: - raise KeyError(f'invalid backend: {repr(backend)}, must be one of: {sorted(_EVAL_BACKENDS.keys())}') - eval_fn = _EVAL_BACKENDS[backend] - # evaluate - factor_score = eval_fn( - mask=mask, - pair_obs_dists=pair_obs_dists, - pair_obs_idxs=pair_obs_idxs, - fitness_mode=fitness_mode, - increment_single=increment_single, - ) - # aggregate - kept_ratio = mask.mean() - # check values just in case something goes wrong! - factor_score = np.nan_to_num(factor_score, nan=float('-inf')) - kept_ratio = np.nan_to_num(kept_ratio, nan=float('-inf')) - # return values! - return float(factor_score), float(kept_ratio) - - -# ========================================================================= # -# Equality Checks # -# ========================================================================= # - - -def _check_equal( - dataset_name: str = 'dsprites', - pair_mode: str = 'nearby_scaled', - pairs_per_obs: int = 8, - fitness_mode: str = 'std', # range, std - n: int = 5, -): - from research.part01_data_overlap.plot02_data_distances.util_compute_traversal_dist_pairs import cached_compute_dataset_pair_dists - from timeit import timeit - - # get distances & individual # (len(gt_data), pairs_per_obs) & (len(gt_data),) - obs_pair_idxs, obs_pair_dists = cached_compute_dataset_pair_dists(dataset_name=dataset_name, pair_mode=pair_mode, pairs_per_obs=pairs_per_obs, scaled=True) - mask = np.random.random(len(obs_pair_idxs)) < 0.5 - - def eval_all(backend: str, increment_single=True): - return _EVAL_BACKENDS[backend]( - mask=mask, - pair_obs_dists=obs_pair_dists, - pair_obs_idxs=obs_pair_idxs, - fitness_mode=fitness_mode, - increment_single=increment_single, - ) - - new_vals = eval_all('numba', increment_single=False) - new_time = timeit(lambda: eval_all('numba', increment_single=False), number=n) / n - print(f'- NEW {new_time:.5f}s {new_vals} (increment_single=False)') - - new_vals = eval_all('numba') - new_time = timeit(lambda: eval_all('numba'), number=n) / n - print(f'- NEW {new_time:.5f}s {new_vals}') - - old_vals = eval_all('numpy') - old_time = timeit(lambda: eval_all('numpy'), number=n) / n - print(f'- OLD {old_time:.5f}s {old_vals}') - print(f'* speedup: {np.around(old_time/new_time, decimals=2)}x') - - if not np.allclose(new_vals, old_vals): - print('[WARNING]: values are not close!') - - -if __name__ == '__main__': - - for dataset_name in ['smallnorb', 'shapes3d', 'dsprites']: - print('='*100) - _check_equal(dataset_name, fitness_mode='std') - print() - _check_equal(dataset_name, fitness_mode='range') - print('='*100) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/util_gen_adversarial_dataset.py b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/util_gen_adversarial_dataset.py deleted file mode 100644 index a69b0212..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/util_gen_adversarial_dataset.py +++ /dev/null @@ -1,447 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -""" -General helper utilities for generating -adversarial datasets using triplet sampling. -""" - -import logging -from functools import lru_cache -from typing import Literal -from typing import Optional -from typing import Tuple -from typing import Union - -import numpy as np -import torch - -import research.code.util as H -from disent.dataset.data import GroundTruthData -from disent.dataset.sampling import BaseDisentSampler -from disent.dataset.sampling import GroundTruthPairSampler -from disent.dataset.sampling import GroundTruthTripleSampler -from disent.dataset.sampling import RandomSampler -from disent.dataset.util.state_space import StateSpace -from disent.util.strings import colors as c - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Samplers # -# ========================================================================= # - - -class AdversarialSampler_SwappedRandom(BaseDisentSampler): - - def uninit_copy(self) -> 'AdversarialSampler_SwappedRandom': - return AdversarialSampler_SwappedRandom(swap_metric=self._swap_metric) - - def __init__(self, swap_metric='manhattan'): - super().__init__(3) - assert swap_metric in {'k', 'manhattan', 'manhattan_norm', 'euclidean', 'euclidean_norm'} - self._swap_metric = swap_metric - self._sampler = GroundTruthTripleSampler(swap_metric=swap_metric) - self._state_space: Optional[StateSpace] = None - - def _init(self, gt_data: GroundTruthData): - self._sampler.init(gt_data) - self._state_space = gt_data.state_space_copy() - - def _sample_idx(self, idx: int) -> Tuple[int, ...]: - anchor, pos, neg = self._state_space.idx_to_pos([ - idx, - *np.random.randint(0, len(self._state_space), size=2) - ]) - # swap values - pos, neg = self._sampler._swap_factors(anchor_factors=anchor, positive_factors=pos, negative_factors=neg) - # return triple - return tuple(self._state_space.pos_to_idx([anchor, pos, neg])) - - -class AdversarialSampler_CloseFar(BaseDisentSampler): - - def uninit_copy(self) -> 'AdversarialSampler_CloseFar': - return AdversarialSampler_CloseFar( - p_k_range=self._p_k_range, - p_radius_range=self._p_radius_range, - n_k_range=self._n_k_range, - n_radius_range=self._n_radius_range, - ) - - def __init__( - self, - p_k_range=(1, 1), - p_radius_range=(1, 1), - n_k_range=(1, -1), - n_radius_range=(1, -1), - ): - super().__init__(3) - self._p_k_range = p_k_range - self._p_radius_range = p_radius_range - self._n_k_range = n_k_range - self._n_radius_range = n_radius_range - self.sampler_close = GroundTruthPairSampler(p_k_range=p_k_range, p_radius_range=p_radius_range) - self.sampler_far = GroundTruthPairSampler(p_k_range=n_k_range, p_radius_range=n_radius_range) - - def _init(self, gt_data: GroundTruthData): - self.sampler_close.init(gt_data) - self.sampler_far.init(gt_data) - - def _sample_idx(self, idx: int) -> Tuple[int, ...]: - # sample indices - anchor, pos = self.sampler_close(idx) - _anchor, neg = self.sampler_far(idx) - assert anchor == _anchor - # return triple - return anchor, pos, neg - - -class AdversarialSampler_SameK(BaseDisentSampler): - - def uninit_copy(self) -> 'AdversarialSampler_SameK': - return AdversarialSampler_SameK( - k=self._k, - sample_p_close=self._sample_p_close, - ) - - def __init__(self, k: Union[Literal['random'], int] = 'random', sample_p_close: bool = False): - super().__init__(3) - self._state_space: Optional[StateSpace] = None - self._sample_p_close = sample_p_close - self._k = k - assert (isinstance(k, int) and k > 0) or (k == 'random') - - def _init(self, gt_data: GroundTruthData): - self._state_space = gt_data.state_space_copy() - - def _sample_idx(self, idx: int) -> Tuple[int, ...]: - a_factors = self._state_space.idx_to_pos(idx) - # SAMPLE FACTOR INDICES - k = self._k - if k == 'random': - k = np.random.randint(1, self._state_space.num_factors+1) # end exclusive, ie. [1, num_factors+1) - # get shared mask - shared_indices = np.random.choice(self._state_space.num_factors, size=self._state_space.num_factors-k, replace=False) - shared_mask = np.zeros(a_factors.shape, dtype='bool') - shared_mask[shared_indices] = True - # generate values - p_factors = self._sample_shared(a_factors, shared_mask, sample_close=self._sample_p_close) - n_factors = self._sample_shared(a_factors, shared_mask, sample_close=False) - # swap values if wrong - # TODO: this might give errors! - # - one factor might be less than another - if np.sum(np.abs(a_factors - p_factors)) > np.sum(np.abs(a_factors - n_factors)): - p_factors, n_factors = n_factors, p_factors - # check values - assert np.sum(a_factors != p_factors) == k, 'this should never happen!' - assert np.sum(a_factors != n_factors) == k, 'this should never happen!' - # return values - return tuple(self._state_space.pos_to_idx([ - a_factors, - p_factors, - n_factors, - ])) - - def _sample_shared(self, base_factors, shared_mask, tries=100, sample_close: bool = False): - sampled_factors = base_factors.copy() - generate_mask = ~shared_mask - # generate values - for i in range(tries): - if sample_close: - sampled_values = (base_factors + np.random.randint(-1, 1+1, size=self._state_space.num_factors)) - sampled_values = np.clip(sampled_values, 0, np.array(self._state_space.factor_sizes) - 1)[generate_mask] - else: - sampled_values = np.random.randint(0, np.array(self._state_space.factor_sizes)[generate_mask]) - # overwrite values that are not different - sampled_factors[generate_mask] = sampled_values - # update mask - sampled_shared_mask = (sampled_factors == base_factors) - generate_mask &= sampled_shared_mask - # check everything - if np.sum(sampled_shared_mask) == np.sum(shared_mask): - assert np.sum(generate_mask) == 0 - return sampled_factors - # we need to try again! - raise RuntimeError('could not generate factors: {}') - - -def sampler_print_test(sampler: Union[str, BaseDisentSampler], gt_data: GroundTruthData = None, steps=100): - # make data - if gt_data is None: - gt_data = H.make_dataset('xysquares_8x8_mini').gt_data - # make sampler - if isinstance(sampler, str): - prefix = sampler - sampler = make_adversarial_sampler(sampler) - else: - prefix = sampler.__class__.__name__ - if not sampler.is_init: - sampler.init(gt_data) - # print everything - count_pn_k0, count_pn_d0 = 0, 0 - for i in range(min(steps, len(gt_data))): - a, p, n = gt_data.idx_to_pos(sampler(i)) - ap_k = np.sum(a != p); ap_d = np.sum(np.abs(a - p)) - an_k = np.sum(a != n); an_d = np.sum(np.abs(a - n)) - pn_k = np.sum(p != n); pn_d = np.sum(np.abs(p - n)) - print(f'{prefix}: [{c.lGRN}ap{c.RST}:{ap_k:2d}:{ap_d:2d}] [{c.lRED}an{c.RST}:{an_k:2d}:{an_d:2d}] [{c.lYLW}pn{c.RST}:{pn_k:2d}:{pn_d:2d}] {a} {p} {n}') - count_pn_k0 += (pn_k == 0) - count_pn_d0 += (pn_d == 0) - print(f'count pn:(k=0) = {count_pn_k0} pn:(d=0) = {count_pn_d0}') - - -def make_adversarial_sampler(mode: str = 'close_far'): - if mode in ['random_swap_k', 'random_swap_manhattan', 'random_swap_manhattan_norm', 'random_swap_euclidean', 'random_swap_euclidean_norm']: - # NOTE # -- random_swap_manhattan -- probability is too low of encountering nearby obs, don't use this! - metric = mode[len('random_swap_'):] - return AdversarialSampler_SwappedRandom(swap_metric=metric) - elif mode in ['close_far', 'close_p_random_n']: - # *NB* # - return AdversarialSampler_CloseFar( - p_k_range=(1, 1), n_k_range=(1, -1), - p_radius_range=(1, 1), n_radius_range=(1, -1), - ) - elif mode in ['close_far_random', 'close_p_random_n_bb']: - # *NB* # - return GroundTruthTripleSampler( - p_k_range=(1, 1), n_k_range=(1, -1), n_k_sample_mode='bounded_below', n_k_is_shared=True, - p_radius_range=(1, 1), n_radius_range=(1, -1), n_radius_sample_mode='bounded_below', - ) - elif mode in ['same_k']: - # *NB* # - return AdversarialSampler_SameK(k='random', sample_p_close=False) - elif mode in ['same_k_close']: - # *NB* # - return AdversarialSampler_SameK(k='random', sample_p_close=True) - elif mode in ['same_k1_close']: - # *NB* # - return AdversarialSampler_SameK(k=1, sample_p_close=True) - elif mode == 'close_factor_far_random': - return GroundTruthTripleSampler( - p_k_range=(1, 1), n_k_range=(1, -1), n_k_sample_mode='bounded_below', n_k_is_shared=True, - p_radius_range=(1, -1), n_radius_range=(0, -1), n_radius_sample_mode='bounded_below', - ) - elif mode == 'close_far_same_factor': - # TODO: problematic for dsprites - return GroundTruthTripleSampler( - p_k_range=(1, 1), n_k_range=(1, 1), n_k_sample_mode='bounded_below', n_k_is_shared=True, - p_radius_range=(1, 1), n_radius_range=(2, -1), n_radius_sample_mode='bounded_below', - ) - elif mode == 'same_factor': - return GroundTruthTripleSampler( - p_k_range=(1, 1), n_k_range=(1, 1), n_k_sample_mode='bounded_below', n_k_is_shared=True, - p_radius_range=(1, -2), n_radius_range=(2, -1), n_radius_sample_mode='bounded_below', # bounded below does not always work, still relies on random chance :/ - ) - elif mode == 'random_bb': - return GroundTruthTripleSampler( - p_k_range=(0, -1), n_k_range=(0, -1), n_k_sample_mode='bounded_below', n_k_is_shared=True, - p_radius_range=(0, -1), n_radius_range=(0, -1), n_radius_sample_mode='bounded_below', - ) - elif mode == 'random_swap_manhat': - return GroundTruthTripleSampler( - p_k_range=(0, -1), n_k_range=(0, -1), n_k_sample_mode='random', n_k_is_shared=False, - p_radius_range=(0, -1), n_radius_range=(0, -1), n_radius_sample_mode='random', - swap_metric='manhattan' - ) - elif mode == 'random_swap_manhat_norm': - return GroundTruthTripleSampler( - p_k_range=(0, -1), n_k_range=(0, -1), n_k_sample_mode='random', n_k_is_shared=False, - p_radius_range=(0, -1), n_radius_range=(0, -1), n_radius_sample_mode='random', - swap_metric='manhattan_norm' - ) - elif mode == 'random': - return RandomSampler(num_samples=3) - else: - raise KeyError(f'invalid adversarial sampler: mode={repr(mode)}') - - -# ========================================================================= # -# Adversarial Sort # -# ========================================================================= # - - -@torch.no_grad() -def sort_samples(a_x: torch.Tensor, p_x: torch.Tensor, n_x: torch.Tensor, sort_mode: str = 'none', pixel_loss_mode: str = 'mse'): - # NOTE: this function may mutate its inputs, however - # the returned values should be used. - # do not sort! - if sort_mode == 'none': - return (a_x, p_x, n_x) - elif sort_mode == 'swap': - return (a_x, n_x, p_x) - # compute deltas - p_deltas = H.pairwise_loss(a_x, p_x, mode=pixel_loss_mode, mean_dtype=torch.float32, mask=None) - n_deltas = H.pairwise_loss(a_x, n_x, mode=pixel_loss_mode, mean_dtype=torch.float32, mask=None) - # get swap mask - if sort_mode == 'sort_inorder': swap_mask = p_deltas > n_deltas - elif sort_mode == 'sort_reverse': swap_mask = p_deltas < n_deltas - else: raise KeyError(f'invalid sort_mode: {repr(sort_mode)}, must be one of: ["none", "swap", "sort_inorder", "sort_reverse"]') - # handle mutate or copy - idx_swap = torch.where(swap_mask) - # swap memory values -- TODO: `p_x[idx_swap], n_x[idx_swap] = n_x[idx_swap], p_x[idx_swap]` is this fine? - temp = torch.clone(n_x[idx_swap]) - n_x[idx_swap] = p_x[idx_swap] - p_x[idx_swap] = temp - # done! - return (a_x, p_x, n_x) - - -# ========================================================================= # -# Adversarial Loss # -# ========================================================================= # - -# anchor, positive, negative -TensorTriple = Tuple[torch.Tensor, torch.Tensor, torch.Tensor] - - -def _get_triple(x: TensorTriple, adversarial_swapped: bool): - if not adversarial_swapped: - a, p, n = x - else: - a, n, p = x - return a, p, n - - -_MARGIN_MODES = { - 'invert_margin', - 'triplet_margin', -} - - -@lru_cache() -def _parse_margin_mode(adversarial_mode: str): - # parse the MARGIN_MODES -- linear search - for margin_mode in _MARGIN_MODES: - if adversarial_mode == margin_mode: - raise KeyError(f'`{margin_mode}` is not valid, specify the margin in the name, eg. `{margin_mode}_0.01`') - elif adversarial_mode.startswith(f'{margin_mode}_'): - margin = float(adversarial_mode[len(f'{margin_mode}_'):]) - return margin_mode, margin - # done! - return adversarial_mode, None - - -def adversarial_loss( - ys: TensorTriple, - xs: Optional[TensorTriple] = None, # only used if mask_deltas==True - # adversarial loss settings - adversarial_mode: str = 'invert_shift', - adversarial_swapped: bool = False, - adversarial_masking: bool = False, # requires `xs` to be set - adversarial_top_k: Optional[int] = None, - # pixel loss to get deltas settings - pixel_loss_mode: str = 'mse', - # statistics - return_stats: bool = False, -): - a_y, p_y, n_y = _get_triple(ys, adversarial_swapped=adversarial_swapped) - - # get mask - if adversarial_masking: - a_x, p_x, n_x = _get_triple(xs, adversarial_swapped=adversarial_swapped) - ap_mask, an_mask = (a_x != p_x), (a_x != n_x) - else: - ap_mask, an_mask = None, None - - # compute deltas - p_deltas = H.pairwise_loss(a_y, p_y, mode=pixel_loss_mode, mean_dtype=torch.float32, mask=ap_mask) - n_deltas = H.pairwise_loss(a_y, n_y, mode=pixel_loss_mode, mean_dtype=torch.float32, mask=an_mask) - deltas = (n_deltas - p_deltas) - - # parse mode - adversarial_mode, margin = _parse_margin_mode(adversarial_mode) - - # compute loss deltas - # AUTO-CONSTANT - if adversarial_mode == 'self': loss_deltas = torch.abs(deltas) - elif adversarial_mode == 'self_random': - # the above should be equivalent with the right sampling strategy? - all_deltas = torch.cat([p_deltas, n_deltas], dim=0) - indices = np.arange(len(all_deltas)) - np.random.shuffle(indices) - deltas = all_deltas[indices[len(deltas):]] - all_deltas[indices[:len(deltas)]] - loss_deltas = torch.abs(deltas) - # INVERT - elif adversarial_mode == 'invert': loss_deltas = torch.maximum(deltas, torch.zeros_like(deltas)) - elif adversarial_mode == 'invert_margin': loss_deltas = torch.maximum(margin + deltas, torch.zeros_like(deltas)) # invert_loss = torch.clamp_min(n_dist - p_dist + margin_max, 0) - elif adversarial_mode == 'invert_unbounded': loss_deltas = deltas - # TRIPLET - elif adversarial_mode == 'triplet': loss_deltas = torch.maximum(-deltas, torch.zeros_like(deltas)) - elif adversarial_mode == 'triplet_margin': loss_deltas = torch.maximum(margin - deltas, torch.zeros_like(deltas)) # triplet_loss = torch.clamp_min(p_dist - n_dist + margin_max, 0) - elif adversarial_mode == 'triplet_unbounded': loss_deltas = -deltas - # OTHER - else: - raise KeyError(f'invalid `adversarial_mode`: {repr(adversarial_mode)}') - - # checks - assert deltas.shape == loss_deltas.shape, 'this is a bug' - - # top k deltas - if adversarial_top_k is not None: - loss_deltas = torch.topk(loss_deltas, k=adversarial_top_k, largest=True).values - - # get average loss - loss = loss_deltas.mean() - - # return early - if not return_stats: - return loss - - # compute stats! - with torch.no_grad(): - loss_stats = { - 'stat/p_delta:mean': float(p_deltas.mean().cpu()), 'stat/p_delta:std': float(p_deltas.std().cpu()), - 'stat/n_delta:mean': float(n_deltas.mean().cpu()), 'stat/n_delta:std': float(n_deltas.std().cpu()), - 'stat/deltas:mean': float(loss_deltas.mean().cpu()), 'stat/deltas:std': float(loss_deltas.std().cpu()), - } - - return loss, loss_stats - - -# ========================================================================= # -# END # -# ========================================================================= # - - -# if __name__ == '__main__': -# -# def _main(): -# from disent.dataset.data import XYObjectData -# -# # NB: -# # close_p_random_n -# # close_p_random_n_bb -# # same_k -# # same_k_close -# # same_k1_close -# -# sampler_print_test( -# sampler='close_p_random_n', -# gt_data=XYObjectData() -# ) -# -# _main() diff --git a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/util_load_adversarial_mask.py b/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/util_load_adversarial_mask.py deleted file mode 100644 index e88a628c..00000000 --- a/research/part03_learnt_overlap/__OLD__e02_learn_adversarial_data/util_load_adversarial_mask.py +++ /dev/null @@ -1,78 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import gzip -import pickle -import numpy as np -import logging - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# HELPER # -# ========================================================================= # - - -def get_closest_mask(usage_ratio: float, pickle_file: str, print_n_best: int = 3) -> np.ndarray: - """ - This function is intended to be used with the data - generated by `run_04_gen_adversarial_ruck.py` - - The function finds the closest member in the population with - the matching statistic. The reason this function works is that - the population should consist only of near-pareto-optimal solutions. - - These solutions are found using NSGA2 - - Usage With Hydra Config: - _target_: research.part03_learnt_overlap.e02_learn_adversarial_data.util_load_adversarial_mask.get_closest_mask - usage_ratio: 0.5 - pickle_file: data.pkl.gz - """ - # load pickled data - with gzip.open(pickle_file, mode='rb') as fp: - data = pickle.load(fp) - values = np.array(data['values'], dtype='bool') - scores = np.array(data['scores'], dtype='float64') - del data - # check shapes - assert values.ndim == 2 - assert scores.ndim == 2 - assert scores.shape == (len(values), 2) - # get closest - best_indices = np.argsort(np.abs(scores[:, 1] - usage_ratio)) - # print stats - if print_n_best > 0: - log.info(f'The {print_n_best} closest members to target usage={usage_ratio:7f}') - for i, idx in enumerate(best_indices[:print_n_best]): - assert np.isclose(np.mean(values[idx]), scores[idx, 1]), 'member fitness_usage is not close to the actual mask usage. The data is invalid.' - log.info(f' [{i+1}] idx={idx:04d} overlap={scores[idx, 0]:7f} usage={scores[idx, 1]:7f}') - # return the best! - return values[best_indices[0]] - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/array_03_MSC-p03e01_kernel-disentangle-xy.txt b/research/part03_learnt_overlap/e01_learn_to_disentangle/array_03_MSC-p03e01_kernel-disentangle-xy.txt deleted file mode 100644 index 8bf33b9c..00000000 --- a/research/part03_learnt_overlap/e01_learn_to_disentangle/array_03_MSC-p03e01_kernel-disentangle-xy.txt +++ /dev/null @@ -1,128 +0,0 @@ -+EXTRA.sweep_num=1 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=63 -+EXTRA.sweep_num=2 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=47 -+EXTRA.sweep_num=3 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=31 -+EXTRA.sweep_num=4 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=15 -+EXTRA.sweep_num=5 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=63 -+EXTRA.sweep_num=6 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=47 -+EXTRA.sweep_num=7 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=31 -+EXTRA.sweep_num=8 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=15 -+EXTRA.sweep_num=9 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=63 -+EXTRA.sweep_num=10 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=47 -+EXTRA.sweep_num=11 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=31 -+EXTRA.sweep_num=12 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=15 -+EXTRA.sweep_num=13 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=63 -+EXTRA.sweep_num=14 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=47 -+EXTRA.sweep_num=15 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=31 -+EXTRA.sweep_num=16 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=15 -+EXTRA.sweep_num=17 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=63 -+EXTRA.sweep_num=18 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=47 -+EXTRA.sweep_num=19 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=31 -+EXTRA.sweep_num=20 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=15 -+EXTRA.sweep_num=21 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=63 -+EXTRA.sweep_num=22 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=47 -+EXTRA.sweep_num=23 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=31 -+EXTRA.sweep_num=24 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=15 -+EXTRA.sweep_num=25 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=63 -+EXTRA.sweep_num=26 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=47 -+EXTRA.sweep_num=27 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=31 -+EXTRA.sweep_num=28 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=15 -+EXTRA.sweep_num=29 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=63 -+EXTRA.sweep_num=30 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=47 -+EXTRA.sweep_num=31 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=31 -+EXTRA.sweep_num=32 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=15 -+EXTRA.sweep_num=33 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=63 -+EXTRA.sweep_num=34 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=47 -+EXTRA.sweep_num=35 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=31 -+EXTRA.sweep_num=36 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=15 -+EXTRA.sweep_num=37 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=63 -+EXTRA.sweep_num=38 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=47 -+EXTRA.sweep_num=39 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=31 -+EXTRA.sweep_num=40 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=15 -+EXTRA.sweep_num=41 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=63 -+EXTRA.sweep_num=42 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=47 -+EXTRA.sweep_num=43 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=31 -+EXTRA.sweep_num=44 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=15 -+EXTRA.sweep_num=45 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=63 -+EXTRA.sweep_num=46 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=47 -+EXTRA.sweep_num=47 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=31 -+EXTRA.sweep_num=48 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=15 -+EXTRA.sweep_num=49 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=63 -+EXTRA.sweep_num=50 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=47 -+EXTRA.sweep_num=51 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=31 -+EXTRA.sweep_num=52 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=15 -+EXTRA.sweep_num=53 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=63 -+EXTRA.sweep_num=54 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=47 -+EXTRA.sweep_num=55 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=31 -+EXTRA.sweep_num=56 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=15 -+EXTRA.sweep_num=57 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=63 -+EXTRA.sweep_num=58 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=47 -+EXTRA.sweep_num=59 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=31 -+EXTRA.sweep_num=60 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=15 -+EXTRA.sweep_num=61 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=63 -+EXTRA.sweep_num=62 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=47 -+EXTRA.sweep_num=63 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=31 -+EXTRA.sweep_num=64 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=square exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=15 -+EXTRA.sweep_num=65 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=63 -+EXTRA.sweep_num=66 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=47 -+EXTRA.sweep_num=67 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=31 -+EXTRA.sweep_num=68 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=15 -+EXTRA.sweep_num=69 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=63 -+EXTRA.sweep_num=70 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=47 -+EXTRA.sweep_num=71 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=31 -+EXTRA.sweep_num=72 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=15 -+EXTRA.sweep_num=73 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=63 -+EXTRA.sweep_num=74 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=47 -+EXTRA.sweep_num=75 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=31 -+EXTRA.sweep_num=76 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=15 -+EXTRA.sweep_num=77 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=63 -+EXTRA.sweep_num=78 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=47 -+EXTRA.sweep_num=79 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=31 -+EXTRA.sweep_num=80 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=15 -+EXTRA.sweep_num=81 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=63 -+EXTRA.sweep_num=82 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=47 -+EXTRA.sweep_num=83 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=31 -+EXTRA.sweep_num=84 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=15 -+EXTRA.sweep_num=85 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=63 -+EXTRA.sweep_num=86 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=47 -+EXTRA.sweep_num=87 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=31 -+EXTRA.sweep_num=88 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=15 -+EXTRA.sweep_num=89 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=63 -+EXTRA.sweep_num=90 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=47 -+EXTRA.sweep_num=91 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=31 -+EXTRA.sweep_num=92 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=15 -+EXTRA.sweep_num=93 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=63 -+EXTRA.sweep_num=94 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=47 -+EXTRA.sweep_num=95 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=31 -+EXTRA.sweep_num=96 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=exp exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=15 -+EXTRA.sweep_num=97 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=63 -+EXTRA.sweep_num=98 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=47 -+EXTRA.sweep_num=99 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=31 -+EXTRA.sweep_num=100 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=15 -+EXTRA.sweep_num=101 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=63 -+EXTRA.sweep_num=102 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=47 -+EXTRA.sweep_num=103 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=31 -+EXTRA.sweep_num=104 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=15 -+EXTRA.sweep_num=105 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=63 -+EXTRA.sweep_num=106 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=47 -+EXTRA.sweep_num=107 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=31 -+EXTRA.sweep_num=108 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=15 -+EXTRA.sweep_num=109 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=63 -+EXTRA.sweep_num=110 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=47 -+EXTRA.sweep_num=111 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=31 -+EXTRA.sweep_num=112 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=15 -+EXTRA.sweep_num=113 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=63 -+EXTRA.sweep_num=114 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=47 -+EXTRA.sweep_num=115 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=31 -+EXTRA.sweep_num=116 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=15 -+EXTRA.sweep_num=117 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=63 -+EXTRA.sweep_num=118 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=47 -+EXTRA.sweep_num=119 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=31 -+EXTRA.sweep_num=120 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_4x4 exp.kernel.radius=15 -+EXTRA.sweep_num=121 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=63 -+EXTRA.sweep_num=122 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=47 -+EXTRA.sweep_num=123 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=31 -+EXTRA.sweep_num=124 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_2x2 exp.kernel.radius=15 -+EXTRA.sweep_num=125 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=63 -+EXTRA.sweep_num=126 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=47 -+EXTRA.sweep_num=127 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=31 -+EXTRA.sweep_num=128 +EXTRA.tags=sweep hydra.job.name=kernel run_length=short settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_1x1 exp.kernel.radius=15 diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/array_03_MSC-p03e01_kernel-disentangle-xy_TUNED.txt b/research/part03_learnt_overlap/e01_learn_to_disentangle/array_03_MSC-p03e01_kernel-disentangle-xy_TUNED.txt deleted file mode 100644 index 995006af..00000000 --- a/research/part03_learnt_overlap/e01_learn_to_disentangle/array_03_MSC-p03e01_kernel-disentangle-xy_TUNED.txt +++ /dev/null @@ -1,16 +0,0 @@ -+EXTRA.sweep_num=1 +EXTRA.tags=sweep_MED hydra.job.name=kernel run_length=medium settings.job.name_prefix=MSC_TUNED settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=63 -+EXTRA.sweep_num=2 +EXTRA.tags=sweep_MED hydra.job.name=kernel run_length=medium settings.job.name_prefix=MSC_TUNED settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=47 -+EXTRA.sweep_num=3 +EXTRA.tags=sweep_MED hydra.job.name=kernel run_length=medium settings.job.name_prefix=MSC_TUNED settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=31 -+EXTRA.sweep_num=4 +EXTRA.tags=sweep_MED hydra.job.name=kernel run_length=medium settings.job.name_prefix=MSC_TUNED settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=15 -+EXTRA.sweep_num=5 +EXTRA.tags=sweep_MED hydra.job.name=kernel run_length=medium settings.job.name_prefix=MSC_TUNED settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=63 -+EXTRA.sweep_num=6 +EXTRA.tags=sweep_MED hydra.job.name=kernel run_length=medium settings.job.name_prefix=MSC_TUNED settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=47 -+EXTRA.sweep_num=7 +EXTRA.tags=sweep_MED hydra.job.name=kernel run_length=medium settings.job.name_prefix=MSC_TUNED settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=31 -+EXTRA.sweep_num=8 +EXTRA.tags=sweep_MED hydra.job.name=kernel run_length=medium settings.job.name_prefix=MSC_TUNED settings.dataset.batch_size=512 exp.kernel.represent_mode=abs exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=15 -+EXTRA.sweep_num=9 +EXTRA.tags=sweep_MED hydra.job.name=kernel run_length=medium settings.job.name_prefix=MSC_TUNED settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=63 -+EXTRA.sweep_num=10 +EXTRA.tags=sweep_MED hydra.job.name=kernel run_length=medium settings.job.name_prefix=MSC_TUNED settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=47 -+EXTRA.sweep_num=11 +EXTRA.tags=sweep_MED hydra.job.name=kernel run_length=medium settings.job.name_prefix=MSC_TUNED settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=31 -+EXTRA.sweep_num=12 +EXTRA.tags=sweep_MED hydra.job.name=kernel run_length=medium settings.job.name_prefix=MSC_TUNED settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.001 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=15 -+EXTRA.sweep_num=13 +EXTRA.tags=sweep_MED hydra.job.name=kernel run_length=medium settings.job.name_prefix=MSC_TUNED settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=63 -+EXTRA.sweep_num=14 +EXTRA.tags=sweep_MED hydra.job.name=kernel run_length=medium settings.job.name_prefix=MSC_TUNED settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=47 -+EXTRA.sweep_num=15 +EXTRA.tags=sweep_MED hydra.job.name=kernel run_length=medium settings.job.name_prefix=MSC_TUNED settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=31 -+EXTRA.sweep_num=16 +EXTRA.tags=sweep_MED hydra.job.name=kernel run_length=medium settings.job.name_prefix=MSC_TUNED settings.dataset.batch_size=512 exp.kernel.represent_mode=none exp.optimizer.lr=0.0005 exp.optimizer.weight_decay=0.0 exp.data.name=xysquares_8x8 exp.kernel.radius=15 diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/array_overlap_learnt_MSC-p03e02_learnt-loss-with-vaes.txt b/research/part03_learnt_overlap/e01_learn_to_disentangle/array_overlap_learnt_MSC-p03e02_learnt-loss-with-vaes.txt deleted file mode 100644 index 401f92dd..00000000 --- a/research/part03_learnt_overlap/e01_learn_to_disentangle/array_overlap_learnt_MSC-p03e02_learnt-loss-with-vaes.txt +++ /dev/null @@ -1,80 +0,0 @@ -+EXTRA.sweep_num=1 +DUMMY.repeat=1 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=2 +DUMMY.repeat=1 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=3 +DUMMY.repeat=1 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=4 +DUMMY.repeat=1 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=5 +DUMMY.repeat=1 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=6 +DUMMY.repeat=1 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=7 +DUMMY.repeat=1 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=8 +DUMMY.repeat=1 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=9 +DUMMY.repeat=1 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=10 +DUMMY.repeat=1 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=11 +DUMMY.repeat=1 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=12 +DUMMY.repeat=1 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=13 +DUMMY.repeat=1 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=14 +DUMMY.repeat=1 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=15 +DUMMY.repeat=1 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=16 +DUMMY.repeat=1 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=17 +DUMMY.repeat=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=18 +DUMMY.repeat=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=19 +DUMMY.repeat=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=20 +DUMMY.repeat=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=21 +DUMMY.repeat=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=22 +DUMMY.repeat=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=23 +DUMMY.repeat=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=24 +DUMMY.repeat=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=25 +DUMMY.repeat=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=26 +DUMMY.repeat=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=27 +DUMMY.repeat=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=28 +DUMMY.repeat=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=29 +DUMMY.repeat=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=30 +DUMMY.repeat=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=31 +DUMMY.repeat=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=32 +DUMMY.repeat=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=33 +DUMMY.repeat=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=34 +DUMMY.repeat=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=35 +DUMMY.repeat=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=36 +DUMMY.repeat=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=37 +DUMMY.repeat=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=38 +DUMMY.repeat=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=39 +DUMMY.repeat=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=40 +DUMMY.repeat=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=41 +DUMMY.repeat=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=42 +DUMMY.repeat=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=43 +DUMMY.repeat=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=44 +DUMMY.repeat=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=45 +DUMMY.repeat=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=46 +DUMMY.repeat=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=47 +DUMMY.repeat=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=48 +DUMMY.repeat=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=49 +DUMMY.repeat=4 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=50 +DUMMY.repeat=4 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=51 +DUMMY.repeat=4 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=52 +DUMMY.repeat=4 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=53 +DUMMY.repeat=4 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=54 +DUMMY.repeat=4 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=55 +DUMMY.repeat=4 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=56 +DUMMY.repeat=4 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=57 +DUMMY.repeat=4 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=58 +DUMMY.repeat=4 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=59 +DUMMY.repeat=4 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=60 +DUMMY.repeat=4 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=61 +DUMMY.repeat=4 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=62 +DUMMY.repeat=4 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=63 +DUMMY.repeat=4 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=64 +DUMMY.repeat=4 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=65 +DUMMY.repeat=5 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=66 +DUMMY.repeat=5 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=67 +DUMMY.repeat=5 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=68 +DUMMY.repeat=5 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=69 +DUMMY.repeat=5 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=70 +DUMMY.repeat=5 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=71 +DUMMY.repeat=5 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=72 +DUMMY.repeat=5 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=73 +DUMMY.repeat=5 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=74 +DUMMY.repeat=5 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=75 +DUMMY.repeat=5 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=76 +DUMMY.repeat=5 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=77 +DUMMY.repeat=5 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=78 +DUMMY.repeat=5 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=79 +DUMMY.repeat=5 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=80 +DUMMY.repeat=5 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/array_overlap_learnt_MSC-p03e02_learnt-loss-with-vaes_RETRY-2.txt b/research/part03_learnt_overlap/e01_learn_to_disentangle/array_overlap_learnt_MSC-p03e02_learnt-loss-with-vaes_RETRY-2.txt deleted file mode 100644 index b653a393..00000000 --- a/research/part03_learnt_overlap/e01_learn_to_disentangle/array_overlap_learnt_MSC-p03e02_learnt-loss-with-vaes_RETRY-2.txt +++ /dev/null @@ -1,160 +0,0 @@ -+EXTRA.sweep_num=1 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=2 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=3 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=4 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=5 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=6 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=7 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=8 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=9 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=10 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=11 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=12 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=13 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=14 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=15 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=16 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=17 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=18 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=19 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=20 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=21 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=22 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=23 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=24 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=25 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=26 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=27 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=28 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=29 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=30 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=31 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=32 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=33 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=34 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=35 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=36 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=37 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=38 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=39 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=40 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=41 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=42 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=43 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=44 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=45 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=46 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=47 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=48 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=49 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=50 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=51 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=52 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=53 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=54 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=55 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=56 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=57 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=58 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=59 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=60 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=61 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=62 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=63 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=64 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=65 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=66 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=67 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=68 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=69 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=70 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=71 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=72 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=73 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=74 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=75 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=76 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=77 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=78 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=79 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=80 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=81 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=82 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=83 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=84 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=85 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=86 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=87 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=88 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=89 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=90 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=91 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=92 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=93 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=94 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=95 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=96 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=97 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=98 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=99 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=100 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=101 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=102 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=103 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=104 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=105 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=106 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=107 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=108 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=109 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=110 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=111 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=112 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=113 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=114 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=115 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=116 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=117 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=118 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=119 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=120 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=121 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=122 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=123 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=124 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=125 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=126 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=127 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=128 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=129 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=130 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=131 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=132 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=133 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=134 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=135 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=136 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=137 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=138 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=139 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=140 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=141 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=142 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=143 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=144 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=145 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=146 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=147 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=148 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=149 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=150 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=151 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=152 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=153 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=154 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=155 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=156 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=157 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=158 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=159 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=160 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/array_overlap_learnt_MSC-p03e02_learnt-loss-with-vaes_RETRY-2_ALT.txt b/research/part03_learnt_overlap/e01_learn_to_disentangle/array_overlap_learnt_MSC-p03e02_learnt-loss-with-vaes_RETRY-2_ALT.txt deleted file mode 100644 index 7bf96ea7..00000000 --- a/research/part03_learnt_overlap/e01_learn_to_disentangle/array_overlap_learnt_MSC-p03e02_learnt-loss-with-vaes_RETRY-2_ALT.txt +++ /dev/null @@ -1,120 +0,0 @@ -+EXTRA.sweep_num=1 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=2 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=3 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=4 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=5 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=6 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=7 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=8 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=9 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=10 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=11 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=12 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=13 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=14 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=15 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=16 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=17 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=18 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=19 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=20 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=21 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=22 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=23 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=24 +DUMMY.repeat=1 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=25 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=26 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=27 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=28 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=29 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=30 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=31 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=32 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=33 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=34 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=35 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=36 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=37 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=38 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=39 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=40 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=41 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=42 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=43 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=44 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=45 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=46 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=47 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=48 +DUMMY.repeat=2 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=49 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=50 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=51 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=52 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=53 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=54 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=55 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=56 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=57 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=58 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=59 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=60 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=61 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=62 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=63 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=64 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=65 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=66 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=67 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=68 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=69 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=70 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=71 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=72 +DUMMY.repeat=3 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=73 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=74 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=75 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=76 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=77 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=78 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=79 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=80 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=81 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=82 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=83 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=84 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=85 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=86 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=87 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=88 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=89 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=90 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=91 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=92 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=93 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=94 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=95 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=96 +DUMMY.repeat=4 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=97 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=98 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=99 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=100 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=101 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=102 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=103 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=104 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=105 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=106 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=107 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=108 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=109 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=110 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=111 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.01 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=112 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=113 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=114 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.00316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=115 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=116 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=117 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb -+EXTRA.sweep_num=118 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_none63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=119 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k0.1_norm_none sampling=default__bb -+EXTRA.sweep_num=120 +DUMMY.repeat=5 +DUMMY.retry=2 +EXTRA.tags=MSC_sweep_losses_ALT hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.000316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k10.0_norm_none sampling=default__bb diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/array_overlap_learnt_MSC-p03e02_learnt-loss-with-vaes_RETRY-3.txt b/research/part03_learnt_overlap/e01_learn_to_disentangle/array_overlap_learnt_MSC-p03e02_learnt-loss-with-vaes_RETRY-3.txt deleted file mode 100644 index 7f645af5..00000000 --- a/research/part03_learnt_overlap/e01_learn_to_disentangle/array_overlap_learnt_MSC-p03e02_learnt-loss-with-vaes_RETRY-3.txt +++ /dev/null @@ -1,80 +0,0 @@ -+EXTRA.sweep_num=1 +DUMMY.repeat=1 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=2 +DUMMY.repeat=1 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=3 +DUMMY.repeat=1 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=4 +DUMMY.repeat=1 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=5 +DUMMY.repeat=1 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=6 +DUMMY.repeat=1 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=7 +DUMMY.repeat=1 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=8 +DUMMY.repeat=1 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=9 +DUMMY.repeat=1 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=10 +DUMMY.repeat=1 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=11 +DUMMY.repeat=1 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=12 +DUMMY.repeat=1 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=13 +DUMMY.repeat=1 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=14 +DUMMY.repeat=1 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=15 +DUMMY.repeat=1 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=16 +DUMMY.repeat=1 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=17 +DUMMY.repeat=2 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=18 +DUMMY.repeat=2 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=19 +DUMMY.repeat=2 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=20 +DUMMY.repeat=2 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=21 +DUMMY.repeat=2 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=22 +DUMMY.repeat=2 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=23 +DUMMY.repeat=2 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=24 +DUMMY.repeat=2 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=25 +DUMMY.repeat=2 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=26 +DUMMY.repeat=2 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=27 +DUMMY.repeat=2 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=28 +DUMMY.repeat=2 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=29 +DUMMY.repeat=2 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=30 +DUMMY.repeat=2 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=31 +DUMMY.repeat=2 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=32 +DUMMY.repeat=2 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=33 +DUMMY.repeat=3 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=34 +DUMMY.repeat=3 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=35 +DUMMY.repeat=3 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=36 +DUMMY.repeat=3 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=37 +DUMMY.repeat=3 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=38 +DUMMY.repeat=3 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=39 +DUMMY.repeat=3 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=40 +DUMMY.repeat=3 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=41 +DUMMY.repeat=3 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=42 +DUMMY.repeat=3 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=43 +DUMMY.repeat=3 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=44 +DUMMY.repeat=3 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=45 +DUMMY.repeat=3 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=46 +DUMMY.repeat=3 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=47 +DUMMY.repeat=3 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=48 +DUMMY.repeat=3 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=49 +DUMMY.repeat=4 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=50 +DUMMY.repeat=4 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=51 +DUMMY.repeat=4 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=52 +DUMMY.repeat=4 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=53 +DUMMY.repeat=4 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=54 +DUMMY.repeat=4 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=55 +DUMMY.repeat=4 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=56 +DUMMY.repeat=4 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=57 +DUMMY.repeat=4 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=58 +DUMMY.repeat=4 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=59 +DUMMY.repeat=4 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=60 +DUMMY.repeat=4 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=61 +DUMMY.repeat=4 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=62 +DUMMY.repeat=4 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=63 +DUMMY.repeat=4 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=64 +DUMMY.repeat=4 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=65 +DUMMY.repeat=5 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=66 +DUMMY.repeat=5 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=67 +DUMMY.repeat=5 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=68 +DUMMY.repeat=5 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=69 +DUMMY.repeat=5 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=70 +DUMMY.repeat=5 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=71 +DUMMY.repeat=5 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=72 +DUMMY.repeat=5 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=betavae settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=73 +DUMMY.repeat=5 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=74 +DUMMY.repeat=5 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=75 +DUMMY.repeat=5 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=76 +DUMMY.repeat=5 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0316 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb -+EXTRA.sweep_num=77 +DUMMY.repeat=5 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse sampling=default__bb -+EXTRA.sweep_num=78 +DUMMY.repeat=5 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_gau_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=79 +DUMMY.repeat=5 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_box_r31_l1.0_k3969.0_norm_sum sampling=default__bb -+EXTRA.sweep_num=80 +DUMMY.repeat=5 +DUMMY.retry=3 +EXTRA.tags=MSC_sweep_losses hydra.job.name=ovlp_loss run_length=medium metrics=all dataset=X--xysquares framework=adavae_os settings.framework.beta=0.0001 settings.model.z_size=25 settings.framework.recon_loss=mse_xy8_abs63_l1.0_k1.0_norm_none sampling=default__bb diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/config/config_adversarial_kernel.yaml b/research/part03_learnt_overlap/e01_learn_to_disentangle/config/config_adversarial_kernel.yaml deleted file mode 100644 index fc40894c..00000000 --- a/research/part03_learnt_overlap/e01_learn_to_disentangle/config/config_adversarial_kernel.yaml +++ /dev/null @@ -1,47 +0,0 @@ -defaults: - - _self_ # this overrides defaults list - - run_length: tiny - - run_logging: wandb - - run_location: stampede_tmp - - run_launcher: local - -settings: - job: - user: '${oc.env:WANDB_USER}' - project: 'exp-disentangle-kernel' - name_prefix: 'MSC' - name: '${settings.job.name_prefix}_${exp.kernel.represent_mode}_r${exp.kernel.radius}-${exp.kernel.channels}_s${trainer.max_steps}_b${settings.dataset.batch_size}_${exp.optimizer.name}_lr${exp.optimizer.lr}_wd${exp.optimizer.weight_decay}_${exp.data.name}' - seed: 777 - dataset: - batch_size: 512 - -exp: - optimizer: - name: adam - lr: 1e-3 - weight_decay: 0.0 - data: - name: 'xysquares_8x8' # xysquares_8x8, xyobject, xyobject_shaded - kernel: - radius: 63 - channels: 1 - disentangle_factors: NULL - # training - regularize_symmetric: TRUE - regularize_norm: FALSE # these don't work - regularize_nonneg: FALSE # these don't work - represent_mode: abs # none, exp, square, abs | eg. if `exp` then kernel is stored as `params := log(kernel)`, and we obtain the kernel as `kernel := exp(params)` - regularize_l2_weight: 1e-3 - # initialize weights - init_sums_to_one: TRUE - init_offset: 1.0 - init_scale: 0.001 - train: - pairs_ratio: 8.0 - loss: mse - reg_strength: 0.01 # TODO: try reducing this! - combined_loss: TRUE # choice of loss: >>> [F]: L(h(x), h(y)) >>> [T]: L(x, y) + L(h(x), h(y)) - out: - rel_save_dir: data/adversarial_kernel - save_name: ${settings.job.name}.pt - show_every_n_steps: 2400 diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/config/config_disentangle_dataset_approx.yaml b/research/part03_learnt_overlap/e01_learn_to_disentangle/config/config_disentangle_dataset_approx.yaml deleted file mode 100644 index 87bc204b..00000000 --- a/research/part03_learnt_overlap/e01_learn_to_disentangle/config/config_disentangle_dataset_approx.yaml +++ /dev/null @@ -1,71 +0,0 @@ - -# ========================================================================= # -# CONFIG # -# ========================================================================= # - -defaults: - - run_logging: wandb_fast # wandb_fast - - run_location: local_gpu - - run_launcher: local - # entries in this file override entries from default lists - - _self_ - -settings: - job: - user: 'n_michlo' - project: 'MSC-p03e02_disentangle-data' - name: '${dis_system.dataset_name}_${dis_system.disentangle_mode}_dw${dis_system.loss_disentangle_weight}_r${dis_system.disentangle_randomize_factors}_c${dis_system.disentangle_combined_loss}_s${dis_system.disentangle_scale_dists}_s${trainer.max_steps}_${dis_system.optimizer_name}_lr${dis_system.optimizer_lr}_b${settings.dataset.batch_size}_${settings.exp.save_dtype}' - seed: 424242 - exp: - show_every_n_steps: 2500 - # saving - rel_save_dir: 'out/disentangle_data/' - save_prefix: 'MSC_${dis_system.disentangle_mode}_' - save_model: TRUE - save_data: TRUE - save_dtype: float16 - dataset: - batch_size: 256 - -trainer: - # same as defaults: - run_length: ... - # - 15000 takes 40 mins with batch size 512 (heartofgold, 12 workers) - # - 50000 takes 33 mins with batch size 256 (griffin, 16 workers) - max_steps: 25000 - max_epochs: 25000 - -dis_system: - # optimizer options - optimizer_name: 'adam' - optimizer_lr: 1e-4 - optimizer_kwargs: - weight_decay: 0 - # dataset config options - dataset_name: 'smallnorb' # [cars3d, smallnorb, dsprites, shapes3d, xysquares_8x8_mini] - dataset_batch_size: ${datamodule.dataloader.batch_size} # x3 - dataset_num_workers: ${datamodule.dataloader.num_workers} - data_root: ${dsettings.storage.data_root} - data_load_into_memory: FALSE # I don't think this is truly multi-threaded, possible lock on array access? - # disentangle loss options - disentangle_mode: improve # improve, invert, none, any - disentangle_pairs_ratio: 8.0 - disentangle_factors: NULL - disentangle_loss: mse - disentangle_reg_strength: 0.01 - disentangle_scale_dists: TRUE - disentangle_combined_loss: FALSE # choice of loss: >>> [F]: L(h(x), h(y)) >>> [T]: L(x, y) + L(h(x), h(y)) - disentangle_randomize_factors: FALSE # achieve true entanglement, use this instead of setting disentangle_mode to invert? - disentangle_randomize_factors_seed: 777 - disentangle_progressive_release_data: NULL # 0.1 is probs too high, 0.01 seems okay, but might fail later on, 0.001 might be too low? - # loss extras - loss_disentangle_weight: 1.0 - loss_out_of_bounds_weight: 1.0 # not really needed -- if this is too high it struggles to "invert" - loss_stats_mean_weight: NULL # not really needed - loss_stats_var_weight: 1.0 # not really needed -- produces nicer results, but we have to explain this then... - loss_similarity_weight: 1.0 # important | smoother than using the stats_mean above! - # model settings - model_type: 'ae_conv64' # ae_conv64, ae_linear, ae_conv64norm - model_mask_mode: 'none' # std, diff, none - model_weight_init: 'xavier_normal' # [xavier_normal, default] - # logging settings - logging_scale_imgs: FALSE diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_abs_r15-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt b/research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_abs_r15-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt deleted file mode 100644 index 59cbca8697ff9bf433bd6942f2d2a7b5c1058bb1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4587 zcmai22~>^S`##MBnl!028bUMC`<(W7l1dttQiMopkWw8b5u&K1R6@u#WC~HbrgRON zT@;loV{W0VQ!WYd>vvlB{_b+`f8F(e_qW$x@7~X|_kQ>KzHhJRo$4+n&Ev_-^Zt*i z@YHzo;^#+1CWcvt&J)hFSiCUW#Ziej^KZ6Fa=tKDjIsz`B8-eqPz;U_3t18w9U3eQ zi%E!$4^FgEjN_S_2Fi)&PY@&Lg@?sS{6SH4iH@Booc2ejVw_~4gcxHLClx3qZs#vv z5-06>QA!vm=1d8)h!OMALHOpKOePk|HO>ZQv+e_dmITM?=OrrxsY|!~i9&a~! zgS+*BjECH&)ut<{-9?*97q(E;nRhg9mx!DdB=FFvhgzcs)7zg?Xie#58ur^HjGt_e z%@N}eKWrMVpDm@VF`m#4vxVLJk;G4NMa)-U?Ctf0d3X)2>6nhgJJ%qCZd28BRSbS} zn?{@dN{bKaVfi6{3jLsnUsJout#&73j7DLa#}kq$tD{W0SLB;vjE2)&P*b^$szp+$ zsEf6mal!{Ri~H=7+TKxDwJaj9e4tlDGjPzu9{gol6seMlc^;Q=_<}r6k6DNXybbip zHW+U|_@U5E8;54igl3={np1%67rB`l+catKFq%!bhnF6NndqKU*H8hTwNE0lRnX$@z zF6#Ac*p&>&)u79y*Q5ZS30}C;D#z}%ufQc0S59q(DK1{>qHMhkG#&7TY^o=eb&Xhx zq8bHRUu8%=uJ+Q+-ot>0%<($?RFq+)Jb>A#!=kDfW z*u4w{KM2O!z*LsekjE8gn+RU?<#AflsjT>78e$DrV_NcNC_F7^b;BFD^$QAEfW9lI zdP1EwJA`BPRx@;DTH?;UboN#J-xxYhm0MXMiNymOXjOeF)h?5R&UQ--CE#h@IIx;> z=BuzAw+!=Oxzi6W852vb zAdJzyO+qIgMmi0^(sSqRN}ns^Kd)pF{a%HuHS_1o4BLpj@~gD+$|G|5NdaG4!tg}- z5xt$H0=d$I7}K@{)$eXnkiRz-jJZbjT3Phv%OO&${greNT&8aAB3jq?nJOwrlaG-O zo)6rK;-$sNk$6pGS4%_l#(4M}c2o4n&Gc$Y8@Y9*V^_~yzPphnZf}&r(It|!qw^^F z?KQ?q8j9KH(@-)}h`kkwa1jI}qhkWKgfvrif*;*{HW>OdhU2}56P$FM5q9Yax&6L_ zv=X!IR(J_1Z;l<_?L4m{3z;$8)lqMv+I#mz!QUoxSVc*La$V&X5CAU{_51JzL5-La;bg1Co+7) z$o)bxR`m5HySF$+~>NxgxK2F33P>Xassl1Sf^Qm?k zTejcMv$>J=q*ZZjmpZ)+dqtyO8KZBP8b;Y!LtALX9?6&5AAFg|O-fQl=NS>rJ7$T5 z3D0PHqcwdCV$cY(peNgg+J-9h_}JR=qyYl%V` z2AVFnv9p-s$8aemwn$_69Fev z%l>mu6$&XFR)jjCY;GAD&e956(!0sP zSsxvZd0f}7efD$f44H(`7c0Hx@M4StEa@FNe33w5QWd8;rI97nHBeK1AszHlN6nCd zDU7K0r6>M3kwLVHX{`1U$*HR99CB$Cu?aa6&Gk&Mc*& zc_O;+c##G!Dkb)A1slG}0Lm;4f~+QzcjntITJM3JjJXtJSx-{!AL!l#dwRo*LY;pQW?O}$RY?P-0Zr7mQX60VGBGT035iVG$)hfx?{}3$ zw30k#&d#x`(RZUpw|mr0<8aM81ef)^kk*%phh`iEeQPOtd=A}C`%2+gf2Ha2iFmG~ zfpL0PSluilyeOp*`%bEVT}!6BC*X+rPF%GNg1*aCWbcv2{rx>;Y_XiC>usfn>z6?o zVTnD5r}I^>r(l-L1roIc(U^M!a86za6-K?}A-*%c+qUB4&)M)@dza>%52er7ys5{% zi1cI_-O;Wk+gmYIk{3_POEyxN$dJ7DOQ6Im8BdE3p?A|aDn9j%zRj46H$M)Bn)L~~ zxy6lwJeI;sH-N9&pu)lB9Hm5sk-lRn-Eba^M@Mea-m+>E?$N~2@N&i#FsV`xSrO)Vp_M)4g{ zzQF?XMisKsBhqNP;Q*yw?x?#y8!sch;I^&QUj1T+{XL`o_9;0w;LgkdbuGY!nn`F( znv4SRy`L`;1Ab8~F7I21L63%EwZV0&*dmRKU?@})7AXR=J*dSMJ#6C|oF)V{g8gVNT&_$lT7Rp0p8Ybc@BMgb?mvcrv5LaLTi; zqm)h&J?xOiwY*L6Uh07{zvtuhtHI33&;c?>OfYxPXskV_58l|#lp3rImlN*bjhYTM zh4XaRtcA39NZ@gi7m~jXz)rpf%Y3yNjfdyr>Xkrrn>5n;x<2aGtELIBf1*$3$?S0E zW5i6hgplLo_MNq4VQvo_T~myc)5OXIZ>%!ifFHI+;<%0>?$-B`Nvt7Sw2NuMNJEru z7=T+>m9cJ75=og%g29kpO7}HG&s<0NWG7?%vxQLmCWQz+W1QCT!jBJINU7Ed10^mHtUqmr^>S$xPuf9;&+1a>PGDW{8~V}yIqm4vgU+?n)bYR> zW5Pssrtv@_BsPV_6zBSVS>H;^}~e3x=nfLt4rW~?7hh&+<2REwHe0HmCT6F?B4$5H)ZL<*)2$@YF$=xzIvA*t&gkF( zRNLlZgp4{812ph})eud6M7ehhnDB!P1}!>CC8sY@Gp7)Yj!j|75sGbgaBY(uIwXwJ2b^FEj>V4@Cq?LKR{`&9YTA3&K zOI#<0#V3eGyNe_L-_`ZZ|JTMU_OD)767DZ4mi7I0s=Gvg^H)yed)cqjKkzsEXNKdS z?C;}J`Wv%H?0wI+wE8Fa`(25Tx&V2UUdq9h^>G9*f)Qc6guP-Ze^Od`#O zgvwBcltiXfG^qdmyq@3x_y0c6|Ml$K>D+VgUgzF(U+3(-)_XW}9K(}_$Oi>hD^xs-&{}qA0qF!T{ZV2@D36Ndt@40+~w~xot zK+kmnzWz&tOlAGV^z~f^i1r^K>gn#~>DTYSLS$`xeBA@*{nv}Ke*IngiTaH56L%38 ztyiqy;3qLNOexS$($PUo-=UxAVdY>I=qIHwdR6o%+UghRH(=+A6}~<`o+|>qeb)uZ z&i424^!N0z_FfSv>nCk6*H0$UPj;@K+}Jq|1EjazxD{|xwCjJDIsfZnD~f6Ux5G9W zXER0Pe;o9G`~rurzGDZa@>spcP3EuOZdrS2Fr8j~oFg=~9I;ZmKSy}#pH5@cMpMs0T~e5qNSc0*y37%@6-DF2@TxNXVAo%Dq1U$A$1XPNQQ ziE#eKA-3c6HD)*RB=gyS+Ol6)x@Blq3p-tz&$^QJG4FOOo;aDq-`~oh&tBi~ivnQG z`*wEd;&t|XB|w6|J#X2rL_M?Os4xZ4^ZNkUrE3h%o+BV$w}l;FsSQ89OVP!niz;oo-^`9~V4jRZo$n9~D0 zm>vvh{q7sOH)hbOT{rl`C#|?jZwy9V`e+$|NzB1Hgc%Jx&hC30VE&d);8}Lda_7Q< zm}b(4`>(ygSGUsnVAt_<{AV2ch5hB8mKV6~xe)&6dKiDCC(BzxN-(F#kA&VD?lqpn z#^0@MUQ#uilQR)4?hSa_ zznltUBRw5pdihp1MQ0uBzqXX+`5EAun7`;ZA6|qVnBFdn&1Qf- zlcwSqEg954IEI^sAID3|vuVXh4IEw&&7B&jK}Mz@)Q1JZ>fc*gTx=FDUf4oiyN>`& z-NUA5$wKVBe?Toz4^C*jV-nH|pkd)C^bH&UuXhfCq~oUG_+dDFbI)N_ps*@Hb!~fYwY0EYizIUK_=V$iMhwNqIGi(hHQ+) z6nlo%=!zqsSHazh_px}~bbRyLAJxBYLU9gwN%}q4>v)Fj-~uw)u+{d9Z5n(A(I_sC4{;u_68q!yu$a|3o+{)XYpj7ckfKd#$66+>SC;$kuB z{QR*e{7Q^8`EH7#EHN#*w)PC}9`lDE{rrH1X_qqBPx4^7p@bjSE2jC|7Q&`iHEjL& zbV!@?nO%`bh|?Ja9g05iMJ$Jn9%up?isB$8s6tYSZ?O@eMXVERMZumH^li%X+2DnegTYbFkF`cNKBiFn29Wvyq0-8-rnz zMJ-ENp};1rFJ<&m6AYcKU@E)BE=TvV;~(;v%k_THx44)kjMs&%x<@QQ`zqTM>d2yQ zceBsE4eSs$Fth5-u=$)hT-foD?O8ewnzv|zm{uiN{do&hQVyWM^?3UGH-nDN>MvC8 zFcj1u%L>ZZwveXvSKQ_vkAYS@@a2d5c&y!+K9v-){Q5ycwo^Mj$tqiZdsskUhy14Fcg%%-qh69`egO@ex`&>< zTS?V!v&pEJk*aqpI=$Hy&=5j7}_Trz16kgvowd z(4{(tjusVSKtedCSM0*UW4>d_xUIDLy{w?#(?#Cb#Odxo96#)8;~9MlD74t(l+;cSPXxdj?1)8!`CuM?AY)ngRmT1uw4ww9&5t>wUMNQN9sQ z?-zw@w|1b1mn?C$-SohEAuV;S#Ohm7m{bvhlIun+K8q-Z9ub; zcks>H`&g7DPlqf$DYLMhjMgWyC0+aQ+pNi$x>y!f={Hwi0(d)GoVrxH>8p4hMQ$33 z9t(|Nl;SZy;61}2dIBT&O!lFzR5#o{5rfF zP=+RZ6NT32C0O=xHQs+J4L0JZHn1KSnU` z)+s)`s2r`&I#AW0yZq@saeAse1Ec;<;+7_Xp!ZD~p+=7OE-NEdmgXIDVEQ>o=imvHX**$#c*2VqG@J3k)04X$TiBdIq< zD0yTj3^?ooj?*>Z&w)<9IY*3aY|8nU!wz(qC8CexJSh9y4-y9tgr@T$d~1ji6=gV+ z_Vqmj7jy*4LQ)n>Npq~?P;!d1>f%2i)N=gxVlC-Tjg&F zT}OjqMu*~! zOIE^^0Xy*VrdvEF>o1$+n9qI$|75os8Lp1fB>8%Os#CI{oU9P;-uH)XvireSRv%`+ zbho2yqbpsie85{C%aF#MUf#H^jt#h4#covze2VT}oV@uG-Ozf>UOJ}oabGOaF=+r! z((>cox5hwb=QB9_sF_^vDhm57qtScrbnbI=5Ra_T;JwFc`L=VF{E3n;>+#B=ovIC} zv%wahmv`_p(PDT*JrXBQ1j?VUf+IWQXzI2yygBwF_lX+@Jwv{MN`3;~JMB$d8!{+; zjV(#6n2p<=q~Y;`2>AOmjEx984Re;(ak;7%dY$)zON{XZ6^Tk#0t_7cZnF4ejyQX= z9r>yqq4K-l^t3J>JzmK33u)G{bWJu-ttr7l{dZBU_7DsW1Ddcn0yn?z<|XZa_&Svt zcwomzJbpcgcI+PjOUG35rP>Yr)@)TC9i`4Y`c39@jCW#siK6iAf~COCHc;kMMf#pt zg3EtrVa%CI4EiNS9hdj9sxvjDvdD-oO%^AmC4DH&k)iyNKp)>7BE#J}^w;P${W`Fa zWTYhM%Hmh}q4hI{)a%k>%TyAgs;O^I3Z*n_(ebMDs3Xc3R|Ma~F(1a#;)Rpx=J{q) zT{nqJ%uFav=02X6jYYLp*%-S}i$>|CP)%7iHMyr!@^xM6^ZJa*?_Qyy=yQ~ZETX0L zuL#3*$YIAJ%1B~VW-UXLboy}7MRA%j-IxM`YRE!bRG;1zU2}7uqTjwOlp0Wi8Fv-v zj?@NvciU2Uj$?&m8`CH$CypvZ{itx+EXs3TK+AW=(1I{zA>+=GYU+EwH7A$P_bTU4 zEeg2XNmX=+(4noFUD$t(CGEHOgqg(v9`9y=YO*op-eBPL>o=TaUoffrJlL-s2WIyZ z*>&0e@S@fb3Sz||xy=q*?x(`Lg4uA`wux?`4W4&LUAK2~u!dydk% zK9jPHA;T&2>8|lC`mE|teVK7|rR6XwKN>42rm6}a1>M+`ikQ6eH8=jFihegQVWOWF zg&ugxx~qEW*cKo1D{!Dmst$C@!(_fr!MAu`>!2{@7 znT3kFk`(T{nrM!b_J(0e|zasrX zyJ*61SBkfEpn=n7P(Q!5q#4vppL<%_z7kEE`|u)e&QQaCyPxxU4+&SV=*Ie$s>1yc zHQ`)P6h2)u5YEN9L3vjq_sW}rAH`1N6GA0au5fm{4IjEH6K~hs)0cKf@|h4yLvrt7NR=2? zUC-q^mKmXz==xajAd#dtF_1mHknB}UF}!vQdS8#nP3y~X{aiiDQEjFKo(O_(M=-Wdbj9XF;FS2|j+K zHQ6qcrt@Y)Xv%FVd}iSW_vTN7y!*OPsa4CfhbU9f^qInQJ0^+a~76Y9YC5UuG zfW>mS$ae_+vvvTrbWbJkvP_=(atL&UOaQ6qG`PIh2tBsD(9UWzn0-}(l2%>fA8t3Z zYaxc9{^LBijk$wM7FCe5(s0NK>EX2IH<#?`W)(KYtiygK)VqFUGk7cMoq9ympKM~< z5eR!s)L^cq72LL3%F$#ltPHr$hF9iOqSZExxZK4$_h`ZXcnPSB90FCE0*!xDToCJ3spghG|s5=h+^4pL^(Jo=kF-Tfd=cG3D2^D_eGe0G6H)AXPr5}`RG zkKZ?$PEo%GK(5M5w2$cr@BSWSVOe4z>1_y}T_Gspn@=Sz2iWxis$ie!$@dmaff%V~ zmhtNod;771B_Dr8X`YQ__G3R=GGCqLi0UcL(-WXAb{x3e90AU*-x(j7OPX(^uud!v z#y7RGf`C=vwN;rL4r|8s;SBUt=Tei57*0`>1L^Y`@F2$>Qj&b3&clqPmyV>fn}?7QstxZ&2|V4rzNP<2mD@f_1}G{0vfHm9z;yxdwrPn?0Nz=ENHg52u#l zBPmD9jC!?qfZv#9(34;T9X^90ZRTIzI&2=v8;*cw<;`e7vixtjB^X8AW~WN^!KpbC zQzLSzX?lq{8R-a0?vc zbDF|H;>i^7X!68K<2RD7sWSzSolYqid$`!AMWChT0sb!nq4K*0d}^8mA8bES?(3N# zStt&6qXOXD2|I|hih=kkV%U;sLylveDPqD#iX7#NX*U;vjeIB+wWjl{ItA#Gu#@s@ z?BH~=9<5s)kM4G1d=*>4`%KNSs=fkeohhI}Noj2K4@L0Re$CcrFt+2Y0fhcggC+Jk z%tzT!XgiTA6o1}Nr}T?){HsVDVUmeWEq`#ptRmi#n!x;SbP!xwNR!PclI~SwI%#S_ zJ12XS?w(W{cdbA7x?WG=8KE@ejx`lLHlXMbQ~HNC(zKh`DEnJ7{g%!odjm`Qy0#7r zyHBBq&Qp9o&5UYSxl?Bi7k;mCCuajQTG&&G*KVIeLGdkqnPN$IO*3hW!CtDcyDqBN zHd5`I@w9TOAys!xr9!_@O1G+~qc#0`*osu@yWvg!FPqcd9z#00$dV)i7t(?C9d!4i z8Tf`5@#F8iaHIV(bZ(7AMe9-=b3TQX_ofOfG)@ZY9ohoU>mo(DCR%p)3(XlSF8p{{ zM1ywxWPgjzg%IuCw0*7{t!#6qclZ72LQ@8fNv@_ZAu(^t2$Ee8M)-8QpX$lyag~Ggax{@=km+Xb??d-n3!*9{TcAR@m>jhh|oKQEA2? z8gj846P;CQ^L8gv`<_PI`fnmloxOC)#+lAXjU-(^aWefgfSMcUkZETpeH-@~Z6m8G z{cr|_H271}C}-ML;zl`RcT?job78dUXBOa5M9;>G3;pM|(23_C=^wQ&QjpUT%7&d3 zydMd|+TYt@w%-`|dRogeDJ%+}E|bIR-_r25>@K@e|AHLH$3TzIPo2ca}m)3+(8_22J{Bn=}RWc4N#iIqK*8f=0G$vp%W8 zkat`Lg-QEhzRn${+gi;wT#16|HG6F@ zt`YZXcKUC8BFam=FMDJB`3d--eKy{a(jpIgGgxckP0=>TF||?w=ZP@ONfAr=xjh~H z@d!%_8SX?YPpQyjm$ldws=<99%X8_q;T>*nT;ozKUm!1sYgJC*l2~tQXV%bWuSK+W7AAf$ z$K6*}b&ni~tNp50>Ox9%Yw_h>wJb0uFla~iCV8wl=c5v(vi8y`-6 z!D_Eb(8>LC_{2U7*qfUM`brM4;c5WrZKy*hc~#0&iNqu3LwU!m5U4wI4AfIDf<&M- zyvwwN?+#;yYnmc7wn+n;bSl9@DioHvCPMCFb$;bR7~Zf_rAX~JIQsTx*!f^B{9NY) z&)=+oKWiNM`n3{tX!|)fbbdB=XB}rB9AAil2W_ag6^(aJ5%}9^5Bc|HSZ?p~g9r}} z>9Iv@^P-WWJo*5eANZB!{gX&1QuS!Yns(NGOB>DvKV@2VZA`;$56h{L0sQrnC0}1j zP2UYrUqt~X6dOU~Fct7PJO)yHWAH0zLUUXu1_@EjBUKe5qT@hoUK~v6o(<`Kr!e$% z7cO1gt~@8&oVPec3uRnUmW4=7zLp%I206}R)SN^K#(2rj!n985!`fZ~4g*oT~OBu9hM%u0MNK8U)+UU9dVr{O@^Wmx`nF|5^^&6c(1uzOjn zLGwp8ZIj*##S4Ejxl@1G6Vr9@c3}<#K0ONSt15Y^-5~1O_7ojo#6j&%KSraT-)aZc&fuV@0WOEY2Xj21Ry=}R^+zzRkeFJ&nyLt*hlpp1k@ z{FdsA-`QZ)Fu1^n-@gdr%T35;h!Z>SQID@Y?&2ZYIv#Uy2ehr%g7JpR(CT7Fha6T> zL7q8o|Md`r)3;brP!z;nZDndk${_rn0;NH=IQoeb{JMzn`R+be8EgwX9?HSd`?nz^ z!vaI(S5ff+Q!43r!e*>(V&|?bgYR}P`R2{}_$jRcGsmxB^9;w*?TAcxvE~*Jd8Ujb zSNLN0pazuCXSBfUFsr^6$8HQn80ekJ=F2HS#UnLHYSe;Y)pU||bQE00WZ==|Z|sS~ zVfJoaE*m_;7|whc4>n$MaN|-cu2q(ySu;y9(c1RKW;Ff3N*hNgSyce6WL{IrMAX;bj&s}hXYQzo}Zw&b}< zoa8R_iSpRVv`Y3h->b0$BSxRVV8s%wvXUmN;{CX+s+9#Nxq$T%Ntl{f&bCe6#m*m) zg2DFJnf*XVA@s`;tHY0nSlN^p38u4C1%7>w;7nIpdd4-D&}Yt!Dz^yF7NiTg>4}1& z;zr@eX)B@dWD8x{(M*4gR)UJ!Lt0c~Biz@VDJ0zFf`|A}VTt-DYLrGn@~^D0>63!s zB1{(YhK&^p{1pVn-fUVNGZlS`_rpvbgGoL)`{0kmbeu5n0$JNB2n`#?3KQBV2?z2N zgdI<1g{aqD&=+Cg3imYxlQ1s4nKo01OScii;UQg^vJFP}HIr3T3t26;5>ke56h34n z3VCzWg+Gy7gaJASne+Ng7CZ3-ixBZ&{T8GO%dQm(?r(-zUHf{(D%8kPu&m$5d^CoF zx>X$0&Glj1v;twY2Xyb9ieuDE@cI^I@)6a& z&HsoKJTQWRrITrs$s4Y`HVl*WvM}>h2`;LUrieEw_omN`Pow=`O3C#t;)MPqa7NT{8^j17}?giiJTaKiFDbFKkuyV-z|tGsc< zK{Y(pbcq+ONd)D|CUoT97go^m9@XdN<9dS^-17Qj$a^;e8k5Grm%pa8TiKhkr!y4( zyo8*6n(#$!F+}qF%scZJQ%xHOZO^Bo6N>m|S$+7fTgozI?cuqj0@$vuhb^N3twye* zQIclV@k<3-{L~=f$b2|5tcq`bmWN+$KcKpCDT`1sA&tj5pqF}+x7iQErRa-N;f-kg zfRSE)Dyy9}g^eC7K>3I_EUilbNACd8e>@GAPfe%4(*KZSo(gP^cVY{aT~w%D=|=dTxF!_rgCM2p2)yt<12?6g@%I&Kv~l)h-1=iF45F#9u^<#qIw-)PWuth5V+uyw zDnR<`0t}mbm8sTEwR{xtnd!^zuuNRN3H_(cBE#bK%-K>M7UtZsloT@n&8|GQLUJZM zz3e@k@qRRYvDc-t4{unQh`&0zSQRQwR3J$>#D<*Q$JDdLp?aPd>6RMd@V95#;m_w; z?2JbCEX)8@=EvYf@p6WiC$LUq5_E`nvE1d`;Mlup2+ebb2T7-}{aiOr6zk+;_RH}F zDkq_9RRepYZ36Av=0V?fSC}|jK`47M8>IgD!~G}P@VZkS%ubvF_Y0k{c1sWcQPqt% z_n*WGE#qK`W*oTu-V4#D3gBci6GopshO2vCF&l+w{E@UA7P^(PY5XUv{x}GJjW$B3 zx|MYBnFQ#De_|^)++wp?AJa2f3<<&GK)y)@idO5wdfU$z;W0x6d|j60-4yH16p=i`BV*RWln+I<*4Hz&Z9-@UB8$Q;J#jTJJ~C&Jw- zWAJ);6|%1;L!kN|C>pND1E+*z<`Wg_9aW1Pbk@LAn+WL2ng)w*>A`pvM?OYPlA0!* zWc!Yv$6Dt>@a~`_I6oZ-+5arylJ1dMyihdOT;9$C_s7EWQ}3Bq;dM6OZvZq~%fa^d z#&F*$ky;i=3X|Vtu}>ELKu%f`?0$Vu0!<&DzsVIi5|SQ zr1ZQteukFvIi{h!(M1-^r)A-D!JEQ6bzoevCY@HBjqmNu@XaL;+;A}yQ>J&|;bjGs zx#t@*%nN7Zt&&;apR=q+#{$mSYD0_peKsvpQn+_W5Y#%>fVJg5_SyIe8~pbml--xc zHEZ?Z#hWwi{iPS=8!SiLpY-6=<UpZ3F5NUeD`_sldBj``_0$TSonbaO`r0OB*v~89XO_-=c9@bqr z_q-Z?t?{D06Zg>AL|H-Fcn=xKtRgmEjaDq_#^hcV8b88`UU;Qbo9-q`dX!8NoeM}^ z(53rY;>}0J2)@NHZflNp|8V478{wud5k!e6v4w40NU%@g?+h<8E3RV=k0t7BMqc zL~g#~!lZ#M)UfO$RsZTFXA2#{YT!u$GE#+OQ&UJjs0ha$kHAfkiGC4(uw&+Zem!m< zdoa6$4m@8-*0mNS)?`EpXUu6{s5dnfAE4Lry&$%%o+@%eY2vM^bdwvKOaQf zEw9tkAIbD7GLvkcSA@us{Pna^>jBL!SYt8k~u8fJ9ArvmNOPGQ8z zI?OAwq%`A9s;^C^{`0QW^?@5n>5?g(EHEIC?5VUMC6t~8)Kj5(JFE*&rQ9Q{=-792 zx?gKVs{V}8;W_k2s4>jault%$6$qh395@~{#n;jg}I#u!euN*he^e4ON@|s<*E1(knCwO&*2@a_B=2_wKT>APh?B1qFm*rIzqh2%`wGiM|i zzdOnccHhMt%XiZ5ys_{_#DB@1-i5{0;XFra9T(Rdj{&_;u+6uCB(6SZ2Q9TA#%BzK z9*t$o^`^qK)<>*iat*7HHWZB9L^{Ydci510X;>Xw#~x`pK~0PnOitOt6pxxQ7s{ni z)sgtmx^URJz?C`5wK3^uGCc46C#<^A$P7gMK}YRxp1XoUx%CVvFczSBn;ncXHzny^ zBWT&$VKl&cF(2~P4TQy8!Ta?Th{`8$SvCbD&ua*Su2|uUN(HF?;|1Hi!XQ_7G5mVx z#I45;Bjz}g)_*mnL&Xv>u>WRA+iMPAo?5~@-AH3hoG=L z7Bn^rmXD9ZIZ3&cX&lMqw&kJoOCO#kcr}z}6?L;kjNi1jzV+eN8bQ zC=y|ncS}WDhj0^We>9ZwpK~(o!s2YOSSQ99C zlE>4&PNQFsN5a9qFL9<@7hB^i5Bo&*;RNm5Z17p9fnewX*G4`Jb&||8gc~qny%s#=-san&~ivuWb z>kzs%NfxJW3Wndux>@!fWk{a-jw{+L(Mp$@f>D|>?Q^N&%w-;Q&s2oR@0Nm9jVvxp z8%%{~22j15HO-!PlGkAsQ~n{qn?NscFfc~d5Ep7M?e|$qDS)GrTF4j0;Xqd!I>>3sB>>2<(W=_e_|5pOL-wq zjW)(ld-M2+i@*5^rMtLxc_=-Ma-`V=?<{H?M_|NiO``fD;(#X-|8z94YLcXk2@A7oU`O@e8K6`RzkS_)((} z8v_%`SEN-uvTPw$=NDmd`&N|bPQcckrFeIj9zDG9iCz^nu-SJ{0ojhP1xs6 zt-+GyJMtuYZb?Cps`sc@CQgkPAJXayahkfO9*Z>&VBEb-lzuEhw+&WPw|^NO^6aEn zlj|v|VJ96`m`ldfZD?(uGo6S|q}LX0q`TgdKDoZ8fT}oZ&~~Ff`5Cn5kUjnNSWErF zdnqpRCo6N;62)A^t2OGlwYrYGZ)PZaqYG{0RE1r~#|n8phiP769BFOwBb8aRs9)j& zYF3FMPjy>1{qQ+jxbYc}8K28-cRc2-^ez|ES3?5_9a^i@i5IU1eCv4lQs+SCF2Et^r+?l`3jUE1*DlO`zorNO0jUvMN0;AXQ{02%ZH`O+RXI71dn#w>@PtClpju>+Zz z4!u1&1e*^Qa^H^+c<8+x-Xy{@4(vEb5$`-$%=Q>c$z4DXAI+rR!G823BaRY{(`e(W zvBKQLmV*1KK+1MeqMuWX(fV*E4&U_@(>0{2bi@Wrt#dWxo@_*euZYvc?7x_?O@@kQ zGkWZIh!kyg$n3}~(VV)FR9{HY^yrrulh};=X6VtABdN4$Z#A8+OCj%pTJ$;m0^V94 zh4Clup~+YidM`|-YvIjgXEvGaWld;i)?Li9kHv<0=h0%G7M&ZHLdDtDv@J1}ZlBd9 zBcsnaYU4|MdQ^huG`f)E+Sj7&QH9S~CO#Qag2gM8$f_rhe40hN`yxd_T0I8CE|~MHYnAx+R$cyMcOAd=;SnD` z;S?*3%^`_J4T$*;cyB>3cj)}fOT!~^=SD{NBGvFkUK}ZpEk!Ajek(vm7Sd9s_}HQG zxOTy6x_BUiBqrI?5rY}nkf{wX4+-#o);^Zuat@5-tGPvU3tc|+hJSs02r7d!*_82X zpjP!0D>HY*`dN0g<4*>~8?L6UCJD#_Wq8->XRKZP1`k_Zf|s)5==q)@xYUbLXKVz% zSSf~EEB^9Avd62Z8u6=j4!sGNglSqu{Ci$KZwt}qhwiEH5hFPtbZ{sBx~(Xv7dKP0 zV>w(pITMC%*~sn6#BoxKC6;dA$2CM4@4Jzgs4VFY#{D90ld{`#Qs7{iy{DIZkCUOj zKMVLFdsk{K--3I-*t0Y5Gnl9ABjymZgKt!^AYF|B3ak|6+4>CU-A`rXHPl4B$11kS z>yoKKHM z?5*mDC(OLLdBsp@?kxu$^UoyX_?%|HUJ0>d9#FQrl)u=OT*R}Z9mwFHcy#>$kTw4y^S?X*KG7OJF4l-f z2RoBkq%?Vzd*Z1WC3tW4jNSRv!(ROzf_EE_(Y$)-$F{7q@Vgm6Fi4K?25hXhNn@BK9k{C%gG&{LaN*JmcEU&#c(X zhPJjdRiy-&P+Nj-qMZ6>_Iwz2qm$>X+K87Err{bPgg3>^V+*>rz);iY^roYh4jO17 z+dfjn`=8`rO9&5{24UDTYm9%V3FQ~d=w_TO?YK1%rE_B72LhivcqT^LNKo!SclozY z2fChHh6bId`Hbn~!Ta25*pgw2Cnij!`)*5U*{vSzQ$2OhHR z4o}}E!Ue+|Xoax|H8>zn3X%?ZINp*kdAkmtT~NX?T#jx%DkGa0BSHS|WHhf0z*FLc zE1S;nUB`yPmcZe7KB<;g-R`3Nw*EBo)I^LM(#HLx6>-%fYutZqABIjB&2fqCv~0*q zI(c4>mbMmR`?Wjh5!Z+c$%b@xeGOmksV!w(D^z#uP7h|?4 zZsAQ&I+xZ?Z5bz7(}{ifW2rUPzEi-OL4SDEJHX1}5;Wy;7hM(Ui7)OBrrmnq(fQ6U zw5tq5w+}`5GR>IWeMZ4JslkHuxprz5)yq~#u91PyB}#dfL+?{+sMzxyh3)Po3%h1& z6|11?$N4n0^cJP;EudNJI!NM{xp2Ag1%13zK)zr1ke;j;eW;&FG0Kd3UZ!Bx<1K>z z<^%XFm(l#9S@g-#i{c~q5abrnw}KaxpJ*<4_;iqPvVgW3<{QNBx*F4yT+XBAg z+N$p zsP_29dObHX{V`9N*60Y99e#`HR~`}Z8g0yUa22~Q9>H$=w6L>>_Og|C6oDTe$n5AV z8}g%)^}RaFOv1dGdQCsjdfv&Np7ys4m^~BnaZ&(s%O}<>jNOB{vq4j zsRWCawz0o;ms#G<7A9=_&1Alb`nPqkKBY4(%y66KiQZWDbm$cZ;cr=omk*Sk(SzJb z5k^%j4_jp3vCaiGP&7-1kG{JPKTS0yrJ@WP65L-XI%OzCjZlG-ZStT!QB-}8I?lso zqWEkFH!N{z#0`<(!1t&zJ=fol@9ac+Wb3co?N}-wv-J_bT{?hPOpT%G*EA{m*(scC z|C`TDSBLt{dP^zsCYCniF3&P9qJYzv*~kxG%<1Q9IR5o0+q6&x%)V!_^e#`Rt}kc9 z=dJ*w>vinQt7`Uk&pQV11HkT|O>F+)xsZLoh}L$LaXR;$ZE&b(Hi9`k$7)juI}ULhtfpN2Xg0x@R) zX3SeS3D?xW=F`)jVe!-X)Sz?;A3UCf)@5xx<=Q$v!}%YWyrrJGdvv0kmbma(To%)( zNWlJ*2Ta`c8`GI;&GySX!HcI1>?2#4`z%Fh3%7%Ne|Iz8eMi`{);;WFPCC0%J^%|) zTo~2+8{NK6g0r!Puqi!&hfB9{y*u`3oO=nIw>XoN(=*)OQo~>FXP6PV2_Gc+BO5Ul zb;JsATVpq5hPtBlR7;$7HVXSMe1*1ctvK}8aB#DE$(oLzW<+8jch(pmSoLB1unH8D zibnf^p?rDGe(aG5!^wHFcu$7ozQ}1f^0*W_ClBL`mSmxw+AI=#J{XyOC=UW{2r3-` z!Lt^?aFJ)kXu~l)bMgy)e4Y-D8)RV85nDD)vy`3vRmQ3&nu1w)0$cuNj?iGA!KRfp zFdeCSwuB1U)n}L39VQEdMYV2w$rmbj%0&78e=Ie`BH*+81Q>q)Df?;?%V#cD$LI31 zXxhf(IHi6lcOEH@8M~(ASONwXl(GiZnFZ`TAnc81beiq^fPXlQ8@?{6zDp^$eB>ud=5_y%y(cYsRHP48FcZL>R?`~x) z&s|{q?`T5Q-y)2Z?xMMi@+|dZQ`ukt;jGO{7sNyMGULKQa44{q&A(qHEO}tZ^c2Kk zk4GE(Cr<%_P4wXY!O?)ZJxtHKi_Tps#Q+^sm^QYA`OW5#l&S+aUF1M=xg~xIkE2)P zlxWVYS^T`2Av}>h%93|agP=(q-=6-4OD<=SugDjftoRE*RO(}*ST-ZCMyB86%I?US zu+4u5v+_Y@>~EnC^EhD)V=H&FA0pkkqKpizcJD9DSsF}^`QnroF%$>2PJ@e5B0G$$;f4$kv%L&K85s~wS_@mJK^D?r7UsL7#0+z z1flzeL-BPfkeqG+We@DlMq26DSu!YobpZb1VZ=pV@%+SJ+2-X%Utl)gLi?08wr6tPG>Bz9pq|#qWsJLP&v}BJJ&aD^uk@MArl3lH|+W!%0 z_cYMfMg!qaOsde<|D?!^EC@UNC4~t`chjCPBWcaq(e&_L8fnTq3ctP`v63+t@xW^7 zwCcua^3v8NpZwi4Oyq-BKPCu!#ZL<1XH$i+e+-3r&W&`;^ASByY^8uyHNoJvs^BW( ztAnmt3TMwM3MQibJhSN;6|2_Kj6oO{*yRi)IQu zS%gb$XaAbZ4`_8?ln}(k{$2iebGee3)&Dh@2YLDjh)nA?Q~3YxF8>ds`Tz3NO!nVT zuT1xv-(O^F|96?=(C^>X|Mm3$r|n(&-#7kS`=1W~{~7(C$EEn+B7cjv{xjNS-2aUI z&u^vl-(sDt{=eA&^E=FOkdiX~Kb^G1$Nb;!|K1GX-}e8mum9Jk*i+Fi{%xYTfA1|I ewsX?I|HZ`qy@|ja2T7@apNoqg|Mvf{zW)z?vJwXX diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_abs_r47-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt b/research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_abs_r47-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt deleted file mode 100644 index 5c2190405bc90aa5319867ec05bafa0b528e7f81..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36843 zcmZ^Kc{o*H6t*#Aky$cCB9*z^wa#8wDG{QQp;SZzrBFmlqLk1eAqwHA(nP7ug%CxN zG^&K=K_!(^zWY7jKi}|t=id9==bW?8u+G_guXnxcT?-sV#e{?;C58TTT z$$N{rbcm3lp{Io4{F? zD8;XkYg7efbqsRVk3xCTX}F`X2k6ZNkc)o?O-bYN}`D*Sv z_@ZZg`383L=)HP5#9!VCZhN(GqR&_itiKP3zBKaoe2xR9Z-H=T?n+*x>O>IPngj|9 z3*p;}15oU^7_)ERgt!M!N%UTrPmiHXne3UI3mt2O4JnyfH5i zK<~Fb5IFrfEXg10|tTp4OckN+%5~+|i z>n%)e_yD&re1p65bTDY)d?=XK0)|z{ll=PxT7@+*LH8cC#VCREpb_|N*X6a_bwGbl zGE|iwgiqs*aH61p2Ih)m)YEuKaXAK)X72*kl6|mxeKCm0d&4&WS8(Ru0eIqglK1+r zD)J{N;tI(H(E20|HhwFCC#N67uVObC>B-^E4;O~F15q&QlRP?%GKKwhtuT7)Zm`;t zFL>XgcRspH-%Okx)dmg32>XTYsr3o*ed z@LEEbU$xByYu1iOVYhtV@Ph>GMg!K6RKY#ZF98{QOZt~t#}3>wC;83AtoPjoXlhZw zxCA|vds79w_Qk@aqmAJDD+|mXUgTt3D(Q7^5pC#kg&mq-Kz_Im7KI7M^R|~@@%uFI z$-MnwXmJthtq{g{G=u;5GM@0jOE|1&1>34G!?BDhaJ(%TJPzyz`7ApKs>uVDs8+}v z*a?HI8s>_1@Ej!FakgYTZ;#k3ur4|a<+m?^vU@a4;itiB?!Hz3K^0iU9f4E7%mi~! zDcDa6ga@JF5RxE@b=^krpfM48s~$m!N*&047=i4<10eNe9IS3s2lK=j5HswD_2c;% zY5bVCx7Zp+v_x@%(JAJnkk5?$WKgm60Z8pT0yEZSLglH4AU=iXYb@A_|76HbzYx6URXNV}CP>|qzeO5rv`!@kLIb#1U z6A1jHhsRDRVSHT?JnWaI+wvyNX=N4r345_^ZZd@5P38?>^@r10(_mWqR_IbLhiQXt zkR_y!)pq&J<>pBieb^Wuy59wHkAu8s|1mgp_a#WJxB@omg`j$WA1v5^1z77Bpn~gg z+#?b?uLZ$HeOLHkEQTv$4?zBQFBt#d9@r~13z9NiVe05?c;1^1KcmK>JbxTdC;SJv zxqSqu(NQ4geFQYClxIZ-$`jQ{XaB1J^Vi0NKGg(6RO$XkWSyUmlD{A?r7=>$^94R9%I&mzQx# zLaumZ|0%e(wFy?A`3Tu_iy?f|QCMp(gjYXr!uR63(`)440N>^?^Byo^kFv(nkBy5# zdD&Np)iOmJ=_xq=4q<`R6`-C=aIRgHf9z&8M7%o-&9lc~%kMszlWBoI*($g}_A1;; z8cSx|%-H_^6gDTR3i?j&gJXsB@y+ZRnyRzZ$jzFl8>4Bq*gz>*DcWls5 z!Ec6>@Q+R(&-`IJ43wrqq=*QpiJpb=k7mN}&vH0?avaY8*~6q)d9qpOCc@S?D)?`X zH<)xhg$*%Z;rBUJbW;PI(xHVr{N`~TKUg_{@rr?3~#}>IADkiZMD%}_$IjaWW%*piBR-a7Pl%{Qf$L<_BFqjO=j{; zEZ{O6((Q#F;bGVw^Bqq63ZYWcA4uP>i1Uswg4S1FEdNjsdz{I`uIa+)n2`Z?J*S{( z;UM_x+ysY|4LpZ^Cm^v`8T-as;rjSJ*y*i^lQpa0Xz&#{R?q~-3W?wslnj4uT6w{n z^LY-QLgHs!Jd%su7ZH-hjEo zR21T;uv0%pC{pAnGt=^-|8mvvYw&Zpcd`Y3>|BL2G;6_h!W5p<=`_5(MU%hhOC*N% zilXYZ0>I`)Xx1l#Ul!hmLAxg~lIBGQN4~RL^F>JI?ja^y@da)OFhHwmKg8`Tg!!2h zF!aVQu#L%tB@-2qET6D>rF@bf62=Le9zlv-9JDv(TK#Di#q%?+!;(WByeqyA^R5Sg ze)||KsGfw;WtkAKAcrY+lkn%_Q7ERL4|n2EgTn7dke<`V8)x;DmuZ;;!|Rpsm-SA_ z_VI<2OOAt$!8p)&I|ub5>mk}n0Y3G(0Bs0>*Qt{D%hww!&1Jw9+hE25G5mM?G`I)V z!(YWbxZ8CJu7Ak`D;x&5GQei@#3=wk7+uTVVs5eO@_!<|H7Ze9Oz7Pq#H`8CfX zSGVP;Y`zpd(xs3$q>b}BN@2myvDjJl74EH&<6mk}$I9`&uxfZY>|3vkp%=6;xJFuV z%@{P+v>}f7P+@z*-Wwol(P{a|UolP7gg+ z6>}Sk5}_>_u{a;`bzLZwMihW|)?diiS3rdaTi}A52zI_04^Py}Kr8kuFUrOftgl{# z!~i`^N^69PCPJ9sEC!vmpLm)5b>JYX1k*$7V8bXmcp2Xd>YsyQ-SA3K_}0mb(0Bno zGT9K=5)MJ%??A~-W$2i<0*!djd40PLpv}e&0#<&6>1*qG>z!+0{qPMK_aqZ09_)hL zEzQvA-2eeLWdLIHaZC7WG`;x{W;V9NHsxz@Ddz^9KPrwM+9{~xXM!gKjqqIRZ73-e zLA#4)ppm!+jiy`Rf@Or+zuvLg^%?Z&NfU)F8==-&(?nxck2ke5$n__UQ=_RQ8)&x~!qT$AN8CC{7aDdaurtWOor6{*;u+n9yelHIWzpKaVHe zf0smAL8cfLk7p!(cOG3GFs2DwV&tRvjPmClqKikD(q=DxGI3ePK4jS7=P&Y@d)E|& zyjtPV>SVB*QwgU`KSS}XLe^9{$Q)<XW*l)lmVZ*YMz1`XaDPFM-Wl zr1Gfl{b9>eU;kusCBzBuuH{=QnZ7i_9xdz1jKS0BlBskTuA7VFzz@{Hh zVEU?B_@=>ugz;f3-{qrlo3Se__uJ09Q+ftG`-4I6;7(Be;0sGVD`EAmyKwYd9!U2% z;^mixP#`6Qn{zM2v{NlmcZ;ydzy;aM8hCGNit7d^;=v$gteU8eJCw&@SNl3pjCcwn zyMC~9)3fQs{5YDSozIet%0V+VT`XPmWSM6}uokX@u>@9l{`<@MG z<+6j$-JH(IArhYiB$YgdKF^Y&^&Krtci#wC|F?%*->{oq4nF}CV$*@UoD1z@|1nqV zII4S;MROkfWIUf*SUY7AQ zw1;94e=-m}=Q-kD14B3)t%l2mAA!;udGtOy1$IUE^5Ujf!EMX2puE@%Si}g-*sp;T z)(hcUhfHu!iGz3dF7Wbu2f_GGh5!SZ;>@82&|g2yvshjV#Vw*Zs5cKIPUM2z#{ID7 zo&-f7*~mUGjfKwowQyTc1uZO0@oir}XLIWxODnZN?JZTDx!@V@G;O8nX%}eBEo*xC zp9M1+6NsfBZa{&o7Ddk%qHld~+01TviXD)nW35BXFkFXTeOg0Z&EDLp<}CItXOJ}w ziPAjDa&}4k3w+DD3kPEs^LM|n!BJ;5@vCD6J7^|KO`pCqZfQ14yy?STdrlOk0r!O6lK4fN#y({>2pnUFzXZ z<_%uU;uWCep@i0S3bZWuL)y^*Ue~%R_+tN?SGVmC$d!wu>BafbE0`-bXgBbT+iQ6C z>6hSQ<6{^{?Vu8`jPEXILUxgl)oQ~a*xmIPws%ZH$!nugG&COGR%MZNP#77wuV6Eh z-hq079L}?vh?nA|sA1VwQrR8@Nh!4~KKdZ_J}M(s%?$du$ci3K-@y*pc5};`1hIlC z(z!?46h2X&Dp8BFnv^KKQiz(OOi1g(3bG#eg>&kvWUY%NsdlG4{S|)7q^o1NSLM=J z>9U-EX0!!nosH!teeYr`RTRnCPm*#as@aVW5kdn65uus_w;Qnv~wd6=qYsoa!{nP?) z&E7#gtzg?TL+RYSY+83$0yAyJaZPU<*#6bUpOYtn@#+;oDJ@{REEatGqCj5O1m-$B z!HZ}iP^q(pTh{>6%hkZ}buBOlOX&BghIf0DV5VF*JoMCqxh}_{3(aBGiwbaA<_XE0 z>frIMjUX++CByB?SSx=UM<;~S?!oDF$kT*tm?aEH>}O%@^bwf!E}b>DT2ZxOJ*01V z3co>#qQ|<>`5k&R+sTc6`F$O9!?q&pvOvB5^9;mT@Qtt<3MDPzU$v9Q2wOE&67^-xud7)7l1uFQ z>?JJqt|czsY)_BsbV&G2HTx6Zz?$lBv81?s_K<&)E#7p2#l?)r#M4gb7Vit?FBUVg zc0F1%#)bZBDw9gdQ}8Wo1MSN^+C7}k9%dNhSQ8WEi)!}kKRsc2@?4nZB#SordqM1xAto;v57Qpx!i#gt zaPRI`*dVQn`?p+xaOdm1#+S#yWqv8F3~h&iw%?#WE*t{pCxP>d2T-|XFVtVWfh_Sn z_j$50IV*Ovi}o^DZB_)vaw;hE>n?n~XT}Z9zt0ZNx(pM#+u`G&F89TKGkdQU$;DMZ zgi-G`am^i7d@s0T6 z$gANXK46LAz`cxF01NUZP;0?Icz;9%AKrPvp8uLgw*${}7YvH<)Z1rp#L)xJU-pDb z(O%f6dl|A?jzB?17|7bbfkP{@tv=SxffGl}vHe^El>OWSTW)j!H|ZLzciss3*;l|z zCJgd>tx-7I9yeZ5NBw&PkdVF+@1K;$7t1B+gIqT4RSTywZ)0kz3nvm?KpKg;Ons$b zEN0l^v}i%xU#~(kADn23sH+A+370LD7TVH9Lj@Hymi9gE?I^$Yw5uQEWq}H}mfl zV$!=Wa~5H*xKp>yG4uXgm{=u-{zr}Q-s9zLQqeVLw8xdLU-=I%3hq5v%!wSC3Qg|l zg`B((&^U1%4qNB3)Z_)U=6N_pot#GA>%z%?c{bHwl%NYUr=z6a4BTJ)1KgK91?kVC znC83{=XPFz|FjqLrjD(KQiCc`_z?%oj>$qzj}vBBZi9^v&ceI8R2VPa31_}_!P2S6 z;o^bw@bSk0L}=S0FCm2Wj|rpsUehVZ>jkU-wScD1QKr!2DNMP)9i)$)hf?QBxLwKA))AmFPDvKx6whVs3<9I5PO{8^& z#nh-;N3XN4(awfK`XQc75j_J`?fi^hJ;^2R%>g>ly^TIiai*81mb7@e4*ASmiOoHo zocy(e>~)eX>5p@vscpU__kJ>AsULgTl8U8`5mYi@L{su+<2vy!Q24HhQCWxCfqZ4! z5h&<)!_S${A4__@Ih63f5EeOcCR&N!f;b%!oFTdkGS%DRaj_ykUD^j(=9l2l{;?=> z{2FiK{WvK9yb*-2f8g2B1pKyK5{!yUph`*>TR*%4LuXN(rhfu5pPqrk1>$&LQ3!<= zc(Vr?MqJ4K4j3%TW!0xr*dLRt+@z}>uyy`p$dkSXuFvA*D`salfgLaG#qN)aSnKwXiOH0+!>wKH$qgm?Xl6+Zj67)Um^1WddLw1t z)1&pZGwEKH4}FmKq0^gZ(BB7lnbhMmtdF;WZ+q|zQ|!9KPI%5BGv$>O{n>{?9?zt~ zmWlMPu#whYC?xsW9;7qFlBCL%D8>I7t5B+7irXKvmy)CK>(oyuWj>w_l)hp^H-ThT z?P=TyVY9}4u%B`cu1r>k?Yf(w>tPKz&Paj}_g}%b@;sQX`UI+^ zx8e?4DSUGMFMOY$jv+_?LTarc&Z=wyrOr3p;&&5p<)9o^)J|nNvEo#sFw6}9d9ann z9@w(50-lOCfzL5rwuuYnK6UGH-Z!n;m*AsJwC*{RoUn!V?8qXiJ#MVNDvB*hKf~4~ zZ71o8w37onb7=Y4|FVA%)45?9ny?SASeDZRMogZF;~M2tJ(zzqB?j#?v%lv zohESj*lFI-VO21T`T$k0H1JbrA5@$ZN54fX=s8*%TSjE@apfnNGD8VxSe=7?is^9C z(j8y&gwf4+7_N5}LchWjXuGG3@y$HE<$8iUbHAI7HST10T$EV(JToj?vIa7@i({}> z6L&V>6a!Z(;uzx^uJ|{x!eu8|cC04_+9Xo!_gL=Pu1wB;od(lrkY&A2bMTn;Pe`~l z7d4+w;KvziU|@F}EGu4!4fV3@w2CVG8hwmoLNT0SX#!22vxL5nJ;BUnW-%+<3a(!J zwUyo3m6&(9l@kk5!uL(rLFA7EF2AnC?1nm7vs5>`@gtYJU;P_=WzA84+!}ZhSr50` z+CVL&6XIq{;g$vM&?0;mW{Bj$KfVr3NY%uI8`3x^;Bi;RpM#vj92iw5kCg+uxIFR& zDxeK-rbWm+ipv?w*333g(3^^X0}N36iveaQsNBH}WR|tM5hnN_$9Ti1@M1^_+c(!h zeuoB5*>@7Qlng_UqY@T88G%_UpP_3+8|%+E@;*pkg4U2Sc(M2txM+{?o`>G!#oWq* z^?O%BzDh2L&pZn&{N!=5`w6g2*TAv+4WRt8JcwLe4Wo|~gR$fTcz3fNOuIF4Mb#-d zn%~QHT$iGP#c9m_)pYKrq6&%ZHKGEmy-esrAM7i91S1hopkzxcTcTw~OZ-I8%|x5o zE!C#~ek`R5tJ(BQMwF&M*vV{5w7E{c3XY9@&(dV>u;=#ASnaSFNoGk?Cn(T=z0o9- zwvMcnmNIpnlWfhRem3@&C`raVVPc!DfueYz*^FB}Ay&Xeh)}}Z zIdpfl7fHO*qMXfI%>faO(C>Uj;_4dPH~w3|iM8j-lX3T59U?&7X= zcBV{{K90QQoG!?SyBJ(aDKF{AopGby~r2uEFyV#O8{$gO!kEpxJ@HSnFe z2qm(?$b;NJ{t+zM-ouXDb~4u;pV{`A67)-G6velV62y`lsNtI#x!6u%^3$@}ADcmz zT{4<(sEE?d1*h4M^$9G%KA3NLG@jkhEo4PyB4m;~hDI)ZXQhv_*`>q@>@nY*ye@8_ zq}>vP#o|<3H;OLBePD?T9x$5=eXQzP0cvT)bLF@9u_^%<;p=NjBT~+^`}G8x>Epa|@j7f61ef7PvJah@pPw!OX75@FFacXDr+Z z|BVX-ajis9jZ?x!mTAzsd>v$*%-}6sxB_pko(Ha0f_IdY1Q%wV1EI1woF(!OcyABG zd3!lDcxHieZO=e&ohiya_k+lWVrUPRz%S!>LWa0Bh70ojmoyn_8k~ocOkI5Eq=>x^ zqtLN69xzu9cZMpXd%Qdsk}OLyEAv>ux)caVXoarS@htzlJL})2#O9y1#B;{earA%A zeEGRku6l=1$d?;!i|21wXHhF89mc_wN`xK(s2zD;=#^J8DZ zvx`}9@{cag)!&a+4-L`a$`|N^c$hw=gr{^v2K&N_;bzcPkiT#m^atcI@xW4eu_y;p zuZUqnrV6aPUjzQ?)u7_40HcRg!QAW!B#CVU#o66(^Qal_qHmya;sQAQwno#Bo#6F* zHX6I-!;;u?Zr<%0R`pvBr&^A{zh*=H`*;nze#nlqSJuFJ&lRvwz?-%{*JjHV!r7W& zX~?uU#Jucmw&bZKg`N7!T3Yt9fKMV^>LO8e7R2X`DKSj#;%#;yT$r3EO`{Wa3n^k@ z6sHfu7 zsy!S_ky(Mn|2m(7ryU`OfI8|c-%if|7Lx0|spOp@M7)Phta8~-7W=sZPMhjsI}_!m zvKS^7^OY@BlOnkV*{odP(`1^{p;_Q(x)>XT5W zYd1Bx_z*Xi^1z5FYKTQHoWi~zKhAAaN@FuuX_Mz$U3$9d8$2+{VOfiSN}6|4t^Y=P zrea2WS9Y_--`u%;F9~!%tVE&@CQ{#6GfGo%qjS2O>CL1By7Q-oUaj$?GId+Bytjqc zX(Umr{(hR!5lJy0I2wAZL17Ey`HSQ<=A~glEjCdBw5X)fQ}q? z_PsD#Pu8WSrCQYaJC&WilE-N-He$Qp46%obMs(|g8$0cz&((fWqGJ;T{M)TVbj@gx zwU>2)(~K%uRlFDW#oFS=vop{(ey%R&d*{l;aM6cKOJQ19a*&cZwoDKDx(Z>7gFE1iK(+{cCpr%BL3J> zits49YuC+=#;Q`}$WmIR;7kcmZ@|;B(zKfwLw{3JY3s){>fO7M!d_0M-Mf;R?x#8E zzkeY$OS)4>wg<^Q3l;e8y)-fNFtr(6BkuBK3U@@xxE4uA-Y3%v{WNMyi>K{f%joXW zd9-wW3jejG1I@X=jFbiOZE962c}X0m`C}rf<}#3$xDid%eIJ9lBXSwxcJh9@@i%x6vB$fyU>)5GMZc0LUY@*h`T<(nxa?ZC+o{xp{_OU zzTU&cEQ2`%jRNNIw1`&BH{-YN^yN2X?BM^pz~>*AtfC;9|ClY`Wl-B5J#s$1i(4bMjH&fI)Bd?Y;}-m3FE%}36P`$sP|$2D znm3)wECd|ww=c~8=ti=MOQ+VHUG!sUIlcX?N@4Abm`9fpUhlUh&24k2>NQb_v^z!I z-bhZeu{34Vc@p=Kps^7ul(N=^$_>NG%RY*Vm-~|RcoX`vUztEZoro{A^E@rCjiJXy8_4{hJEcD%dLd&=#SU{xRU zabf?lysU0WxS@j|gLcBgWv@VI$xQfAs{>2@pTZyg3NGhJC(+=*mhKzwt4*)*&2K z$NUDv*m^c__9`nGpTT_Z*0Q9_Us>F4UD~3WM!zRbA_KAetaD2X`*YwWix}O(aw?P9 zzK?ftV-|-pWz+a24*hVp?medcx1ZV9v@z+77c5~~D);n= z)O3vv+oiKH7fP8(<_$Jy=Ln4F>Eil@4>@bGDz+^_m44(-rqbXSZ1%oCXcu6E`DYW@ zTLB&r_d0-`X|{Odof}@1)nt2S9Au+X%Ax-EB&-*E44r=4nCK@S!*{Ddr9>K{k5@s$ zgF$dfs)f2fLp*Kq4~Bh4c!7Dxq5jJd=l7_X-ZX{^Fs&h28)QM)#d5gzO$wi9I)Y~V zDY!7RADlO+;?53D*wH!z9o z1yUo`xj{(eF3Wy{G;_p)$xAq;!^zA`c@c~BTZS>?dFZdP9+kX0!PrTdJ0KH?r{8@C zneCdmtssJX8+?FW3S33suQ#$&I?mjZB2SE8C5Qf>t3cUJz}Gtsz}sA5Oe$@p-#cVc zf1Mo4J94;SV^9nWnopT)JGz6CGrs-Qwkzy+7R zMIqBp&d50jVm>Qkm7^X$fAxkF-m?>Hx4-6X{JI0aJpC@nsVm}1p<{4NtsC+d)o zXtdB(LbW-|$zJ$>z9dHPJehc9>lZn_6p@C(cvN&e11pW#3KsZtW4P73Xw|5$jyE>k3 zYyaVJ_!lhQ3+U2W!lhXBbMcACxlQ^zabx^dm=HRX-|tciDgn+oQ~W3w|KJm++;WwR z$&Y5pj{D=piv@DP=GC>S<#5?cA&~v^H#$2C@ zWu7t^a_b|s>xm2acpe^~bcH9fUKI*|xZ;e~LXaI@$SeJNg6HF00s-_7YR;W!t2J-X z$cfSH!iph?tZIPeo?__bJQ->7J809Ih?~D!qHC}U9v5y7`C-c_RVaKYQxIBZU%Fx#8T5Uw{QH9o)O{PZGF(!?gruOG%~!SfDzx9;8M5GyxD~vV7sRu zqz60zcYT0q)tadARSoUpdbuP)e*f$%9WrkzV3VIop@L))H}{$aZK^3}u0yLi^H;Js z>7^Q5{rE1MR(qc*Yg`7EQ7vrhJAK+IpS{55r&Yhgo&Dx>I_Pz`O8lcy!%M)m0)^tgUG#VB}E;$H_6B_rZHd|=8a9r&sT z-?I~Klj;5&2QtxLLR$yDXjX>@J=*FDIfEY>+;cSKneg3Z6jk_6IDJ-^w&^sI!D{O$=PR zhHHaHG6I6?(QW@=+e(hJB;_JPg;EA*9< z!dR19c-b=!TYib)xqE43Kj|R#AIb#t%R)G-)Cy!z1mmdHR+v}w1irrZg(_W!4{kW( zs*V4+QzGx#dObNBo1V{>t6qf;)gdm;az4ehKV=Td>fE9&Bj8gY#_R&S*rm&#SpLid z@T*OTTu+!%4xqX+(%nZ`Bm44D%<& zZACQBW+_`T$C7re0s5`0v&QtGo=TH zRmL*+ z&mDYfxaLlq-kQ?3yF!#~@C%{}KCscjU2MmADJCW~0x4$7+?%ak%x=wmdNJz{*PAsF zIQx9Iy?+c@Y=6f*Hx6@(wkkL}NEH1YFM!GZPjJ^`9Fm769{DT*N3xYLtW5$$s}55A zCK3L>uvKhvj1@$FdI#2DV&VIi7`Sg6P zJvHYr(&?B=v)_)TEQtyhVQR_G+h5AwzEPmHlc!Nn7$awEJ32nsiQNAI&Dz9AL-!)G zzZpeU9yT<|TAq@%gz5Heb@F;>L`R}UG2deXx-F_<3D-0z>5dz{IuuH0_b;ZhztiaK z;wh8R7;`(wK3+`5;IZq<#nfFQ-6Vn4H%bVEIjheJQL7uul+-A2E zyqM~h_v~%X9WZ$G0V*%HgQib6+#C1;9f!t2{Nq*Zl9vcSp}?7c{rh2hmUw|V%nwE< zM{#^CGL|)0Bs1Qx&-lbIgA6w4lipGpT39YmR|^%%s<57gPW*~*Htu8<$_L52+k)1e zOJ~Z#)~x+y7JHN=O*UUg(e9Oo=-;t`eyD7w_nY^U{?iSDac@S+i@vg(U18j=1729q z7SK1bS#*p)g_g9=qC(5HbZkc|*(+Y9u9RkYBEFagCAN{6&K7DmT}@B^Eh3S*L?dVA zDZih)jp^NB#-YQZY6-Kdxk>v1c5#2hhj?2of(DNThsOQ^SdiUCf&IeAV zt2=CHSNTFZy+j(ro`!NY_xqX2N^_cfcq46_vX@+{H&Y-kqBvB;zb>Q5;8+?dY5kU_ojQ}qdq)OkIQ+qHyAHGC zHgatJg>&$+Jq8!YU1a^U1ocM`o%xYLVf^a+a(cVZoJ4}JGyi4DY>z+}q%~fIwz?l@ zQEJU3Dda_a!ba22>p$3ouR>HhRK)tL-}CXnIFL+qiR82;*vBX89xx-1buRdhtm&zwX`6FN#yyn^4Wr3;ia)(JiK4HCaik~r5VoB^ zz~1mD&~UH|-BFJvC;h{;?`#5ft8J%GAqrfPtR&stpg?bCG_y^bIjrC785>dXAU~5l zGJKH3X3tCI60NFO@&_T>yz3|1FFJ-sTY1r)&CN9MC6gK0h|u|xT9&ooFe_=PWX^5o zlq1mT=H3e9>#OAYB!gkL{qNtGSKZX(ir6)P3uc8GJJLp8=23mDMmOL%%=y5|0 zsok!o!|Tt}?;lwdd^43KH?1OFLrqF`X(7ua*Xc)O5xwibKn`QCQax8iHwvoB$V?AQ zE*~Y6JtcIdyMz{oUL^0LV(L3|fflL9(X+(c z;u=l*T0_$<>PRX_mX^eCpv$&9=>8Kw@-bLK(}lK>@P|TrW&Ml>=9p95>O0J;LWhQP zj45WO4ykAZh1?M6Kw*T!YD(FUg73_r@i;q?Wz25cSuzd%diHGLE-JD2=PUZg@w<-} zQfi(hMa9d|(Ce}EE7Fo?J@OLZhFSC^)|4N0y^9n|4$<>NYsfNh1?k+gqsg8nHB)LAZvlKcNa-p)@T?0Fa7?VN&d*N%Yi4lA_t zFhgqr2U0DvjHy16r_xQXq+#@jtGUp?eAOnATZ0Vwgo{(mtI2ew)t8n`IzrJs`OILU zITc8)qh^O_x_&Kz4t>fZU&%(A@P2?IMmti@S#dhrYDZE^-sEbvmdu9_QSG-b`V(!+ zpRsNh9l5)dWVcw5g0TV>osc2nE=ziqS4dBt;`mWU0sM3|aXMFV9I~Ds+Q}G@!Ncidp}6*NGCevlk>-djX2-KP zagH}{a{>NK+0(U}%(y)bLTcyJbnUt1=!KNBN0kh2e`npN?z8NR^SJJ(a;Vd{j@Ame z<1Ys-iGLD^@?7cW*ASZIew3nqK7`TPZuEI$Bn{16L-Owz)35z5^!ud1Z*GlYJvQyM z@5)X#R)9;_SS=)rqQ%q{yqc7&BI%WZJ3SdX2lCn(#GAZ{EW=%?Sx{P%KF^Zo2fLAh z#(H`)MUS&d$8j|O0m{et3R#6pel5&ll?2!(aA(Bkek0!DaiLf)b6taB*fSWG+?5(oLe+7}Z7_ zW|&ZsPcB1$^M%K=xXT?#h-RuZ=$y=FeH=bm|rCChF zw+S?_ucHvvEhI4tq(j1<>7D#;!9#nWvQ1T+kLCQx*Qrhe^ z%F`>MgGH@S91%&9NeR>$lt>CXqqx46yM^xFSx@V_M`7^T95(Zsz@AGg%sM=s6E}}yZ#T^%Z-IU&QqhaGIVs@P zArGc}w32BZ&SoP=@>r~!DT!;flhyC%P&D%qNUZn>7hW`hS5rCck*nn-ZG&iB%O3t+ zl>>qr)i@fxb|SMr_8e}9bwFp&RT%p44a8;(VeFD_S~F!HP1SnO!e;+qu?r-q)UAZ| zuaL#Hn;Mvp^))J%UP;yVXV{h5N01m97-Rhmyzl|E83Q7Pg7omk+xgO#%vXcAF$>V`zF1b#wlzfgPI6RRf?wc;RND#ts&nm864$! znT9I_`Kz!kG_c*C9xS$^QwNvPw8&6uoLI^&sGEmn3C|dNYA(nNZJ{e|$#l#!k^WwZ zB{N$s7C3j1DNB7}(L>P;l~kD9vO@M~mk;IrxJHS^4eWqtAjo=OWdA*vq{jn;ELP<` zE8R4YrYm>T{d){69!PfZQjbn@c2S?`}&*l4taa-9lE28x+*%_a6Kj%C~X{bm^YLDWxq$k`yAP z6m2S_R78U&g)}Hx4T^T5G=I~c-k68w#mj#@|L_R4#|!+kb^ydK5#e>Io!8E=CzTIVR- z_~FG02Gm(>lm*lKoetV{%i++Q%T#h*ismaOp{&5}H9dWf5toAStGD0*8mPmidyDAC z7F#+XtB=7w5zNh94*m8u<4Ji5`aWC(7c~XaTCELae`-5j5iKN#2Se!mgk+pP$e*22 zIn2#Cx`0HZr;%5OFhk8YrC7yTbjdc3Ch8Ycj-x&s<1&-p8Ap-JrIobGJDB3$dr@$^ z4f$#5a^foXe5Jh-*S*b_ns3aZ8sT$k*uIKd1EMJ1!Hr~&DKXCv#k4Chj?$}V(FQaj zQ!*0Xvo7STwUFLz$mQ(0B`kR$85h1BLZ>DL2=pUO>zsKC@aJH&O0&INnr4PTqfPDI=EuN z!P=cFZ0S!O_Plxu3%6{C6=6%TaL_WoI?Rq(wPIVCzL+K2%}=7kUG?}*@Tg8$84q#i)tKmz z8;XxRg{)>U+blAo?<+W3X*7>ApPSR|FPBk#={MG#Rtrypeb^^!qJq)J)N3L^?SekG zVG+``pbZodQ$XWaXtVw`PIUA~B!vj~Ciwt&D!pt=H`}Ul(bXp~<)IC~>TnyRZK=VV zR<^>v%AFP+52On_*HYItCz|p_gZ+D2K-FFw=+Q-_;JIpaW{Cua=o<^~YNGFpJlIaZ zpCEPk6$`v|89imqX>P3_WhXKkrC>zfQg`8o!YOqA;D###RoH!NRrckaI6FM69{=hl zk?%1JEJ;Xa8DI68k4H3jYq}E)SZB{VYNT*$R3fF_{Q-+MsIhSdlUR#x75E01!wTKc zkT=_q?f%b|U!D`epEUJCpF>}vvUn9UUtm%=i^$2VHk!X;c!fL&$w z?CQ!-Ai2wknKyf|;E(fg&$sPlAn-AEQ4TCNZYB%y{s5VR=O>dTqS1j7Z1{$5bh)cb z>!Msp_Max%%mEBj7{@+5(q|qE6`8bwDmA_CMswq2yxw#PgWXl=Ym_Susf?rR0{6JA zJR09iYmk+2=BmHkgL0v|+=-OaP@u7ug(QyT@14nFx86#y87tJd0~^!P_0DG;^+=sA z4UfU43+rL8OFVhraibVHRg%*#74D|HvCz01J720&M9~$Z!?!Buv;lEyiTK@=^2qc1rPCfXl<2nSYxKgD?qp zEomt8Gm>I<{>|KJk?`E^zJRM+(%7eY-=XuA7;D-%i0#O40Bsu|+%4iq-Q$@&g>igG z>t=pt>3+T=XgPmNG@gGh@R5=qZ210)dA#M)aDLrNS6<6w6YmhSoj<*14}`Bj%}cwdn{fAoMO|DPlAHYbhwEm;k;Dm@d% zNn7)4_e|zpT%34$vgR#)EctzVT=*5|7x6=vIq>6pZV8N&DWAG_fw0Ng&fnd)hqqh0 zo%a~KiN6)-${X$r=U;xB$7`G0@h{8LiKsWL(2SsvJ}7Ovkb5Q&4@Rl75w^caeK;$;M>Of z;#V=jW;fl2yPgKnHfkX&{#pz6vyv%Is+EgK{{dH{U6`a_4}6}l!2-qOnAjA5 zRGpVd>3VsXu~e8_6lSuhi+A9R?+@;2NFpA0LuSADEq-m*BQYfx`e-|Zd=>4mV5d48 z(`zfLIrtXl*s9XzH$C_~dKZojDn%BnN-n2e$I zA#LG(wfXjLeAoC6-zcck>of&c<|NLxZJ5c5Qf<-MLX)ICTTRmndvSu2a&6sQ00L)W+&c)~_ zlY^@Yd-W=o39|hO819-Ej_ z4>o(WS<#N^7(Xr#2X^W(FftA{R-=VLDoxIq`(bStv9Ay znQ$L)s)ap2=C{7rZ=V*_!v4NtTzQ z9m0+}Z)F*dP-(?i#etYBJ^;;^>aaPni1b02bKHibZ%PQ;888^Vf41T*a|zPwSHW$< zvp79+BQ34nPP>X0lBd9YNL@=t=ZlOPY|r4lF8b45MHgBk>|{=Mo6?~t?v!L7N2#NW z37(0we^E0j(;2j-=puGf>(x;W= zJ|v2qW8CT1zQ@1|U+0`g9Q~Ovi$b@X($<}Z)O^B)j!XGd;n)3~G%sSAXLjM;)*-Y) zY7y=0-cGAMH_&L6AX@QB6EjUEN%BxLJ_u95GgBtBhRueUI$%SWCl`^5;6K_l^$I*n zU5YccpQGdTQj9y0jDZKFsPWxps@!uH#>l0E(KHn%*(zdBQaNV)Yz~uN7mTTX%lO}q z>&mNXc)5XkOS(`Z%|k3V$iF*oxV7E-;py%hu81 z*MTHz45sWTcba)B9n&%$`HJls7%4f6p8N`?7lVUHeBwG9HaM2f`_CqF8_gQBi|I^4 z5{2jlk;i?cg1_c;@R<*VOk71tv3;Ce)=pTS*?<$<#?uSo`+Jp>L9TqVz-q;kT+3`+ zG*|HC4w59fnF)B?QyC>AkD_ycKfP$Fq(8$y;t(BV5~)`(W|bPH-xjpr6`hzM+*8CZ zG|^eTpJ4uQ2Gi@;VXHd?hB=`ST+~JE-M$H=SQyU-3iqpTf}ZMXYK+nIU%@LKNmhDn zC*YZOc(+QAIrw(aw`EgFV%J;jvFb%yIEZ{L&fz-G4j5x{3-<`zS5lcj?NK<5qoefj zzl>~*LC#6kyni$0>{>@JB3F^Ob`V{#|BR0R5pQk& z1#j2}(e}=jG;;Gg+Elxl@@10fV#H#aTGkD}xe}_E&Y;4yNcs~xnMS>|rG)Fk99g%9 zeu!VQ#Qf}(b82pUh9G$MuHqcmEqEMzsD5!laiJq+sZ#Yek8VZgD?I9FL0A1u$u z&)y4Z!p^J2Zr{er3u^4E_&HqQJBYSB_2V@6ZhUufs<8L$priAIbE8z6>DTUu??rm7 za$7zaKOK!r)8Z+(KbiM?zKwsR7*50gUg4C!2)^*J78n%}3zq_K!Pq6maNs}#8C*7} z5voU#$`4`Qv~raH;f>3crC7#xUu+ncPfAA}sr=tMGz!h+a>6ul&5JyI*>DJ7c1bel z@u4(iY$O#|uBUY$U8!}qHZ{xCp=PcO>rWDAwoakc`M{erZvgez&Y(?qH_$RSL4R*L zN6lMiG4bH_^xu^oWPBirv_oPkDI$t~9bQX&0@P{b31fbpzXr)luBB7&Hq-9WvD9Ul zM2p)6O-GpNpPn~lFTKwTOw?YQ-m!re51T=8i9o-byy)MpP+B>|iIp$oSjg#G{Ig73 z*ip`;9}hN=?YA|gP#Z!SL7xGH9R=@m3=gJg;S`;0&i2`6G<;xB3ybsU$a*gxbFz5*fX*&B0%a?%Ky|h@+||DXPd@Dg-LP^lz9pP`Yq#+S zB$n~9J<^ml{06+r7C7;k=OFt>oUJ`5&PsIVv##n=N_;0t<5qTXpBMRaE>F6-;T~@6 ze!m7={UDiz_NLIiX(K6m+#GZn9m`2*+~M}g&qh(>Fuc(31znLd=(D3UbloA*&|-yPudJW0%V1KGta zrfVj%1+H)={cU$7+ZE9mt^ANQZ*IZc6P(CId?p=roK3x3gXrg=ja2h+HhJ~mhq{1$ z^q^)H^~ZbAHR1F9-DFMg<2~sJFKA^W#t8fqa@tED;x^ZBc(p}~cxgs1jm{*ydIlZd z)d@$Q7~qCdKNR~}%h?XZaQq8Tyn0|H=`Kzs(&rlRP0F!2oV~H{S5MfdVIg$Ag za`Pz|IJLuX$%7!u_yy;sB^k7IL&A}I_&IYB)3R@YeS^%He5@{8b=aB>a+}E}S+jvm~jz%jmag3w<7& zME(KUG}iexxjp(uo^Pj7@;YgHWH*`G?F5adN6?IhrIOP97J5A2f=@AaBCg$+UM31V z)eu#>vsRwgd0LZXN*?J?jprqXEahKCZY7@yRXE4b9M3;8#~TX|;({6dc%Z!$^V9y( zZr2oYxhA3@cSD---jt4gH>Bk+>hMB=rJ&=s(29xq0!tG~czzBoIKPPOhHN3Vu;uih znj(FCT1l%quamCIRT4KZptz1~`Z{wjB?!FO=!^O^dh>l!UUZGlITuoqVIjSIdV$(U zR?uR+LVfCY0Uqw7mtp5=N^l9C-d0SZ_ln5-WdTLpiKEj7)_muMIJ)qqfF^G#rhAsf zG}pLeoqUykO{=7=^;MLzP=RckmecdSQ6%FNK;=i~(Ea2!^x{B1orrCvnj@C9 zZCfoi&NU=84RczwYZ%o%75=umQs~Y2f3$LE8Qwnj9bfD^hGZvcYt=(HsBGR4{xgS=vR>J3&95cqo@Fl`3fR8%6f!#Y%EJ zAY_+Drjy>YBw;VqWryrYczF0J9uKXc_17N(I@NFio> zn6aGig0Hkj36a}_#Wx1uN5obL!N8^%iCq_p^vaoAKh9-L$J~ zJSE)NhY8mm@$uMfj1C!0?sufAAjq1%80}64HzH_YXaZfmv7F2nThR&sub3Uap3@mE z%&Ak{h4a{v@<*GHs_FzfHFpWs6l9S0u#5B}K$lssokb178z`(z@H0pTQHe0OFK$M< z8lysJ?8Hl_s*qzll6BZDqHl{xOJOZNd9{IDIS-Ogmt|8wT%ZM_40^7-gsNUR(P;$} zs?2qwJpzZY`-lk}?;6XsJ^PBE6D{dzRTy2Em_SKySJ7J4S#;Mzf@zPCq3~03bW0~2 z@9cHL$bfygOWcOuIqs%AOQNu&Vt`2r&trX)I!QcMrnlvClvOg4WU}^?#ne}9->o!k zI;eulDKDU#OJH&}#pv@mjeNRY`IsmP-tb=#p6&61n+s#X|Pky z)`PL3J{VwMhyMhAX4z*0`u?dJ`$x&L*ne}e=l%w|H&ugv+S_9VR&krgY(mK&lJuuq zhK^793D#bwwBdyZISbs2?f2QFpRP-4ve$9Lb#*2^RhGrs3;vCxIy9wGjK1$yrkM+! z=^#s`fxo(%GleD@5&bAMA;(c_^m=g_p0Ks%%X-T&SW}ILFEb^} zK90sunnEcKGpIU%(V-L%mSRyz8IOfs_b*49AEZo;D&q9yx(N8Wa_ZiN5b7Ux9;En$7+Kk5vt2mo`99Fh#(3Pr<^dNf? z)*Nnx@d<*5bdLdr3{j;SmbECgW-jJC^q|A-&)|1giTQ4P2ID$&Ve0Okpt;`{y2dWT z$C(nml3ofuAKL(Hh2Md~plDEh@60C4>##%9(m{2f4-A#%SXQ$<8#CcAcmLm8R2`{I z4avD^ta}pp{Xe)TmjwUdr52QCW4OIX^WoM|SzOoAf@%}rmf`6H?StFW3DsMmfk&NJt;OWdXFG@RWNsrHq z)7InnVeW_bI8mV$8x>_RWV%r^YTB9mD<-B}#63gNpBe zb3Y>8nE&b7jOG?X(0__7RH>*{!U-Ad|fdbU#%Bw^GE$gnx&$8SR|^o`2}|*MJ6^j$x?43>wfG?zyuy>CC>2 z9V+46hj&7rP@WDtOsK;v22FVCcrU~{2wda+QDicGHW|F0Kna&r$VsQ&w0I#}gR$;W0>mYE9W%U!1%gQIBBWG6awMu&7O zq-nF6Dt+}DOO6je!=ItT9%9Z#j7?M%_G`{G5a>no+(*%sqI>uxLW$YRHDc!4dfeWk zgBRT7S?r2nt|GAZ0J6{p2+IIyS1Rhg1=mzC-%J}{D8R$|~V5v4zg2zob2lLxu*V)Iw zRp_&b5Ic5wcs#z0*C%~PMioB`QC_DLJRZH`%nuEtHG7)SGR+Xr;~maC&kF|){eW(D zuW{?-KhX0r3NIbjq&+X)DPL_eDWMLD!A-nAO^Lgu>%!`U_ioRHvGmz&2zhv$(4L#_ zblEAMe8lck8s|w65=?L)mf9FRb`V z6)H~BrsQ{%$a}s!MYw8E^vxLbe@AvMz6eH zazo$k1EX7K@#}m>(@OQpXu|E|omOHU?Qt!Z6G9Lo8UqenIW8>!Pu+~S#?53SMTh>ttQ4U#$?4yA;vp%TL^80hyylgvWHE152E>dF`l)r%8Bc9nec(P>)`pik(j7h#TVoM^; z+1yX{kme=Ju8a|91|0}BgO5Y#Yzt;ye*l5jyugPXW%2mDs6`aii+&Rmyw*&IKc}ysv7pLRAfm<=5S%jso3D)f$|yCS>%MJ zOtE|#i}X!asPBwE`X^A?!eZidNUSH3JT-uG}>jJ5v`VN|VP2v9fYOzbg8724Y z20XHiWc^O(AVv2GSZUv;fZL6boqQAg%#GQO+$MPHgiPhuB4)jJ7Pp<^+$u4WW;dRT0W1dukB@CZ9+!>Nn&H-k-fTT&&qpUG4e<%jEPMv*NYBEddM4r60xJY}7dPdLjjPa*EOB%3p93d{8FKH#9@FSnh4dS*U({Mk1u8Nohvz4AR1dU2b}vf zSOIZ9(npNCyhVV(kGhJ_pM%P8G zL&A>z8(79*Ner5-+=2g%9KqZzB)E4w+mK}&lWT((h5xvMvw9l2Nk3fKgz4Sjl30bl zPcI^F*@xmnu3a$y8PhzCXq3TT3RyFn)c*?i{gzsclxRoEjrF*$AsHv-cC)AFlvsD7 zDSzMJgh>UwXP5hT;?Bhn@Y9YCT)L_r&CfhSpSPo^GG;Gr&oZL<7GKcT{2D4f&P4u1 zDK__4;<8d&-u${rO0ucl4;f*-)?cpFN65a*WIX|bF+JJ7%}25)_u&r*k) zvgv1nSixc=+#{ccM*j@iP*X`ZT=2WOy^q3dOIysY&W41or$PK(BfOFoXF=UXuz&k# zcH`|mcv0>JI`{WLidY}F@OBYBOS9q!dK>8NuS;BuqytlitFV&~hh#4srtdAr;ucym zc1@4jd_01+VfNJ4YDb%rJaM4us&H@DV^^*!P;Nyh`mORr0|iaojfr@Bk_2(G-_UA> zAq)16!StmT!k!|Gwyd8=UM1QBcYYbew%M?rlt^a%OPefjs*(s~D6meKtlnDEi=}RY zr{fS=Uy!GZXM3?;;FergS<#&{;|a5eP`#k%ompmuACeaEcSefv>hW^i{8E!5^aUL_ z)r!iPHtA>g;$l?=dNJn^Ek5B&x=B{F$!sWz*-DeaBsJ>2r$wnwTUbVe9II|9#}NV- zuKaEuH7Ts1eNQbZs~{E=Wwe<0+i&O?C_zf+ci=*8b=24Q!37sOu|isb-g{`Ucde4F zb8%^QvyfdP%hF{P*v|E*v0X{%UD#AWrXl;u<;@6c zU9y$S+~S8G<7d*OViC2h{fT7;PjI=~U@CB$KsydvldQylLbl!)Jnj%inZq(^)1^3? zd14-QYG{-5d3V&kqRK34CefjqiPV3Dr@GZ%RMxYcM)t>0)B3ZdDX@<7M{7~z88_M& zA4!|~HW38}P-&n!oygN96Tx5G;WUIalgwz{<^US1wTW~Bgnzf!jrw!6==~sBYP)!r za@t~Oa#R@o$`fXoyFA5TaHg{T&LsXojeUIOjtw1Jw06OKdS@L+gZwf{S0{|dgnhv> z)sgH4XG0_ECQ{DN!Ni?xLciU=aE}7e^dWAfk+A?T1Z?G0HjkumxBWE!bOC)H98d3x z@^EI27Sq*JX8Tj0qssTWI6p;}^|l_t&zH&t2ASvmmq+pHa|8JPp_V-NwU8?GkK@w2 z3ph)wmuTJ2(>}j^F6GWX;e7mt<8NkD|D*flz?YG^pbMO^i$I%6SFq4FfNIB0A;Txq zB>A8nLuP2xjSs#wc&95Z`0zutLUJ$-ZjL7RO&R37MDT69hmqKUvGhi254KHO%nrSF zC+`KbDaX)@dev7`v{?eR3ioU_<|-YPF{U+ zGn+I6-D&5RaMmVd?Jo}z^s`ATXs$*kE#8+-Jr|=%q%oK>*Y5^==Sq>Db4lW-7QOuP z62sI42DoQ3&8rTe`(YJ$*k>h12>$NdI%U*)=04ed$)?kC-%wF*A1k|Fz}YRfr>*=; z3^rND4gYZxCEpj)=}DISocI8KUFTHNc$R=s83^UHfaRo3fS0d%=Hx5K?1mP@q3gEH z>|8!rUF*YJZ&PTC_il>Zr$NIe|Hu7Z?TvQ7#?lA<@3^(X7&C?)<_cZ+po!ct+PQQ% zEwI&PoZcZ^`B%szb&jE2!Q1WLWMEUXNBqZ083X>rtF8N76 z7^7@SmWRS<;?Nk<+9{$LyAGkj7$N_u$A~N^8&LkibbOe3M97me!5tO@cvIJyLK;2s z)es1ED)3c|i{FS~+0qMkN3(I|=)v^T z;0qT2ipI6BzgSm-9hCin;7-=)nL)?E%@QxAJ{1LT&Q3A$?{)C3x4n~7!fs?3^EU6 znT-US*V_YT2R7i086(hMQi|Ow`vt47n6cDG;=Xh*LSD%Y-e9H)GPi18_$bi9x|Stl5}I2 zXLZ5B8e5T1dM}I^cmq=T(rnkHcCfFJg&c!dknF9<26R$I3HJsu`J$h&D?y3XcecZc zLxY&jrLPdb-h>rM8L>>QPB=YYnpNMC;^aLPssF(-{HGBJ0n?<|6f0efQ1U~sN1CX& z)SA6(8q4(Oy7C{E*fC|!n9ZL!1V1kJL5Y~5n0>&C1s*GdP|qWn;-*L?GgP^rbRD+u zjuOjSbPc)&nX_TbN3cE1{n$hyFYQ{s3#&aM&wfjO2CG|zaCmLHkYOs$(p1K??;Rn` zK}vXcE>>oKEj`e2p&p)}G-nI$y9)QGuW+r914%XuUcA}`2X8Ae?TIJB&)|;e(N{0{ z+o;E?R^&oTcZR5=^Dj6_&Ie<`+gR3U4j;s-z%Swj81EYf_0hMWy2OqxT^t2o4+gP0 zBV>db$xq;OE7e;8q5%a<<`;P*;*7Qcx~exW!SsP zW|%X>Qqa2`$^AtiTsgEJKSd2Et7=#BuCgY%U%lv5l8E82?GMCk(4SN!!9$|P*Z_)qYn?>&&Ih|3~`gLB{^k<(Xj`z)O|#T#P7;b$BOqDp4x;CF`byP z{WP;M+QrrM?!dsXJ~TGAq0ynPw6xlYZY|q{lY}g`F$RuwM%RKoe;DJ7hg(I~4zIXA zac%1LTSsj^)5*Ez307u`Xm#^?dOqbJ{#Y)-y0e?%=Jn;UXo@@ZnH+=JYMoHlro^6% z6ce&s&Ovo(2~@>=gwfmXz{xok(6?#<^R$``)gcz}KIH&J2-)i0BSwN>qAF_)H)W=K z)1giN6F2GaAkq%V#Hmf=xWhR^$RycV&<*2oihCD)5VAJ)!q32^(f6?Du?0O?FUM+E z56AN{1{CZ!msU-jNO)ZEajuC)??W7SJ4>3)d+;7F71iOi^DTICl@!7K!Q|7dDrmf0 z=-rKAn!JAw{wHw~*B5`pxAl^gXV8L1VjkEQF~HrDo6mpj`N%EeXJY;DW-MGHNv!`9 z{%}5q+&4dL74~QKL0hPOof_@(8BDUFQuJbb8(z13fFoCY#LIu|+1a*{+{D@#EW9B> z`Oy>T(L!H(W@kW$Tny2|dlNWBSx|FKE$Ve&gU1CSaH6>uPIx5X$v?(qEHQ-Qh5DTp z`I(roYcNHBdChtJ`T!}iX7d>+5vWaPt%W)5 z;|_e39LSdZ7Goik$1vkvb8upV4$27r)u9O+ar=%#`2KV?&J%n$Q@ys(l7~0B>5nw< zW^ojD*K8B=r4(>R=qUDdim=BYXUmtDj1as)Q`r0>d2~Ct4WAv{gx^fGa7junr#fN_ zt*#K}u)p_k@N!{~v2Y`ft{IFI{j_nzs5uBfG?=<@mPn~=W&tY%{%oEceI2SrKQ5*V zvqK5|33g(+y<-H;I0ri9POvKzmD%m*rcAbV1e{#HncW|!!e;+j2*VdjGgbSukf-w- zl=oY*s+41})llg1I5Yr;Ns|~~zFTDSfd{%-XlEmx0IAAnU{2;}=Br#H^4eVtpXSaL z$xZtT)tYf&|56`pRy>8~-vcnQU!HB-ei=>(T0x%aS-44eK&5af3o7{zL7xhtWJx+O z#dlDA%#hvvuFDSVNU?g09B9q^1u2)6+3T@3Y|W`d+_*z8vA6pPZj)2Ty(1jilOIhG z{!x#$9A8r;6i@pxWPZz>qp)V@_*kJgp_yp!1NQUfXcVX3Kq1P%~mrbouW;;3) zSW~_{8yVXG(iMlnxa%c1Yi0}#8#UV8-BOge+BZKJS6-WQkL$R?jNk)&VJj(g7S$75-TjZ;T+ zMRQhj`j*2nMO(=D(2`<@Y-Y3T={`8`!B3R%lP24VCYYBYXur$O!Pn=_oK)6bSdpd1 z#x!d)4NV8;Vf+~!OypS4M@KfHegKZFRb{S+jiBW1I7r<34+eeP4oUOpiguj+0R{7g zZn7yEkoPhcl!d(GAR))Y;-VO+wseAXe7~qhIvF;cIto#(DeyJr6Ic%U0cm4b!wQEs z$aYd-e0mwI8luCFO=y9Y{{CoUGlP?qxdI73M=FA`}v~*V$)LE1M6PU)lKV>MB{uH%tPh~3? zzh#+{1~|L-6>14NUx5{q>1)UUmb{zFl{)vqggPzCdl!eI^xF`mBXE*y^x3e*r@5Rt zX(*PLi)or?q3W#@x4(Wq*2xyZqwIR9w91Cme z{c5TRw526C+pxlNCOw#=L4NP{prV#OQ_qrNVXIu3S&A{8&Rs_#7CV^ix~16ZXiV!i z1(4SdH_8zh5zc5gisd(Pg_FJ6EFnA7fA@I0xZaf-YBo~0!fpzApDSc^x01J5FR5-! zp$)t9Y3tLgw6(jE)OCvlFKQCyZr)BBS^N1WxkUPYB#CaVDxyctmE`1em8wkhscAwg z%~|`6oFBK)2ahAPEPOXzAJ|B0^IfT<(~9~9<|1vtll{qT=JY?L;pQ}Xs{ZOmyJ7>V zCRq4>&jg`?p(i_?xSpak$5FngBRl-*B&cZJgSSu8kc@}WhGR1+ZFvW#C)toBtfA-m z%P}>`iyi;30Zy!JfSPp`u;$J^xZd;$gv<-bSSjSYZkK0;Qm^4h<4g!--yr8dJtlFn z5*)@^g3{?+I5c@STaa=eqF;7GZcGtaR0p%Wn)1v|Y7j|@AEFs0k@U{pjQT&WBWLxQ zw0uJ@&Nb3!@K}?@tslv>h5bZe&2-Y9Aj@Xn`HxvF@xmjcuizoCh4}5dGF$jfoq1*| zv*0y@nNt31PBr2@CsF*HbN)LYO#~mG=JLb1@`4o^+;pK%v4hO`zBgKG-9_1vpYc<8 zJuY&z#%dFPp)YAOU*h7Ai9fCI%b0q6dhjETIDH#E1W)F3?j*alWg0n56d1%AhtYjZ zG){WH0CleX;+oXYaku=txas9q4C3WjYO5mix3pplcl+akhvk^+?uAA*-LOiybCkPJ zrw(DB8TscI)INO=yB7(0IYW-%coTQ3kXuJvQ_bj~_}}T zl&^jXT-hMD)qNAYGOrvS-}3;=Ygr=h;K+#) zZ0L~raJj`2WoNl`a za%HDrv8T{K)!l{`E@rg8`YdN>aG8xU-2VSSMnYwzd) zjV&eM*H$Rn5hwH;zOaI_-0AQn;|cWI9}#H@o<8T{LWk2jq5C(p50FC#= zc8i6XtN#6xdcO@@ARzD2U!B8I=f(VYa^_?VhP9oY=zs&8L)5BQ215< z0jAxV21gr&uCR7XQO-9%QPNK-QC39_Bp<#F(Q$^%ry>Uo1U+AR-bveci$cKh<0V)o z@U%1M%Q8u`Vwm2NO@9urrd{>^XgoCrqF?{Vm}@0;s}CmC`gIhvXbt>XS&0^ALRWHa zDGgb=pK@Q@QT666n7QW-H_opH*1QGUFmxD|?^dCQUkrp@tU6uZEKb&|%_u!{0okwW z=CbCNW7z&dxV+I3LbDYPUYRR#1yJ$}J!*jf3s74Z-1`#M!;MWx& zxWA4I=tj3Wt(Fw0A@&*+DnE>DQJG8!8_~^&K$4` zSg}h6LLU9Kr=rmd<3+kQ`@m*c7I^+S3~`GSV01x?Xk@Q8tg^2bt^R%xGNu8W<$oQr zy4;|>eH?rGV?CTvih=1@Ux0Lr3j1(T7tE$sfbOfg;Ph-I6eYQ{kM6=5IqW%{9WF}^ z2H_~5{S+D_bYW3!4fOg7dmQ2CW>R95UaQB}ZLZ{$Z>cJ(h~ z8zQg6#U6e7lll)84By}gp+l)vL4{5~|A8gjhEeI%B{X`@GOpm8;AemJ4Oia~vJ$l~ z<6jMFHlf3rjS~D{3Rex8l(Yfs>MFxZSxHj*+lOhJ58=fh{#=9J5>ov)j2F-b>Hi=oxv&9r02U2S+pj-O2#D7kq+P$~9(&BOKp!Xl_ zFZjR>%qoMCNfJz@VGS6Kz7Btegk#+sS(;wj0BWC}0z06~QlnRjGDe2OufHcm4cljd zN4*Z~JM|sbCRu`h^(v^2S^&2k(jn`BIXpTg!%n`CWag)A;Mv|Tk&lzDUGCt0u-kbv z$e&MzPoqlVSg??BDP(blMj12HgHiCG&;e^1sS3wqvtj0@v24s-efIjC1e4t)a7@v{ z8JRYb*_@~mxhcyD+3p<>c%TVPXZGVLA?qT?H=bNJ9mC9L=fK469ALN-dl0)7jed1- z+m?u9+oA_}be=rja8#l%;y}SOQ)zE$3peV*QH=lf7LUlj#g;qCIP{?|GgtcyXOvy| zVU>U2NSY%X`!X3L-*#cp@OKz{)X4=ZK4s(^dc2NlnzFz-Rq?8{SW*7!*1D1Z1BO8#Rozs``wUmC@J zy*&x{{Wr0MWfttfs$R_al8?tsT42n6Z@8zv5AvU0231E9YwaZFsWXhdIOm5V-%%8@ z*O!b7Q|QJyVLr^1q%Y3qbak_mObZ;q9k!vs5He2 ze(v~%f`8(pIjwyoMGNk>&=x^cPap85ZJ$R{L9H)Vm2#|2k(kx2Sxnttjx7UySh3m@ zj81E^1$**QT)Q8Y>Uu#_X*=69bf2hs$N+rZFUP+8-2=H1AgY2E3 z;7fbf4Y%V8;L;r%W-qfB##?BxFQF5er}SxfG5sxG_SsMO6sv{2w11RbWx*T0E~O`X zAJEH@9aP#U^mxjYQ{$yX%GsYxw;t@_)uNIq<3b|687}PNPBqc(s19nfY#@{4r8Hra z1wU%mKMK^XruVz{lSy_D;zW1W=)8d0KlWqCj4B~ulfWqWe1tD|RanGaOSU&LnWdc* zvfy<-!Hjq3;6aVhFFd^m{NCPxyp75%=JIDz=;%67?>z`RE&E_)pwQ>_=NZh)OM`3b zPhr#Q{jm1;FHzW^B3LkOKK%Yt0SXV~K`F8oRv+#GJ&SEH<^B#x_~#4bHJ!k7#$mWQ zVg+1r84Qgnj^GqF42stH!&na;Hv33AZ1p`0<{BaJ>##2Cd+G<5VuW{2v;{oTIt4ps z#6j{pF*c`5iTQ<$W6M@Fg1gdVcq4dMl5_?$w{3Q8OH&%0mORUqJ{ZZgk|mhVS~XT# z)CrQ+hAi@@9Fq>-2BG{$IF~b)DSZ*+^3Dr=v%gDm{WeFk3}43TgwI>;gEkAh{s$a$ zPJ{0%6_%6T2jcxo{Gew-C&}mg@VxW_wCYJSt=D2q;#38M&)CgI3k>GQ&{Dj0=>T#s zKeD$y>p-!`A9hB4g0H%=Onc!x7;)$s)E)ni#mK3#FXD#GEx3gHTv-5mbH_8^m>#%m zp8)SWdfB^fSl8Hxx-8Pc3b~JxeI+_&EeFj6y zEJ|1KB^8g1$ox69-)5nJ3)aq@v-W%TrH&+LXA{eAkYWj$)rXQDX_)?78cMXCh>g81 z(krO;znK=fZPCcKO|s-J#;_zZGaN>Htx!ark!B?~)3KUT9NigbN;9I6^zB19uA4vii>!SbjQ>fOS zMuo^0zuT=uyTS)7-?A6WT})AUz>yKUDiMiKb(pv@72Ro-@Vy#|P$_%d>Dh~?;sk70 z3}qr+G{}W+Z&GQppDA3UP2{`2hlW!aEJKH3k9jGYk}u<0^**%a$0I&$03B<8Wmknz zzo5!cs;eqZyjM0OwCW)8$_$C<8wC<-{t@fEWXU;YWp=+|G3#D(jMeyJ!gb#tM#L(m z7V6$02GJSMO%mp1Rs`!z$)%%s59H&Lb zbjp$XJzF3k#}RgG1*ZC)g=VrCGxHpvr*2G?=8q)n0^CVQ03BzT`W>d|Nsu|(dl8%( z3Hj`3ERz^T3RmdB?|2=CTUKJha4Y6W=?y&pcoN|+REbvcdCa1G#3*N73^#p=XP#el)rJnyMoDWQ-lV@mcinZy+{1u) z@7Ex+?YEFt$ESFFfwGg^V%RE0Lsqgyjk|Z=6Eo>|G;;Si@?)bGiK6YkA(17+T`F<6 zS&C1+)6AUc(I+?dX_3-33gqTDvxvmlH_#Hs!*jVBw@lxZ4So{Mx~-Nb>YsBFO&Q$0 zwlwLO@dR`BIN+1`4T6%MVs}d=W+I`-_6`LBF_p0t>VPh@&>G*JNVIy<>^(+)- zEJOXZBiQ#a2T6Uk$ffh5zqPKx!g|U+9?-+{4Ra~W0WRI4H@2Vpx z{<#c?UW|jw^kg(nj-Xuaa3*lo9cHWJLv*e?!i>6n*h0PD1dGiSFg>XcT^bJ=C8d5` zih9TVOq#ICpbL{vrLj5bC)qC+GUSOFWt}*vl7ua%v1jEJa{u-r+#2qSt&d(IzjFyW z7<+?V5oySpK2YRd%*{u(d;u1ujv|Q``8ddG5VhWZ$_CdX!zyI>odr_#dI5NMdp5Dn z>cQB6E|krFjZl#aY8XYXGT4OexKO~}J)c12ALOD~n&6jCf{{*9aL)Q3ItHI1yE_=W z=(;)``4o1%=rlWHbt>*IWymg_p|D_-Ecfnn~AQZ35;y`09-npaE2~3d%b@v1m~Y1v)6*u zw#MP|r+(bOQiOd@LoLRe^rL24BKpFT@UbupSq+AG-f$M_r6CwqFpeo`wPTcvtFZ7K zAsb&SlgJP^#QP^9<5>=J4jsb`o3qRaM+FjV@&R`JKftr17!ISiqyAwAH=@0OTP7|L z8Z}2?Ur{zD+(==p?=-^or6$?6&5+c0?!;QdOqRK1!ljtqVm%GaxT|W8C|{c^e0pXg zdu>Y>0^(m{v1t?HBq-xbUzeZ!HVVIeR41mvZCE9zO#;U1vpb#^3hR#FL+4C0ZsV;I zHm9C)K58zrWwJmU8DP;yGR|k#djeT(&dv@s+e}Fk`bG~fI_bX z<2qiQh+7_DobG(wJ8Fu~oBnt{;t*x_q`>qCCGyU73EW%mFxutyFqwH9#s_v`M#3(P zjF^dwYgVCMRtJvnp21^#4|d1ZV1G+ETVgSdZ#%*BA2!!;rT#s_2XcORt(uSWhpMdo zP+8(tmWQ!x+}S4eI4+W21Pf58?d0UJ?JwK8$bo6!B`*pHIGe9?!=~@`2h1IP=Nz+_xzuoK4w9c4d=4 zceL*^H^t9{m-GzcLmsc>gQe{_AKhYDWXL0cMc{( z{oP%-Nm5SUxnocozX!*USQ2fgAw>U20rcj_L3UdhlPZowYD_d#C5j+H^}7RU8@cGo zE;PNTyeE5EIR&Ni)6$Uw!C-W-my#9;7FXRkvkiRL+tN=hlAxs+LD2o*v^<_yhJ2`Q4OC{z-q zlm`6_q4B%l^ZoM;&v)*<&wcKB_C4qB_niH%_g!n<1a%f+ z$DU1-++26LPTaZ8(_y;2kjwwE`HO7awGqC*zMuzwr-Jd6cng`BtdJBO-&Ziwb@L`~;s5m&83#`<*IkSM*G?I4krl#%G3MT)D?|nF8+Y#Z z7Mm+s(qm)99*JP2Ice1TZPbkv1-Zb)K=@pVU`Kyg# zJV?HF0#SB~A(941N!?yGy4!0Vd26FXlVnyC|7!;H^YRTr}f)} z>jjc|?=F(cqVHtcy%)rH;{tNX#fmKP>LF3_w@788E;;KRMt-}zBIC7uNprw4(h&B8 z#0ESk*K4kk5i3>cu8XDQ{w`7KYG6y2zEh^kUJ=C5ay5wwIZ5`L{w4QPdWrcdA-YmU zh<ALkcIiY`urer(u>y9Tgk7Se4kG)B5uNrA}^&pzb{p8)e zIAXFif*cM$PY&C}kdtxwY^D${As zlSXn~{w1j$oJn47d_c_kkEF=7g1j2nPsU|yQSs+@NU+L6l5xJ8TT9GWOcKRACPFBdvfp@o(-dG$6@WWCU4YRx^O zGx;$&=cq)omW&~{=O)?=9UCGOWJ<}@l6D*CMaJ}3&QUTlM3nws@Ps_yc7r5J-6!s& zZjpVrC~=4pp`OtiR47`BZu-E;)XV^4X6sKX?YhagayRnTPLY0zPbc^D^=RyMBR0vd zgZD2Q!*nIvNUMx2J@8bB4nmz=bh<@K(gVqE*Pmp*(MfWuRF;muAxu;DeIlt%r9|aJ zEU}&0L#&r7(R**hh}!5r(joVWc+4vyjgMClmEv}rox+cZjm01_ctzyqY5~;=941$8ji&zZVu{M_bHw-A33A?kBu&~bL6<0(las5y z5;e`ImnMf!$|4;G;*=8g@~yolf|P=$&vZ>WFK!s zUN1XFc8`)K>w=5P$1^cxugxG)_pTtn`hJmZN@Dcdx0~ec*6~DV+zIj|Oq{FC41$~F zXH5J&m49vi35h@2VEOwaYGkbWq2X?f-5LxF@y~RB+&8lOSR}DN(o25sizcDjqiFDH z6>88dPP^{kB46C*ldsKXAP&ne{z)O zC_Nz--xJ6mg$|;m*+uTsU!-+|99`AjPhvNy6Dgw(BD`u6m9VcQDXG~cblzJcru>qm z{O4hlyz~S4w&*WeUam*$nghweA3It-ZXErSeu9i$r$`SC-z5gWgUAfCH)O_?GBS5= zAbI!n1##B?M8un>(r+vD>G;2A$>-!Qaq5oy#x1^3Z3-2X9<>GWBi6(6~ zMpL&N@94}MU4Qbq(@QJ4ZJX=47vFohE0$ll zmy7#2?UMssc1aJHeeol=Fyb{=V|0nzE471rQ~8{3tldD5RVmV-aW}}Md-n9by&Rps zbQFDlAc3gg5~E*+d&p|ddNT4?JK5eRO?yW=(67E@sVgTzYhx0~=JTL+QU)|;?NPkF zBFzU*SLY|aBz(5mSNJBhVDYJXd{U`~2&sqryB3_f_yrEX2!Ecc^R4@(`OvDPNO!cL z4tLDy6N_~6(EKOKpEyVYTqu1K)j~#ImY{X3Lr7%jH&$GELsS(`L39j6RzuMf#MYYsPZWawc=s?FsH4#nsU}CG>Pb98?CJT0a zA_}JB)PNQd*)BWUnR$!sF}_Tk9c8Jl`A5=MsY@5~PBc&NESXqfMejY0Bn^Mk32wh3 z-CG5DL3$@iFE^!wD+ukQ$dW26UnQ;=EP0Gf*MUZLNZ^9(6ZMpBvQSd z=q693#eW8g*a2%AerGMUwin~d9KyMj&Kukzk562#w=9be9nI!DTC@G~b6H5&TIREN zCkr(>%--6CGL1V?tlaZ7OVLSY8>@0zyH^3*qke}yD8I)VyqlQZs26N)djo4JC}7LX zr?I6QdO1~veO%4Q6P$KhJUuFB$HQS$5$|BwrGK;x6?3T${mLq@zj%5t#@S+&U4uLH&fa9XCqnEJXuz_y_+*gD&cCPk8-y<#JR@wbu`d< z3eCBrN*nwf=@SQUvUpX7&FxhdbjtMGBuI8Fbs<;C^TTrV`U_zqs?<&X@m}q%0-7TtbDfZ5GjRC0#}jXSoC%se!c?u;rSGNZ@Rf|;VUe*OTt7g<7l zo<1S`vsxnS*hzAZXAz%RV`|@go^FN)WVCd!p@EeDvF zZU{To7R9c7iD!#^(^%&~E-So!nR!Q-GM)MN*rfIvHp8ufeGqxYM*e-pUhjCvQVrg* z4fc=NzNwk)r;i1b7}v(_ZQsmIT=J2o(_8S~kTf0k^Efe!n7wU^5> z%;!!;E#Zz|4x<*uBGg;Hh%_X>BH?Ruh`;dylA~-#4R2l``}F^jHCuFOJ*P}(kFudN zo#d&?qE*!UuoPV)nMq`Gl8IPXAkmnUNUSeZllj3Kv@b1}tls2Db{L$dhfLPeM=hb` zddv+XGDCyD>b*s>eU{Q#^Ael-{t@*3pfAnYCc|aQq;e;6-f;hQXtJEDsZ8wba;9Ch zjqSPP&))kUWBj#K?BS;rR&^|!WmH{cm;6gv|E+s${;fLpwY!mB%zDYDyy#%f9o?+` zY!5SjKEy8E`NE|By<=G|iR{y_UF^y|Qs4HG-WfSIj|7AFKT*le7 z%NQ?Hgwk^tFv%qYgJx&p`8FDnqQ@|O%^qmAuI8OCN3!J7Y^YdtP@%7GT*b*E?ww*H zy87~Pb$1fKP*IWJ?mm&9Xi$UjSQVC(zn59uJHzJ0b+W9>pO|0jARB$Xhs7j+Vs}E` zvd`Qr=3oDmv7%a*zoL?b&bY&(Ixn$&&s=7nmclB>oMN&0$Jm2M{w!5!D_g#QDVsT9 z#VRiAus7%5aF1vLS8Axr1+NOBF$!6vY~Ud&c{7T}o!m~&ZRjHtG#-!<24hIlt|hc) zUoQPGrjiWa$tA;{f5`m`3zGHX2AQdoPA0}bCh9E{s7YflnK`6JT~2KviAjRIHhCjO z%Q;##<`Y@*yp3f1m7_7`8ZnXb!;CEudUsmB6mE*!7WJ~S9bnmb83IH zuGl`dZ{`<9&%9^Tom&|uHL@%FYS`4h73{NRF)Pft!1RVPSmC93rfn3-zNz`M|E{_- z_1m*qr}j9uqg{xdRlm)Zyzc9%=ox zkD4ifnn0B*e7r$Ijc=1&KN)&p%p0Plu#(7y%qFIWcS%V@0WowPCQknIN!R*z^6YaU zIsA7tp?<|gb;%TxvG*q#Ve*D-YEB|?wo-Jo#(KJ;VSqlgisG_2yyJeDYBNoP8Emk9 z3v)LOWq85!(I1UL{({Z6G9qQS+;FJm>bG`(QwWY9IT8Ra#tI*v109Gvz zuy5c2{?t`rbYUe5oJ%owWeG0p6k^u%T>L3chwPnn_*lfDZG{1(WXEGmVKCD+j^{&9 ztK!&AcYayZ1~$D-leJY=vLSj37Qx5ib7(e7?%5%r*c5wqj>fFjqcO!t9Xd{m@CufO zScn+3#Qv}k1%0eh=^JYbdBfHOHnYX0HEgy|CDSP{W5 zID_bHcu7{8PN6RA{*nDMa z=YnvSWR=2F^z)hcyBo})?LIr2^o(g5zGfjIA6THlcV^i)%=k~jh;5d@_eCQRe?S>8 zT(mIFWfatt$71nsb9`yC#K##kFn^X4Hh$cOIypT|xZK1fb_jI8*{q`rJ-ci(q1JbKLrYa5wpaW#9p@CMWUa)F&Zl*H!ugtOQC z{n+9c%UF=8IeRwrkMkOp#0AGrBkEZ#7Q=5vu`L%B1pw!KSS#ZHhP zN6N_X_7)QI%ZfUC=n(d$nOsntM8l?Z5YfsvMEvG=^7W4kSrmJklvXv71vB!2c-Il#;hB_rmk@Jt1^TVql1{!BLqCsk;5052a0Z`bn8QLV_WJ4;)^j?D z)u|+~D_gSIy3Wfie?lee-SC9nZ)#-^Hh*F-l>cK%k$>3U9#L$~l0sCkJl^K1VsoxG z^z98%*)|p=b`o&K8sm#+qv!j4d>-73)GRxEj@blTQ8Q{b^qN^$bRxspjhA(ipdDf9 zNS>bv!}@9HOmycfrna!X&W9oC{Sfz5%Hfh&jqW*h;M^MV)3X`R)mu>}-v<3iL4R-8 zhPAt1;d($bY-uBUOzV)lvKq5*mqYXQLkxBu#y86s%;20L$0SaJy+k6?UZvxOh~K7y z7Qwu?L`Wh0p(yST{btu!^|JZ)pICSGYxdip{9#f zXwNq}`b5CZD)2wZq6KoEj>$Z0Wxk`k*^0nHw)BE9%oj)@vsMlTpH$$JqXm2EQFt6W7Iu|p2r0Bd z-!FzQan6wEmZD+eW+)oEWAubosC;mm9hFIA2?xj0Bi;M??i^*_dqpVcf7%|88?E^X zgEBNi{u%a5)*@Ol6*uoZh1-W(e9CBoMPM_qvlU%6?f5UP6GO3GFkSZ%iB(-_;5%{k zVLRSiw_?Y$X2joVLSkbr@|vDvKr|In92iVA>+06Tu z4$F*f;G*@nbIw^;>CxWrWbCJlB=P)da@9XT(9=C8*ZWIIpqezb7*$4c_1(xb9~By= z+)XMTz9;hPdu>upDIMSbhB(cAL3;L8lPS{&$efTbWaET?WM7Of&9AylYSr4wy~Gd1 zP0-Knvuz={1q-OTzAPQ7FT;&n8OV(nZ|Bs+P1)XEE7Z)GsMw)+<5cWxyg?+>DPzAD{h?aiOhk>s~rm*NwS z8u6BD9T=*cfcsvTpmzT$VnUyxXX|U6{QDNddp=^R?^oz3^x)F!|8PpK2V)+5!JCAS zaBO@Fh4R<1Eq#V#(hd0h=^{S5n&JJMcC7X@;{UrQ#hdStH1HD&`sD! zR;|2+x?MrcQ8OANdKcs8whd^w<^cb2M|8a6(fq;&3*Jmd!l6m1*ft)1f5%{_j~?7N zX(B^i1<%&YqDe~v`WkYnmMjU2rTdqJl7*v%>A*k->GJF%o9z|o9Ek;V#PtEP^=BfP zIi`jv)YOujo7Bl?pDfbhh|uLRWkfSYi!RbyLEib65btBQbl>TTl+;Zk zKbGvKy}lpG?kZ)PoPUOHdQ7>3rZjHQb%+}`k+SX8JJ^FI(QL9>Hj@mv&gQ?bW)m8l z*{Yhi?A6!-mRTx-x^O8}PL@a96;*t)(?+DzC~QbJLbS;QDD+$6SR_My*EEz2&cf|4 zOHkIn4oTnq5Tq6g>F@+-{fWZD%2fENI3iP9n9kpN9&cVn^WtU|oP*>LsqHf4RiCKx z4u5k|`#1-GCY7P4uoWLm+tF_F33`XV;&x>phPMsFq~sSQt^c9u*Du5$9L8&lepny= z3hBO&=(^bsJVenK?KtnFoi5s(hUE7=DGI=AABD#wk3B=2Q0y`hhcdi2t8i z_%t>ZwZ2g}axNbKsfMC3+7HvL*5Q-fQv7)4h_;7z&?FqNd@@{1Ct$6VF>1%@OMHW>2>Qox>Xd2xzcLYu0)#wjVVQOR}N^gXyQ0W~_Br&s|SXdd7 zOmj~vKkJcz=dUJlPKD%f%v191uMCxvtt5*dD^efJk#wYC8EGH$OQ08pQc@mAg|aVG z@3AVJc%~(H`^qV9dR`~D=71gxDRyCfGlE%)(pk3M=Mpo^tzv^-&FpCPdnSMU2YX;B zhIb;e*dwV7aV<@}lhegC10yu`nxL*=BDSrbihq|F#_}^k8=X**u^f31H{!*-{jg97 zhW({zG`qT>@%&fzy2J`zipqT7Pcir|;gGJpoLyXQ!(tO``Tn9KEOOC!UZuo}pWgWt zrP=Qw;rIplxnJ;2u*P!&hmhF*7amF?e1N?e@Ay!HuW}RP`@akGqrU%zSNRaWqzlGf z_<}XQUj+ALCju24@ZSY1zJJ$v{+NS5lRs(8n{411W-eiwN)&}Q;t=JO`CDhKFgx=z z^Sk7NoGZ~N><_}dQ~NOH(niQPE`x-sGZsFXiJ^H6q774UPGur)q?+L9W<#uetBbXR z8rbEgjQH;|I3FyA5zmI$kWUwz^Goc=$rNTkHJJ6jc43wpquHtD?VRzh zIL__NG%oCh2G<{&M?;2A(+1%KbX@9WdUmP}bqQKQxX#u41-$&gPVKvi9m>R`603x=*E{wpR(S z|7xHx2l1W4Kwg4(ixYWmSEw=m57<=j<^P2Of3q6=&V3& z)?0_S=CQaL_Yh}NMfe<#8f=lxW>(35e8D$&etGEww)xa-gdG0MzAjRM63xO@n>Gy0 zYr`nN_Yk_=ixY2t;cLAxA5bRFORGrnzPDw0^$tmXPJ=kV?YA&rk?;$Nb-m~}e-GZF z4XdWLVRmR1w!c(?$c#ViO0*LuKdojnJKcGY(}MF($zr`+4Qz!(dAG`kcr`H=7e=qe z8>c|53<|=QI3H|%xd|T0E73b|F`6Da!FcNoz?sLs-l-TTWsYoz@pwMg5Rc_`A-hZi z3NI89JYO2l1tK5@!|c?QPIj>S8GH1vit!Bv>||>^(|>)0nS|LhjXEXv>(D#y@TGLl zEGU#y3){`@o-v(UrHIzv5*CL}0u!-r;a9g8BlZ~M}@c6O25(@@Lp3~1wAME-Gs zclFpre+PEfW;+v6jAL^iTwtLY_nBi-3+vwYk!9Te%^nv?VA>Bk94}D=`KFEgYNMeH zBh)N2!!3`=XcFQuL){)j6P;ixvj~gNEknQeI*iitK=az2sJh~h$ozviHE;}aFCAbu zG7Sf36Kv|N$Lz(`Xeyq<+bz^ZL$n$Dxy6;={%s9ClinxL>Y|Za@C!N?FJNx+0p2$M z!6R}A&WDBh#vloP^nGc5SKkOe>zxwcb8!T(xmTKBy;XvLv0a!CZ5l#E;eYrv@E+F^ zUts3SU)bVt0-L%8#~pc`p4#ln(<2kviU3`NluqHN+^vQO>cPGe2rj}mfHsAUpdt2>G%D^U`M2gSNp!U*3yu4-cFCQJlE1o;bsw4bZ+ouVg!_EbVj}yrnQEOB zQxfnkxBNS~K!+0U--a_>;r6B6%@5n?@u)Vk|KUE;UM%p-9Bd>8LFdRT_fB$9?Kbf< z$|eSWlHBN;3*476FF5fjLM(K&4jW-_#%}9bv6043?7q|*mg}>fbzTc#ssAM~?c4(P zeRefdPibb-S$*tGoG6ZcmW4!}3hKRdPpX;0BA0ojxlPpF>=>J z2#;BTuy5-yY~cG6?%GQ3GgFW994m>(I&jWJ^U?YmO^`+f4f zubT?Lc&7%>MvvmN<2Cro%_@BDa(Q0txfFjje&CZJ{|Cp*@ZpMjylsdU zOsrz@T|WVjLW?1$! zE=&}r%JTRxULAhn`gnNB2!09^aCpE12dZq)V_}O_K|S|FYYzUrT8wMAS3;^}J*L7P zU1d8kWa^7M+Ycb&OaOYKL(sz_@umATb_(KH7L$vZo9FP+VkgGTwH16$Mat6mcsM~2 z+sFwx7NJMCPm<$ZvnTSKo%DJ0hq8S2w;zZq?*z^0!{;VZ-ak%;chFPfgX7it5578l zjifQZ<*5#T^sqW#Jz9z1oGQbQ78BzW_Vgny>^*i`4kBB41b>_8^JUK_^7lr|@jHh{ z(dB~k<*f(>C#WOnj8v2-@>qkNV8Us5usavd+u~s$ABQ|azOo%3iq?ApFe^R)xn3Vg zw(fvN`zDBWu1DQs7aW|t7q zQ{LvzDnk#k660W&TN%SD1JAI=_)K=A{1VIfUd4iPnppAierC}vimXC83|MGjp{zd4 zo*3hb&jh$Vvw~I=!F8``2wFG`PYvhde$Qfj`L`0n4_)DU)&n`Cc4F7ZJ+L`&2+R0D z6fF(`%{zuC8zS*QBnqdFpMffiNB6f2F!Owi@)t|7_**1wSr`j$yn{7Gv+>bK2Gvz& zJi=@7@VYwhlBU2LkX~3V{fb??g!#!OlDv1I0?#_s_@q4}`6LAc{`9JeeDVYXe&N!Q zysBV7UMcWv!zB5(-NJmUfCDgG+l!Uw75JJGbpfYWiyftAd_u1@L`!Dl;j23^@(p9R z9z-G}V=1PpzD1E%J|23-L)h&ME*M7Pr9mXR#vMcV%Mc7@1>&LfAsl?Z8|GzRi1pus zT^C&8SiTYu-z>s0`MG$U>3}$eX;3husCT!*hXgtb7ybEoLS@(8#Y@(o8|jl<2I_# zr-x0y5uZo@h^S@-DSo$^*vjK^ zZ26aTX5W>|j=#IcOv9_#g@#5pp!1p4uMxo#KUrAks-x9bA7!ORn73vE{(Z5+k8T3l z6L#2naTabh&cPzNMetd%0{++6V$f(4k_CBwv%4>jeBF#TVV#)PT zz~VUmD~v?C%SlMjk3-F;B%BM#fRoc1^t%MW-Lo676RzP@P_y8ANbc_t6Js4O02*S5-V6#pQ>;yHzdHZI3D87a&k8Z@J1mNcT1T;*}!1|^n zyn7Reuwy52vpW(gqQ|k%FbtoT9YxcW5cCBF;#~9*yo)`6>u-GVEx{ABRW@Po(slS^ zxg23F3-N5n94weM3maUf!N7^aaoS}3?KVaIbt62VrjI&J4UGCDi&^i4;WDe6CD=DH z+g(*`SHgAHJvxu=k2=S)L*rR)Qv}PNBe(~?yE9GAWh^AgiiN)Zz?uHcpo%kYk++Se zM87yX&IcPy(mPhsY9S?uQ5i|m}p zO$I@|@!6!AjrjYPozMEkl51qpbYBIF`tSO~6(6$*A_E_|ZBYSvwr@D{3Bc zgO=dQ{gtqJz8;=$+%f&^b`)Or!R*xiNOkwe^VUF^+zy3}?s2T@Jr0hKf}DUa*>gJ@ z#v)0W8=Qg+>8ltYn1rVui8wXS6Wb+PU^Ty5z!^M-=bQm(j=q8BpZV~b+Y0T&LlAp2 zjK90Z`OE=1zROI5S8vthccqNwuLn-#FDKja$F@)8Z>}864-Dw>x5PB~i~VwZ{(5o# z&ZM8vIyZ!ok6JM~FCV`b-oW380qFMCBS4}WCPU50vhu{Qj6|G2oP?%@SMkI+6=+I? zS4%WTc}2nSSQJ7mkHdBRaXi=%iuQd$IQPyUXP)mz-fmy?joyKf+3r~JY&~RpR>H1+ zG3p=Ag-@s>qQ$4oTgF-(9g63FmuttZ*3#ot1?A$gcWoxIf0AU4)ZEz!{+jQ=v-O~_tC3x=E8bJ z+PLH5lkGTI>kBvOgUE97M^bJOR&5T2-I=3^Pl|x+;uF~G7K`@5ILvlVz}bi-bXjDf z>ds}%3M#;w+Dy3G9L31^dfcyk1`EGNIBu@T_3R5+V^oD1C)#nLt_O-IhH>(ZINw+& z&xd`~x09XR@j3n z0uPspPZgZ~FW}pTdQ^%x;*tJyym?&@=jJd7UCl)DjsjTKT}GyD78+`k@IWU4#-8y= zv5&luQup%mp=@(Q%X>h(zH082ns(3brxv=*>&S&kJYKgY&quJ`Jxc|y@tH*3oYJgwIZ*tBf$$l{)5Te`*7K<6-{N0sC|DA zCF!*Y419u+p$BkGzJ&#Mvjpon7t{Rn@O(oNlqO}ON$))B#8aSqAPGAb#KYv@NfhYE zV4ybwCo7KOvrZ^{Y6Ian-yaUs4q%1i9@q=wv1aEM%E?8)F=cI((yBZ@M-Jq4R4c8RCabo-)tn51oNx^zIDhtB3L!tO& zFTnNlk+>uhiy2=};d@B}k^_>Ez9|(;E~i8BX$F=RJvNk&kg8^8qG2X@-u*JEZLD#o55$f}A4GkJeY@hvsVYO^svtl56I?@<_&? zFk8;A`@`|{kvaciiV^?DTAM$ptjIgv73Yh*eq&!vFKUe52|NSMFnjg@$FmY5e9z>@YwkxrfKJ5d`|{`-b{y=hamsoOv0(+1ZaAnf~P?&uAYm8 z@yz287vS3b(?R(C*dG=hhu}PR51hqzqJH^SEU<6`KW#OX|62-O0Zs-7&qCz&We$zgP@}7(I>?yP1~T4t3N;M5&!zKgn2t^qd!>=WDzXZg z)wF=cTBit8w4g4QgInv0uj<>Nocw_>3Qxl={PUMKBV!hhb97ajdh5!iv6VluS5{ zd(+QAeM2&I6wl$IQUcQIPw;s9 zQ@k4U6f(Qspg`&a3e`^V{V6g6o+7mDF~+9V;+FbdBz(Az z`-`taOa2NH+AqQ6$OQzi&V`V67N+TBK(CM$U^kbvBzsqFxl5DnV)bK z_v-LW5-$@)%<31D=D%00ljr}oscxS^JIWt(6Y96K#?mwF<-2U=a`-wkTJeYlZh6U? z%0IHEoj;go?jIJ>BZYTvN^myQz>?#m@p6+Ha{Z>l&cGIuO0%#;WiAqAm!P+A6@KJz zKy~moBqe!6`te>&7C(%N`vG`Y7K}wHVJLcj9Md}@p;8)yE7MM6Z)XCI`y|8mO)93; zr{k)CFWZ%w3#a)4FYTxzl&-xF$(L9088@NSa}&}Y_XRv?9THQY;KSbr7`HTG@4y?J zDiY{%dHtB}_YY6zO7i>-MgH~;9X>mFEWcjKil5XujbA?K%BK}f<693~@j^0Vc~9w) z{Fp&SK1W)T|04HKaBchXOQ7p@`@O+Ffd?Q?t`V+5PY@Yfhp~?LVU~Uqvz2e+xyUtm zd0m5+zym5H==1C!hv-1NxX4lIuR|}y; zSpv<&@`81xiFO}dOdn^2q^*|F`^=$TW+nn=%*ADig?Oarf}@*U(RO_c;)Qp@z%`3juEZV2@F61W{N#XpfdsM-Gj{D*oxuzDuoMVhcI;U%U`dyfOFzCfzw2kvGF z@y8^j_?3}Le01ALUNgsp-~DU~|14?-zeIR5-&Qb#KTtD;U+iVVuPz(Ot36cWcbZA@ z11&-QMh_7a7K&++o~GjJ#CQE}=4R*bp>>zYzrIa-3%!Z$G3a~TGK zMMzs+08Q;Yyf+r;Ndo-W<5E!6u~$pN;z{u6g@$|CBVev~*kT_^vw%^;_A zzZ2CgMrWoBa%1O&vgr~R*ecv+_T@FKVnI6_Z_vxmiT`GWKP8b8CX1CPRPlX{E`lTt zv1q9oj#v`}b=t!w!WrTV7bBu%InEERL-9d(j4RlIvXy)AXo4SJWCftoCm2h|9)*TX z1ZE6GLF!p70^Cm{_Cf+ug_E%0TnfT}rK8L;3yGg|QLbA6`GpsusdEKSw_e9!$1Uha zmtuuy1^kUGakuy()}4NeAeSbL`}_i)huYxL{Q=}$4_a3a!$w(zAO0%Ek5m@uzMFLU zs}siYfyA0muy){G%C_ z3Rze(lrHEGQnB`O63QJCa6R!fTsFnxc}El!qQj9J7Y6N?V3f@az!cxZ$QjxV)p#!` z&DadEczZ^oviy>}02e;DfG5!j{513)cV?zOE>SE_URU}x*;@l)DEP3{qMegrm z(E)FmqgX8~H!cu8VK7`G6>gFb^h|?5hU4i{Bf^4_VAu{=$9|Yc1~En zX)y|qxIk;{8m#&2j{P%sU`Oh1{1ot6XTSTS_-ZgT{)XWdjeyeIC+T6^xhnJ@ zuE)UQXXsw>0%vc%hD^s>%(&Vu;Dvj!RpU2mvqgE=o6>ydC>8#kg&v*3W}1)1&l5GhoFfH8LvHCO_RYu7Q)?JBB8FTwpn0Rr-Kp*xg?$J*y1 zWRi+={YiqF^bCTyIE=b^67rHKaM33mKX-*8Syqq_1=^*%}R=Q|5gi<5)gZpYK?*X7#`SLy_M z_zRXd;|EiS5P@Em6kOh`z&~0O7du8F<+%x(wwU9v9EV-^958RfLbQmvAVA3#**2SC z_i#H@tajnQxc%5F9RO9EU`)>m!|~1GI4g7l8M{wnWIft?70w29d zHVz-mgNSV*s{Jp)W$#tI*i?+q`6aNex(%nj6=+Vn2SZUoJwNXuW>!AM<@wEs7H-A6 z!8h2i{2m(x|GDefIsk)>e~}j@#{b?f%ZvR|;bkZ4^TxAH`8Z?Bn+7@YPoDVjjeDH< z!X=bf+GNU~SJCHBnW^&Q=E?GV^TqhBlm9|LZU9v)zamHHJxbdAlUE#_T>2DwZ?XQxU>o1mbv1=3>QrK zy%6ScvyewAj@~f`?`wkZcl2>#xfZtWRfT)96h7?{!Q_v9%xgp&^9-(Ie=2XYuq&sS zNym52Z{J+H=7BY>iT5PWLNv%F0k2jrHL~3cyV$AM|CsR! zNo+2X$5m-v$Y_x_};52-hI1kgduRx^$-xq3lpn9?=@;>=O?&p5& z%JoNwz{jAY6@~*%$6>uPN`T|B7_%}C3$C2O=23$6E^!Vux6Y%cG7BjcdH7UN2(JZ~ z;1P5cVPZEhDgP!a@0UXOYZ=mv?jh-K6*l`m#Q)&vy2G)2-#6J?vMHlTSw+$Fde5_| zqT$ol-cs6o7e%F1+Iv|ov=z~ylA@GCA`KEsQ6b6i{{H@OI5-@R=Xvh;zOU;#*K;~5 z8@qz@FfQQ@JZ#@#N!UjWa;t#q{qK09SC5`93e-qY;q)`wwESnt)d^O77UIO#7rl6; zW-!%%Ea$~{gE{r67pG5lVz&Y-nlu>FIYOIr0#z7btH8^{>!Gaj9Y*acV6x#OMr?kE zuYPZkcp(p;hvcBou`GDlKSGvc1`_h_plVAh5;K$0J~#;(o3Ej}hnP!J5^#OJsQ=}$ z@L3)OSKU+S*XtPG%sY%1FZSc%i9HxozY_}%h2uo$Ex1!3f=}Ft__Ec|o3jKjD(2$q zyg=MJGzB+u{V{v)P@L%B8#REV5iL4nu2xvbFL*7qD`M3aFowyKj z*-P-Td=|yM@at!kfPvO{vD2)9Ti}&Zw z3cur~>dekY?iClz5&?qchU`*_mh5yq%w;l+p?7|hQ_Te0Uy6c*#j z#rMd`FNgWpN*u4N#ha~-_;g;8S>|f=U8uv>_l;Qi(VD+voatu`dcu#X#;e&*&5u*& zfF%jeRCcrBx~oQvIHkjTC8}JItjK9CjcD+w!`XM0FbgP0b^UuhK3WWoxO@y1bKm(s zIoMtE7^^xz!qLe4_;mj+KE$Wu?D=HGeN009tZR5}dKs@R;&CbcJTec);(Xm1w3&Vi zyPb~Vb;==>$@U}VR;17^?8HT}{!>ndp}slh=8q`Kpe6tSU=?A51vQr4cKlG)U8l27Ph^1dgfn!?G>^;lZlO7#$OUVU6>#W#Do+zFmzikpizP-UQvH zVW@2028#lLYxqP$YS;&@ABV71ItFvUQ+O~X3OkkKQ1k8_tP&{_8oR!vzL5tl7^pImhG%R}EAZ;&;;6e+Ji!8QI1rs~$9dh2gI zZ~lwqK1%fJqs{|!b@`#hn4W!Ysg&WuX{X35J%%y2XdQ2l8AjVp znxada@#<{vqQu1TX53ry8w&q@gK6>?814LoORGzV|JnS#WqD_7j6pc^8Pw5yQ3(VRm zdmoZ_N8-S;ohVfg$8EzfT#ViXtMHAYHCc^CJ}VFyG#`nP0Z3>q`2QipuxCU+?EfXO zbD2BlF7Jd6?>nMug9VaH4Ujld=;YGtd_>Goln6<{WNsyeh*8{GobDM z2v6reMy+)YW+gmFU3DH#S`|QhS1}gHf5hId71(pR8bAGN5s}gWqc#fsuUeVirfINd zgC6(inXu-$Elm?$+2>nNs(cwvyZDXl@nbmab$T)7nk(I^Z0Y7`N-d!YYP&^)RZo=J zq@}tAyx@`2e%R;Q+l_p+H$K>(% z@U>kU3c99X#=9geRlbH=%S&jQ9goo~&tpYWEcC}jp{(HqM)f|52Zo2>)^$JR_je=X zXao#ehr?d*8){uPVcw4oxY2qo>US@O{+A$}2%3qX!^U9fz(Keh=Z)~GqUY-BilIXL z)u~l`tTZr#uV!n3Z#T( za}=&Dj)O(@dHjxy$I%&=v0w8V3<8n_PLKk_eQCH7Na=gD-1_5DMdvMo_7%V@Di33?) zlLu*jlz*{okbi5Zih-K8_*rU?0W(~MM^27W?>zA$WFRiw8H1W8f2?Z?#Hp<1IA9wL zv-DsL-W-ZsmqRf;FdQZuccA@(NbD=$2S*%6)%IgpI_)HqcAtU8&^RorIFD6f379YD zp?>$SL7^%M*8VB@u_g^M(tVsP%E0%LnaG)w1$*5bRC?zkXW=Vc>s)|j@3&Y#{T(J| ze#GRkN}QhZP0SrXk*nB zk#FX3jWPUpq7OF(x-odJ9Y;2p)3elohL^SZ?wT5#B9)k}slWvmjnLHk39kp=M7{ok zi$^}fB)tTu_r66$n7AIhS2(Vji|YJ|M?L~C{w?V ztuB|Lek=hgU(TbeZyc5`ID_a8Coxg|7-9w-MvKON%<35lCqKdetC!z2h9*X&M zf-z5M7OWpENBFKl_opk*yPKTB0tDS_p6d0nW_P= zGd6fF>w+ilo~X1RgquQF_Imj!=nIVgU{U~n&RvK#kJe$4$rc#phhpi%ZTQn`EB^Z# zj?F>4aJJ7rc<(xd5u&eaFgby_F1`t!X%U z=^mnbWr(j#bbXqIOU>E1+$9&Qy5zyarU2Wtig0*i31)nM53^OD5&W$hmj?bo=*!=b zZ*M|PJ4MFUDD$qj29FBfUa^lMr#V^D@3B3{9dhTRTYdR{zCUj|hEa8?KZ6?jvg;0a z?zv^p89goe+1HR=^>rC+rop~flsU{>ktgpqVTFD@irW4_K$ow$?)w?zZ@tIrloEKf zDS~a=0^~d7VWn0shNNYq>*p*KY|6xq3mLenb03ZE)8JH}jEaj%(3^G*SG!+E%%6BP zl%2!4mKd0OMd9uwQU8OEVdUaNSZT6PaDBUA)gZWzkz1iPWE(8K!my$|3@Rtq8#efSon14ZVE_15nI-|e%w*? zFFA=SjVN?*h(-H~bC~!b9>G!axB?IkDGhzQI3te{R z;8S8Q#wfhP&i;KISSbGP{stqS zoT|sjf11=?uF8A*N|b_Ipt-UUPEo%QYV}?8p_OnMQUg5aq|fI|T$U(Lhi z1G$Jw%)tTq6CAO~goefgT-|;THD+m;-Z4dJ7lg+*R@DE9%P4G!7rLW!7%q>+G`%Q{ zczF_EwU5DR^+9|WeuP?^omkyt3uYAVz~$^6STDHF$vZ=lCT74xTbIDC&ni5cG8t)4 z2jW@FNDO`GgJ)~qkrD2M(_LER31dFVe?5F658eDyK4PVjY}n6LQojKr7s%(RwCVK) z$$EOa^!D*lN$2f7smJ1Y`HadK`F`tT@_pMiaW1zPRHpQUX8C`R9?!y<7;kK7UI%~C zg9g9fgzcBsV(iNuINtgo-V0r2T;$IG=VK=i-iiG_2k~_35nK^^#)(~{F}+gY^V`nD zV@^EMuUM3sYth;&G1rgK-Q2t^h@{#*W3z>^ee-fCGWBBKrv2+yuobu0x{F%;jis;G<11} zTDToYA!Us|R=cHdk{{it>Ao{=K<74r}E*i$uPr~i- z5#;q3`)27*?3x;mU5_I~hQoebA1|J%z{3XytVOS>n{d!|v*7X9;@A(-d#lXC`HKJ0 zklqiARmA-7tqJ?|qw?Wa(enJt3v%bgHPY-eRni#gyJUOskTgD|qvUe8gVg#(E7@bO zE;8#s{p8IdZt~Q=i{;=7=nvT=vnJCWALfgV@=w#*MenOtmcjx1Rei2TsDTZ%lDN>6+!p)=t z1#7FI>G}g#oWxA$--N1zEl5&RV*eYe{BlZz7BM<}xy^tkLruA1u{H00>&yl9o_wO} z!xbwg^X140&Ip~%8=5{;?bw6wO&l3=#G2CrOu1mT0rw~B@a9ep`fpR^Z#^YmecXcE zXPXdp^*4I?{6M?5)yNoC0qq^1pm)9$gx@K-(vgU#YOn@1q1kEjLlAvkNEM!;E`V;jS?gIh$7?e338gM*HAMkQFvP zSt7sf-Ce%mp|^Z%;!xT8qgt|hUoYuSo~`6owpWtvjg;>wMq_^CAY>rb!uuRnPe z?l+Q`#Wg|yNH{J(3&M;^y`g`091@=l$GstA*Iv$CU@K!t?jY{}IgQpsB6-_; zI<*o9@r;R#V@EsFca|*`CYaOiKSQ=VtH(rLExvPA=Y%?CZpu`o(U2B2$o^oh(l616 z)ga|sH9l*9M#R@onB2P*I|ARrplcD9h2+Cb^#2v7pJRaN|LlHcVZczK^E;gZ&78Z? zUz-Y}Be!tv^9@XIyn?e$nCSzTAw#Qx*B6R*9wK zRJmlg1{ZbJVLx$h&q*|5Rzx&5ok*?R9VQ`}XCOzt)Y*>Tw%Y~1} z;xqnE_=JT$O0oK6F;-lBgMM4Z{NFtfhlb~ZW!X5@^cWKlKSYa*UfYrjv@_oe?y ztxxxpbszdj+A#8}=bKj3CHJz$lJ6r)Qgb}ubua9)?46$rLwvQVyC+@VF|ts;|GAfJ zN$M%NP0?n#wW!}O)w^M#sAcVcO3)U4d+!fo9`Rm)V}m{*NobY2j1j#|(iL3vO@vO& zC7l0r8J(YBN2>QtEYQA(D#WrKfXjDZ$=+rkwYL?F7u;zt(@u~T)9nXw{|8pvbpkIhDcJ!_)mVYM$gt*O7WPzjL!|s9 zoP zeigQLs)1>@pLjR9Ug(sX&@f1W5Be$5G)aZ6AFI*evL=h)=+NlAKJykD(o5Zp4;`(z z&8Q=rYP<5-sGd~P8BX_MK|Hti2;chzG3v`O4iMgI-FIF25;OS)@tpG-^r$=UolJ565m}(pX=!uJWl7h{R+bw5Kw5tHj?{0(9jX1F zInq-x?<);4@QV4FCwm!slye+LvOIByT&1Gi>wNRhUS;=F;3?lE;JY6 zYl6sajmg9NC%KS0<>2vEv1f((e705wW_sSk=|!oye&ZG<`(4BS9ha~(^PJ#)V(=pB z2!=Tv#?(C#ICbaOGp1d7+HWH(p7FXI4lWpI5O@$|pQCs`A>G zFkL>8Dre0(6uS$;1-#llJejrsYpDitKbeCl+BuFYB|C2t<+ajgy zRgm?)lPHUPd>*NfI-^*57T4Z*;MIE03_8s-VA-4oZ0H!(<%HO@+&w`!IYga?XA|#)NO# z(5}u!&r-pQJj}<(9I@Z4OAt``9#4JB;BmVGD@If!AK$U8v<^ut>tT@GBr+EiII~A9 z7Pe90&U`iMmTT~FSHZom)8)#mt@-PiAwPFF<&0aFw9U5V8lTQ=Y1N(MH2W~CW+eS@ z&!?;D38p=p&le`6I9}?*vu(PwUZpc-6}Id&#F9QEO!+0)kfx^vCM?&b0koM?qQQr$ zYE&;%X4eKq23>5yvV76|f2>D--#VQ4{Eke~|F`K;f#q|`;B);w4EL6xa6l2Zgyo~> z?mT=yl#Ay9*;p_p3sY}B6x!5u+|Wvc(eq>|th)gP(@Xf$Jr-|Gw!-xOTui?fj6tKf z$hE~hsUY-)rP~PVDCU~6>6kzB0eb$1O zmf*XX;qnbXA#Uzx%zj!41@~{LS|+fxJ-={xZv#>*nxP-3zyNC{K5ebS`fh48?5)Y6 zGqt(*jxLX}HPw6!x$c=URZY!lcDo(RPjq0ZjuQ*+crZb$9}iTGruU|WELeMztBx$> zyNuCHyVjTTsUBSS%8}(KJMeN!JL=9h=btym+&jpShv{-VQHb%A~Q|{IL_L|qx<|3~$ zqk|keXdnmJM&Y+ZU+B9n;-|ZJygX{U%Rj#g;09A?`I`&tuWPch)?6TCn+S{ic%Pn8R!DS7BWGam~EzJc_y7&cc*@t=m!!oMm< z_}EGuEUQM(&EIiyc^w|Q{6^yJKf;&&56^xpaN}$xrV7n{gUHOtRn=s>P;IuU*5w|P z*1RnI)O{x#({H;O&C@Jd^Uj(U%kA0!n=^~UdN6LRH%~?QbIQcUjGlFx9sL*c*T*zsHvKwiGO9$4!!N0D z$v7pZbywty9{(_3c)*nQ|3<5%I&@R`4z;z_sC2D_=FM`!XncgmmQr;4R*b&W-iSYDB&3|Vs>5&J}5o4FHmhK)z3kd5*W zKf1`*WCpNRw;u8lThqLDx-VkyD*}Uiat4z{-k`eOL58d;@Y=v@Uas)=N(me*`+Cq! zHr*ypaw!Y-TrjOnnzCWJwD3csRB$|9>NH`D?0D2GDP&P^>`IyKbq5cJ$<6UpaDPsV3Ilozj<%0WAJK+WtB2A`2+riB`~E%X!-uFqhQ`x1J& z&ynw#1D6-ikpB8PPVLKsde_%N?_P+VUEjiP@H@yF-{Wu9C;S^;fdSrCxbOT8vuBIk z8|Pn8O0Gv^uO{pp+XANEHKN^H zQ=Zsq!C#T>IH}f_o31+WNUkeWr^=WyZvZ)G9Cz(Xg~Hrpy_(#cqz@0O@=|2id_ zi1k-(+k!e-lkfvI;P;4MsIU8u-(lkH3a>(Gn+p6Cbu4W32h1ucfs)Xf-7+jh>&35; zw^Pi5@1Eo0g=fgUm5nzmbMa>MOE_eSjQ1z6v2vUy4D?-OKYb2ix4?JL?wLxd*CVNT@Byo}q5ccS!{?LA{Bb#u8b z<&GaLsj0+DTQ*FUa$clKqkF%S~qHx;#cqkMmcFWNh8k)_T5g~0Mi9Br*b|0-wtD64b!QGvP4$-&iqPhg<=3|juV zaA^M$Qv55d|5PZlM2cbSQi`*iKjLjkIZl86g5bDnbSkaE(AqjI{rVf9);GewUo(Cd zv_S2)A|DD5{aeAyKFv^LcMDCnX{F5_8hZ3|Z%v4OX_{-qrOu{YJWklQQZMgccKIPH6LJOkJ^@BBOEUstRPZfTQ68OSrMSeKb0>kmm zNIcLeG$g+Xh?t||h8|96v9k?=6ougMc z$W%qXl!<}Iv)Kk+qlY?YfnhB7hk%{{6fkkr-A9x-8L%H_7z&v?-T80 zZ|14X!ajbJt{5(qbQ%xxU`!mYtvcb=rIjsKR1e10M*-|u`4SyZ+VM%2Av33SX7jve zc}`*-zH4{l4KH(8K6l~pyeD|u{V_Z{-bDY)7ZBPt13m9$Li6Wyd=kEihS)dwIIRTr z!5@T2s|?8jqK7Z5L{F&(Pqb?h_u;3|t_tt0z@)YYv|zcLBD2kvIOTydT@R`<=B&_L z2Wv8HlQyH*>(a1uYkJOZBl0bc_;a=?d)_h^THCg)IBm^7OFD4gZ3k-Ty7HZm2bGk2 zv#sM$-ds7A%BE|0_S#v-^dvK(cD`%BBaNYP0+@@*6ETeXG z-)PRLg{EA5!HApKhM|6~`C+3jJ4b4><6cd++AFeL2B>n4iwYmI~TJ zx&YlHH=)#5_`l~rLEk$rJaWSfqiP&z_q-09hyRsli~O%8*A2Nl$c}5w^6)V#fOXzO z(EXkb^s*0oEjB!hHKhmX5_M1N@YGgjFw$7&ZlEbUG4Zsd;POB^6{{vI?rJA{`Py6R z)$~{z++xM8(Gl|h&dfun3*>X`pP;3t6ytTy<6>+##_1c1+~M8GI=CJE zZQ8MwvlZW_8naFMO`KSL5iUg!@#FqW^r^_l+!^n%Kj9tvjxNLPbLCk7p%RXMUlHl? z9o;U~;>wBNm~y58c@E8RQ&-^p`HC#iQsRyq%4~0>$}l~3>PBfWHC~HdF6nUAUOm2l z(3%u8eA6^zetU1q_6`>OW801^3ar_^qv-E4I&zh%6IY#gqw*RVcck{Cy2fx;7ti2> z?i*MYe}NzVt>;Ki84Th8_6kgEi9{+j3pKIgKBh za_I(R_Mag7zZkLpw*+o{UgUk;(xT314f=Lar(U=!{Z1-#wx<#;#AmnRPYVX;|HYn* z4e*QojrF7IQ0ns?p59-f_fmWg0zSiHcp2_Ieuw26?{M{LK2n-r;`GXg2pKN$yzaMP znQqKnPb&^{Zbys6?Pxf;3m0Js-@D!T?xmOqw+KGzM;5Hs709bHB9W?h02AKL!<(F) z@;2YB_;7TbR2gB;aplG~8uQr1^1SzTMo_RfEeLq9#)@5uzN z7Hb+Y`wRwudkcBhdVJV^0#l-$Mb5=pEKzuYJJm6;l{aD9u$PFQA^6U-TFly{$MU?V z@L2yAnOjSdyiMTn79U}7vkc4Ie!&>wbJ@3{2D5h7!v4Qsh_R|iR%Ijl8#Lo=VGC>y z3%$)TC9XNG%!oaz+%I^v!+9FiQPtuQm*d&||DHR)F?hU9muO{!Cd=2p0Bim`S4tz z;QL0=^7cSRRe@R;Jm^#7%C*s*`Ae-Mdk8&`?KNw5Noz}+trpZ@W5)b^V-9aN_KV*VFBzHHKB=1fhhYIFBXY!R--Tn(`w1NL%#)ghufc**G7G%| z$;ZEi+g(lYd1{tYRGXRvP08Jy0~!a|o9 z_CC>k%%sy>YdHjkRXA7O3UydfHXlXO; zo(|&*$DWjuMt zoe?6FW%bq0Ty(c1Zy)JE4+|R_#JA(na7#{KZ_ZJzOliE(n2)y_(jdTqufFQ@qPR}S zEFIEKn^vY;j2Ndul~^^N)l}t$BxQCtRpOD?io7*JfrmT%Lst4~Cn!SdQ$o&A4^aje+Z@$P&KSNE?l`WbfAN$qJuaNvbhF zdsP0-?9o4OD;`CP%(%e2kX_q~hPZf0v(I7Js3hcVyNkT)Cs0;>A$+uN(Wg%}mdz@` zo>t|EZCwoAGrutX_AA|ESFn`O$nv|p^pq8fd6U1dO5xgi%mH{$p1CS0Xx z#)?)J?7Z5NnmO$_#mI*4YVCPI^mStk9N24w6O9$R(CUsmgNIA}Wz&oDt^-)TaTq&n zp2)Gk=5x`EO$@P3pxx|EG}c|fmE9*Ya{4gpO&&l^zg~Q*C2?njI}0mZS?uq`)vp}5 zGTe^yl-e`xw>7iRwBzxUmeeUX=L`cg-ZwYlzZ*szC$4k(7{MDC>$6$lZzCIZ81O=y zna!F^7_Y%CUDUbuiz+*uR$=ubWsXV^+A^sX^KL6J$*l#hq5m*XU2p|2nlaV%4+=E@ zzfnBKaJO;tDGcJdr{ z=EbA;YLR7OaTk&EyVF8=qHGOgCF281lHty8(&`aulJCc8$$Z~2$?}EnEac>6l@k*hN)q%MNVEGvONV?J-Hf>Ov>@}_z%>cslwD{jc^s8SuN4y ztnaAI=SNkjC$QEMHB}}b6B#ClROx7@#>tlI)cl|>GLJO)BwUMQR*Q_oZ91%-r^{1W zdOV=onyv)~%&;@$`WZ$%)Zc_bBTZQ?_Sr>kOFod=(wA18UTe({@wQwT(}BD8+VfeI z1Giswq`s;v*B81m^O6T0cga}b(2KjC_UF)7L-LDVy1K z%VPR{o5~q|#&Cb}5Oy>fz@Tou*q9*Whr=G+mFq@x(JziU%syw! zB_-C(FtDPBtSy({u@Lp%oY_lEId7^7XD%}0Z*4<5eKp`IwboS1)#I4$x^xk`fQuWn zd3u5tovSoBw@jUPTtrT@j~deoR9SLJh3kH(Q0b=ID#1c=_NY4t6|_4mI&uyZsi%A5DkZnrt-g z&xd87V)2fF3iwU@0h8uhOnKOV!J=j#3{<9foe~3=sd3OaEmp{M`A59hK-NW{Z|n7_ zFSsH9%R>K}+*)X&4d~<0hUe7{`JmE}f8QB#lA{TCX`1rubTi(p6Ejr@OLp7Qma{~E z^TX1b#R0bTebb(mk{!1ycjUYo4qPp=bCaJt@@=CtcU%=5oWSE>_w(SJbuwryI5Y^ICNIUZcUnb&fbvVX4t&gnRT-ItBvl!77L zJ!=4eCiG?(BQjx=jKjS=xT?jC`9jMb@Y0#*4mi@;tTUs-9fWVcBahkGF)yV(SJ>II zdyq9xwYK87y={59izWM*TM$8Jbc!|M6l)W{FEgS=wjo108S3{U-ENc5kUwt|}(G|JDLVq$-i$%xPnEFGBdaBCIT~`mgFAa!4SPPYFKhQd_0--h~ zXrGl2E!%AT*FGKZGjGBFO1!{PkKu&IDTI6~#o(^Syr=j{cKH5sX??F)sr+V+G$X#9 zY)FUAaQ`7usm&uOw2eW9&=08IzJ^YJQW4%E6Nk6I#ASiSEUf;3vNc~Z$ge?UYBb^B z7DYxaYQ;If)VV!WleW;KUbMI$6BC;J7FzdK=G-&XocBcTLC$>(9xk@zS?{)-JGLEz zJ6O^AtrZI!t!dxMmM%pi_pE;hPD;1q%IEehb8ukX?oM=kBWiAcXU=`$!nG}~Tv{Rc zw$bi9^1d6N26(WdUk|=;mNDfRc>ZfIf#ddL>BE6MtvQ%6`-ai6?HKlJJAsL}LGF+9-Vvx-S(DtdR;AK9wT(RYEDU8ch@KpyICR#z%=!B&xJQLaYgA-cRRato(EGh)~Y zbI$+MjzKE6OqtzY=p^j;DaD@4M|5P)Q3q!A>BPnEooN;B$f;YMc>0Pn2ba0f^N1^h zExYpgCDE6}xHCk#J2%bn-~h88oLwdH-y1K^6Mm|GeR{Haaxbd8_F;ZWU#6`0=IRFn z=w9s0fu2KIt~{K*PmSW%<^D9Vn@Ioisl3uHfWb{ce5=2Nkv8jTlp98$>}Xm%4CCYP z>v%qEF|Wr2vDq|$$HJ#_T&Icb80ybs@uQg0YB1K!xX(go&fDh1 zsJV`G7dVOYpiZpV;2``!9T}EsPmck1)HvOq){QoNd$1kX2wuVKhY^=pw&v>9y7U%3 zgGP`lExsx8cGEwh{i%cAsS2cJmBLN!f4O7XxU)GOTAsHMZ+`)WE6*b0LmJkJOoG_& zmADX;CpGR!k!BB0md<#Y$kuoAq;hEiqB`7$QR4;3_a!2^Un>6dd4zzs&tWsa2Qvrqn!!;g0y#8k z4rhFw&*9q_b3nyXo*cG{FIums`+-e7trJe`_A&g|C7hr77cn#vH}Nj$J-95cmd%FlcZzYZG3Q{6|fBySj_=lF3+rl=Jq zgP79JhYt(}aM@9B#^vfM{3k3`P?NkaK%?=UN>^&{uLd$N70tE4_NZKc)q(gw4qy(Hh;|1;Px67z7c)%tznHY z?DG{K<30hmif~)tXmjp9#4fW`cuYyeL(7YZbGi+4twQ11@Z|AMHnJaMlBC{U=1cls z_VQ^RzCh152g`>hqtg|kwOD->c5!#$Bl0Ewwh~z*my5xia^bO_<=Ga&zwTPda|;%8_Nk@3-)be3b=NRScLV3O3uc#t!PHo}iC<=I=Ah8cT)QKb zYcj(aV-(IE`n$NZERG&pyZE_dIBS!_n7uocV~%X*tl^uvGjJ0(?G9#t&0w|~wSlqU zSM%Sm74(T+%7UJYX}o743t!D=UiDnMSOoE)Wgzpu&SZ_rbjqhq=6@~|8IwDX_9y*C zcHd~$+#JcG%;9{!V;F1P{rKhkU@l1XWsm5AT-UQdSNZhgq-VW3S*0h1vW9`;>^M5r zo%{Sh!0!tV%(RpD{(VR?8*U)G>++rEU^9vRRl;CUqOXR%Cgm;@e_)>iZMmC8^ z>vq;<#?Sgr+%?6%xT*Kf0R?vRVQYN=w%;vcZdH?2o_7nP_`!j+# zR5y^mV&7l8JB@)0CsRdjB5m)FI=uf> zjg`BV`7Bjvn7(~O!S`~E=vIR6BL7i2Ars4^?_j9!RV3X`z^R~Q^#CwH` z7B-?+ts)OTRiV*J4W230=Eq5GsDHtjPp4RNt7Iz@kvq~}(~%ynT)3@6S57+Y#wLeu zw0Yj0i^h4f{i`109dj~HI_1U1y~Hc*4wKQ9J$YqjZ}$7xhrd4d(GXtS>&M}>!&neFf~xIDamdTj92@7)7vbZmUnidTsHqG$n#qTk135Kw4ma1# z=bqHX3~5-(TLV{da_u_iAKJv(ZNjG7((2B$BRu$BRXiUn84ZMvy3|78?V&xXy|ow5 z^zTDG(Mv5!@#f%_1Gw(DFXQ?RrH{*SK2jXb%GLf1m^M+&08_cMRRDLs4dQ>V7jyf! zb+pb1qtlgWj(HKr&i~d?_xob@Z$Fn|Z38&cU>ZjY{7+$kKh-af`ah1&J08pZ|KmjV zDA{`yLTSo%-`DG%rqbS1DecnUTeL(AAxeix3JoPnC=G=urBD(rL?SBd_x}F=Jm+|n zb6@xM`Mk&L^?cD|?WrVAnn=C>xzYU;SL(yhG&o}fSz$PNrnpe;f}xbI3Sy}Q<*P`@ zCwB;)_j95}9|n==Igt8!51??P{`6mzBb}-5OZlI<&R^m{R^IlMYGp@8I=$(1fGzFj z9fa#eYjW1>MbBcaNT{%&hB{NaTbFOvrnP zDNX8aMnC!flJu~kp2n8c<+UYM@E+y?&t}c@t;v42Ep6T3n|myKdey%VNh<1~wECrsh`YdlS@8bhy6 zkEBmIE;Qf*=&XZ;+KrrP5dXc2af9ff%|P1lxj!`(ur?%#bDAyt(9B9Z+Rpwiae*y` z6j{?--g~bZ)svbkEa_#91&z?Rpm5IC)*WR=uaB70uhk~>D3iUS-;C&qvk_UV8IoCw z0bQ(P{x56A6|=auKc`7rzADtGK#4AI>p~BLWoc}9E3QQULB-7xJpUEc znmGUSR66oHl99OKB39R3#CkrzZJ!HtQ^yy|%QPjE2Nep3Jj{iytt*5NBmYXIPP%-z ze#EBSNaod_!|>4wXjq;K&oK{SG%XMInZ7He#?auBb!@A;o20dsD`(t;h>(ItO zI@I2wOZRj2XxexK+HGY-!F`M=nthQAtj(w;$edQXSd!rtD@yh3MKe0Bso)Uz!%lYO zrs+Ukn3pmj){$Cy_G;qlM8Uoi3RWiS>pzU%%y*^vMH5J4?=%XUI-fFlt~VrQGim$y zlACau2A%h%uU)oKtTlTzX7b#Rf6jvEzrnLy>2k&}3Kxiu7V|D>sS|Y=4WvIcj`UKQ zGp`%%>4LHydGVaT#=wU5cI!o(cUsZ;sg@Ky)tsbyW`xTo6fGFjTb@%Mm}o$?KlSLy za6Ph;*JTZgHsx6J?7vluj5M_Mc)w=C+_E@(*q{{ldT42Gj<9L(cv8%(W{Gu%6Yf|xLYyzCj&tWO^ zeTLFUEVa@lFWpAT+Uu%9fKoqa%dKmK0kN&ZN|jZ@_E=?#%~PdG{r*F2%SmiVj%IB< z^Dr9jLiOYm1WkT}$>Bxt+VT+zD{2vO`ZLmRG+^h<8YHdy%eg{LIBUncwXbqywXrMj zIJ?sKzpSHO&wI>I3bg!yB9)ja(Oq?AQZ`YcnY{1ZctVW^O;x9qMZCLV50#H{cS_CE zqJ4_m^z5V#E$XUAUAF2|<2XY~QZlCaMJCi#Z%Se9=G6L?zYl&rDJaC6)Vyse_m({s zNgQcO)d0#2b|RZ(2^}5a!o9mIT}zrk--gekvaXA1V#G$;vD%lyo?a!x4Zc+KY9n=j zvxvG+oJA!$6DY^nmGT{3sO6)C`bRm@n`++Y+d0zo)AqDHi|;INYxZsSB*$}>^ug4E zPHLD@sg()&d@>@ft%hW}K%azWU3xG>hc2Aarb&;r=oa%Imh-N^-c5scc&gK(O=_g> z|Gz#-gZj!9n3xRSI%+@X70DP0$pPE!z$OV z-GD=BpW$0li|;c&vPZWF<-AV~Soj29o9^P}f<#2eMx*2XNvxIm58qF#QhNaJ%En!H z{?hxgFz8&6FvI4%;MDTXx!LZ6u%+w>R2B`UoK?&%$V+7B8WeS?$!1uS%9Ois|1L*8=JU>WU03Rnra+2=m1rX8RduaYp#gDf6tAK|>uze21N)+m z-O{3SZrY@GO^3?I>5-Fx0mc6{q|c*_Dfg-=WoTGXgsc@gruCxur?zxW!+}!D9I1f! zcupP8l-kpU96ephZ`l+&ImMlJ-&jjIYkX+zz(i8p=|e8v){((ncakfgLKQBqw7#zk z@p6pz&q;GN-B$H%HOLL8>zq%3k0S08v-_grQb*O%Y zHYFa{B4z%)+jtLYVWB~C8QhEKsL+%H$}~eqiPpC(kow`Sv>X#X}jo zxV97h)^wnvrVR@RwIluOU+iA?55}zdTKujSH+z0Ym0UB-(#!Bc^9$Txf53&!wUBEn zgHkvM zh*{Za8DosyxpFS2xctQ=jnZxu!HzYnb&NBs+ik#mTI2J_zEdUs#iSYbzb zIX3iaO;4I`V!`jODLvU_OpdDzsdI!rYlL;_&ICTA7HQE7&F-X~q)90O>eO$!8r|XN ztu#rQ?C&U2)3dHL@|8S|FqET5uViW5aTzMfZO73wtHGe13jST|5SP0dO9ttU&LxsP`6eW_A^CfHa z`P|@{Qgq~evg&^fq4`m`{+)A0P50r8G!n%_`@nNS9HyNuM0np=Ogqtvz|dFNdt8O| z%2nxXCNm3|G46G{41TLWqVM7|Ok7-v&xT)Mko^t&d;P#^)p}ff^9$oR=c4owdu+b_ zh1ZyNjOZsr301OW<1RdHGKx%+mcGQ);CU3yaUV{;l?V?r@!jA-Nu z16o_nx}8{l<^@`$mDG(cmupal$eIloRl1U+Or>f{l)poP?0(2o+#EjRVr8jwxD0K( z(vDKcf4Io{W09Zrovpu6v#}m0a%<7^!8g=*en$P2O3dN9ZS}>E@Utt!go0e0JgQ0a zSQllsL4};ni{LrA6^rCzF)FqYp+RxTHDV4{!g*ws>_xRJzmvP75N3W13*+X~*o+~R zb?Ob;_KucJ*%KpdJ~>iwbFOvn;$9%+ud5Xzq;f z9*nYQr|`|;oj8g8(RsWtu&KKY`6sN`i|&CZxhF8`?lD+RXFapTj7BBtlIq?&IPm!@ z>jodgpl1=n{uIM>X*u-nm1Ekp&sff$<$tVGD{5xH&48a+#6BzY8_jr@_=o*!Z3wrL zAyXS!GBam?<#~C+Vg+(=Q=*sbhh6WbO2Mp$7`2yc$2Z-{aHKXV&DJIVG<|wjWk~lL zOh`MyoYcDaq!*z!yd!g<(J2E+;JxnT{E>9FX$rYbSW1d|`=~1G3f0%|rOULGuC+|z zJAWh%pDUqi`GMp)+<~;O+OQV1C#7ebQ!MAC9WyYZ?qBq2&nR6QI$N72WOb(iW_UHd zRi~NBs&tb5zm-Nx%<590_&f4csU*jIMp-JHC_@Hk+pr`54@@P^*b&@_v)T1nsQLqP z%zu4T`~~yss&F;G94{TpvEGAu12wPU68sqQNmo&An+nw?T@nq=XfS`yN7f$0;9Vy; zuc8MOjt}Dehj>i+{SoV#^D!;yojCu)DOBz~1FxQI(L?5#=q_(g>eCb@saN@YpQ|ew z{-LMj_>^>^;Q46hS&>!3Ch9I(kZd9u`|gBL(G)@fnU|2)5-OedpCxK$j>j(6e=5rr zL2+*{+S*r#3U}I3^ZR!3M`R<8e72=DM+1a>?@Py6lbvRfgRs7Lkl1kr3j?#EfA%To z)DvZpLnJHesm7FPPt}gZZv%T%7a`{h0S{&OLLSGDTYvL{(ivUT@R zLD4o)82v3xNZDB@tUZ}2NYb}SO!m2v|BVL8G>ri12aDbi8xBzBk73xP{Zr7|#QKsp zE9z>sPdvxjWWD6KNbXJjja`3Eq5D8}x}3`V0>0nY)iq($yL|l2eU1jd7x))kh}rJ1 z@Ggt9W)8o@?w#-9v5mj0>nqXs!WSIWuEvAnTK1;YV^RlmEbUtmG5s&rUv7uujZVx> zkfm;V@>Kn*E1hEh-9~K{I-#vbtJ5^dz_2@wFzUgxe;wMpU!P(=@?HGXgl3Popqo*> z$T7yAK7JoSZ!$sEn&apu&;P1lu`WL%mb7_4qtJH&`C}aQeFe%h9Y}sJ>?t;`7wuNE zpbNUDG>i5AMSQn69oC`SK|QG1vO7)8(V&yP)TkBQbAMALpXaR2E|a5ck7Vh|appku zY{$)0e<8WnjG9eN2+;e9b>X#~A5jh6wl7#Qwi5S3KSG~9I_|IEVY9*;sO!AKg~NsD z9Q^{xv!A1GbUs>6{f0ODf4v&|Faunju5jjVkEkYGX*?uZ(ZdZ3x9t@@t$WhIWahr_ zm?ViE&Wwyt2gq@@9j?^{N>AASluR1$M)KRoNp6q+AOy&^3HP5XN+K@)a4z5U(fQOq zRbl_`--4m>FiF*kGGWfjVTh`oEnQ?)Aa&t<>(HYnNHaW(?4v^{_lP2`8LlEpKQxq{ zvnx&Zo=d|9ZKffkqbYRlW~v=Kmwnc*0i=yDaC=QYLNp6e zbmbMkjeUXKamC0AQ$fZ=ZHZc7D7wt^$D0j{Xjk)N(Kv7@%^9j8aZOO95xhg2q;!^R zG*g@ze^6Y~|GCus)gr0&E6G;@op&Jjp}k=X?b58L&RO2)V2Q zGxRROR_hN4zw!}|VkI)~f5F38z5~sF;Mvc5xUXr1PC_$Ioo7GZk9PQIaxKWU)0R|b zBv&fZgHGNjY*C|WxjgG+O-alxAF=-iX<4^{`!BiMy71TZf1ZA2X$T6E{$T(@4lNt3}6k47v9XA(e(w z>DRrtrP9>NQqLe?$%a|A!m`6Ml0_$igc}D^1xMT!bk1~@7!4jRG(W8s&eU~E9iSt< zbR`UP9ym(c%5-p;=3?XO50cowZha~gCNg5b3NBntu583oK!H(63#c|BUJvQSI zZdzp^(5o7)J>PQPQ&5NbSE|vp`zw^%DzWP2N5~|;$B_0C z+}%(N`x!;dLwt!lOPIxz_6#ztlZXv_g#PTqoz^7_bw2l?zLw9+wqn$!Rb#Ac2J+kw z!7@D-VN(x^|JZZ);K?3pNt`9Fd~uTUW?M=o?L8okTl`NrF-Vifv}?mDvWMjA+%SlZ z9g-^sf23}MEhPnwVa|)nuM1<22MAkdy%a7?Y!^JQ^%0KNmkZy=R|}sxq z4((;EVZ3}-8n|=`&042Ld%gZ(R33AHizM*dA1ih%8BF7Ow_fUT8(-``BX`~ral$cg zEc9Lj-@wB-x+E6MZzsX|{%zQwOM{!{eJm_^fcS%taZ2$S`|9=Q*NZTPxk??) zr3m`WKE>A+IIx9T_|v~(y*_J=ynfaU8MT-W|lF{%dn(@b zlwdHNmRby^yl!?h;eaKT-ZCbyp8Dils!h`(*!Shj{%%o?^_I#sfzKPYK3&PppbN=z zEz-BJ6)t;QaLlKX`LI8sTUCRwdCYPzWd2@f1tS0Q|Bnr4EHZy*ip6XA-73UFS>BhA zVh;TL$M{tJ00k%SL+xD}q>FB2-I*k$mdE0X$`QDCuHo!tZ%m(hM6}5JjQ=ce!`Jc@ z?Fbr7SE{4M-BTqP_Bk3a`ELBpY%s+!OGq>Oo;2jcG73L#k8dgQc>f?*5)yw!cowr- z@E(6j2>)9vsOjlRy3JIS_~_dT2d^Fze2a7>y+^4?S9@zpomPg^ijHX{wt9*>ai69A zo>gH~)eibRD_QvY-cqvcWmjbOW8dBKC#>J)b5PS-qU#a~%aZ}H=@kao)n_4A#BjdP zWyG(%hIv13BGoSq^F7k>>2ww*-_Azm!zZ{G@{Ad&`Ox<(#M=!;7#a5#lX{fF;qeE! z+kL{7MV}FR_#5V;7Gss_;XStzzjwD_eP%1Fn5kAXNR9@w2HfcnzyBG`O%7Be&ooWi z_(h9W9MYwSW=5p(&YXNM+0t02ffTsag&O-$BSRT48rT+2ZK-Q$gUdA9;N?P@QwP!v ze_MLQbK3VMhBTbL*KORt_c)_TVa(mwVyHqxYZb|#we??2<>=sqPAL1gV(R!7XdG=s z-!?B9_-}Zk`xyaWDzI?%2W%Q#2Fo>Xu`#p=&zZaM$(XtO512uq`xK#KHf$`i zP&CH@&N+x+%-WT$$9Jg7)~-}LSLaMcd}qQ zAwih#)J0OTM?tb?@OkH6EkA_g*TN-xbkZdyN<6P1%6LkVFLCvEM$M~_O2vb&Gq&*6@FLMN$E~EGE~;4mAVFW z)xnIOjAevM&6m%&$T)2zaT z&pg-q`yPE$N>SnR7HiKIAw0GK4HO#I!Ijy237a?rYk(uf;u z22Q{it$6fWmw@1~a5T@khzF}@;NCXEkMG8GGwhLckSa|j8r*VS5zOC&R(}}bWq`eq+Cpw82(8Uewy?Uj<4!184{W;v@8e`HfXLB z{9^NkwEk>y^J@D-6_m$AO0F7Jh?Xb=`% zdxjEE3A}&D;90Ogp6ag0{k5O4?rAtIRwd&H=SF!KCLulL8cyopVE*hayo$bqANTK~ z>xuhFxR;4Xn%Vg6_5|KPpCOw)+y0LWIZyXB;^{3Ey-ML-@*bYgc~_BG38y(&oti=k_xN(NgYpx)bLz8Hw2Iyd}BSp;Y#atZ|Qq$0*hg-C0CUdslN2t=| zJS7_3)|K?VxX!SWCAB~8n6aV-_t!O|lKIM;lB#k4KbEKl*6FjPZh{$PK7&`Jk4)W)bzx@uf?QX-}@&HYJyR_%k3NmN^%qdhytg`= zSoH3^fHh~XqPX%5{^(ajKLqHBBH% z6>ED|2hkrTo`qCSpbB}BEQ9*8Z;@vgZ;V*?r9;;2|J%`%J-?yKBt4@*R)6G3e!eVu zt9C#L`U9`!jga%N$8h%=91i+|%Nr~4V;S?+7BUyEumlBjigDke2y!bqSFu|@mgqdg zMbjs^zv%(|%QCQE>mKeHr7>6cHhWWVp!@Gc>~p(0$+sQ)EY{`X7hum~0ccd1I8l4^x_M)rdG?5>iNlP?LG9WByay>e;KNNsW9 zqgD)QIf^AaS7WKC3(i~4h0pe>Xzm>fpQ-b3`J_9JjvkF)(;{%Vk@@KRnVsnn^8YzV zGFIVm+Z&EP2F%INy@<7AE@L`Fz8n4~VVvhp_$6|t<(O3Fhor+~et2k<<4d41vkZ5cSJH=fNIDz8V&mT$G(Y@_n@fMAo6SEs-|IwN zo-AuO<>}!-CH5<;Qed$rvoy8o!A|x~x>{1%(mqsCF3{PkadhUxV$vCUie@%1CcS`h zG)-PY-mJT@ce3QYk|D*ZYg55VP1cpG(snPNC05AO!3oRfnW;(1s zq{8a`ZS3rO6BCXnAwupdHW^<=woVK-?~FpkwQ&CapM~HNh}H`sSY&Y-6BW*(vMB<6 zu8hXu_3luxmyl+hK;HWn*DF#|?H~+Yptp z6b^-pv2FAr$m}@Dv-ks;Jirew_5Lt(jzOg~1Sgcj5ab$MFV|$u+*Hjp!O86=uAWUkEP;63rT5x09_7PNHI42&u@p2%W!*gXFpZy4t+YC zq($-Z>h$)MGS90O=y`D$YLaxq^js^xxHTg$paC!QYoN<@-SW;#gx2u;KI%QJs!Pyw zV=>OZFM{vWLTHc4$I6yx(73}nKTg@`y(kNR{L@+YkcLFf6m)r-4CA_MaN&J6;%Ibp+KcYje4v!? zgTpI#Lvy-6qRm2>0~?H+;zQ=<%_d$qdXgJZ=OO??>T}w(|q^Y0_a@kjPqf|kW(nZmxA}$ zDXqYz4z6e0zoH|gj=5Bg(0JLx`BClY(_5BWHp!C-@Bg~6KBA_v8|~B4qtPwweG2ML zwi}(O*v^&etlVizr5^>Wy3-&dSGwfE^Zt_Fq@rU+y-fA!5YGv}xTsMX?`HbN$&&@& z@y2u7vHe2}0&g_pz?wRwrhdab?rT#=F)yh8Jw8ZEp%hY#;&(*|xL$x6V>zEa_!%~- z=iqjAHtaRBu=4$VIOnF}t3e7IjFRDC#QA^263~5OJo<2cOmsyw^VlNM<=|Ny84-e| zamQh|+z+cq4p)nAHp zbDxV5##LfRVy8Iempuke8-yO?#zJ^C1F^Pa;5&06{O)as5jG)k%zhNFJP5z8$5FhA znM;ZRxNUYAMW>Hrrq*fHE(k$PMmS!_M55Xv8g;8V?{f2HJinEIw#C<>@husz@>0-d zbQc;C>A0Plg;gopX#MpB5|8H?kdTiA^_MVtSOkT;oJ%*d6u)=rG@$bjc6^deS)y%KU1zn~K%*#ql5^fv-7{lpyA z8f@YIb2KxDk2b!?eU!p{Eo<7OubB;3h&`|QeQ#sWy3SJ^=b26SPgz)2pAPAI&H=iR zf&kV+uJON)K+8lNwZ9C-A2A54j=}=R3vhJ~=M47{gmc!+%enycw%}X}_W<~=4M4z~ z0Q`+V2y@O*9MWeK8a8f+5WWzX{*FQIff-n>KvI)1CE=d4xlgB$Hxmit1)e z-==8OeFF{JHb9vMx5|^8!1w%8=4WO%L&2p1o7$P3c$0gM9-rXt_yJx2lwg_GTO?@} zp`p0|dxU&+UH%--b)1`XA)7g%4{&fC>z@PDVE2S`-}t`s&$y1CZiyJp-1L_07z90t zLc#D0$lMlz0nbCRb5IcceNNz@(J>gW;mpLBM^Vvw5Uq0$!>Q{&EUj1%pN-4-^IU@E z|E9yU2XJZUI6Uzif@K!{kk@XG*Qb?mBdJ=vzO7PRQ}jmc{5V%KBY3|MIcTzQ;y-P{ zJLa8`6*N}}nfXib*jz4o)%;p~_2`p0JFHopyhZ_HxCQn#TVv4?N1n6spWot)$Ess- zK6Dnwteb<&4lDWI-GW4UZ`fY)hMC+!jCMbWjn00Y|Lu>1zfZzuEa#dRha#aeocVIB z1N|Pwna`Kt`h)WV_atJt{dH`9&*%RK&XdaFey1}X?(ta|^eG$Vm!HDc;5o=DA18Rf z8RYvKhnU+qiuWK@>?KmXS&7Ag)i5lt!~SjT{~!1l=WJwX$yzzq<}1>U3uwG>6z%Ryit@8aVa;$_=RbfZHTNO~&eic{eO=>QHTro;kupwm z2FFyMANdF2*8hAqWtIvaZG^>4xH77=WcVDt+$3+t}||2AB3^i!;q~v4vpuhVb(hjsGef( z=dw-6kKc)h_FR8_Jb5&G1n1knGCskDcH68F7!EHV)EN8l-DqC>c|s#Y=4IFi}Ud^p%7BvBJ>Ma8}78knz za5`OKpH$q_=`_l97(JTNpI$liq&XZs^P_)v`lGH&GR);(bWxVxYqlfa={Fwc)Wg(^ z`?=p=u*>Ho=tL>9o-_CPC+{Rm3Zd3BAFG&mG`2hI73#9_>01`89PeY>yt^1>pMsv0 zjK8O^gYpxw9OhKaeeWwrv#FDSHXSGR%drx8o&}m$#*5m(oRzHznc@`C4)5 z*H$t9Wj9Ri)Q4856}-y&V5yuV)cmAa&DkmcR*b{a&oklSz7)X)tGM6W1dsQ-F#q5_ z*o^RjW8Fa%-Z_e;vjgx_^AuXA24i-42-I!DvA2IDmaJed#Pe7zm>G||nF)}YaSf^b z{0(oVpjGcK=J(^eNGpqV!4E+zpCDk|Gc4ifZ;Aq3X)eNQ)&p7a{`*~*N)%Ufy}h43 zLt4L(w(Sp$yUI}PKY0pU&GmknCdpjWqfIX@$#jn+Y5p8a?!Twfp_u)2z1uWOX&Or3 zb~(~MEi0PZq(?s^G%1yRu30jikyh7<14IA7HsBZP7u6zq!Z)agRKoA`d)Vy#QjLYg_+5q{?~BpYy#8|#>47fEZWyZ z3##3JlQKz8hRi+QnxL)uQ)~QZdKyuDGJ_4XNhk zNrFc69wD^xpRoRwkuZ9RC@fwZAS|3wB7LNpARcdcDIN^@C=T=bCC;6#g3X(HK)c2W zj|bTz)X5&}bq6A-j|ey0QJDH^GDh_EfV<2xoE@+Z+nl#TqiF~JaL#=Q_X1+tA0!rZt$;bh`o}GdbvBWx8?JH z(S6wQ{3~HpHawPd?sVKU+_{j4?A3)ho>qh-%B7s0!kp3s=5h|G#?In86dq`Xu%eyW zd~&pMnG(&NsX@x-dSt6+NvBL4sW_UbDP#&=`@V;&ZKlxZM53tbzNDyaNiocLDVJ!_ zn+eQ@F_I%Ew|368ZHDrJItj3g)7Be-5LIkCEIv z8~z(O7oBsT<}OI%XOx2fJ||=Bwd;7R%ei3g@dz}z1Y4hI=IKYm|037HHKBMvm~+t| zokEHIainJ*AA$c+)T;d$5C`>;1}k@&S2C zwp_Lld3uP@Z1+|AV_2N{dsB|6olznRdUfJxkFL;x8Xl);!`0FP0WrPM%3RmV?7?^- z5|VV>@X}%?X8pGiX2LSm{aT0i8JjR+=MI=G+>aVvA5;__LhhJju$Lak!z-uoXk#!u zIp-tcSOnyl?;_(Hh4-nknB&j)-+=@SsK17wA<4K~l7fifG*o>_$FGw-56ORkUJg&N zeIska9P%MoUdVH^Vr=2vsLtmKcwYH}x2*q~!rI_kWy?d{MuS zXwJjA_AMSB?_zP(F&cmVL?U2l1ZE3iSmzszq8!e}Q9h2{)qXJK_dPl10B0%f$DLa{ zaN*7-JUP7{(tgYE$Y>#!E}wzJ)noBGl`uWd36C}Td7QOEsICR%Uo!LXx;pxPlSA*V zKSV{v_hRAH=i(X1OQOg4uhI=qj|zF`6(p`piiMdS^Mv^u`%7LT%Gr79XKC}qbD~ZB z15u;vYf($RN^H^W6fb2f;K~aPxTrHbbD}BaDtqDL97hBl8-mR#!?EJUL~Ob+8?$v5 zVd$KdurytRjE;@W``QVWBm1#*rVo1f9fn(vW6X>@j@)b9`+eZ&|2zysGa~S~&jnmQ z9?f-1ESk#Vq3|UUd#7B-U5i_=JemTfOKBMSG#%of3>=)1jj0unu~z>%Wbfy}Z%Gki z_Lm^^bUD8doL9y>-v!JqjQ47RZXf0@2XrON)6BX0qD|*(%_wi415JD+rCyE9Q=@IP ze`8J2%1W$>Do2BT3Hp2S zPR^P&2tS^oROboj*Ja~iTn1$Kr{i=?8ZNx%99-{YxGuO3N3LgOs^U=|7mF3+qd6Py z0t&KuHg`G<8;gQ*xaKrGz8uGSgJbaD%sJQteej5LkjG!ziC2R+VPm&7P_12o1KNu) z?c8h(jhcwSe#2oDJOnX`ePJ-k8mbDWm=$Y)3(gvlYA9j?_jmQbxKH;g5^XvAq{cs7 zbpP~K+Q;^$kZfzJ~Gl(v|ar;z1csWba5#b73nhKwubD+F{bJ~5DqH2m4_PTAx z_m4a9$lV)I_l4})!b)eOUx9GN84ZK z1o>P?e`zx2mfuE5L@Ea7-(!nG7PhX=#sk%-XxjA*TP$DVKwvRu7?+`L%}4a0&sb^o z18x0(;}mh8%6@s$nygCctf9X;!i+XewC7o_gj!g`7ErN~X8X9&RXYj&^0B8a%>OxH zq)nT4sglQbW=nW>ppmnf{I1qQYs_Z^UH*s<8Kv08_k2WpA)a@8hBMhuF!Osh>+G@+ zHRT>|-c7}!(J83kpN!!>)Ch@xxpST^SpJSIiM!Sws| zk*jBj?TI}(l z_`-bFVf>wS43mePz^lH2FwPA|$D>fh{R>BQuSoWhMWJTYCG`7t8HMJFaH+nAq=CuE zzIYqu6{%3?`matV3+qSod^GYgzX$n9D|-dA&Uf(r@E+}_K4EB6H6H!=g?X#mxQ65z z?YL_FQ~TPH6YHO+F!ySafeM{ZkfXla z+Yt5q7mE7Vpx>oWIQyRUrH9_3%c3F_`sQJeFO0xsk8zyOI}Hy`%tErC2l`K0jxT-JAu)3UtkS*9;gR7(9Bayi(fV{`ytoSs<=d!aW`*IhYv|J}f#)}uaUh6u z0ke58!@iaV({Rk*6p9sG*A*>3jb}?v@a&S$`3;A0EXEhs-@K8xYA?!0Y=K|gT5O-P z46_b+V3_|b1do}9sG4!mavOoy(*^Y2?|_U+J+c0n3C8r%Ms=7Pv}P;8(Y!-cxL+?0 zI`UDxs`FF~G)xyir(O|V3l5625^JQEN=3rO<%xp%F+)MK%}6p-IY=;39xYiXewOYG z+$Vmz9w|EHUKe*w%n~2kyb*2xeG!9RHgcY_EDkg%K<2zU`h3;H`zIziy}TF3X7oe9 z3BU=zQHV|+k0V>BVv;ZykJou1;pYnIuUwC=NQAQBjWI{2rS2#uvkAwox|4bIe!el@^jAkUj{Cg6Xh26k@%{vI%l!`UH|C>{r zj*h-h@yqB1;ysHH_vkeOeM<1@8EcF(YEfp|g0A0W=msf~lYe)zt2L&l+ia;KWiZWW z{zu>L%W27;k(63Am|8t-Nu7PrdJntPJ6G1)=gZK>NcK_Os>R|vtRLJ_0;5*uzkBm+ zp(7uMPCn(Vr*ynOmW~fcQ_(&24$}2+;xe-iGroq5rcQX7Wda=i3Z6x z#MF1^L?LmHcxhs#G_WfqyR%CL+e8(Ke4dQ3Y{)*LG*?-&)%&@0{CqD_)iYE~`x`56 zEJ+aq7w~M~p-j|R{8cPcZV_+K>J*zME5c`ZcjPS9!}ovYn7gbG6kiWU!;qmkes45R z&mE86KGQMj^E}+Q@-q;oUIdq$-6Efm+RrPKC#d!zl_p#S24ON5osp3p~>98$s^KWmY#^)eoy7-t%P6$vm8sPDj(m3D^|wit#c-q59d0 z>n;cGjm$ByLJzn9cEi>|%BXtYAhiK-oOqDx|ks5R6} zGDg_IAm=t%Fs)0K_PVlATr%RgIC$eZ(X%984AFcj z{vGy8G^%_rT55b3=dW!NZ@iSnf3+%bJlzfQBMsr1W{s`2j(B3`jF^qX@Nldv_LodV z!u%O{`QLn;s#yX}hgGPYy8+wPx1rc|56ld`@phOm?wvY}K^cB{=yDt?7N>AY!{tKEeH=#AU1J=hBXqaX<`mo4|ChxYUZ}9`D=b)i9%HEx}<`SJbKY&(- zS<^3XBbuw$jl2UDDDesVL{B!c#^4LShL>ZeJm;E73h~;w02_qo_-F7C1=DWhQ1eYp zths?7U2j0$@+$TYx(x4mF&OqM8WravVZ<4hCB4INIWP!+o}7fS=5f|19OLuiFiaYK z(5rJlKIrd-+WV~-`EvuVJXwhi6-%(aaUO0u%;YTliIDbiMP~tjuBV(4o8$;1e`~y6 zVFat^-Ef+Bt%nD8LG$b;QM%>3_$i=N{5R*7cuSfs`o2gM_o1y$FNe6H%B-E&fv=Mp=IQr_^|1cvoh_i?=WtZsa>YT9Nzg5t zfr$R|k+XjZl;c+8-lp{^zrF=7gLY%+qy4z<>w|B8hw!uEDE9gXKzwit%5Q@3V@4=` zMuy{d5B~fXMPc5)i#!X6gF@Cd=xQY5hRk*3XD8v$ge+V-lgl%weB56A0&hB=;$2ZO zW~x=kQ^*~QU5RfXv74O?);lc1Il;pC&3^$nJb`yyTMuEzVIN!!+>cyg7xbTR z!D7?(7?!&l&$E``>pFLQ+&dFxE)!wI@B3E=7o6E8U`cX6#P#kCZ+|m&%7W2)5#0#gnhAVdyk6ww9mbEkqr>*5A zHl}?gH!SZ9qpm#^cINsD4bg|BNAx73diSm3&SOW#-}^$u2WknT&EOOl%!maxmTM=0-mccw`2y>S@y{Kuk4!$dT+%s{I5 ze1v3pqOxW=doR{PvTqAcYwkqM!2Njm?!f)qml9YArwRUNg zCKXMZLn&#{Ae9i3h$badB_TtTMv7!Cjf6@<6s2S+mE^g8pZ9*A_r9O^zjyl_=bVpy zIs2S__TKCIuJ1r|PX>~F;vtev4k6RBaEj=Ef=cF}qJS-D=$TX&F*)&6CD%-r zX^Zt924b%CNYVJ$c=5*fxuQSM3dpb+DHaA9i1l~1#U^caaneXRap{C^-g9lERz7z$ zD=p=Dad~u<`&4U;<$4+9ITmhN^o?`n)q;{~tk)&lKl?Jt7ha$V6M2rkNi21rK0}5M zr)Zba35xFtr@Eq0l6-TB7S;#Sp;Lb3@99G+FLzU&>`oF=+$lSKJ&kKzLp2i1C~D8MzK8XdKQu|v`m-)R62`hHkAR8NiV#VY9q?1M(r6n4-cZ;)d}> z>bVd?8n1(>(kGDo?;oJv<9z6K?jHJ5xQ)`!ZQ_|UE_5kt72Vw6NTv!4X|L5BQjf8r z^o0{Cs;@u?){Le0`KC0%+>p#p=}_^b-jp$3j^`Br#_WIwESy$>FRwnv&gyJT>dC;~ zBhoSMNDOvchT!AdzIcDU6XquC;I9!CaTy`0;H44(e5oD_8>xNq_O^MI|cP z+LtD(45SMg!^pjaYlr$+P|agTZx&4=4>KEjwP7wzu&^ip$4h9`nN_sD-G%d4Hqp|h z9u#odi!Q(Op*#tWYm5z~>6`}^Tf@(d|KY0SAEC9XM+rZlp-ZZ9lsPMdp3I6O*_~%- zu7w*Ja-8L~Ng;i2drF(Ke^8D5ZMu6Wl~_y_&!+oD>(u_x>lLzMk%g+L(_c&Ms?isp z9U3l{g^mJSp{aCD8lU~cTpqjDjF(XM9VpG zcxTLUto!4G5>q!|XwNL%^+**f=cxMe z3b7`m@!b{&)PJ%bAIiy)-N2_s1^9dWOPs0QgiG#7QMb7= zl{sp1{_#Kx=x0bpM@>lkjs^9d%BZVgA~|24PW_h6ru*%7bmYxK&MjX-tG@G^=As*& zaoA3UPF^(l{C<);>Bno>05Y8zMAo(kX{32Dl^2K7^T^{=ta^$*JddKs2QSbVt4Mln z?@K%EKGV_JH>ls(3Z9uRFB&Qp)2L>i)6`Hx_58RWu)c#5{z{8_E0x8B9FFf?(GgFq z8zQdgKSH$R{)M41LR7LGB3@0>5yuYgE56;REFM*s5&KgIeH_Eb|0|(eYIo?ZdogW& zEHA1>a!yv;4O(*gGpWq*rLDG+l$U*hMvsZ6DN3j4@P3ZdalYcOoR2X!Y(vAVL2UDUrbpocBFH87A+h$ood4-Qp!L6xs{Kl zdTSH9x!sUv9@eM!yPEX)oHA)`lOkJeLW{{Sag`FU#cFP1LPs(V7LI{?h z_ro`;uBa5f2w&VChYAxKgg^QZ*|}NX;OuS!4Iv5O^(Pauy>nr(5{D~ zsXWuoTNhH51xzVnOH31X`>#iI$~Jqw2M@Y4L;kG=2I~y28)@FXGqJ_S7x3 zW9BZ(AG4PVM)=WMhX688I79_QLn+qu2yHwNNg> zhgU}1V!s_@G30?3UaGkv9LjNH-8H|Ua&IKujnQQCX3A_qb2S8>{|!_ME`eFUqcV(__a8W!HjM?3F#Ja@evU!+UZ znlY+0sa2Em+VttxU_)B7+LR1!%n9ABXv~)hG~&io>eijb`z;Ho`14}=kg$p%&xOKP zxRb;CZS-ls7o}zTQudfY`WF;J%grNb+yq|pB_xyn<#S}QaSN^aOJryrM{mj-$lfH7 z?zEb4&GE!w2%3>?`OG}F%Dfaa5C8~cJ z%y>VDR<;I^@}34dv@MRtof7G`8qX(}Jx3lllIcfZety!5Aoana6crgj8tJ~If7pxS z{5`0k$(_XQF4VVj71t{*A+3oEsQ0FsB$dG@HH$2G0(JRt%@241CB7|YL2gmJ>dSc{4Cp4M22*;hBhH6#D zsH>`p^CMaX`DF)%{9m$6<`sPN^Q>_HbKORl%x&CKIUI?WZ>x!R8$*`@FJQsLDB9%XI zq`RCEuOWGiUmLaQmw~<*zFkS|ncq&Md5+IcyWh0+tF$<+g=g4r<29eNhIlnrOH5E2 z#63^7M9B~hvG)|7Wzt(wRAw^b*r~s1NWu@=^}LOe?3F~FW5T*$P#jHfCvZJVWdxN5hLUzmAZ;J&OLOyiKd!}% zDym#4CVM3&~Mo%WUaP>m<4|4r%5)OVX$xMc!#f^lhX8Mad4Jy1yFqV4X58 zyCOr)B^_9i^A-mRg*c)<6|yoTs5>$VD|3r5s<0Z*4rs&p zZb?e+Q6l=@n|t$U(f#0oWRg9MAi#{02almmDOMCvBhrXh(`X^DX{*0E&|0ZgWOT`y zMwxFS&-XiNbg(z=+Y><6BRT)uER;^fN7HCbpsghdlonVhr1M zybK}de;qXQ^$4*;_b~l=T1pe9m6QLvDvI0og~mcBb$CmOR>Ks;e)m*(=774`%y}B~ zJJrRnUsc4T>I$NYpOkoUd?#JJ_l5R-tfJ|?%jxj*QYtnKp^}Fq#7#?okgzg@6k$K= zFABtR)1%^HPc^Y&VYcvt>t)=EE^rU3Xlj=WCA-5RG|4f5o~-ny2*X_*uiiw{Bb-US zb_MxrI?&sS*<^1!o%`L2q~$)I9zHWC+l^+lX3sEkj2=kE3;R=LXm3(~szeP}B`HO> z0~dO|!x>)+(D-Qz`p%5N!TY@NM2;h_s0Q4#%?!;y=-^RM#1YPtXhPqFn^O0MZIfe! zem+Zuc$F^@mc?xRkx})r_2yB_3f*ZEvVOc{t%F8z5 z!(&~zbg~@xds8Lr0sUx4gAT>L8%$EJhFq6rMiMF(Be% zeB%>Q>uH5oa(nQIR^z6oKy6W0R9q+G_qoHEMP#|3 zAL~3XdTo_8^$TEBRAWviTTH2de=e4922ob74lO>|mx5RJqN{&o=~P`e9uu2zsA2^M z=RCxBRVnD2AAzQWwqeno$r!X=7i)4PG3ryTFz3S)LH=#3U|St2g!+UDK@;W)n_3iw z#r;mO82yPX-c_Bw9^?;Er%PdO{s_?CCClcWUk(G~6`7XlYDoKe4}|8SZ1e7eZ11?M z?5@3}@N4TxVMaFzQPggCjCDx~_BMj)<3u7YV(Y)_Go>t3=wCC1TvQ?Q$ zOZ(Z=1i7Wu^^kMG)_Bmqzysv7*N@CpSCCQCF&f6XQ48Xw#GkIkRGpQ8-ADF`9~IY$ zsaXXmGF#Hy+k@9XY0${+7is<;u8Uh#Ot?o?)or-kFMVhY3{lP6enur zqwVa=c-tltt>5g!4|^x$8=bzWtn)!Q+jL&IP#z#ud+!oVXHO8SmzfKrvgCy1Nm)!M z#gWxS4`EuHec|ZWIymzO*|*q>5EM2XE|y#X|8srWvSr;6`do?43ZKYy;#Aq~3%{WD zsvL`H?Z+Z!ZfBcDoMq94&skTMvM@EuQb;N}ChR)eBy8C>0PU+6VtrNsZdQ#$+k;o| zXL|wK_ThMg>qp#{-ic2J$dJ-5C3?TO4`t+Pk=!hOQkgcCV)__S*VxfiG?epyw22ag zspKlmq~<+~$ToH@Nq_aE$`1!~zNR_Bh# z-M034o}Ysn(`6{is*l(mU`g8teZ{HzjuaOWPD)%abJW9=GM+fm-l2#R z>zI8p8f7H|5fkn4gsUFrgwzQZbIu93)2sx`Zz@92#_#N-^%1uBi90)Syf?de<|$-s z?SclEKd^MN0aJDR0xJ%wvM$d~kiDA$2V7&o|FH*K6;lbnK5@T}UG1P+r^s&CK7gHN zn#@k6H+w685dO9YvlbgA_PJAojaqbx&DbRfHyz@HXIcM*EK>mw^xBF#GsCf;eFDl{ zxq;|eh{MV&aO>Y%tW0S`t1FTerlvrW8miobryt#ztV5qh52A63!>Ht*DQ*5ehIX9e zbHS6=q`1z8rfEBned9{fv-hSLU#|aiJx%RHk5IuVj_<45(Ce39VZzIJO1c>?dhX2< z9yBz;9`&JO!55wjEP07OViGNXkj?ABDze_r^TBMIC}8_nS~0wX*1P8!gb{L3`TN^-SQl_x> zMY!m|appTaZ-G>u4H<}>caqDoD6T`lP#LxM9xfi@5n! zDC*g4#rzrL@pp2!kh48r(EnmB==@7(#UuX0oL5zlve|=`NjAWT+IVpK&Qjl=e%GUG$PYB}@4=1NW$E>JWon<-n>;l8lZ7S!&dUuY?RX}OiSTnLztR+obLs7a}L0rLcg4WJ0;;Cy|%}EjR5oggdYJ4!gkqASaj8uB^={@!#$c>=o{%77_m!}Fr!aL#f#OZ&P9w*PK`RfWCS7i&E>@BVuzu6z%#JLTDGuODz> z?r-QGcL$vPYhlo?U>NN42Nn#^V+Xlbzhvw~Az`5|-i)!u@E$MxksOYa!_VQ=A6L-I zCr#PwGYXl<72+92$_Q&5HY@n9`n=!^!x?Kq}?^ z7;V?SWa-(9K8DLvPl6;ZyVHrvYnt$E%xa&)z!)m1SAXGIkhyJm>@z_Pm3jGgl$kUXmGhXtQ1u8sXr%mr#+r2EM0h zvqN6Lp#0Wf2p%;E;y+5U9EV@v89Rcxy?tSIs?8NP^j{7slb*rl{TA%pt3Is$q&oYl z`3*kr^<%@AGiGpfI6Jt@h=shmz}g;J3&*cq7ruQ~!cQkG@I$;aDjoC3Bax@^hd~04 z9+iO+H*#^G(+m9c;vGI)@)@seZN-$rZgl-CMWwb%v?NuP+qgt2dOCa);Au zr%`m%!isb4XOOgn9li40N&V(olbgFUDaK2)>XK@V-B3@9hB=7`ZLfo_HrF)$i=>RD z)->t5i)b|F4aykrr;Y`WX!w~NO1@M`$)QiE)S;Bt@a#yZw6~Nux0*igt>)gr)%2qQBnmhRkEw(PC*@`)2XZ?uIhIw=S^K1N>?jmj$CJ}cGrboP2=X+U}2`8Pz zA)(w0wf}pxNgBbl&pOkm`xD88W0cKZcI0__20i~ao)W7@(Y@N?WK=wu&Nk}MnsyDc z@2^JXx8=!clQiY{{l(LVTQR$y&yU7bh{Gak$*ch!Bt;Erbr2LC-H{__op>^LOsS6J{aUI4%y0 zic|2~h}*cqwGjJ!DZ`NXw^;kV0iRy@jzf0-#u)=8>1Qu_QeLD&b`^cdaDg^G&KgJ~ zW(=qEs+^11I);ulPN9vCwlwa|1{%F#AZ;1)8uNU^1bK}*VWP}EL5ur|jkL89Prlcu zHtr|6f6XkCyLgl)>2ke}(RuDE%{4wdGwHp>L+Yi?F~?*65; ztN1w`v3WvGwS`2c4{6rfOzK|E$IUoT&HcFk=J`=Fv7bd-D*eRR1A6pft&Lc(swU34 zo-JhUYZSEhM+-xgD)9INeTu%mk*-~sLEp@$(!H!Pq{w^jQ~ZX}Mae;=@IjlD9Q%+_ zp$f$b^7Kqriu@A(U{d>c-0IYTQ(sqMNmUsJea^>h&AVs;shG7X2K8eOFkfDolY2D>mqun^rPW)N>twOkK6r> zNXlvmb$6enqGRXj(c<&8J1m*zcczh#`yJYzc#k?JbAN~P1!Ttgc5hmXXzS4;O71A2 z;NuUetLYw1o_dFT#-x+yf@Ioo=R9o)JWoSxBPeCg5VC(_M7Nv+u-M>~DD?ED?+q?u zd4mha)oEjSVJ`OJ+m~t*Oi8&mv^3M2u2^$Cd)r|8!Zj_MG&IRuMV&eo70Gk4ED4F- zn0CJ%wFWk$a})hq)3Nu8SUe$p3^Q!}@#7~K49K30TcdSw z_Q^&8Zw3jUZ4`uK8pGM!qy5>vN!4(sLXLrz5z8}p37$?;%wtIiJS}?)DX#VK?`Y|_+D5RRXk#_7lh5lmE3$bR1{~)-1CRH_g3A_trn|t5#r%wCrmtHd%{>HO z$QrP)(SM+y)j%dL8^%_BQDQ!yW0`UFEWsc+S7^@sC%pPH6mj-+^lDv)^}0SdNeG?b_%E237i?KfQ6`meih3*d;FwvAou@YpT@mh<4IyQ&jZ)JLE&_V;|KRB+BKU@p5CY8&}_=(y#F0` zcWBnz3|js$jh>&2C%b!TG}-7IDNH_03hM6UoER-kUU@{Q=qVF!X_g7^`zWx`dSjYn zU@v;>-=o{JRm4r(>Zpuk2pLNUQphqTnibK5-?|mZc#0eurby7^$}V)wZ^heFzanw1 z)ZJTeQ2SXawirCZ)8*NCsgrYLo~QA;I38o-BT@ZoFdCfo#TBY9sOfHlF~-BOB}@V} zQ|<}x!siKYiD%e=mX|PMy&@Y`Sq_qO8X?V8n!VcQZ3Vx)Aos)xwqySy=47tJ+$#e> z8f~GdcLv-TsKQ!51wwX`7q?60&cnsW}c zcTQlRLTcffqdglL?+5C-w;*HoK{hs5PSBkjD?A9P6jprDLZ3rcXxL?kIZM{#+l_u` z6c~i5!%yO&u2|0fOTybrui{XHZ1hRb!=e=>c=~K9zTa7enrU^|Eb|$2{(Zyd%ud|j z_X|eW)ZtFgGbs0WJ9Zq^z~H^bG-eXr6SFMWT9$4J}(T97HI9wIh+PDf==YC<0 z^@$|2I23C)FB8{tZNT1wAR4cIl+HQEP(;cFs=uE?10P+ccCG7#de`Z|E$;m^igW1? zrcn8#3)J^_G!0ZfN`2o1abK?;wDRRL@!0ARTqT!86)%4>C;FvF;bY@Eyu*DA9E^YA%Ry~8ecESi2>XB^7I5FP zC1t2FxCG~#J;aW%dsq^C6$g$?#H%-BapTUDctk%4C-m8mQ>EQ7a`i&IbJh~as`bag zpUQ<3PBFr}+#!OY_7Zlo{4CICe~3M<#rALWhd>j?%-xgWpy?f0HN%00)F-k<)j5#) z>IF1xmt)JGPKVUvg)nLPNsyK5&8$N1L)szkr@G#b9WXLwb0Q>}`}|Rm-_nQKjOxo; zmJVb`l%!eW+Gdb|tWbUF5YAPgNgp2jyz&Z3lUGHxAn9WS5E!Yw27@a?L{=&n?XJ7-s+PT@Or zTKWNhtgOeLiH|YzMFNII;qrdc9}fu(kXnw zW%82e97}^F+B7GDbMxZJc-}dRlgl9;?y=JQGgoen93hYsDH>)g^NjNvTA@@(&UyD~@bE0EGq_6| z(>WI>KexcH z`3~&TVF{GDtHBy(s0uxwGT8WCM}$Sk{e>G{--Y%|lW_NiG=4pQffi1c7`3VlPqyY` z*Rgx3M(OBPk%TJ>W3j?40;i`R!IlSsXd$;3Q}1p|fq zK1PBi_D_cpt5len{tP@DX0ob8Z6>?yHMsvZVu#eW!o@T>HgsPXoRaH-nU!PMH%Wiy zbGm@p7wQX9?e;=muVaF_RJzc%Hc$9F<2A**R~5y0&biR=^JJ~|TFiRiHPEUxVGR!b+3*`n z*=WPpuy^HWDAS9A4Lx!2D(@N`*&)eHkLWRttRd_KNwZNCE@6`41QaFf`W|gP+$EB-j8Sjd5r?FsO#c>*x9fk zPJ)>f--1VLzCqV@HKsAm8tyb|Gh-G60m~l1;Yru*7Vj~44VS^}55tyx}o z3ABvc4;l{?*ltPz7ZYEovvG%BmgZ2~rpn?T1wmzP0X#eY1?GLpg;w1UP;{~pCVp3B zmmj}@g6<|zu6h8Hqvk@W+Ie`rb}pOf^cM=_RasA{KAZk*9LsNf5BDt=vNxHxxtDtg z$W-2j2PQkAd#*LiqTvV{&HmV0GSkP_;e{iz2Q- z?BpQWSC|drZOBUB4Pdbwu7Te1i%?k{0a^b#SbF6F>bUipB2~tV ziIPnO+%si|R})d~IPt3QXWIS!01enEFI3qrhiYd(7*kyc!()bkUGp&Z=i)#neYhK5 zDt@zCuxT^o)_K8g7jxLrY70Tjj)VWLSK!xgJk!jSWNJgDSxsFq415s=lH2BiMN=#| zPmhFK$8SK=yi#az^Z{Q8gd1IR4!$Mf|_X2zvAoI5D{mdyx>-i-rlJpgKcm9Fo``~wE77p{ zX+B(Ba1*?IlcD@?F}QE5fqCsGVaJx4%+RSF-gMQDKa_h9Od6iU!8viTJRaDPx8_WL z>09tlk!6j0-C)6yJgZNGy`VtmAcU$&vp#>8fkERnhkG5FfUxus3c$;ZdCws|XE;Asc$LHOU7+<^Dgoyl@5d<#`huhunmf z>rB{-tIn|R65n_IWuWR+JzRJ_8(I%4v3`S7V3fljFfAJms;^2xHoq9W6?I_M;0X{h z?I0MZAA>ChYq+QB9@sf32^#;rhf9tlS=OclkT~}#d>cC&WD2f;+@RMm;`9rcbG8K@ zaIF1tpaOf-vIG{oBtq78OD1#e3`qHB!<4BeY=A}_99QgsKO=ORao`mgR1gL6y2IJl zN3`N)KMA5&mIZ`VQUAq}Qe91k086xiL91z@%)2JS0}u;ayJt1;8v z;ckx_3w_=MEe;`IrpUlLB^^o*0?6HvVq;zJzylZp;v*#MC z@!=&f^2{}ueYXbuqwbFnvn_$r@qVC`5)FU5%B>O`{9%=kB9mO92AIivZP&&!C4U`Q z*tZEbE;|NioJy^fiu0jh<|puZl@B|5pMb8IA6EWqE_}O#VONF*Tc)uSs(e(LS3x5T zJO32IrW(PzA!^KN^-q}euQywpsLni!A3#iCJ-nH-2e!2EYwApS1%uo>8)W`UNc`t! zH&a?hLP9}7;{UU#NlcJfy>0D!*PYIzb~t|Mlp9-{1PL=P&uc_|L+} h$^Ym6(7*rumXMI-H@lg#a{qZO&42#0{{OS?{{lwQVTk|$ diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_abs_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_2x2.pt b/research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_abs_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_2x2.pt deleted file mode 100644 index 065918166559e60992cd2c3f7099246b8a055950..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 65259 zcmZ^Kc{EmE7&Vz`peR!%p`;Kc=iFz%kwzJlMhzmRlBqeBDN`f~r9y~CDnpb|L?xA> zN%bq5MWfP0-}|lapKn;-JFL6zUFW^;I_I9}IeR~QAE((;(h?H#@)G~gOIc!|gnNL; zYVS=eO+4KYjPb)L&MtXv=H z8?bDXnWDeM=+O)0#P<&rw{%~%(qHm_{X}v4I$!qAsoRCEmR zTp6&^bE>z;21S2)!#Vy68~hdL_$wJZ&6bnb*1Tk!E57yr{5t)AyWL-6=>O4e6VvHa zH2$wn|8FCZFmVr`VAz-E4PM0;?X>2f9aQ-Fi|5&axI(se{5`g(NR}JFKE}3PRpYW% zioCw`CmVKSCVz6Xo5f{4VYQCkY%br!-Y?E%tM=ruq=`4!sBc}&+EkPOU3-ZI4mr=Z zZhy?)Z%t-O)||U;lH*53n|W?T2eVtVn=MT@;zRRqF$;?>Cisrve_ZCV%r9S=mrZ|e zI3^^*@y3Z=D_C{>F@xvFHB^n#mnlRvnk2H+2TYIAGcsQ zV}s+EVrn_NaM_fTD3;aON3r;o@7N`|V%BdT=W}1}rGwFFka;y;)U`1U^A^U@;^p#O zcKKU2Eb$b(h*ry~cP_Ed33B{$bp`9TFSLBss?Wa{+wltB*KC@x4rf!Iv)I;LR-J3Y zgMAipef3d%U)EIq<=0nM`bLJ|OFqt|c6?$7m0bAPn}KYHMOPWAOu2x&B}k{()wR!@dpV zlYhwY(*I5{kG?&ut|5-8Z7XGg!T+&YYy8=-NG0y}_W;Y1*Wm5-evI|$&pP^cvdp&$ zY=O_q0MY&)@Sy~ z>kaeX8_zZdbui<{tC@s`DYLk9oEbklIqBw|m&~za6mxwo&-GiEurfzSmJue;PG)5> z(&DvVADOpl{4W3lFfhOK&rip#RwTagnT`D~Y-Jgt1z+LiTCH9oD(3n$=HJ z=MOI)Vs~{zShwkZcClj&&&epXObhzQ!q>|1B~GW9di$97 z+AJm&o5L=pYV-8>dVI7vZoB6O^7l7e*j-Tui%L7qE(|f?L%vqC?10o?P=t!6PM_Piw9h3SthWS88H__7jf_Bp43`408uE`@ch^3yJEnJv%n zkKe?m@9JhJ7kIG6l0Plgc5r?tWiWrxQ^Eq8f3Xc*#5MagP4b)T&rGv*`PPvVd?jT0 z;kffmqCt}HDA40aR7E`GVjTM-eujf~s_~iKr&)940k+~}Hq-2_U`BrX*vcFozNh6Z zJGf>xzj<~Xe=^L1rCg3+viCDtQxn+TFDIBmp(c-Q|H-^;qTlTyl_fCSmY87bn2s2XDnn+1&K~3B2=x@eqYPh?S zuHW52Ew&+aW5y;r(;Z9^KZ9vsZ6f`sJV1ZQk9^ZQh5k7=xtx1Ai%Uvmf6rcFpJKyU z@wP>L+%9w8|9lS}r=`q!so8VZxAQXl?_^*8Wos=v&_?|Fq3`VGY)P)wew>{wG2_x799h57 z89eexyO2`7o<^Teq-_cZ$k}l_Im?7lPTdA-UMoJvZ=emHK2+)GL7DxWsCuOh4gWWh zY?{Z?7hOe~XD>xJ`{W5yIR}K!my&|d({uc;j2wUYY98Zf7w{5mcV01h72CDz8Z(TI zV09~Ec)jZd=G8Wkf8CUT zr_K4aA0j(p$aiO|%kC2fZEU0b1DzqOvBmCR@O*ppxgCI3unr`e+ zq=(xE(L}=`WUH@DMZM$c-!M~>dTvc2Ll}K{w1Vb4`4Emek*4T{(CagofBJHY$(D;T zgSjg2lR2H=UUZ*%B`I-*LqFMs8=$=pY`penZ`OIZyk>9`5Dm+}^Oc$zs=!u;>mE4?2#^*uTgG|Zp=XmnFtx4&^5IU2qLG0WBvagV( zn@(Lqz{qdH_fvU7-mr6b#*^kX{{-@(EiJQOxNQk51LqtR1aI^^pG`MRg_VTp|a%qd2m+nr$ir1k)=5UbC-B&YD-#q#`Pz;(7`c`%>z-&+3Sxh+4q zOogi-cjFCaGQx7#34)?kw9w`8PDs~(D4eP>pfQG8v~`mUeOX~keh>Vpd!!H9W$vPn z`*u_CyjW72y_d$1kEfIJ2kB^BJoy_XlIoKnx;r|C#z;j|^WhXar|M6g3V(#b7I9oS zUc;$@NUQEWzb(yWs66CsD%(Ns*%dMWz}g z#lP|(Uh}D2*k-$)GV2b|62}-CXBSPIN_NreD~YtVE}oSCB-7BQc=|mmp3K+nrSuQG z>AKNQ8vo0WwpIF4w%;sL{ozbWE41m)FGJcd`JZsyrcKzsELiwH$U@j#AuU8{hw=1z z6M2ul0$=rG9M2TbH*3catgpd9zOqh>Pd%#2E4Pi{yM`^{-1Zl1%)H9x*+^J=`^fRc z2e+AFzh)+@Wysr3=&?-`pRo^80>2Ys#5X&AVp{d;eB$ZbOgK7+kG(3xWuDk@r+ESV z*o7H<^<6`L;ZQ8M(U%mCN)8sDl z`Y_21#U%fH$f>V}Xhl5U7BnKSA^}CEtNDuAdVI}pbG{|*5DT=o5*fK35`8&Oq9+kr z{AXn|Tk^`Bm{|iEo_k9RHod2ggIXVc&}=d{EVJUTed^izvVW}R!XRE|HJmT69>TYu9M1py$9dhU5nS7>jlI6n z#vGfKc)!#{_D)Zp8BD8ZZYE-kPeE*X#y}oeae@VY&*Xzu-MII*VZ3|zOupXx0JlC| z!w+w$pD} z5^Xzslzv&K(^BbN8Ww+wQtVC>yLyp!%|Ao-Wa+KChYM`tGg;ad^ z2fa>sOZ_h&7NVXB?3{%qHpa_}rbn8Hu1r=%(Npo9h&JOk7U?)8CoghYm;obuMZW*P z1lF7|gRW+1!7|$gw^IY4skjQZ*FzvNB^p&iw&cPcqqOd10l=#G6FNqxRd%Sunt8jT8S`*4Pi8I@8<^GS*xb&946 zxfJY~PPR|PW5bdutLhN-h_+Dk%4k}B)1Dqr@uG8&hLJ*$9?d$^BJ?aO6fR$g7wUUg z3#(oZ6MpO8=a)~v<4bnLbBTL~eA@j@++oa8?mhV|-{d!jYhG36tnd+=akZWW?7GNq z$=Wm15m(rn2o=s&KV)IAPO|)CO8k`OEH3%WnLAhgWpn#4;Eu_C`M(&>qmr!o^!;0T z+}d;8*zhQycJ>9gzdAlWqavxNG3(OCQ-I{oFs}fsblsXIy$|C{=Vp>^6xjP z@941zAHE2Eif+*BAEUXq&RLp~I6(9?d4njqq#9pxt-1cyfqcOE16X!xJC>{5VKdDo zxb~rSL<@IgW&JLU^z?*$XfVq6tiezr6lJDia8rv!=-F6w3F3TT849a_P`F)Oi*NUX zv2Nu`Tn*j@y*s-bCX6*_(*#KOQ=Jso?f3#C&PIc$*Uxhnj|yH^LDIwPAAaf z!Wg<79zx4*Z>BY~mQ&g#5BgPRKtVrd(ozF?TCYEfhUZiXt>GVppD!bX;LdPi?6PUX zrgtL+Gy2B+XC2~BeI9d}=(W7v%#n|M)|Y1#8}jP69y}_yFaKH(zORk(-AC=%wnQVg z$)}s?1&!h(Do5}dyF_+;RUf|c+)RFXm>eHUub6eBIj=hq!_N$M;%4bjdCY|!eEjha zzImRuaAB^hpe_+Cn4HWIoLwq}$`xJ0KR+cp*x!{#?U_N9-(u*fQV{*!ok3e~@1x5# zCuyGCNm__vI^$SLKVxr@($r?s*m#F_JpD{-2fw9VInwxS{gISEDIx8?435c-!@i&~ zs7hE1g_>X}H%KAr*m0rzz89aMa2ngg#)#IMFGF#(9e*ia_XqpL@PyXa%&7Z3Q@tWB zTKh>))b{8tB&QySd{;D1pUB1}_e2=6#z$PsjL8b@sw>TZ+Unjz( zE*mb#qv7pw9Fmvcf<4g_tsEsI^0wW{R_?yeHazjznV-=7)A4|$pLvRZAvZnp|w zsPD=z{$9jn?)kFwA)V~!zc_ZZO@jBH9?Z-i%X8N&Bl)%N*UYec3O};gh&#OUnsaOYo?u=Z_|kn;4NFf^=Q$X;MT%?sse@1=S4d(~JPQyNJ1 ze)H+8@@|^dzJr`U?4@aTDWq+lPW_MMQbA@W1sGJ2;-CvOZ&w|C-gcgB$JSAlA(Sg#O!y>;aKTP>;vc&IHV_i$`vD3dZH^!}9tBXlchI zusjkeP6_yUF$lxQEXULpQ;>R6N_2u7i<%9FV9Bl{?C+Nho@`UYgy>C7>!%fOl{P1> zFY3HhJ85-*Ti3Z)fN#UDclmCTFlx0*;(dFl8ZvQ&ETXlh62UOB; z-ArlEw4Sg<|~TX;I=1>OCDn?oStxJvP(M6jPEmoJVq6eJS#aKF!L#Ets|3 z5LV3CFKE075yHJ}goWk*xWVEfLg>zN{-*Ufm(ma6`;z1N#pm%VYUAx?E2C4jXjag{%4*@D+VG@uTt9Jby$6pP32%t@9(V z{&ioqPiP5^Z}fyEo?(Kz)DYeant}5;H8A0lw11UIr2^Dr7poow#N*-~V z!mP6BuR|4miMdR#$`7Ss_w{O3qts;G_FzJtZ@5GoSQVuP0%DA_sR~+Xu_^{#=EwB1VCSPCB z`qQn{Rd=7flbdKya2;u1zeIbll#``eHsuwZrDNYh$lf8D*d6hjIdcnb^BGA$gEXn_ z-UnfkL4{zM6e;wdG*X}pZQ=E1IUymrfj`yx#I5vg`Pb!#d3J{dZ+?1!FW4*M3qtnu zUyALlNlT3{KNP}NY?tI7V?JBnGda!%TIMnLRr)+GVhf+zW59!o9&=@rLM|ygTo{uy zSU96GScrC>F4TFO2@}VM32S053RP`&LeE)wS~*FHmaUpY>!i#`ZDkN?)H+dA-T}Jm z9ZMRaC&)S?lY$o&QvLZ$Zpr(x)L7zZ~T$GLL{ z(a+-$qNS5zkh2G|OHbflYZ6xJ^q~!fPpDtz6489q5Rpi0xX5|6lW23yT9L%ntLS_$ zk1yvG@&29{GS>vd$IS@4RW$IYcsg|742Ag^4GiBRg@l{^F~;~WW%%_$@q?F?FMOlA znnqeMxRC;Et0;SF8C|n3q!rV$s44y=-E)ki(5eIq`|U(6k9JZ4%xOoe3oY;IM|mIn z(Z{Q|g-s5ph0oc0gw*ll1N!NcypP`$qol`D@WdG84nXB$pxBYi1va|SJ%8B27DZJRIis(1n=3*u0~g zF8x-9ioP=L`M)e5WH{BGL zX?h5}JBYt^dqxtzx6pkd88wrGuxHsZbdAe}P;(sn<%@BmD+fDol;KwYTsX}u!nL)> z@j4jiZhvsn?9gV*BjN?k;ne5V^qW@t_xfKnG_u(WP6Y*s>L=3h#5k;A^EuM~Ns53B}# zpqd#UXxy0p$aMT8>TbJ3yA7^TXUz=?E;>stcZzA!`8{;AD}$CMtt087p>)@OB28Ou zLK2#i^kv8+VbY&;!7R~O*pt8o+jTlZ=_y$u!)3T|_E04MX3)%C*BSBmT8ntp{2T0W zSS%a1K8`J1@5T@Rn8@d^Imdc0l(Maj4eV-c6Z<($i9dQZmS^N>@@-#qg)a-yjyl5hAZZ)BVKEV`a=tjl02PjBq2UTx9 zL91@1(!IQ5a!NTtjr%LeXCKmOJAS{@^U4&z{5!nOyAaQ;2?vxqx{Y5~^}B+p7>~e&!(g z(Mi0UnTPYn@z}1t4>3<`Fg45-8`ZC}b+!HYBdZv$F?Sr_KV=}=6U;;^p6!sT9E1Pv z*a_b*MNuzb0-4z}F?@#x)SMXlzg0$6o)&HmmBBY#6?BRFcjmSd%`i&gXv91U_i?PZkpN zh`C9eWkXgLvgJ#x`M#7N%xR+%Upb(MMVD{pZ5@Z$?&}fUL@tv*R^1@v)(sT~%+3^2 zTqA@5qpt{iTrUXm+wKZ_A0$ckx*k0nI)Sn@+-Yj?O7g5aO6di$#NHK=iTW{eo^X$z z%GA=j)h}t@hKCf^(m|#@->IGbrHW)pbRYXeOLGTc*nN4(Tpa@cO?|Ouygs4^X<%y9 zc&rZ9K;9uIbhtU-<*4llKe`mvK~m(EIFmoEFTj*H<|2`^gXny)EzUnu<2{3QdCtzW zkTTkdbxZv)G^7ybuBTvjK2OZeFGJF@2z8;w&|Fdr3!f6)EV&26(Iq(fy#}Sc7&)to zaBx5whKrhv|_`4O0d{Pb_a4u!+k&HE-0rmxgvTp zN8E=m*J<;%$7GrChGGtUpnZ?N(9G$bbgx$dF=P6`?t%)2i+Nt*qMMqH zH_Kt-`FPwqkPXF$X^`l10_Rqrz;lUnC_8u!tK16^8QOq7^9!JH`x*jHLNz7Q4 zi>2vl*gZWPse$pxx+IGk6D~nNibPw*wZ1MVMQ+pCzzN)e_zqniRz!Yp%Bj(3KXvD1Q_7B=B*pfVNX?#nCwWlkTOBf7IFUR{ z6iKf~l8)Eh6lC4s3eowQf++EtAWVfIVsXN>121{vzwdl{$xl|Oy^Y_o3uVSSl}z+Q zg}dCTW;$)>SyRaXp8uzUHMa^p|D%nt#Ja!G^P@y?GYu4;z3CF(=-v{>HEGcHZ}Rjl z&zL5LjHg};4;p;YlbjF4P}r?lijOWNUYbiPCmLwP&r0ep?;z=WFUYX%2Yo5-BtQ9H z>iH*&enAR|Z|aXz&-+5_s1D4n2IFah9;z%fuy>0E-mMsmm$Rm$$H5ene{aEr2fMJn zLp;UjYn5Njt)1p#M z3Y$?Q_9~ngI;@q1Q^^KGV{<=l<8^?|YSiHNN3O67rd{l7+6s0=V<^Yt zR(3xD+_;|$?~V-?PFju<^q)NvO1q%`c@GgHMTJ9QPmDN0K4rkc5Js7(Z!ZAf#4go2> zGEU>jN7v=!nz_KAL%gcLh_JceHzPl-=$so!y#dy zBbxH{3ACn~v8dP){GrDxA)Z{D-KptCKy`Sy%mgFs=jeM`4 zfnYi2jj-WKhJXd5>5S=XVe`VdlyO6s`q}u=M%}q&{w<8&E)J%qwlo@`mPqfS3aP6% zpJaRPQtgR5bf)$z{d9Xld*Ad>hQfEMij+aWa4GcB?1#H`YIt6%hRY{4G1p2BZ@(E} zMD1{dDq6sKpApQmY>*Uhj=ql8cv)u-H}8dTe_#X~Pj{Ght-&Q*NyIODLJC_PX^pH2 z|NZ3*W_?x^nb_CiPir!i#~;Gd1q`G1@5a_E7vbV^8rKfzVV-U^%I{U-!lmn&8}S?^ zlk0G6pQLERh&sIK`vME!Ux%f46_)+3!u17t=yf`c*S9a?^RnHLiL=CY%|pn1mW-6E zxA445QB){-7TfF$xpuK5Sr2?dx=SQ+H**a}jB&@O%|__Ibs-A=m_tk23QC@4aJ^;& zgV?dq*O>_G3B&Q)Zxr08szGVL1}a~u!L?Qmjbr;FKwAdS+~x3l;ScKA*h^Jk+Nk6B zSE^oemzKpf(y-2Rw0vJ7l}GKT?C>LWM>UwFs>8_W{2bCK3Z#$W24t|nk%CXY5Ppkm z>uUr;|8h_is$N z{Kq3{;!ONI%}_ec0q#B1F?NIx0@MZgNBAQBNd!K;*Z>WsC={m667#XklvS-P`ta0Q zv~%fS^jiO+pyb8S+*%DI^;RUbNQ=S+H7M5b}#!Lje=finYE1tzx<3YtDJgodpCcFMZS2RcTvsPJj zAhndLtY=}HSrp#K1j5EJ5~9(*_&!{~+n9B*bDWO0D0}pu$sl!TCcb5lhxcQKwFuLv!AecEm?IYI2W=cT!$iDNP@6mu759 zBmHMN)P83>wfx&dyE5EptkPQ2lNe9qcP=8YPm1KZdO9siy(WB+8%jD=`-SCq9trNN zCJNtFw1l1saV#$2Gz&Yf!+gqy@RSyV`?Ve{mH&^$X3`+YPq=)E@1*6SfTMh6mqCSXVQIP`xA>}WQ_VV!AMWNv}H zCI{rCPsjXGD^MOk7KggMu;f=Ta(0J;$PgUL(A_wGJ%~`w@CSizaDQ2$G)Z z@Na*DHNP(7y14C-KS^k^zKjvC>X2SlkA)cx2nzXzr0!<8?H?!#DQQNI<2M{iYry@K zdUyxdVO!c|JPuDrLTo6u?!Sx|1y2yQD-Dv?Bk(k!l}TQu(w;tfd-|lB@^4X_X*XjWKjz6fX>S+bb;0SjMBj z4CON}_vfp{zN4Lj1=Dgg=QsDB6(+dH2vvKv>1*n9;bq%wiXAB@#vVKA+aPngaz2c* z9GuCza}T|h*+HdmGU(BY{q(iFg2r}A6_gYnbyHCdf=K-8mZ82e^p$-pGUZH&WD zo3qG1QGm9x3dlFrpd{)lh?ZbGW220wc%aNt}4e(Iiu z(eXGudz>W3qS?@Es>Z8fj!>|ELk8minm&a)9%5vUHYxE<75$!egj^{r&0Q$nn^f@Cl3C-WY03?AG;=N0!**c>Mc zv1IUlHEOAE!n?WNH_J1N^;7K(4Bu-i!q(MBqmUZ4S+Dospm5$Cr}dYHFn6mGv7 z3&CU*+5;`&ICld2eYL^AHI^7rFay=MC&2!e3$kUWA#KiLl${VT+hQY*)diyLpDX^l zEG=?G3Q#aPW+q^v4Nz0(a$Tm2B( zi&}6bQ%Yno|2ZT&4MhF!{|DoxQliI+El@Xkh~}v`AWA95{kU3aH1Me4FVr!udJQ~fh&c+gT7Nf8-V;r)SN1;GY4;C$&h%nQ{^m^SxK7LA)L`wVWW+S0z+%Gm%a&KSR4OZ=xla)5)S? z8Vz-drY8Ykn`iA3YZ+r`j6njKxh|xpgef8+wbIJhTV*KV@JqiwOld$WoIjsGs zK%>h7P6pGlYW*a{<<5np$5gb=al`C`Rxs!cKy{iw=CydiXu1ntngk=edN%ZL?#J8s zNMu{@hO?M!jn9h3E$<-wrd4D`{DX+!=%;cConR=>;j7-Dubpi6*r~ z3@Mw9_FcOmo#2AtDlbGn_s7W)enF5EQ z=D2im5@rt@1)pLgoJcjmbsYnEJl4jJ*de&!BgTJ`ikNmm8MDKCDCVUM%r?BElbv5_ zWK2D+ozg~K?ZxEaRYQk%9Hv`a&rsLRSlTl8nAjIMPn`4PsC~B~Rn6W;=U*w)yuEgG zNBV-$5ZRv&{tjVrFlIY7RQUA|{di!(IkrVNndPLJ(%bD{goRIiNq9Ax7K})x13Dh` zO#d{s+3Y5zE!W6L>Le|8xkU@b9)#sKZB$-)m&zXhp!bol$aR4nY6GNTbA1q$G6%pp zRUOZVYr^NcHeQArz}#LBMUBSLmp8_~r)JP`G{vT)*2q03;9kQl^feH{e)=33Kc0qV zu`BT)WgfH_t;f(PSCqJIf=;#%s@!(KWM&}dh+@$zT7j**ZJ*vtI5bsog*U5+QTQe8N5y=>_BnRHkVc5!I@S;NOCpZ<4t|#v3DT>8unLtEO z-GSeeeXzoG6CTfVg}2;#SbcHDueY99zkfP9&dkB4)j&IJ@p6uUbMvN%IfyA7W6j~^ zHVy+XiN_^U4<17ea4b+8H^o@^bh_l=eNmH+zzBy9KRO zdbpT&CW`$c2`8xizjB(997z&%hHSIl>5|kTvQf0Aro+B8&8<`D$TgvH4$UmQ_zaUt z7{L25VD$d!ZXye#s>nx9v2n>@r>DS#(mO zjGj82qB!q|6w_HvgF`-&(yOO5f=gq-8!1ecABf{#{jpqJhrhjg2=17Rd0dw+Zf_fl zXX0nC7hnp9O_PwYZvuLar=j-~;5EttL*u4k{E7KEINuHx{k?E7bpcwk0`RYG3I4?I z#KTwXP?;Kx0pj}E3HeyGOj-k*;sbE`9){q*DTvt=hY_-;v1>;X_TD;y>ba@ddsDoD zL#7zd4&FlV^apsC@(^>+%ZlFZet_zJ=AzSS5Adi+R^%A|5D)r3K<<-UnA25+0Id@A z+NOfpp2X7E2Qa_z6h@ZBp)4o`(R0JG?n@$8V-4(v#A4g(^;ohY7&msWgTka8=y<*q z$;yGK`aK`!=3Y>*nvH!;^Dueq6r2ikz?gf0)R^fg;S*5ta1vIJH-+A1V@y{ci?<6$ zV&r~(bkz)j-5pKTzwC#IM+336j}$D#b=>Q>xRsjbUZiJZ;>qmCZQA{}kR~?YA)D9bw9KW0 zzKnWAdZE1(Zq`BL^!nmrjT{#H48p+8eeveGCbZMFV34ebNHrt8me)t{MKet4HO8~z zNyt%|fU37P=pHM=K1WBWl}^R4jSKO9nH|QpctWGY1#eRPu&{M8<|YN(RePI!v*T!TL=o-v z-$W}-(#hz%GySd!rXgeHC^$((Pmh@Kq~jOasi~$sR8fxW3@WpnGsU0jjB=unvz6&q zz1S~ovVb0^r%}wNT_ojGNu4`VDB^ts#Tj0qjV=%A@2gr0`PxkzYhF`&svNX3C9rFp z8g5TeLF@oc-1smEITm`j_EBFP6DDBKjPc9P44+?_p;N5Pd1_jr*-Qk7hYnD-n~Ac} zMX(d&^eM6{aqXrnw$-kK`cF6POy3TnLOc%w!(dqLgUxTkVQBA*odXhZY3K%=TXYbv z5@DEKo{B!+k#Jd=4bQWQuyekEVy$dM?z)Q1^QEX1?xLxv1v$b)==@X?>4rQO>-AGa zTSh%bL#vWV)$bwvq*`#v{w^*=m7-$XRg{Eg;kxn#}NaW9Cs^>Ta@` zHGtH9IM76cD9b;|ovd#7K<;9{kpC#S%96Ho<}%2G77tXVWQwKB$5zqK8M(AzXAJF; zxJk3lr%~hG`;@n}ns%;uPQ$<7C5Nit6eiI@pARTNHdPYmma3vKMZEt(T^nh8)G;{U z07;(>pz(Yhe%v-iyPYvUh%L#J5-f1+#7qojlQD3kGx}VzMUSf+8tk0$`Gpt0CoP0R za{!d2JaOpnPCUK75^-Yx^mnawfJ7wR`~uKw(36Y6Z(pNZEeAOnkDLBEpXe=S2SCk?@b0g z!2r>1m=7sJc=Z)T?n#5v(0sg9NJK$yE@ZYuW5DAhxFyc{<9DWDzi12MAMeBGA^s>Y ziGZt@*z5Tx49YWBBL2-zs7>*J+3Emn&Rl?vW?txNafCyr8`59e!qjdK#(GZ1$tg3@ zI8TiG#HEAOdSf_Cn4(+47^lh&FnpW=93H7d$!REt4p+gOZZ)jtl44Jp0_q2TAhqhh zH0tL)^05Dp6x*+nbIt?eC(_7xVjYd{6HBN1=TfEOS}LEukLeQ+DuE^DOf z_zyI>|1*kGm&Bl;E;8BQ9|NuA(RZmjb|`A1r&J#Se|2y_-Uvd{SQu8ABSqg5i7zL? zj;z1}XTmvZCR`>sfE}KXj#^iQt@eVM$pU1J@2nq7*bq?%P_5$yeaD_Y`@LZ#&cbm&QW853wIK zk#Ae*PcMgPk%`>@+%K|=Y(Jc$j>cG8efka+_d7-tC$^IJr6%g^`9QI{kEq{~ztmGL zj{DTUI2E{1uP9aO|IqIlO8ch{~(--k=Ea{Oj=e_DyKH6d^jucv(vL}Hh&Kd#8c;kmF8 zx6UQvxBqthNlZn=h;S4fPltAz80$Npf`yp(&6-ny6O+^7{q_o?oeMDg%1zw5--2g5 zo5Xy%zev1)6i!$H>A0Y`nT3SvjX5 zM8{yRS31UtHJi6TQZZU`JML&D!mM^9qFmyjdCd=>5DDk`tB}zZ43p7b;`M$r3e}fl z%bPX0=H-h2>^%^6)D8;I7a?e^HP-(Y?+2BcjCo>Na?u@gOqH65+ifP0bTojjm=Ejt zYh#1T2!yDsf_+xSv{}+_{*B$cc80xktYUrR9Qd@SSDF3PV@%g9goZ?!(aU`Y zX`7Hpqe4pP-Ks2l_3{(B{=P!aNB_|-zpwP$rw=B*|4!Gc2EaO45qc&%c=dZQYBfe- z(FjA7h%sF3ClmDeSR&>6B&>s+5A*RiW)XBlmqF&b2ii9I ziMjYH^zjsPCDk?Pej5(s?HeKUJ_c`YZNb0Tc$}LQ4AHzKyzC9d;QztVb;srSz5i%w zXsM8tNEsO=bU)qiBYTezLdXi)4N|m+_Rx-&_AY5kgN7E0_EttyC=pV>^ZWkQ%O81q z?)#i`UDtaZ?NIbt+{4(nXuRll$29RI3?K5tbgnFvO$$NFjsh6}jK{{P3hWpzLWF>MO)iEWu1i8DEp-TC`cjXIU&?taOJ5fMtjCDRR9R-(1*$<~EccxjyUM~L5 zKBvBigubV&P`)=icru86rLmfQ2J7_c8@v3&2lip`2zzOBA1gOB1AGnB;XPX# zvZ2E0c`Fa2xJA&kQ^BzfOQEnx9a>M;GB<&Us+26KY>NoFNR%@1St(HhRw4 zW8l0QB)+-8&eR4={Jjw(K{*+9s=4KL#GYj#n19y^QKzC1u-6q@J&CCP<_*y^>8Mc) zLh$Q64D>(3hR9MV-%Z6QY*AOJViAe-zbN7H3E7?c_^M2gkpzuRNkjThonR#@wj5sP$ZHJ zogm^Ej1hH5$OQXC?T#I~n!VA;^}BGfMAohC8c)@F0RyBMjyZuEicA2c@J%Fh4E^5%D=V zrOE@2^f2A#ZdBB~qUvAIL zuZUxwuU^kt6Yz*J{`N|PN6wbTIt^CzvL)-j+mGGan#e|Nj%53yzOY+LGuRAviaoVr zf~_3>$4VCdWF_wlLphWOH$&#arhFDOm#DzMd^P@Y)R4JW1w~SHc2!*mw+`U*(#`n& zWeW=9_af)y0j#-s49irt@vKE3m+EyPAZ>=1m8Xy>Vhg>M`cS&=fSZ0s*h#%|A@aru zZSlm-J9ltaz#mOMmU#6%7>k2#q2nKk!W&KyvW|!EG*9d?O2_QxK>Xm%LyS-)de4_& zPEt{u}~)Z@=>F{1Ofo^oc#$cT46cAgg_0h4tIO=^eIv1$lbW&>>{xRe-y z+@Yr!lMKXqa@fs#vh6|%&_j3C-|Ni;c~1C zPP!SPcYy=N7cRjylX^Z6pF**YDZJmEg!*%RWbV;MdHylH4LyK}<9l(`ZwrbuH^MXl z*hKMP*YNdtAFBe)D}Q7OZuR2%~z8!(34AVInVfX&7hSWrmg7 z84GS7wpQ%|>$2LFtyW55Pewgs&p&#@vNwg$X!eRNruCs>F~G|7 zE6}pBM1z|ts<|CteaRLT!%oy6V-F2oPjFA(gRn~=8ao`(=of8?l`4U^XBgi7 z&B0(o0D{(JASd4&GpWYo(j#}YpNYbwAy*jf4T0`6N9?ElKey*SY<=$u9#MN#xjSRs z3R`r`I$&CsDJr@wuy6BKtf-?%b3w+ z?24zZ>^fIH)?v91%X@!-$&@N%9P|U2MVi%&aNR`>lj91Ux^`bS+29)6P5qynHs-LB z*TUGez8rSF887A7^4SMeTj2eR8(&xdV(0GVMP>b5L^z4lIZFj8nF`pawFVOz4EQqE zVI}2O56G^CbNhPSJ+U2L`}Sa8%5H4?d>9WNAHz4bGgx1y3tYJhr(YM~zvT{0R_nt| z#U3XfU%|kHGw7lN{4IA4{-fED{ka&k%h#2n{zfv3H#pg1HlcBhH96je#khFAA53p*GJN zLw5rq({G6qs^9x1bO+1#d*D6iCOR1xOsC&5q0}DNk}p#2ju}i`FQV+*)&FVE?H-** z_yrxfbR34%@jck|aW_ofZ-dOO^;Cbd4i6hwApGDu7{@R$HdDpivkT$btb&bp;%IND z9*{%4aQXC$ZFs?rojrW`Iq{4&Sjvap5jpJ9`Y`rpL?&y=@60Z6ruvQ_KJ15>bVkW8 zkTIU2$s8y*=6p(8!>B*)XO<=3U>|uOV68g?SOfI2S54lsPv(DR*Pd0uK+s3l(NP*! z8gsFsSq$CwvS^c2MdvkT?DE=*fnVy-oS}tc-#zeqzYl@tI}kDdFiN`*;SlZJp4D7H z-6cI#MqY!vwh@dn%`y1jZM@~Whf|kqU^(mp{YHBTCwQWq&d47Nd?CNa1BE`pu;%te z*WU=d+T?|I+Y(@~(Fdt|=@@+CkIW4@*hc3q0Y%!^Cq?4)-g1zfBzTI~V(*z;Jf+-h z!`K_RDu0B7{x1mU4dZ0CJc;WZ!SqYFN#BDJ%r}%L5@y42Tl5QY`X8|@z6OV@TA{l1 zDSjo^U?L<5>xN&#X*3ef(+VIV6O5L$95_+mPT}}tJk|8U!NUo7a>xrSy&^GdwkPE) zgK z^K>3K35SbJn1b~=e6L~<@(DoMjn z{v-Q%nhM_LePt!MYT1djUiSDoKX!x1J~lJ{I(z+d9HWyrpb@Mr&(Y=M=2+K^Y1}R2 z<*=DUn%Ajbt9*tL`<}OtjV{k&Jw8;hs!>ajxVMsBeMPjO1ceFO^l$EK-56Zkguz9# zh}P03?9{nHE+sco&VLpOSkVM|&2EUCsKxP(`G~z;i3{Qh$XHg41(w0spqYzjHa_qb zOv4{OcRa|C$F4Rq8}82fY!(h?WJ6@3|wO?o(QrHi_hLy(@SgU|U#uqts2R?>G|;>+Tx%*3Ea-^<;lB5-M{VE>FS#oF1G?7!=otmM@` z7U@PT&zvsJ`G$R(S&`fv?ejKFo!B?#xe++WD(7Y69R{zxeKJk{7 z&mCiRzAeMa6<2X2gK`K{iV$7B4P2wDXi(!|%=!RaoDXA5qZytZJc4Z{I%xWF4Qhes z5UzFvNiHX_>F`Mezr2oh&yCUc(}14ycX9Ej6%r{9Yff>SZym14`0Pk=KOdM^xS;-C zAkt_a#{T2MNc!mml}(SJwcii(-^Rh~L;yVKey>~e0LOkm#r~(k@F*z4w9irKFe}HE z_wg7x{svavwv_dZxkAf7LrXx#zL)hgkO9E{lR!Bj=qFyVifkvKgYa|U>s_H z3aw2K;MM*Zd4~e9ZGAk(4gIii^CKKf_JQk-U^Ga2fcr)uiqc%rL%E>36i3XzGPX-{#YEk;O@xrzJlZ{YN)6R>VO4x#2Fc)IT#YQJ9tPmvDpwrk^o?j87M zYNKKB0C>bT;q^ill`7k@|L9`e5&g;*O;S9;c^MWO{A7<%f5#QiznTw)oH!0y;hgkT zQ;zn@d7PEhdz+$qpOI?~*3g|@#>AZw;VhD|XCfCma#k-|!>)+zWxZ)F5^SORX#P~T z(ewlRGkP}SK990HBxP{=@IEZ~aUT2sSfR9(!OU~ZVUeeaamq!e<|$$NGQjEM1{jNJ zJs{!Oc8s?2+7U1@OL(bf1D#W>X~8oR}bV|y@NNy zzO=8W^RsF&8h6-Y;z1Z9O&xH+D;ftyoauZ)aXfu5BzR_E+5WeAqfFTs*4 z3AkcfiS?cZSZZGnXWbrjTxf;MT`BUpyA9j!8j!lGHni@QBIIZ*R`2b>U}qiTT?-I* zw-UVn;<3P_1Tj7#5E^?5E^j}|+hm{#UersQhz}o}aAP!@dI20Dem@*vWo@Az{}2m= zP0_g5539@XU{S9JCOS;f=i!LM5tN6Jv%{y=M&O=)2S+q6VL_`)Mw;xPG!ZR0p^8 z@CUXrK80N`X^UOiI@orymtAe4%F5b#a!yQJ$=Uh)GBekC2jlnp7NaYat|1`a$kb{J za%S9<uD`XTD=JF=XRiJ z#$9+na0Az!Xsp~|gNDen2tHwmxbO?`%Q=euDYr10V}!_9`rVbRVYtu&$;2KJZ|Mx* z?Tj7TPWU$MKK^W^ee$?J_oQ0;4Y#$)cm<=nDOZ=gYpqI<*-$6h9(Hx)>qv+F)Ru4VBb7 z4Ed%Y%di^v*2PntiLNcfM=*bxkECs3*zQW_jm=@G4o`vO_F%mJ6^FR30jRN$L{OYJ z#w_WaLVNzg5`P?@<&GJ7)c=3b3Ch>|@{cbKDC(F)Stx3P@QiQ#%CRL6V- zJ5(rF{n>!_8D}xu#0FuO(RgI%2HTT&k;u1`azP7GP__&cc@J3bU`d*9!Gpg&RqV*L zV9nF_UNGwp)i8=TelQzUR5>fB33JZKb~0Ly^Eno6vpGRN|1@qLyUQ$(Xks4MJ!g)U zeP^UsD|4>=2x3~QzcC`Z(VB|(H#tk56th;eF26Ut3QvDseEfb2Z5M7JSttr>U+mGP zd=-EHbHW0OX>9jCgNF(?p+n#Og=shN-uD{5$DD)hw`=GMra^nRO_9CK2yOJcoVag- zxlisvi0nAPnqduHZKf18fVh(YT>H+Tu zb7HU1gUjx7$dR!w{N4EpW5F%xbSg%~+8R9WPC=|oIiflvuzRSG;+8?Um6464?YVuQ~w{c-Ik)fM-8;&Q7c)e^blOU4~^58T6A zVuJAi8(jTO@m*mv94NaAbFS-nJWCHFKd*tQzXt0shFGn46Y|lgVWDLUXNnVEjk`*9 zpAK+c5Cxsz*OB?*G?c#U!f^+k8G>za5--?$_S&3hvGJP8&y+b`9iNzaN9S{#V{S12 zTKbuP|Gi<>yZbUcv1v?9-d|>_K#`NgQ_L7CxoHIdrumvFYdJr^Oe5qYOXf>xK+P_d zT_M%SY(37yxt{TWt-pH_O>+eC=cNlPCpx8h+u;=ZW*j(pj1PCKrs12#Eqpv_hmjfk zpwFWX#pU|&tFVL)_0?Q1vj%6(0h8|>Vb$b;y)qsMiuH%ibZ>-nhal~XA7X=|Aa*_w zSC%FqXA{M5>K@~vb2uWrGjRSw1U7ul!LFKEs9Y<>x#UF19DIfD&uPdWc?Z*JS=hg{ z4cBiJfG!GLRjcwg>MZ!UT#G$xS)SZi^ zn&VvLUW!1-)eL;w9*&&U$2g!Lj9mW&e4rY#8kZ;tkNV+A8tqkAd4tEqA7>}ranH*G zuiiW2`Qm$cwZsO8OKmXaWC`xW`e@BKj8hi-kW6vH44zw%**Xm$IdpzI0?6eYWe*i{ zvk_j-tn*tzL@nIk=TmUA}x2@|tah6&4B%CQm1W!%FKkN|@RL?V!nn7-^g5h0t6Fy&$Qf|&3=s) zLefTQHl#G2(UnNT;v1smeKsF)Ir@N9c^n``hLy~e4$H}F*WxfoOBlToLk_>#7lx#C zGf{Tj9NuhA&hUvrX8Z9&Sa3v(C5iVY?no|iMwh!cjtAU_ z``kb*xa^C(#!%EO34*F|6a-g>;K%(W^b3aJAbqAHV$taT{uD3DqVa>yY0PLmxQ~`% zH1aV7+^S%qk%8C#wa9D9LA^sW=AFvNK6;kBqZIGr2NCaF1B0C-i0=D0u7Uv;D zB0R+5-!hUl#7*`ldys7(xyjC*%gMPZZepUqL(aXLgtYueOjwN|XL}8P?;V8S?o#*{ z_8>1bACjicD14rSRSf-HmnU#ctirS%>1e4ag~H4P2#FNn=l5u=QhEyMY0)^Pln$%4 zVVGK;gl?k{n0iLRU_lT(OF|KPit2&51989B3sW@r=WLoY{wTUbGSnS8de#^Sc7=Dk zIfU~KVO~dl{fT>7-s^?zrsF;^qF6>T)uK&O|JQb9e@(%`85ovw*VJ=c%N{mbMs-NH zNc_`P$#E`k%#Lz!w%6*+u3nOq4NV{-YCS0DqF`1 zHBjbQopNS8CMB?NODX!}PeXlR8^;<#FgJLC&wuw}%4z^hCaM2ju^M(OBH5}VW}G^Q z<(!wd^H{kPFEzX3^4YN8r?BwIZWz2givy0Aaf51{hox?!d$}2I{j^5M{kwRhbPrG0 zS>slw3%m^NpfTYM(nLs@*B)aNQdnDdMvXnp?P^-;1BObvwR<>4#|)mGK0wD zaUwHC2N75*L+%{x!^zrS%(!Ft4(hde|T@jgL zjbk(J;nE^=+@<=D8C+%iCEn;NNz%{H} zvl8X9F_^y47e~)~;hUfzTDROswDeBy$>=Zv48Q~}Cr&f?3)A-0PE^p#Jz)Pl`SwWoM z@RPV*{N(H4bYh#-54S6SAx-g~Gv7yWGQJwd7{v44WvD4=$ByUEq5h;E0nxeex=;(v z4VjP;euYyTX#YponZ=I;Xgqk1wL|f^6qADsCNU79{YpOle$%ceVoy>y6s)7cv3rQT zz)+N11|rZg2#LGs%){%43roE4m9A^$foSv zj1$gLEUQ4u0kgi*I(4_f& z>n7viq3Z=ciZ}h_aYu@{tnUngmg_(15K8Lg>Fk6H3n7`V&O!37Are}{Z zXNTw*Gecn$^Qvlu$xS}b>_6heNXeaLzRo|v38)Ceo?m{jJL=BSe{_U%Yo{HXH(#D? ze4)<>EM3eQ_ey6rHU4HgpU>fZ&1zxRzO90(-UOmfMzLI{KCusGJ_OI+dzc(>p&IEx zs1$qS@ymx$dl~|x%@m6heFR&}B&yAh!H%wU+}#k5?fY`UADDo5qxpEFn}$<6%CX-n z1KJ+dDE^&|BM;v}AS@53Ni)p06hbGv9VPZ<*f`dSu*=mDaQ%Ws%W9CleGG3cn;~jH ziBGc!F>f^walOPx_*3}F<{c_z`Fvh-eVsS4pg#}Le;4NQlM~i_gk8)-q}C1M_248; zjc>-!_hTqlt)U+9FPNrMjZ;Eh;L|Td_?Zp_T`I()`^}g|{~nH39ZvXX!=bAhMbQ~( zGc1RcL>jv17vLt7h^@#))NhU=@{IQwlbGk>Y~u3_#Y=2esuN20Ne>HHDHthe092{~xU z1btq>iF5wJ>{Xu2k)U@wJm|d-H?A2(uh$&f2b&q`+wT}ts|mQwwIv((tVXT07st(e z9_PP0J7j15#*KnDJpC2`^Pj)j8!1IJ_j@01Wd&lpu|DM)>|qh$0o^HUTyt@Phk`45 zD?Fj@=?WD&KU%{*FjyUk!rvZPM0wZ~VqQ2u9*JFcK2W7|e6*e)wtG?y=GGwW5X!>8 zmQV!JzOH;U0$na;;OvQm)%I%GzDdHB=q9QKOo!jhHmC{aLDIP!+SgxW=DZ=yJo_E_ z_kO|oKLxVZV+voU`4U6FDa373AePR*aQXOmxbhEyoP3R!THUy}B@cYmzc+R+9b@iI z^qz%8?BY~YUN8>3Xnl#BiohPPeC&(}#rE<{FeC_it|_p%>xaPRc&a7#!Pdk`C|i2r zC*^@*f7bWqo__fCqXK9Y-#ko#YV`Pn+)IS=$$sSi9U&QM0K&Cy33H(GY zubH24G2WOXEy7X#(ZIaxUB}t`dpW1CyodRA!GPmhvx?dBlZ&G^LzI)_Il?Fp%x5}^ z+B7Hcyu-6OkFbN@rQ!OckYOa};O%A(S?;S#I?gY`3~M_kCpiVycUa>4=K^^ovw(=Y zKfuwy$DwEwhGX5O(5Q{YlPm7f`56ld`rNMn3&Ee@&(Den{e{&Z?tuEVz1aVGTr(!1SI&#Bgb((KPyC(z6q1`L!#uqwi@|kCPt_>mrnbr&N(y%j5*3*qvu4km-t4{+`cUKiy+ZFMk$9uRaIgIy2L+kv)?a`a0S$EWmO|UA4V(R< zabklxoCC~ptSA6Hd;D>1u?NUpUkrZtf#y7VCPewr{w@Tdn8&yBM>tdwh?8pZ|I4>+ zqMAs3qhS1rOvjV%5X`-u4Xd6A+>0xK2pfw=rBa%~5)V#DHJx*k5vyH;So3rgsC2?a zIvcg?dQfn*2!m((P_Ebj`wOG+bo-51o@u15eKB!2=Owr92atODGc|HC>3+pUJki@sQLn#n0qeXoVC& zG&};r)!8Vu4#9v*I@aG0#tP9SL_Z9|6vdtG3!+#~G zCU-5((Q9K>zvsd%XeF7;$4e3|6BxnaHpbcB3vx+T(AD)|Cgq!$kMw@0UGEw+(&b&4 z%8FoyQ;^0Ou;&>O*@c|2chVdl)i$cZyM%2nBAoHPSJ*RMisW({H_`e#Yo3*8 zXHPh3W9Laj9CP4@LChQoZ-2we3Ee~Zfo^2BKf?zre^hmZVPRb)7AwZ#j&uSzZxiu( z?-SVi(mJ>&2PLsIEmETZHRrRz<6H_&zC5gZUkQn&&tRZci%%^t@HC(W%BM>(a;F{E zRkZd~UG28(ui$1oh*IVa@(+z7{?9vfxsG9hRV$WuaFN%%J@^qYoroI`!QP3VZ2#{c zX66W!>2)H+rCpS~xvxevh#;vS@gc{w1c}KjHF8mk&L{06#Mwodh?f6DNgzM@<2Zx@ z)9K{W!XD}`;vzRUwW2d_464QNkalkrqwilsA!rb>uU_Gz$p?hfb7+%oJC>d+fsua; zOfD27ac(VEUwa0zDC!AIe2OL2rPx!Djjn_Oy7n`$PbCMpgdZc&ve)^5on#@`Q?w&&S&VC>BiN-dr)3d%kHnII>}OgJl|}HM+ddBuDqRH`)oS9UFtr2 zFom0_yjLVimkn8`qFcQiS#r-X=C_n&JEmc7?pAi0$vrqK^+D|QChQsTVONCuRJ@eXof?o zBp$+%kFaoQDvWC*AUF33N*7YS&15brb|&B^+=$W?`3AGg7?r@Z{EKyq{J?wF5)QQ>aACqHzq%_MldHI(hPxhd5dYkTx?F zLX2mUH{QWS#eXJIKc_;p&kGR8_q5NOGo8o|b|a5(98!{%xJ=iN;mIN#k^79>=6P7+ z(hNnnEF7Y2-lmRp90;uep9$5G9I3#ux)f+HDMq6n&1j?<4q@{X;Irx}Zaj{Jn${C2 z#6)1m>QuZ`e*}-e@%Y>p2D#l)NSGB)xoGMcKM{uJ@dr2|^AH#F67YK0Q|LB$fRAFj zequB$Oq@Ow%_Nk+&%>W7PaM0MiNEnLv5KGe*!}+SE2p}gpiK~y?8EhU_aMc)o%NfR zhWYtR$$R>aPEXNXFXa(tskjm|E;Gf5dkSze;|_APHR#0ZcXe+BH^u8wbmT9#=(k`O{}jg8bYoXN zFF7sQkCh($r1SbXLYo9hFYV*wZU!%@c`Znq`hQ`eEkF5U+K00Fd?duAo91Rs;_I0fWaj*V|Ezla+%ZCP z*XpoOe*hbP)L?1<2VA;Q31{*i5tKi+rrhw17bR3rRtJx5C7A14gREUIP&Y<<6^%kz z@|VzD{{n2be*wE``8d&@gOfbe|6QF!Gl$ACygdcG9|X`@pZ2xeC^q;q6~~7wP&pQb z)IEvVtUx_sl>wOR6o&*MV{{)rjVd+<&HYy3Uo#I!6YZdvPzd1*`~+9&**0#^a3y(i z;`Kf<>(;+!OlP?=NAs;YXGU(5lRl%E>)EV%{@7x~aUX{2feSEw?t(36EOGmXBL*tm zu*Jp#FDaH~v?&cr&%@ALnhI4qkIy)iiUEpcR<3)7{U#Yu{*jHQnJEaFk%?ztvM|1Y z=HBHz#l<~Ym?N5ph5Q+qGA@GSR3_aYrI7iQO*NPma2d+M?D85Yb>w47A?=+M3a}!v z84YE{sO4@)%K1`s<0E98OOVys52ExEDw2aZe(N9%G4)|Y{sR}S`3rJ+iSD*v zC>9nbU5iD@K^+N#*=i*1x+GZ=6+o&@BuU9nRnlT9L3o#ol1fEk@|JfTAvwHcXGIUP zn*QO+>;^<9jU(LPHBB@gg!HDXJEVEGkni_ia%PZ=zI_i{+Lv(j0}U}i8Pq#cp-;lfkZ<$Xiyy!`^ggX zyIgRC-v4k&@i6`zS&WZ?ZJMhn)@90VO8k3Ha#UB0G5Lo-YZOIRGKLn$jQsp?=ICld zbm;Cya=B+Vcv}65m2(E3F+AAB%lh z1y9`;v|Omh*nb_6S^o;EYd+#jLM;~DAHZOG9WtW7Vn%Wc(w_Xl%D@gth)&_%toK-P zjhi^SeL&YT9?~p01gV=sWZYpC3h}~Zh1_)F8!k$!=L(a~%M#>Hz6^0$Crvii(#!=L zDN=RHorph^A}>7F5(kbn8Ga-~K5$Er2o+(n&Qg>dlb%jC4+s+#fgd<`T8K2B9D=eY z53w8k07i$KBsFv)eCZUL{W~Z(H-?&;7JQob4Iby~Ff(!hyY*@zGx7nQVy`f3X9qqs zR3X==1qH9FPsyU5 z?{eBtl%SDfNH5%JmdJ)Un0$YRs#U2l+8TkIJ(LeEr<}=tZ(Q0GjW9wra8kFBKHVPv zzG9RIqxVIC5W(yi=BY>*Q#w}0T(UXAtjp%++*=jKTPq`w zU4gQ`FZA4~A0poaqP!2#Nps16t0lsoWI>Dip0XUCW13zL%IVqmfscBZEnm~mdI`ld zO*rwd2A7T6p}M{ik5)AzO|+Hf47MT4|2>8twxU<_Gt}JM;O{mF=bi1e&V7T@#ZIJH z{ldQTZpZ}w!StIwhq(`qEFot-NS&-Kkqul=SUm~QAUPx#@!9$#@MaZ+ofAC_@OcEpg9b$fbHd~G;k-H4Hj26)7`qfEXQR}MCzsOlwtu6m8YuIJDVq}pMf92Cxej;OwP z=>N<@OkDyZRpSt`@c|}{Jy2%kM72YdBRh5#7n-R)s@5K>{k0)bD-M0jzc}&fEc0qW ziF0uB7$0EA5h z!7DEYQxrd4YMqT&@{h4jhiYfy@(^8F26yoiJX-!1J(Sj17 zzzy3ba0fS_Tf7az59{!CMiKf|N{BsyOUmG#Ue-!qvt?+=Zy-RdXdM?< zmms-BntV-?BhMP=l1PEINTOm3DV`mfc=j>5TQl)nEzF4Dy9qy+i-L4xoY z36M?^F%nbDMOM@GFcJO>E2{a)Z;JoUE9EB*FZ$52mWRka{0!rRlSrKX9^cHyvCF&_ z8pWfq9jEWIWEcZq>fpbqpXLG9gInh#W_Q;iZ$%dZ8tO1ebpdT*4b-F7gjKx__@3E> z?96t!3%3#udgRfS>Qx4208rHMr^F?li%5*8&mt}IC5tkYS_5ipS82=uwL{+G5RqeTFM zc!|7J75vF|!qFH5c*wbc--O;Rwk-$+U%V+_6^Wsk81NNkp>HAsIx8u+67vamSNtp3}LB{E{imkq{t%efh{uWg+sUN{HBp36s;*_bf0llRV6q zB=rqattR;fy*OUF< z)JePbW-=(UlI-GhAk7LZN$Hi%WOPWK#QwLQ^!;8-Or_MwQag2`Z>vI7C|7do&tj5r zMvdG$pg?*JRfv?w|DPEvNDr+c2J4rTql@Q}?)}Q7GkFdv`m9JcnM#pQq6}Cx zr99cNQJmPz$r9_sVx)okzjp5wC2GGU$&q=q-z*R%zdeP?;|?K0I6}nZIUji?B0%yB zC(#znO%&~iF(o;QL#MlNB(n?FIj=EE*Anxz7*4bfs2gQM_iGkTAO?-kBJoqfhkE^k zpcG+;30@a;=Ndroxf8zEs$j$YGA!CIh~>Q7A^pgmwJ-`pNZubhh}UjHaB2^Zm^Q#|e=p?T z)I(m9rgHSxVTK^>Ia3>Pv3?v2AGXmfgTFXFy$5TAxyYGY{mA&rO|o?cv1kDw>HIVT zV|t%ayxA1)+z})_r+G<*pddN_cqVClB|;YKNs>J&Qe=gWEIH~RN2bK-{Wn8P$c#DG zL^)sy84{+r)p9xFvPPCDCrgnFXC=w(LJ?9HP1h#H|DJB8{o-0dlKpfFmt=WKe%}ac zwD^dhDaHMHXFOY@Gvf^yPh&lR)UKcEVmdtA(w$LC4QqII;0M zxb;K812=S$zpyzIhOh0aq+`Yzuk zUuO`DlEujweU^{OxO_wp)uP_&Eezv6VdnLI*vC+9&~0uKZ6iV6o|7PTipoU4brCs0 z_gb;+8ghMx7IARhL_#HYl2xDglK%qsljcu41qB% z+%SM}#(YSBU@+lIxKAdZ29T1S?qp2ggB(b3B%H7I#O#+9QEiDJ+xe}@*Pr%e=!GLm z*YP0B^xX-kCV)J!yHEOCgGtUuA9B(vfG9Tl5$id=#B;MZnHJ_oJg0dRhXF^TR!)B> z!k$dK>P)Uin-j5<4&=Os0ogHZO|Gb2AWPzJ6U$^>(zok6iGU92>^VoSA3RJt7ab?@ z*3|z`d017IouvQDCZcjki-d93kmg2p!dJG4EZw0@8XiiJu|x?{n!!yn;-(Q#t^t&} zexlx_x5%0C77D@nSW4$0Rx}wC@^Pqs9tcY(Z(MjtIZ$0odbg_wG%xRi>j`%#C+&y% zq5W`@xI|Wms*-0U4DpZt&}=L>d^#S29aJ-X?mrVaQGMO9HMXcNaD?LL0L(YN55r}F z2yT7|3CmdAIvI!1FFA0J&!9R3>Q&3j2YaL%5``~e6J3wmjw-C8J&r?VEsBQUL-=?j z?jP?!)utBe$!JCDk#>ByYC=|e56z!yz-j}km!j*tp=Ah?)9Y}6^5#Z`jj#_Jhs2#W z9Bls!!R_6++cSkzSNqZOj+=yk9YD}ZK2oJSir86{hcBOk{vJWHGme)CED|K@*)vJr zQxRf#SCS-jN)hLEvP9fXj_}W0NnXh;CI9IA=j1IRL2Fi$g1vHN_d!|G>?%bl`9w$r zJuf_G65&)qlJ3Gwbj$^b%*`ouF5)G=qN6Am<|D&ugRoe{Lu#G-u_Sd0-?eCuCpiJz zMeR74JdPsj`*?bB6xIoKFuY2+<7ExFtk{bs9~(fEU{JZH374ssuw`W{Jo4$jSLuNN zl}22U>ZW&o)FN1*3(qKjX-sFK^X)YAGnwW;+vmervII)v8OWWJi__(CXqAkEfreU}ncgTwUekP=<#dKpET2hE zv`UiOYeY#$$Q;sfNQ~61k|mic;^eHeJUOZ{iwq4YkmVLqM6vgFBG6>@s70!g!1Bf5r*$&Ht4MBs)BDacVLInwlglTT~OZsYaj zK%F{yd1*7bxqc<78n!3LSFa=;mYc~B{}<~nk5x*D;FP$z*;R7moBH6kLU zL{>Sg5#9|7WP^nYx#mu5H&zg#3>i}OeHn3;pGU0d9u=qkh}IWHvT~OcIWnX`bob37 zdu`>(s$1gZG+kG3=$+irRdY!50#P#ett83$HIuye6eV{kjvQPkL^3Fkw5W)WjB*Q* z(TqvNIdBsL$6=gx9YsBzC#5>N&@=j)YS8QPHNP0-bU7}5%RoAJHYz{Tx$bx*%v!uL zJv#{P*><=w-vwKj8p4kDZ!har!1tvLA17wOvvV8%+Ip~i7jtuhUo7X$9BJZc*-#GC zOdazTD3;S`gy|Bth;g)o%8PL9{ObuJiXWA}55kYPF*rINi`?tkm}ij&tG~rimd=By z^h;zOqTWXnnoE4Q3SViTUy}P4XQex!=ida?yasG>Y{IwP209D3;@yurR3Gg^(7!tP z8h%9Zs(NsB_hId(dVKr(1yM3I`)X(u4)a@)Icpq#O7EeqHi_vJUp0~AAsg=ZA>%ba z5tOAKJIYt`P`{nUln8n8hKq>viIE^n+UNh4AdM#^$R9Cja;!p*?5BNx>WDhgQIjK& za~;WPeK{hzbS-i5pG%BOTtZQ6Z3A?LAkIMDpL(GEo;K!R}Jug*o2_5cBs)>I-X0PkwF!#FTO>BRta9H zyu=%dxr=y~{2xc>9oN(Qzj4t(QHf-a#0SYJa!&VkG;A3eiOf<&MM5arqM^N1N<~Y1 zm(tQ!qM>0G8JXExzt{Kom&c?2=&jDV@7K7l=Y_Weskpl_2dAXt@Fe>>q94${Xnhdr z61}ljD-!kBop6nOKm8-D(O+~4>l_v%f88gmOrul8{fir={422sAd zY=9WUvXCv`hP|QA7{2u&EX2$y7nC%G!|2(G_d+!3Pe)kO&zqV^p7d<2UX*|}vr2ID zTrLc$*Ymzpf=hGmqGA~7<6YX2@wXAZlc~Qy*NPW&Td~!r1BKMf9x!}Lxy01x5AB7) z^G=8v_Q5%|3xyRQ@nqs-ESUTq9hT1!<~x83>fI<@`v--QFJMnP>a1ZeAt@;;d~|q; zE%PLVtKVKjgwB;6)}O%EjS%W&25{pY&1zqW3ZXWmg>0JVrId^pHpEgdUoctd2~ig+ z4onpOJLDy-IW$r5J*h6drT*@3+C;(l{&-;#CoSAa9xXKe6&2!gM+#Ra4cN) zkB#nPVS5RzGs;nDNPfeuCDdcbW5wERjNF|7#jZHgYlfnUJV2V=7Z5@^xm3L)=+r%k zOWT|g?7NM2WutI=#4n7>ILj>@Naixe7IAlUZgUSG$?_E^*KlR)1wlDH1X7{Wh@RjK zt1BlEvc>_T$769i+!u2V$WKE484sOn_&^?&hK;xID?JVu%1O7@or+YN?QD#$Le=Iv zd^>*|jjO7m&{c{LW9~vMs2077HTYKE0L8R=6jPmI(^ms$pJuoZu7l6beX0^VRu*#n&W_W@HxI zSkZ?$dXj=hct1KSrG(+1MTGrU(n3Isq_CPFCFF;V7sOQLh0^U`NdEfLZO> zVO0wW(it8bT#J!gsh-cMhqFZ^WG@ouDXkU@obEz9i*}g%t6}o%HY~5!q2x&w<|$Ty zqg|Qe#XLApjmM*Zg;m4VOg!Z^nYu%A6}$}ESE<{M1-eFv8$`jF&!7m7JQ5c7+?mv8^T z&88a@^2LOzrtf$%SV|}omlo#GoF`3zW&l>Bg@ctrf)G1e5c$1O$WrnCCivrK%(2;jP)T_ zY9h+V(T;CTHHuPF;BJ-#9opliMg-yW_++a6s^Icp9-1~~;uC3bE{(l~3AB^{lU{^R zon<&Zw;26#)x;nug-?A0PWO~y#lQP7T}OFF)*U!SJJiWHo+AH34Ys@XkansOR&sBk zLEe}j?)}(*t{I`De&M!GJALghzDGQR+HP^-&GA=o4VMraUVew`{o#U20c9EcNed#& zMhh9@a>9cOHR|(63g%Hh!qeoD!q6l&!Gr38v4Nunn+wu{#&t2_>Z{>`$1Bot`AG=1(CzAGuyv|=A>_x-K z>u{ZH55@h&Gv4il*xq8S8zLcSM9T>2j|(`xZVkTi=4f81n9cnw7)$=#YOcp{By8$P zKjA-HkocB@idQSyq}$|&fAo@a-L$YI#Ftf{4MZ#bEw0N~L1JeXD*U`L;8uVwpA&KY zdoEgIQt{l&36fS`IJ-X|h5Hk6!!ML-o+}tqdllvMnRhl`MT2ZCK8}gR3bO?A8bx7= zdkU64zXnb=$@Mo^Wu0|iIjksOh9U9DKTr3Dd(;VkJqL{sjd-EbJ8Kmw_&e+ zHcpW)v%{ z9wP+3&D0}=4-?Gk`vPOH8ToR z>*JyP>nbkV#1PlxD(Wj^P`KwRZ0B8p{OeFO#3sNcFdvnPUc_Rjn*9pBIyevmkS}5?8rew1iN8tG^$U4bsM;e=l(?#1*51beIKY{dL@!Eqqcfgg8dx zre`T9E z^X-K%vz;Q!Y9>8mQ-^QE^5|n&HEt*tFIB*Zn~iLYK0PP%oUBON@zs6EsFE%>x&yBo2MNi8y2-aZ zSa^{77IR!AgqIamCwfZ>PPE^-HC0A%iK4zuZ@94VrJr!%z;L0sZjtapR7N;fI8wM< zEhR*L{(~EV5<;KfTeP?h7Oc<>pRa=i!?zuf&i#t2xA*Z={WG%8H{d__K3K@s!*hHO z{`yxT+oTKo`Esn-)s8p!O0j(UJycR{zUyNRKL5!_kyZtE=;h$RL)!Ch%p}hz?RU*n zp^FTxv`)mU@d+@uw6gn3FJ%+3|!Na!=@=egjEpm(L-5IYG6xml-!pYAAm%!<-xWE1B!a zKh24m*K#W_iE+8+DcpI5$6V~+MVz<1Jr)y}{>N}Je%+nn{Pq@6VYIyp^+9>yCUmm? zcJgujB(KMkbNEBLF|lV^n7?}}%rcgu`gH|@$09b+#Mv0y`>}!oDvh}`UVLz>EHT$NYwb5-bAa~LG4XhvSB^Y~8@SO=&#-JXX40K5cPkv0>FLOQ zABOjtNtiY+1QGIa$mt2f6z&>6t_;M6zA(5w{2#|77_ImHu*Jq7XO__{bxk1J6^PwK zT0I|gTfBT3Lwd1Jth25_`cvYq4WEXZQqnt|D1fcPc`Q@<%I&ln&PQJ9WQXM(!oB&X zLin#K!WWZJtXO0SThbt%`c;8ZG;c~Xp`G%GI0TaZRK6-4vya3=+9nQfq*IWYm56kz zbH-<+P}X$;J}l0_h%aTR4#~p6fhtT@&BH{G2E2HaPduq+m>exedQk_qelA1Dfo@cY zQl?eI8;nY6K>F2pXwYrL(u|*UvOPvv_aNcsyk6`hy{+AhA0Y6AkUbzJe3>RKaJ1*S zzh}5m`Z_=mH5)FR4_hS6UnVW=Svo?fJS`~UglatWob^{A{c_L>CaRuIyzHf9S zB0d;m?Y2-XlAkL4+hi*IJj(_CC~Dv^STl8LG_Rwja&QD>iYjq#e0)Pj&d*g)Gsg1D zO1?0DA|V(^+F^!TBe(kFXKwk*-!M~l6h_>ekC4~f`GHC6`7@*?JCpYlAFMjbOBsX* zYkxAeykd>9DI2laC>UF>n_^j`6ZU6&L1K$NF79)K+hpQh$B;gMxCab118|@8sej0; z)wq}R>jM#3&`UY>G(%(y{jiwW7yRh}WThnIR%i$$q_XkQG=g+3w{Y-iH2!(tAr9jW z=!~m_w{8;ba@ufTmClOD4%CYjVC=YOSTgo5tZw&_p7awY2Y$zd-P44!Bmd~@{zB;e zf4IGBnjmiR9rraq;q==+?9jQ3XJ?<`qkRD+^E>coQ91%m+AuRV37=BykWSj^FL8I! zekvMjF}L8>AAtm;Z2Z*?A&ne8$D0FiTsHxc!G0Ji9fNQ!Z>a7fpM>aT44W2$qf5Oo z(J}yO0(+0eYDAGu%8(_1)(`-Gk z6CZ#4#4uAQ%+#OG4-N9*FQ%wKaiXK3A^RIagRQs+nvXc2eYQA2{tEY@zL4pa;7iOE z`N%n|`0A)l{FW?R?!eZye3@=Br~Y3Umm4sdS7<8d_Upalf-Xt$cMm)AnX!}|Kpc~_ zCBD4hbSFMp#F3R~PZ1iHy`*Ph8Gj{Z20vYM9xt1>h##4-ma83Bhgqk7W4n0_6JPm) zEyxeW(x)z{QF6zu8^Ji$;7hv7Fua`^fiVg(P#+crGZpd+f4UCyxO7BNCiR$+dAJsl zfKHzxbO)tkZ(tSX93?H&(>iP;O;^RE`zX>W04a#D$*vgR?{s0!^g9@Q|1oS;>X7mH z9V}Ng;-&vr44?iG$MycAdf*K*_K6CDi?}fNqJ;2euCn0AiwVLsU%^aOOt9OdEEv*r zKHE)PaD%Auo-{Wf^Z(+=frpqF{}tuS8{sd!!^O|FmfqE z6d*wI0eI49%Uq}fZzs`A{&LXSm}`tt<5HL-Eg{mp%Iy%T|B*g>UofAfDjI$tkPxmdB$x zqZUbihW7bA;MY6ii3H#{>mRv+?Xf zi%?1T8`Wwi**J3I0iKLY#w)Q#)Ysm?{e5>Z-!Ka08YO5U29T>Cc}AUSclw6v>AC(G zEfEi!DcR)CHVH+avPRfa!2-Q@KOV#csaLb&TaTzu2wsn`)06$>#`cd@Bf&| z+4zp)&E|x2ryqUe4qaKu`+Rlh(oHAx7c|xRD>Ct1&{i`(MsF~mua(a&l6`}BmyZY= zs-_EC@r#7tRvU!2vfFr3J(Wp&nrrNymJHqBGQwPM3s_|+BX-?&+&$|DFJ*679twcM z2_Kx?7lMk7!O%{Kp!xC@v`mS`<-|z5|Ck7kUD1FsX+lUJk{*|f*FWQNU{487HYZ@H zcR3`+rb4BkW`N#VxNm(A%g^LtA??H`e!7hf&0PqacZX(4eYjm+hrNcM@iw>>jpInC z^!qh(wT1{48zh8rG+*(3s&a6I+zj_D#{ax5Qs05QAH$&w{9&9u3VZs|?FR!NAoO~)KQ|*^@ zBoU*WN-*wkJY@Ie!lU*&uI$Uich_iylD2uGeI&*i#KL>p74)qqKe8S%TffqdG}{OI zzx;8`jJ%vae%LhWI)cy4Ltn84itow@){4m(GjpHDhcA=Z)fpv7-?%~O%v>a7CQlba z*B=p<-+ztMoxNPiHgSGy=|bM0a&EO0r}1SCioC2$H@C}lfV0nD&e!fy;l$sL;1#}x zaXm+FaILc)InU||yzk*dyj<@XUhnL1o(q!0+(Q-EnY#xwLY(=}r=`%ppcJp%Hp08@ z4O%~{3emE4Q2G_k>h4Ea^rbrR7 zMyT~^s_z}(x%w=+)m`zf@I2O%H*fNMCwR2@B4UCIK4%4Dh^QO1XpWUQ#|=d@uaO_! z16{8ZF>uEli}EtyyvYwT@p*V?7=}f;#n=-UjT-VmjiY|DHKqZNsJ0SW*^EN>a+K5j z_ej@E7?4jVv2wg%HMb9W>Tbg4jeWS%G+uDN_6p(=FCly93C5L`qhfC}{$`|Ovs(k^ ztck~O$0|6pdAETZqTug97?(}dh_T)Y{G{%xRzpt^Le)Dxqk^A#phvJ?MZd9C%h6bLT{oku21lRx?M0XAMwYq z6A{?jMeNz(F_a@l9<1p}P@8cTr^VCZ-V=jb&3vf7ipA3Rx1dk&QEyNMHrb?O#ll)N zDN`oSqRf9g<$!o-?1O4#M?7^b`zxYdDTvOWn`wYwx zK8+R?PS$w|^*@IQUw_XLL`fTQcaf;z%>F^~N)Luu_ro@zk(hg=%`v9g!_jW!=-xv9 z^@q^>Lo=3a@(sCV!0vMmW~@(#g-QiXO5@R~RD`oK*C9)DnVK1~c(R7r&O@%^hG7y+ zdcz>1AA_s1Avky_0!7Rpt}}xvug3=+BEG09yGY&%PaJlkY^ET4$c%Ew-~n6YKRAX^ z^)-0@YCV$(Dq`C7%->l>a}(0Ot0&Bb_OmM*3ROd~!rx7!#9%R-WH?6^@SZQ+ zv(OY8y0&7}K4tFRsp0%yEk{nx`0kS64N80>=fLkbwBd49#du#~BCi(M!o}oo64o6E z6%>lag_r_q;qSjh&}quc?NDG|Mv|zSGJ&o26g2ug$MWZAL@^cejNZ%p%le8s+2m;# zk+?q`Q{}87cp6}L(H6X(Y7IHw0wrG#B1z^LcKTXjAcA;1`%YrZ7e@s2p22;4H~4Hh zhy16W|DPkSHr)@)N4lVLXedO}U7#m*72U2bIM;gvdx%l{mRLr&|MNjgZ5GP62P1lQ z0qs&E;4fZIIY&2e@L4^WQaVhRKR}~e1^y;ILY~A+R8H;1AF&C-ZPVA-bH!O`?S4(1 zlnH{*&ld<-`4SR`AHhJP0_AvsQ9FpszK`_PU$3KiLOHZkPK3rUwc`%gjgiXTLkV3A} zcMd2B;!koWJ}Uslu?iIMul}MQkv@^_31#d-+gqkc^7IZ-F8Tnfl zlRxH374drCV8FZ!gYXrk&z#W$^;@W6uzo1^0 zKfZd`Qp0=bIj*u35tALzOE zz!7}Sw}FrSG0GaFUE}mq82!T)RjFrScgG!9cAm#i6>m(>ut&aD5QYXjz~A61o{3X7 z*_vyVVder=|9E`8?|}tnDVVJ1kMVbMaQ}D+@=S^mnRg9tp;cJ*DH$`#pS#Jk45M?} zVV(5?rL&&mh@XP6=pfC8yljPSpSqz(8sYxMPw~m|1!}LhLoc?BSj~-y*^q)26;+U( z7mc9*Zex(z|NQkiV9We*Du?Fs1s-@fg1mw;^m*o8gGi(!)axRkJ>3!Yfq^jE??BmH z-Z))GJ5X_V=r1{gdDmSrSn3p}dpp2HIEHrtr;#!K2zJ(+BleRKk~@!JvylaSM_EBl z*BZ-K?nV9tM-1U9_w~VHO#bN%_xI6Q_rL<9`(9_ac87@AqIle@6BC(&CTbxuy#v_$KN5oL<>XevzCy ze|+I!{_u+e?%2_LoMdpAhWv$Emc)zV^t&;vV9XU}6*C9pT@wfe%qCmJd1$oHobk6RqoR@ZM5H#Y)>$1`zR=_>4g67TqY z6!}L;tDYZ+x1Z~<;eIk^O>Tz5)*PH$*F~8LWw_JYgZ08g99#Mk{z>H1TKWUOa%Tw2 z3xDJEZWlpk@^9>ZKSNL``;MzO|6uLhk0`$P5avNWn6aS@VTZfm!RO$`@@5P@kqnQI zbvS>Jv@83|sb{~2KNE`(U`hP9woDvviom+&n|L)m48}cisQE7l84A(R{pL&Tqj1F3 zUB=GKfyjO1fk5&`A7XCEtGO;^_7G-wXbd}adI;vNuVo_)A~Z@) zKj0)ZQn<{}#k}phv3%|KWqd@#SpL67^Lf$F&$x3cS2&YXYFxlg7tVK|1YZz+i7zx- z!cL8Q$+(Gz_#`-j-AG~&bY8HV>lLtW>NghjXB=hGZa{I*0Tcz=BYOsqSx1y{CZETZ zxytDMp@hND7+PaBp}Kq<2DR?N#*KzhUv2`4cP3ybEfGE95DXR0LjU+Nd{LzScIABG{0?y{&qQL%4`=+$i$!QE{qB^BYb)=K!P%Ku zUr+siTRytY!=Yeaj;YfUaA#r-Ds~kiUi|^mFLvVx-+>3#V}$PNPVBKbD(p<`#OXt0 zglcra*teTBYY%WQs0eTVtAX!gdZ#azBOxmsb$|1bPIInvp0ryx^+xv6M9jSJhTh^> zEc!*W3*$(9r@s7P)fFtDGfj5`&BSWXAZqeud|GIOt8eIBA95Uve>=ke(=nXzJOlMD zX7J0RJXQLgB)&d?t;UAPYTFI58{5E?HPI^x9OzZT2^VE3n^P7==v>5sgU(EQoH~2} zds8+*DQ!G@);dX*opy5O#h0{mz7u|O?zc4g z4Gz;d?!UL(qE+gA+L%w=+y9R7;k)Aai{?D9_4${^{;)>YJmo7Z(NjkBA#SZN6HQcHME zu*Sv7$I#Yc3(FJ4jHP<$c%mapBt7tldZB_qU&?^Ggq@RvappAbGzN#`Y`7oQMN!CJ z6#%uN@yI(8gwVz`1S*HZ$u%Drqr;$exCHSLF-TueJ7vpwOhE$%56(biP&+P0-A4K9 zr}!@4jHQSFckd72^yAN@Wtc9A6#hWaBYQ#XKIs78PZQ>EBmYC;0Os#{gS~T_VQBUg zb4O7Rozjlg+2paF+d#AZ1c>aZ!t5cj@E^Dh^JcowJM*FWH-vH_($N1k5Oq`HiO(MZ z?a(N^T;h+6s4&zbVFl-l$ChfwS_1ozjFv9A73p7?6KuuF0hU3n__UTsKaNUHue;h2M7Gm$K6~vyN zffXO#vG`Yq@lt0lJ}&;o-c4&_9+E#aI)0tt8?-C=5X}%i;rMs%?4JMlv=Yk6`&hYj znBN;tnd{fE-SO%4d4ZE5{SSIk$kMze(%detd+s?QHg7M#WXB|&&cwT7;57-@-} zk=-AIG$%JSs3gF>!vo9uQsANI2l3imq$U!7eOno12PF{ir5bx(ZsCLo>DdFwJ0abM zMhQ9L-N-g%O)wFJTdnAjBt30LD^iX;gZHI-=%^sQ&5jz#C?=qo&i(3D;aD+{ixlGL z*Kbe7-&3B*pPhhjs%}V;kA?Vp7d%@*GX~0Kk6RD~m&h|H{YOl9=VVsisz?h!X{-tal?n>deJ!8b;x4mb{Vjk$w+D*fh?I?CT}tu8YipS3!NPH z{^lz-*u?9)8f^s=K&%#-7}%&Y}F-#2x%2qx7XucaPzuXB}lxh8vkpNg!Kc z_L{9PyU$i^{=gdAm2sr^Ig?izgUPok>oigZe?=zZ$(@Ce-7pJ}cCEyewM)=BLJJf1 z*JD}7MqHpw^Zq4Upw+nrueQ^@kZ%Hqzs6A4q?~Y>;|LBs3u{*!w9%Y+&0t4(j&Xi(Z`6J-T%g%be2o`mDz4!Q%?B?jC%|+rvFApmLpCyye9^gZG7m$c zauM-5qmd|9g#$N|Fm&QQF z)b=UDpJLMbhJVN0?av{3s1edO8z_d) znT5=XQAV1|5S+U5jh#}aEOy)R2%9?!J6_Gjldfq9)S|qc$HZS&T8yi|<`Gjw4c?;a zXv$uR0!dx;tO_rP$-H@>K*HxnQu)by#U#gh@YTT+H@{jzkt|r9$YL zDa6N=XqYw;2lqePYpc`~5H`ITWiy{3O`{$! zh#xYoupE;o#K4qv7acv6eLFY{XXu_k)tG?6BPsjZ`Z_j$c1Fve2-MFYhP4i5KJ=c0 zc9Abu?Xtn0f1Wr(?|b+H%1G%p!+URM(xD$D{jCi~4K{@n)#gLQ_aJG)J~Ur7pxn!C z#BS6@y3{(1n70Zo8B6fqgz{0At3bSI31nnBEHPGv-N@LNGWLl<%r1}l=ti*PlySHD^95G%{xI`c6Uato-Q_+F{kJq!e+hR> z{V?Y$*~aaSyUm?Z31C&N#!S=h5*vR!h3(u`z{c#n%SN7;#COU+k6IxEkK&QI+ zcyAM2d~69@fmrIU=Scfv1G7OcVDo5)QRIdxXNf0&=Q65ui1$LX|AYk>@Mm&3wl8xh zHaDI1Cq1!lW)kiW^~2yRnNTGC!z!BXm4ApJRzW%Z&%_~rYz+?SWY-aSi$6UaV zq|5L!awb0XMa((xh@hp^59-V0T>_Ife$VEcz1j=o*i9@ zR>zgFS<7I?FNb2|Dl~3gj@XvH1fkk9>-Rp-CBS8)LurJQKZIVLuG z33Fe5o)umSWI_6g%qBgGwLj=%^=s4F*5iNK_2yr!q4f{Dp74cbPZ@!1QBioE9*5bn zqlvwwj3U2<5K&Ua(8H9I)us;oT!J~%1=P>f#@b8Ev54*jHcbyb(FVA4XB*|^9E5b< zZnW;C^G|&bMo}&AG|3dUt0{vf;~4JIT+=zs3bRjMhSQ7F;4}j%SI3Sxis9sobRx|_ zH0#a?=@m-hdniZ5Gs*SNx0=m8|!7tsVcoVFQ_gaf^f8IiHZOW*> zC5K_!<6wGo2%c3-Vp8-scC6qxE4lKE$$$RCJO-sPyZ*QA8-I{D*zSvDVB#(*iNIwvRnkjDRp_*Sl z*A-f;g6RL^gznA0=n8S54DU-wOt!(6SI#uQK84v!&)|0bQ9RK;hFKwa$D|=;1je@u_O5l*rV~@gi^Z9q!p8y-+j6VzV@=lcNy%((hp2k{W)9vzMn<@ z>SZH3hvI4EV93oIi~eRwI1HHz@zIJXR+@(^lCz;pd9J(fEJE2T4Wu;yik<>=Uavyo z9bFh5+J?F6biY{~fYRN~xH;n(7B}pM=YAWM&Ns#B00-Etv4YPC;>vEeqU;pf1u35- zjh+wwy+4DHdb)!X?NRuc`sy?{OjNu9(*;x;zDPmb!vMUe+?24*Q5dgSihYe)u+OeV z_`^=jt$l!E4QXN7Y~pjNZWRJ6S|CS$z+kmzygk;5o0YYgl%IvDaiy>$&DxkVIVe#H z#0#ZV99`~(#XE`FTj_?Ri;-AbL>wyeJcw;O1C?AKOyN(W<+LYkYON?Y#tpALk730& z>b1K~Fs7RFP$%pmj>R$Tn7J8CzZ-+;>SE_@%75=&iwBCU@UIQnEia&by9R#mSH+nt zs!(YA4<=#CU>%Ckx19pXQBtVP8iObE24m*ep(vpqzF^cRCRx(M33ZyJW*xl>7h zAWy!(S%^&i5B-K}_+6#~4CnCd&{7yCYeT94b}{8%{tITpc7mUaT+)!>KU#)%azwA1f}=&C39pfX07u&4*y z{_PN^Ea-tx*J$BRF!@Nedht`D1%9`Rp>0!*V>H`obuPk*b>V35qn+i90IWHig#TzS z^LJG&HlA|D7wrffr2WPJaxfN{I$&R>H@vOSkeImr!kAdIZMb&*9;)LzH!Q z3?|LSc(;2$zFY1@>X+@X_0j|Pemy?D&>`;V3M{#x1%<^+(OSF|@tN}>-=m7}tNz3J z-dV7JB#(TPsi@dD9FeP~q5tR$)1ERIBfs@CP1!*>nOek*mV9K|#?j1VYXRHl;m@{A z2xc~m7cpNWM|L?VjhkIm#<|~H%9{$8IlrALT(yZTA5FP<2dl@im7F)L$vn@j1|=}d zZ(eLLKmC}iUA=3<)LM`HV*Oq!kh;e&J%p63zc_(mk0>F0&TUm#-cFAC93s0;;dNrARaIj(tD+ zi%s3v#`Y(@W|dqs`>Ec|s*PjWe(`%u&B&9*M#M8ToM*4!UuFl5#y$6%)@!%(%qrlB(G_v2Z3~*viD@%h;YZkC^Judo0-Y zC%bRd$G%=3iu_0s{7D-PElFAYOPhofKgZ(MrkOB}nhB#$6-;zqfZZEaAY;TRH+=~) z7*=B-eHq4P=;3bTI*5Dhg8u$ZkUwdHv3qu4zP%OpTr@`Ay7TB+Xo63q_d7>Dp{lGK zT$Wj3Vbd zJbK@Ph3BP&QsN;s%dHVs_qXGRvy@QV)q%jA5$?U-w$Wx z;^Df8>d&%pJpJyB;;LX26xl$t+6TQaDescTIj{59+79+n~hlfk?-8>(jVNvcSrdZKZo<$&L6os=Kq#TY3Z>f zd1BQ3y4Z zzW!qo3p;n>!0q*Tx8Il;Yun(_VUB#4-MH&%3)3YgP+sDUz8{nyR^$TL&y=^c#ut|# zSi{~f1j`DXFtq*(+(_d!mUz_CzbKECxV^efG3X8{MO<<|`869TOSJ==UE1(HT2kor zY^Qu8ZDHq>c68K93Vrr%uzb{k?b!|3U7nBK5v9a5jKP?Y4A=zu<4IvGEDUMyID}@m zpPV4#5rPqx))@VtFE;%-ioH!PNK&)FKkDUA<{W@*zAdDEc03ZTB3D~(&6R$2YgfuP0iU$i(U7~`Rw<>UZr-VL3MGR|^!aVWn4MZ%!qz6)u@;3$7Bb0z ztu(yA7OWV_PPJ`dK9>)1CZn!%?`}TfwA@bc2FXMDuzoSVrbLJRcXlZInCigh9NNMx zL!y{#%~|%~V;0-+z?<1c*E8AESu8!Kf%zRSW}kK6GPCYBroZktTe;yq6JAK-{do!O z9U=#rVky*AjEBnK37DR$h#La!gdQql%Zf#K{Y?dv8r9(_rH1K!I^bFan7rG7$~DVT zsJ;_>W7lER5t?;e+X4}DEA$TR#0}1lbR0&I8n8#dH|0s@Uc_SqQ>3Z*KwQrbQI5ok zx3)*Tb2#M3dSOi*X^l*;V$V=wbNqkGNmv`n;rUAml;%p|s(K%*IP`~| zUigqz5A9}Sk$e*MR-+=GK+{8}{=ZbSgjDf^4?+S}FGY2U9J-;Pxwh7k~%H6Fus z6=0U7glZ{8n3*Y~mEM1c)&)3Wua3nt)gU`_B`QY?7({np>O*bZ-M1Z;=hne@7@eD0 z+Y#b_0`1H8VDjX1=&n72iWmpXFtWt$Y5&WTGJ}Lp03zz0aXmK}zstR-|4YOpg+N?+ z8V@y(D_AF&N4mfq49Th`pGOO}J5eL`P+VB7(F&E7s={E$R#eo83rqL3;Ri8+f+}jE z<(iGOS$XJ5y8? z!`E|&xV0O@=Ud~FuL1n3O~5PbLVx3SJkHU^w(%RG@dvnbb_K>KQ2nm04(Ia=@JfC@ z%-iQeBR~nG#v@?(NbJZTg~@e8uycbb(v>>dg3<37j#aV`huYZEo;Y?X z`W7?sh-Tutc}#`sw4g)rZ2z8J%vC&yWqU1Q8L>xL{nytT`UlmSZT?{1(T;H+E`42k zLZ*w`;@H5s6usc?Nvp6aQ$A_zN<7LEmuR!68$(&sxBX0|B9YnJU10Zqm9U$)sRo!* z%f#BVnQO~aX8optwS|0Te}}$cGD;HIHC7Y{J>+2ZP#T(Z6rhtn4!J=SpnpmUpH-F6 z9;bqnx+=)1QN^@?`AB=Lg;%pRuy5U3{COlGY}saHd9T7G2@}kT+=g&Fb7&H;UFOY6 zXjNN4JNz7MwXJc(!V_^8MsN=!E&Ou_T!uSzWSvp}@CL+p2SY`SJjp6oU;=vQFJ)nA z2=UH||1pIa=UaM)3f-PhF|Aik_*+6;*0)22KR>%rJ-Q24;dh~zmWlY5OpH4giq_9J zP#PbEuhWPn`q2qp+o|V#N%^lH-bl&Zj{#jzY&NvUURRps6VGV5BF$?Zw;>?I5{~lv zl!(R$~h!hX{E8mNJHHG~jYk8|Pi-<7(I<^eD{7R#O$Y$Sa{_$V|+5Facsz z+r82mhpPx@(=<^;-4Mt85k0JI#CN7u+Q9xZd&Zs?X0zD`YuK8EcozSol$E@3 zVRNLDnA$O8rkz3OzXh1XB}-=6@kQgM-2zsB`Wx3jx{Pc3s>#b9F5%|yzs(sRa_0J$ zO0lwic^djF)-cnNqnXB+(`@QTC1z0N!~RrkXUCgj*sC30OqR4@kA1?~U;Qc;RGZB< zmOW#4Xnwr@#b@^7z-#u|P>gb-|FPp&WZ;)D6p`D!d z_cd%qukkS`R=_f)$7bRl+dLGX#riUj3vf)ub)2fxgjwFrs4n*xqtu%*`k6X!gn58k zOa8?j{>?afS`*WMGrx;p1=xC|9LFi-VL@^xmh1YVvOV)0VcbK7!BAX%Hv+LI9`E!m zMX${PX#Cw4mmS`Qqhv37Jl%u449qZK(*~3vbZjxh%R211 zH#EkmH=4|+j`hLprs2K{P39*v5&IcdrzNM1spg{Cyj=zZcK-%DM&Y@oM;d;269+rUkXGz96qX9?S|YA>AjJ zv(Xg=_q-xfUwwtF?3qLN3<**BDI3XQPa)bs1;AhP1}Esf0_g2&a5DNZNQEwB*_x?f zx@aF%Bo{z(T|E3Otz`FrTo^m05w4MHc$VJ}+wI%H{?{mUiF|~oyMQ&}(IZB!KI);M5_Nfz1%r=nQI2^@cm>3DUR$M7Ow z9J+EA*BVyh4z;^@>}nZKKUIh4_dP|Gls0^`U4SPu_yPmXrt_@4U!rt?08egT8`J(g zLDh+MD3(x$vqSHqL3Aai^HR|v!WaDxWMllLWL*6I1UeU_;_l~+dwIwa*FY3{Jy?yV zQ|-}ZxhJM>+lPYp0sX}{;D&sLWj(e46NR(O~; zZm*t>{<`Wo9VVi2pE71#P(;VeGN|q$jyLWKVC)2891Z;lc~|~G)R<>r#>Q4qRt=cf z2-E)LK=%9!P&siNcAMoxhrJiLB6bMcm9EzFhyoNsu(s&aJRm3j12d!q26CpmI_V^!kEfux$kd z>qW!dH-3=pnhuvE<3ZJ|7KV9c@Kc~0JPey*XmAul7I(omVL?1}`#0D)NTX$mBt9Q2 zfyrUx(OyCkvyM(h6WK{vqNRhizqODg7&ES-HVSTFZ&O1DpO3Rd@I%Z$xC+%rHL+&w z8mz2w#7hp=_@jz(J9xgB9()Bh^v=c`mGWqi{01UNt?^^mE%cmt4pnLsSvK7f+$~s$ zZZ4Pb{?luCQ2H)@if=&a{ew8v(ST0Tvb?#>2QZZmqG)OZcAUF|wKHl_F6 zfFqd9{HdP|uv~+bTe#nJImSfwg5r6mH+wysWieJmy|o|ei+JH#Gi%&@mf;eP46hic ziF|d8U&HE{)hUay-){Lp+Bq?3y|@HKKDTpU)=z@_@u8f9>m@EN&5+LLGo*=)rDS-u3Y7>N zOV4y$P(l7!uJlnPr+sAxT)8L+GpBBWbTKtBw2guFJM7?RU=oN`_`~TrY4Fu18P?so z3=TS3@MTF0n7zIamLs2`u%{1t3di8?hTl+bD2%0xh4Ge;9A-2BuHhkBd?KlY7Y(N=%ae`aTES!<&)Z!7Y%9W0kEU+hZ* zW_dDy{fEsMwqh9h74Mw9&bJGrDvw zLjC8~=&51Ee4`ekq_IAFIrDIq?rfAuorfBgQ&CAs6X!Rn;6fc`JjuR)sb#XvUtbQ( zCJSTCJwe77ABEfq0kll*gXd+RV2;{-cs8~b>delAz;G$VyibC{=2ST6;sd8ul3|$7 z7H%Jmf$>w+;rxgT9IxSrdyljrB|M(%=#1u)BrWM`_)GF$8qvp(^N9A7Bcx_UEIB7G z&Q;yh=Wh4DU=^suqQ0$<4!lzg)cs}@+GceESI`ce!u@xX@x;rO@Allhq+#*1i) z37M79JtWPumRreNwqX>1)_jLUPuJrPdJCT|>B8O>#pvRifeThtVq*JUeCfq<6vy4f zd+a)&J@qbLEEV8|cHKcce2JygZZd3`-Rnp0qJ3H=o;;L+VMmKGtLGJd_PB*ZY@BIV z_yfl68pZi4D|uDoGCWbWEAYv82__{T#xTZ7ULqEOiS-^h)X3q?6gSi`vOxc6E6mj; z7<(*4=+HQ6Iv+Gtq@`3dfh8z=;ITj^&`PJaKdZiP%=XHDu+>bQys2< za}Sr7d6v6TKMh(W-f+7@ZQxgr8rWxeLDi5QXv_`=O9OxC%{T+buj1gUQz@)DbpdKq zo1n9z5#|ei2J4eOV4e0GX!RGctr2ECKS6e`h%@YC0zM-0DE3wf{TGa9xt`PU;J_5j zIAp+l<+bszwK?kv>a(7|3Eq{ThXxnc;;`5(Tz7mET8!O^dz|?(?O8i0x?01sR81sik3yVDieKFoN$e;K!0;}#Z~-ozrNvEAA75>3SJ z;o&lV-dL7*7kc_7KDc-rL+0MZ;Xlkr;dK$7jLXM$>E(!$VR+V^;JAG!SpG^9?q@pa z?p`sT@y0?du63fpdCMW}S_inzJFLUkgz^xN-V&# zhqG{#q9LjbPC>JV>G&*s0zTZPjMENIz|`Og*d8l^WtD=g<`coqdc)wd{SV~7>Vfa6 zUtnAPL#Vvf4DIC?VB*e8VC8rmJg=Vy(My34nHmP$U)#de9#2@;uL(EKuLSvrZ#Zks zS-`J!9(s=y9O*(9tZ9#4Pkk)9#kl8f;T&sK-j*cuz0&a=x#58 zHJ?(!HLC$yUYCJc<#V{U>jA7Y=!J(i9WeIw0PL;h!(2XoY^xJt{3`)W)mOmbbrP5) zG8wf*GeJ$zDESNdh(;fEY1MkpOXgt`!)m{{XJ-XrCQk4$Q zu3m>TNb={hHqD?<0%lRM868Ap`xL7AC!B0lD`q*!o5-tKt>l755_y~~K}+V$L; z2Qa$@e(tyqg$(x-ncD@fSHFWrX&?Na`WqDEnMcrdQQX=z4nM4sK&wVo)KXBu@`LKw zAfSxY7J9<9J_CbhUE;wIMVw&HZLTR0{6 z4UT8)Ybqb%t(7+!_r4aiX=x3^Kd z%t_Fwp8>gE@wileE3TiwL$WXvjor?nNTWZRGi|L&ogOYfwFvv%wQ-EI5iYP-Lg!Wu ztdNk$Im4=Wu|Wb2633xzkSOkr6Tyiif56pl05;A34plq4A$n>*@ISl;n|xcK_h&Xl zF&<6U)gur$JsM!^7Vy?_f+ubIuvJYPT7AXf$7e;*-c`@7s{73yJ08e|&%VU*P1WQy z(`Rs{%wJ{4%sLYL^e1`tEr8Ik&&j>N=|s*bk@&RjAP=7FP<0DQdd5VA>)6vnbroVb zDYX?`+nC#2)4`Kmu+e+&P;tFfKzBzn>iQpIpgFyKjad&ekJDkz%?ynG*nsMWwRqX?KHXRo zPfQuVC*mydnk}nP@`MP|+*piZHTtHRsc5(^8&`kJVY>ffeD&rA+PYt2_(3xpJ6&R4 z_ct*9Sux(aoP#Ev*{HNR72A}~VHVTQ2<94L{vtJwWwl)uQ#| z1}w|W#J(@-pgnmb_*A(wkJxC|6L3aJ5qq>66=S(k)3H5WhV?5aVGzUS#`#F&gq|@t zFh&ejOa~!q%P>6E>xY8;UVyr%5afFsaz>BB z)MRqM(}Ovmsv2&|xe_iiy@_*H6#%g_Qy|U90ER`JK=7;!T)S`vj!lXItJ&osl$Zf~ zH5(!F=N%Yt+69Flo|`pR*gUPAy%dTZHA7)z}hz86Rb{T*B3N(Uex;Z?$F|Yr#A(7v5#u z{xWubDMS4O)mVS72oEq^s6o0ExAyGD(S?UFKBN#Owf>OzD=Ue9oFxivw&wZY*XOzT zmZJLNMRaj1KkI=y;3yk=`ixlPqZ}JtHeVU1yqUv*H6^ri)j&fp#&cMqh)I2{rVEk6 zIs1OW@m@akGW-Sg3L_xp-v`6OPvK8c7gU?x249sX7$1@jzr`-Y=9w|zzVHl04Qz!~ z!kgg!djqI1nF0?YgyU15SJPVQ$eRf6l4oBxn8m1}E+*2qQ`&u=cDz+_9VsD|a4(PERKYH%){JuK<{A z$MByCr-3_n8Ll_x!1SIP&``Jw4I#}ijp?=LW<77HwJo{f|l>YPn`JO?@fTh-Blb1=TpoC!29-OYu|? z!-R@*Szc2SX1T5K$fl+2&YJ6!Mqi|@|D z08MjlsQfVeo$P?iiY)NQMrHiFmSVevECx-Vh*xF^;j%J@)pU-4SC}B`FMJ30wta<+ zQ_o;~Q3veW+60!*>Y%XvDr{k#iFMIsATa$D4DQJR!&`n}vOfWSc5Q?#_du8{J`ZHo z^xVm& z+ee-ppGpG-3W$RLMdLDM3tE4516^+*Ma40cJbk1=9pn`$8z0bVvzzEzk@HkGPM*86 zk)KmaoW%(z8FJ?{W^ug=C%E1sPb;Xylp*ssh zin5W$rQ_uZhp_N)67rYVV?tUxm?Q(u%2LO-UAibxkxqn-u4C0JHQp_|Ec)Tab*tP9V3lf7s=qt6W<|ah6FkX{sw)=e zwqpWG*SV>5{AGsm$o?W*nnOsBwKH*&$R}==+i3rao%GD^)l@6*HWkTepn8@noP4_| zXK`;LC)c6J$$Xu~jW`E#$qS3QdGi`L>rQdFdu|*&9kd0h!}?J8=@9Hyu>ni{M2K@e z0y{S5!ikoWCO*cU_N0f%!?;aw`QN7p9=gyev$b%d~v!3(&RcB>Gb3^CfWt zh0ijbpEJwTp+v^G=K+Lc2IL&0=iqr z!j%s#+}E#pTvkB{_x=vy;#!t+1Gba6r_U9*p*gbLuiZ8DSV;$sGMh4Rf`m%xiOlvdnS(7#qY6cR~T-e5d^V5qj+W- z~ z_O#NChPR01bE*Ln4ztSBd!iTgsH3Iu+NrptCuiMAJg+3vO9)`>dB}VatsrG#NmxMN72qH z0tm+mGB6@Y40bH7D zPUkHXq6a(#=-4H4aKXC>s%r&+A}agFBzsiISQM89t6MiFqmz=9&DobgTI9y$hNJ7vua}Ct27PX|ET8*9U8f_ zLW!KQU=YXONVsslpLA-%L~g284owKpq{FrT^z#oxI-t*^2an97{OUK!n++0lPNxT1 z`Hr8?J|#&v8KsepJFk#=pB|Dw_x0%)3oE+BRgG>nl%i3gGigY)HC=mKogNsQOF8Ln z^!(mo8oFDa3u#i|Tvi8h3hG(hQmZ@Mq8B3^?G=ULft6r&eG#ZMMS)hiKiv4420I*M z;k;u$csOT+jYb(vPOXJm%j&^vSu12_HUfX(C(yjn3(wZDzW=*H$Wf4Bn5{5A5t@y6 za%W--(h);g~}+c}M;@;A1~4`gN}c9ryeM6}nMDRxKY# z!=;6IL4P%PRb$>^J%{61{)b&?DlW$qe6TbbZCU1DSVIzS+?j%#JyLNh zcN`mpv#{SQ5*xI$(E8h3Tz^S}C&u#EoDSS4FFL*w-=(os_2qqX{f7*FAm@mCBWCi< zcjRNJzAWv$o5}rUefHR{KulS`69vU*VnFw7e8(q@RvVa(O<({9y9F_E`dfJLbO_p( zG{IE0$FOBqJ(NnaYerEScv$6tQ*J)IeRvEmPE3W>_YT3Q%oyTu6*rHbvp{^+Bix-=ULDrhBB1rq)m^!RHAEq zjH!LJB7L`3iAPle$1dLhJ3VCRDqs4 zY(?*w+E4|#8d}tUnO+(h!&w$j;pXry=6*d4;*RA9bIzi#xm%NNaq43hzmOO2I9u-o%p3anCU*5r@R&G!Dqza)1h#bC=X^DykJ~A z-3?us&O0f|8Aa9yu)c8^%H48@zCw9gEG$M9wER%*Q35LMoJ0@Iu%(~p-NwCsUOdTj zQaqUrMQ~#052F6L9(0}8Vxa}=KP_s2fv+Fom{28ncRzsGk7=;+$z?eDA_0`1WJ3Fa zBhdaj7TnfGL#(eq+}v{zWI7xnHOLvppO^^(MT>ynQWUyRPl4+pcew=LUM}Ng2={hZ zFxM4m%IVrp=b9hAq9qX_w9m1Lb`E>diz^+e>lQvLR`ZL@6E>hL78R43DZW&=S)QuT z{6%8&juELUGct2#39--lOb)dS8OP{8B0I{y5hE878salTCIpUWF+l~Cm%E$JeH%zG zT$n>E*96mc(=NKd@GgzLuf=^_v4!J17|D4Tlyb>c&pA|415!E>Z1SA}L#&|Z$YFT* z@F3i*NrBCpDe(Ch)AgP^0|!5!hkKfN;IJwe#5*d%P_zJQ@EYhAc46S#13-!ZY8S z#=I0Oh|&BV#+N!4!j-Ox_(|T5c>s7~kU)e^|Cy-1eD5`-aB{8^nIgyou%s6E2! zH2H1#JHQL)h1+7=<%wvkW&!dE&1B3E#>$wShVL$k@F?T{W_(2&mRUneedVybM3+~c zC(O%8VEl{|w&da`=681NJ!}pc0e6)$2$XvS_ngjv&5bJ{P@D@>eQQAWNhs9^IA*1pPoDffgaRZ5PvFZdE?)3rBfJpFsycCAtZh-{`%HU?M4m0(hbLTgd zab(;vZmF6JH{_%dQjd=Ayg+SCb?B<75V9=iCE369 zr}62CcjR)L&qrKB;(Vz}(cQ&FrU+T!^PI>CP zFN-)Po6_^W`7@mIO&TmIONO~Wiojo`9PY@! z1*J9bK=4Hqq^>uFbmsSeWQ{VlKMuTkj?Dkf&xB{JkVP0SN;5>hksY#osZ-N2{B^Vi zpQtb`_UrA~u3wBHFM_Z>P!mJPgy7i15InMBFZQQ+;O~Y&#^H-WpOt}l=Yj`@oMn3N ziV(Dr3PHH6$$Z;_QG^y_*~je&9Y=Ay$RqSO?j@^ z>kB>oGmDNk&!H*`*0lfJcB)`VY5aR3D$Hj}i?Titnx;Y9Ee42DIgh5c8Bn=sVOqFm z1)0(Pm)I*R(?#BM=#ceu(&KrAnDeKRbghR(TQh)qJeovpGsn`Q1a-PJQH>hCQ=kEN zj#2n^h^o$9OkEENQ-Q3VH0j1DHE5p6S!M@wCsez+>xZ6ki!V-x`T!+p`)vbm$1Pz~ zk2_qy=MKJm{h(bV90ry~!LHP!u&DYN_aZ5;)bXUE;7T^K&Aw&fXilpIPB>hiH3_d}6a>`>e#X*whE7;+U zDjQz3=osEE5pn8syqXMk2Ga3zd=Q@M317Ms;aR|8s8o*vyM6wU(w+bUBS#^yIt8pt zj=`8%#;esm0vYeZ;O3?PX#3^{>K(fvJ8lVl+-C#knX1q*bvjg!dBWufba7+r4{@JN z^*OzwztsDh2@Mxu7HajX)I7zNzH4Q*ePj#y!gBBCM^WmP^qqX-RH$ip8o6=jIO&}x zN1w%AC(2bVWMvfNXO2vyR;~-EBJU)*+NsZcHy@M50qulOOP;!aJ4d9gX34VDPB~AYQZuw^{$C4PXr#hzu+-slz8&TCeFM0j8MnFp zHw(G??VmX}`_J5wC;@OamV}*=6QH$938YzS^-bP9*d93(G(Vca_D=*poZ1A_IakQ} zVGF_yFSzt;(R8=PXL89RhvN7N9;n^f*6c--G$Nz7=M`G z01vX~hpL(1Z|E>Cm@L9e&8x!G0ziho9wY82^9bL@LGtBp2x+=B%aZXw6U8#IL54r9^1Ihc%U7^^3f@*-ARqo)YoX zCN%hPAsO1?M2E^25@oS|vXbT!!|?T#wS;NP;^V}2$4n}+x|&vZNO1{B-9 z{BSW+6x5TnL8R9Z25&5Y$W6Xn8u^3V^*uLf(d4tlEoPK>Z>=UztPhj*n`cqCTs=DSyqBnyBAspiiCFVrBx`mW z(HiChnKYxC^gPKU0uAGE+(Btx{Gd0_!GOn8GWvxkQ$FFFZ@00)-$znxhFiV{N39BZkGeW!`|0WeL zf0Bdqk?wzCPFr-P=+Vh#ByaHmd3Ny#3H@@1HpqYA7R=fWFWwvni-9<}`6~d{rTc^1 zb8jG`!BE+@2Oj8chyAl0;r5$l@c69>y#HkY&I|OQa)ShPx{Y$tQ%g8itK-~sX<_cb zsycdR?rWlCTtrrxPNG=4g*v4P(nUiPXnDm^Vtno>U47;TnK#0tX8a;lp{b5!u3}ni z$9H5e*G{fB#!~6H1tj95HQlSUjJ5`a(9(w^#P0HV`h24x&ANM>97yyg+gGHL$;ZX0 zL*NH;Tar+7hV{apr({!5EHT{YVQ3R1NY|{9qk+yXMlS@0Nvqocd1&>4ylze*8Q+bl zd)aDQl>VOBKtO2}|+85#f6oO}znBrz@O)bC9@nWgcG@J;O@`ElK(b@OtP z>7q<;ylf*X$v26rPAw@)Sx^6nJ}0Gva&(iiEg34hP6`I^lih2-lIe@}zdcNhMBkG&$+yYJYz?~f$rrN3zl${esHN)pqWCpC3X@+i zL!-}8m@!Ta?{B|P;~LWm3ZxP7R%hDuatoPmJ4mipHj==3wIoSjo*r91jh5^eqFMtJ z=r(&9s_AisRPBC53I-<8&w?w-!JA)*!?sy;x>guD?ioyD4vNuf5B?DG1`9Ib=2eon z?xS%=VGhZ6HKy?w9mrA1RYcx!Dh+O^BpZ&Kkj3(%^w;b6WLs+k*%-c%#Jh9!b>dR$ znY@fnXz(D(PZj8o&2sd>x1Z#)LKmr}e@S_c46zqVF?MKPYV6Vd(ztEhBANz5OOkjdFWG}FrvAA`Hrl6?-q$C{mCH3`Q}lb|CPPBzM2gW1kD`h3s~L2}U57-#WdaorQW1)6Z?q|mu%-9@%)LNi!9J-lMdCA93U?zDpOmRRI+i`O4{lVbl~?Ja$F~Z4BFI@ z^M{g%ndJz(7D`aV8Hq$Hqlh>)`;iR;QuN2#!(?pREaJ7hfs7HCAqs{0gwAdz7RF_y zZeTWXa!(|GLKDe_U@5v{Y8-J?Y$C75R1jaiV?ldRTp_KHb^#0Xj9LWU_vLd9Mw6(RNSYK zXur=O=Wm1%@tT+9ewZS)7&t{POY>8{VQXUAEK83B0qJO6NSAF4AjVlXbgOM;C~O!ygM2M>c%QPECuZ9bIB!3et~=mF;$Q6}_x&Wvv4c(Iq@fngh@M1O z)l4Aw8n2PiSC-WH)dn)pRG03JYbHhJ62wMoC4E)4gZhpO3|st3#k6lOT_x90=2>AbUCZfY&)mX(zLtA zbM6n3rn(+tr>sQ77k7|Zt{+M1J9iS-xt{cD?jtp`Pms&}L*%ze9Wg#xPPA)n=$5c4 z)Nk8(`f8+!EOt*NeNXzyg(qd?jLao+J?9$P;nhH_)@#z-1_RpJAw@eKVyMkhk+D*B zk>!G;e0=}RHcJHr`S`|;<@+Q1F)74|wA~TL}-T(ijFZR!? zH=o^RKZd>9KeNqJ{(lz#{rW1(RZFtj`aiSuKTrIR>A&8M^CquNI=gqe{qMd1{r?*G zKlc2_*8Y8OUA_Ok@4vq*ssFLB-S7Xq@4r3=o2A0S+W*>UvcT;B-dx~ckN#^}{;%~G l_9yDUFd)L|foV}m@Jp0<)@(b|rh=}m~KZi7rEYH?` zZXTZd-3_;I_1|jXyUTltl_byl|KkYebMyD%jxul!@b~og6La0?zAeDhd%LT@`yM}^ zeXjeB#P;&cp6x8meZC)evZu%kRw3y}J1Z z>=m%%UEsf0aJ4PZY+GLL>!|Hf|Gh%9xp%p5?k8UVy}~El+efIc?t=hNU zeV_aGMV@Z{VtYk&*X$Mb-z&Ce?-T=jTVW9ur81i<+(-Y<)BgWEZAl*0|6|&Q^OjgC z{$G>+-z<<Vv^HoO5c4{pa~L+bXik15$OIxcn$SvfbmkY{m2`l zWj#)QuZbeBM;pntw7JwWxq;9hwWM`TFv-=vN}d#cA*%A{2yPrB=6ik--Qx3PbC?W0 z*d#zxI=e}|VIpx0NhO=?W$5zmB}6X!G`UnPMFpPUCk-MTdR<1H23a(c1=*TZdjBN3 z%Iib4j}Ma-S&OLopWWo`9B~@7h?jgR@+DIi7}KtVY7%tThq&j>Cx;tc$fb|H~)ZrIaJ7W~Px@QBDITY9@lA3IDEO(uxw(FAgP zw=B8dUQT9IKPEp*r%|ujcgd`S>Ew~u50ZFXj}{b&(^Dz@lvjKK717BcAI-lJQJzc0 z#BGus{ZdOFhz${}aU?(Q@Y1-CJtTTUgZ`BhqU$5)(C0oc$+d|Nl4HqBqmGsn3t=Bp z|KkgBs;DP_TxL@d{yB8~*k$t9S%MZko=KnbOeaRFW;D*{Gx_cBM$PyW$=HVhBG7I} z`=3^mK7nRZ5IB|Alva}Q_&4P7GX*NY^#SQz+|BiiB8^`Ch1hO>Lb7H*B%4pPl9qNI zdN5LgrpfBl?j6eX<_iw}XEKA{nh{I_=H4W$hfS!2yeSnkk)gAa8pz`9_sIi6mbN_A$f+b&@(pCVR~LD{KblyW zD^N+x??kFCfei7@CMzXXN&C_+B2#EUPfR;bOjO!Q!{KrAXrn)Y{8VyByNN8 zKSR>aETcWIJIMQ6KzDQy+RDnvw4QWw_2*0|X^I9FcX9gQ<~O1+IE$@%AxH-`RavQl zI@Pc@qCz>*5V=ST9#tP9_uq=rXA?*VY=!C2=946Px*v_LY#>Yjh>?q{mXX=RLgbiN z0%==#oa~#|NJ2jFQg4ZTvbFXNc_*GnLOxz5Ho7k4i98SW_u|m>lvI-VN`ksc4Uwo# z$z(kpB?0~eMC2QxOHKIbiLdtbz2P-tm-3t}9yX;0;pa)=O%ZzIWF7Hxcu8Cqz91uo z!n9|LG94QlCfmCnk{1SH#B(5rl;xYzP612$E`K#u`TmYP98{vI<=05#4t=`*bQC#r zWI53kxlghqLy32NHYvC8p|>`R&@DHo(U(7DY2%WM^s{##acliZS`W>qWBZFO!Y(UQ zk4avOogv+%^VV(BQu>fwSf4^B_rD`UWi4cv)ob!cPm7*TlcdI%lE|s)b7`-i9(~*L zldS!0Nb}cubqs3_Rtts@nR3}mKDJ8?x zg{aG%mlo3kmFcCUEyPMzgAUBEAiuu8AT~|mr0wl`l4mf3h9wM>c;Sy^&%=X+sNE;$ zMVm>FWEFAzeUn^mTR`_;x=Fh16)Dg33^Hd=HCYm%fss3M(co)2wsx6f@&-NJoHh*y z7v5lImDA9A{~XL5H^rij6)4Xgw=+Z&J7mL%n%8^th<&?>h;tVy^iQJqdpM-Jxt|F0 zRFLt89&$E1lZZ;!l6yWPbUVhA%Q9vpp{SIEDcvHkgNMl3Ghc{QP%AM%JDYx0?jqkq zjOo2^-^pUD%fvUQj&v^5p@Nxtgy*{eU0TlQhhj;Z7kQcJ&7MiG?wU!*eh5>?gU5(Q z+H=xoV?Zmi)v0HUC=IV_BF2fPR9vTwbn>1i4}(PM5zUw6TU$BdY0D&Dv!4=A&tJr8 zsWPRK4dehn&|BI5q;&8DF&@4|t3S!myG~z8M!-EHx2T`6q)y^!e3ksJUQ0joB$G{b zlSEnUHmNt30{a*n+dCZdfC@s6Rbf?1U@~Hg`6kaa5~8v!lG7!AZo(H?n_kbv%0RA=TR@>p95SFBDzE0t}SI`t+US{_aAWwjH}{gn1h`$M8E zwaNL?P?BDqOg1=|5Nj+ZAC@RnvBn@G($5FF7T#EPVHk%WnQ$(fT61*81UOGLC-AjH zKR!HSgX~T}#{L_}nSc2?ev;Ol)1@XH3H1?N5#x=iM|ojaMg=KZp-gYg@g_r#d8FcD z3u#{&L)@?Q62-L@q+CjbUf=ePSS{hF_Kz!Q^3SbUy(a-uNBaVnDdl>Dier!R&y=oPio zWT1X3HCUEGjORTkrkllS^jUfO$mu3|CD=!1mwYDWWm>e(Se3fA)RQ++a&)j!oOVk+ zAoFUKDX*6Wz2vPzKc_txQ?HcY?Rb zI+tRSXLf}odM>B8Bg4q6UxVbW(<=-4@%c0~^B%qO=mH3b2jXq@YBX-Yhu>~$qUtXd zoKmTU3e#VL#XT)Fo34iKL7J%9cn^6+tFgZH5Z=9%2u^D|sPH03IwpUMY;33{5|b0; z(2_k=$GnEz!EDmG-ka#Xw4q#ka~JDrU(tJBWes z{l_f$(rqT%SjIdaeP(e-0;oNGNsiZhQlH9RBBI3Tv*dY{|8fb@7abw|JIl%C07d$7 z;Y_;ao&|Nkv4lc;D@oY3flk_K&}t!9;_~J<5;xy5PK#R&h}49Wbw`IOz)d0+$w5eQbC{Dy)#oG;jsZ2Z{mSu zpM~t7k{=u0JQLnmzhN+FPFKCiBzwQ8(tx%?$hUuhs{hsF@#UKMM^1_UIq5+o%S%c2 zx}D_AeotcM{Ful&Tq6Shtt8{AG|lUMLjt(_Qgov)RY%6^celgEJUcX~wZv@QBIr}U z01?B_Avj4KKN`M(8}^B?bX^fx=~?3a2X-i(`WieJEMfN_I8Y7qx%9{*ebQ3-orH~! zlW@KR#34AJ9O+B4FfvIa!xfCysc)bMswJuHh$gnW)T4^)1I!FBf-6Vz$=g5bbWNrj zWvy>mj*Kqo+3#j6K02~8_kT?NoE``d$iXkkkIa7eTXyw=2<*Su$0F*In0wK2_Q;~2 zZQze#S`GfJ=iw1nE&r1pwXdc+r(Nh>8y`A6gZ9fVv*Bp}XhJqZN}X^q!X#%{8>5qJ}~=)~|`Inx9Jd?opyv_K|dKZwR${ z*G#wRd}KjwX3XL}Ve{ex7@YXYG#kU&zn&DPqxqiM$xMgok9yeoB|S`QOcJ^Um4Vl* znWeoDW|{{=S=&2*xG}y17Du_V>epT5Ny>XNOKAX_caPwSr!t(2KkuLi$tMkbw@A*7 z9&-3o5Y_lEkqm~EkQ%26GM>GJJQ4duMo#pTzsI+N-?tM)c1|GOQmzBRrUH1>Q4EiW zKY+bnDX`sw2XjMrps^GW{`)T#iUJ?NZVNFCHWWmQVlDWfK93$>@yo*YtQ(}Jg_7%Y z8;GOHghlV(TJmZ6B#~7}A(sB*WY*^)GO2Qd_$DBo-E;>7Qe-$Y_l)3V>nC{sahMF7 z4U$feZLIXzE=ZZ<2NoL9Y_wmDeOG?ThS$gghRcAxTo2p7`ZN29)1aldlRec+XPJeE z*c_Yh?AtUbLad(koJ^qq-gz^ z^y(>5X}NjS`1~Te_27S0KXHV(?_WSY*kkhVnbrgQGBqHzt! zNQk2vHCib{qas&Ok5D$F`l2t!&m;38fEHP(0F=Ibh0{I(E6 ziwj_uMI78r$$^NUmw4=5w@ZQp;;{^W0K$@w5!} zbgv?p+hR!6VSUxoX5T zf7md!113;|msyu?9?Q`G$@F%+fO?!1WQtCQOQs7UZq6>4IOhQFEne{HRTwC^xI?+6 z58STNg}kVBu*3EvJhu1-!`8tN@hglT-8e{mnKRVDWu%!6SlQx-o%?y{=%+3cGph7An#-}@)`tU*_|1vth1MW+lQPz3Lc#Id}^F+ zlh?7JVL5C1VoYl{o}@8lOX$elpF}YE1u1s8MQ(ld!|it(FsM2k1EgHwo9}57UhF|- zd4j-d_#0?^{s?D3xp3#I54Z38!Yo4%cxD?8QNx~Ke|a5D&e{#`$_StQ6+xy$9Kw^= z!^`+F_O-r}sqf2W@*`&8;c3s-e3oF%z8jeGrDV1uQ-hscQB2oYy3vzcwAhJ|917FU z(i>ta+_eDoh}BxEEINxyY}BB+jjkl}HxGRsTuDZ&+1BRzo4QXb4eklYg6+7Y4OmVp3DFOU_25@uPMtCUh3?}}&z;@wb(9yAl zJwLX=`JZn6wM04-(SP= z`-~q@y6!sgZ_`4z?RT-$N*fOt=E9f9KcJ{T9B(-Xp~ZOx^r=`w9qyhd*=J&KXPP`` zqG}kG{<-4 zh~CL&C9MMWOT{pG=PKOKcYr823%Ix64KA&*1;N)r@HTZf%rM&k{X4gSr^j5VZV-jW zLNUmW@&tb~5g;AytZPmqtIe|k;9_S5vYgEpKEs-~JYk==da^$r6|~zhhThV@#fqjZ zXRbdfd$=H(HkNd9W1R-V%>nv1{I z%Ho_8t&kM47S)aTIBgfm)$UEq&4W~-=mL?g)(_2kA%S-7`09+FIpVE^3; zsGV$unB<>V}sp%zH)0+)J z_OBslXErojoq>b%qoBZ}82VnX2IKqcAhlEjzSZf$oR((BGiJ$ty6G`a4?mpxe2ei~ z#jsBKSxl==p6w7^LJgae$T9w>2Cx=+DF$#{6^x){Eu zpN8BG2~cU61_KU}uxwo+>?q5Go8tA{y`hSATl!!$XfBXxkBP>UlVs}35|CAs;;hVH z$hm4Q#3|ah67qhZBZKE25SA>8%WKk6^LQo7C0~M7PtwR~lPbb%HWwXo!f{4nEYgWK z7!M4GL$h_UOlA-l)#&1`H78)(t(PG2Cl+t=grk%ATy)QmAg>9ggEkv3pw2*lAMw3J8;KnkjSOqB)p&JK}%~NloqIB)~tH4 zSdtBCT7|H}_#|9^oB@s|7vS7L42Y;LhB1jaxOhJrPAcz$ERVH7cFcxd&VsPqT^3?u z1K`?Jc_>=^iP?_6WF*@bvPPp=dLxJV8yK+}^Ovysw21Z}w5LWzhIFQZAuGC(NmbUq zq@K=asg*<=T`+1y|COzvJNI>y4ly2jTvULDYspcoXnyLa(n5}o0UZjHqh-@qlIO8< zG%&oJT;DlN9Qj6Q9f_w~uJSYOqxm$m2if~oo9OlO3uHrN9_5QZ&9b-ovItin)~b}j zf)dw&tHvOEIsAvk2dluU+5@oVwmeLWnF757#_;)}6PP9Yfo<6Z2vJFc=?6G)V=@g2 zp5}ut7gMj1$N}rSjj&F&3>HoNfZcXBD41f2J5S$&Q;PycFvCUW-|qzLN82Gl=HEVHDrcf?u=$;Qsd!s9t-E{K-29_X3w*eIA&K7lIm=~E4APIch$s|RzsXL2}Sb3M>x@mq@?m6=pW_9iGOE=1>i8!Yku0Y+tI z@aanvDDYl|-jphMpqvk%&j~qrC>9JG zs2bFC@WP*hPwbI{BS@}DX7kVNV!rV^Sl0HFtoG#-+SR^*ev{om;}~W9-bM6{%{a|# ziKO+KAL*^%&&kI9{ItVzm^`s4B>8<;$o4l=Y4Y}qWJy1z_H_dEMOq4(5qX9@Jg|-` zIW`e-F1Gjo8^WfoGGUyPhnZphZZ39fV?wV(>FGDcRBPlEt6KAg>F75yuh@EKJ$IO0 zUl0lF?(4(*qsDOF%nta=5@7zV#W4Ml9oW9{f{@3NAbtKk2p%is@?UqsYsy|YcA^{t zZq$RaUO&*&uOP1D8yB2Dh0mAX!s%8cRR80O8GZ7|o1q3G(nS3M zVI0q~#)}y(*luKvmt{oI`Gf|BRqVt6z8%Ej_g;9h*&O5FjxbBfDy+Ht4;N0qjwvnj zcrW`bsmsZr<-eM7;uYmcJ2`U}e^TUhTAQPL)^j3QF#{6frsIA0E%tS*pY^d(m@*4gHr-b^avN;3F+;hTj2pZ}KQJ zcwx#$7awM`zJ}43Q)A@unT0g&p(rs{NGGC46sZ2jdDKTlkY?=_q{?4{$>xJWBz0XA z8GFV=kftq5>|SFNe0L8sYEUEAZ-jKbL=V#^W-!xInQUVs5C=mMtRmY|DJie3y-z zPn^J~SO0;ja3N{8>n9Jn`j_nshI7`oW9anvc%*V8%El(3R9+eeZM}lgB3E(I)))+_ z2*MrZ@wly(hjTt89y@Obp-gQICUxcDjQv+oydVvw>Ju>i`9_?t@gDcbwqs>6mxigC zBogZ+sq>#hbVlw!@DVzR&DPoIG-QOIxtipCrcS9w1KfCSiw0`Wc%t(Y)N}K;aM^RH z(w>IDXDMPuumX;4mc+kngmKB7=it9z6eS)A;G9Fl&{sSJ>qp*#=(opE^0y4WXncX$ zqa{G&uE3iIS>Pnx3Z{+!foRzFll5Wh?iWmJZarI? zn8X?!dsy$vaZ)5UK@5MLA>}c_t@`JRuksq$LyRhd|gW0L*Q;g@*3=pw0Wm@W+n9!Pf=21Bhy6S6L30!Btg7z$}q zG!BHR+SEIwLa~LY<{(uY>LTJoQ!(wHH+p;*#6>}?@uK=347?qTkMA8vxuuTCAI7lQ zXcxLGb>ieZFEqHyaAeRC=O~2X-0)yrEwl#1>3R zMO8p=--DfZa-r$!RoGFr5kUJk#NOumc5VWk%1r=KvtW31&km#xEr!X1v7ptk3^KHg z;pc!pjC!AfxP*_aQu`Vk*wDo8uQ7(v=;!RcwKrRzW68We&VVE1XW4`JKsM^InK^*~ zNC?MLm3}oE;w4Q>FO`t6r5mWaSR^@?yol61H6_n89BBH=cZ61H(m8f#$mu&S)VnT} z9+Hv*Lm@qO8%^2Xi%v}cq7ux@GGQv-+u7WM^{iwZp``yNtGVvP?xZKP9lJxp@SZgE zNKS|6=M7@rEcq%OQyaYab zGN4VW29oB+!_X032)*deD&~vfs?}!b`&}D-%q&4CNQOq9RG~w6uaavQo5+H}THtgvO6B$$=1V+#7q!= zn+{=rE^>9A1d#QikmH&H8{{s)A+zHURvZcfN1b6tsUaNOZwRy`65LGZ!}HW>aPf~M zBn<|`sM2M2)MhceQku)6{460asE&R7aF88JwqVE4s=*sU7q+)ght*ffFaZ@wXgwN1 z4;}ZVuCa_Qn6P;|RrM0!~RC=>6 zlv5kl{ctDyTXlk6IkpnIe{y4hMLL`M_#2B33WQ$0Pb~ajCTlb*XYxjh78!J4o8;EggF?j311MESF^8yYIoJPZWff$r*g<7IsAhQtguahsHZqY@9If1MrsesN3l45*;$5?Iu z9olLo1TnS8!BYG-Oi^rwQ`{Du^KTFX1m*E*lQ$${6U=h^0Uy$NFsCOMHl^1B`TGDh1~SC16mR37f(*p~wC-T#N~V5rw^QPdf`*J$He$_);*4rEq!UIq+N8 z#gY?}S=g6ecBAnCTx%X-8}-xJ!Ih_3K)F2}J{-y{aSK}(@5mxD^dPTkfVOnyQMEcx zT5x9(<-4v$LmlVS(cxUOqP>$4Cpr50ek-{Xf1QnKe`e4z#2n6dv00XvVVRRDJb8=I zX}kjX)Jj0<)Jk||;s87{k?^Lu1%9_h19J_5kj)oh<+g5kG*%8(t`Fg;)oX}Q<-xMp z4=`Z%5%kA~!O=(n-#y~T1$(BT$OB;vjjV=aBjPBybvk|&mBS08<8W*f!^)E@FyqZ4 zywoL!KNQU{ImQ@=+vZ|mCl6MhF+kC%3G?b95#Qc;dT%wqSzTMn zs0rzYN(}uk4VNB1fL1lRu=heT#wL&9Sg$x|NLqkn=lBVy_ua;|O|ICmI2(M{=;9PT zh6TF}aY%<3^A63$Ih&2~+f-9rn>h``k1s;Lo)vgk3b3|h0(J~d#p7$H<19yU%xbLx z$wFcL;yMMp=knv}5BzA(wbSCtUIR4|l> zodVVJW;iGj0l^-Q@cy$6IL#=5IV)}8fC9p!Tw|ENItRXJ4X`H}qm0TAvf>}t*#Msi z-JLH)#~ytmjj!iWd;VrptgS;0OlsKGX?NJOXCX{2)|~C$=nnpe^4Nb|K2hyU9m6Y8 zux9&9<|ER}6tB+(w>?=PAgvEP$ui(Ahp>^$)rhS)0=nx@fQw2xBtLxwyCkkb%FlfG z@csb^NHv4S=Vs`f_6~R^UIROO0ah*O1Cd-l-0Cq5W@fkH2USM9jM>P4cQ!`+Ue_TYT5Y7%L{!@e1eFrHYlaB2oEBOW{+63?nKOg?n(#67+vvI*jCA{jC53vzHA=!l=^>6?rIxm3Wl1|W{*$ubf zwt(Wm3(${$1pNWGVCUAW5VpJuJWbN!YVHYm^(qK97FEE#?+6d}$-sB6-Q}|`!H;2m zXgu(Vxz1{5w(TcC=wUPC%fH7~d@o=z&$hyd8)eR(5vk#rHZ%cFzm4FYP_^pD1AAdOob9REKqQ`Sz)Dx0wikC3~LV#%6ZJ!l8A8tlf5; z%_%p5hwE~{=H)yHVe%kuw+!fu`|zpyB$ORL2Ad31!KttkIx5m3W#?tMYgq=*#cRRH z|0(1gcnP{Un}Btk1}DR}FkgBM3TAwQot{bX<=Yf=)>Ff(9C_5c)D8dL`V7%og%E2H z1&)u#sq)(opl583a?dPrbAB_Bd5dt`;_diR;Xf2R7=qJ}T*sKdMHt_940~2YgXg1V zcyrJTH!MGe|4J{TT97Yp`@IzTa|6*tL!6VUc?cElmf;aWUp)8yGB(USg_j?DVeY-< zs2LgsJll?ASY#0nIbFxK>q5}Cct1LRcEe=fMY#FO3;4HwA7-2>1HT?)EV(_x{e5llt2_6T z1?JOf(F+OMbmcd5ioDNw)FYUW&3`QMbugUq_{MC%k22meA=vx(GAtaL1_>Wjpsv*h zN^TTGil_}7=P`lO*`6?6>orupPJ_ZM(YcRS=$P+ z*`GjjXAj_i$slVt0;_xf!JXa_FjLEf&R>$Kuc3$w3#a0)6+NInQxc2T3ZU<{VK}2s zVUSM*=UqhPlefZ+v)ZBShAN&D<-yN4W8uNp3DzrUjY`_*P{}VE)4mGe^t<~ojuVQ` zKh9%#L>ZoP48yT;ZvWTUFsy$%o#SyW40)*=D)fipp5tZs{_lCzxEhMvhW24!v;dAi zi$?c*+@3LOE1d2;&J?!AL8d1UE}X4~pNwC@_xlSm>J$eX?L|7Yf!|}15(IYco~Xff5NhO9`u#{2J`PGLP%~8Z06d-rl}2<|7nE1CmzFr zsA8C>od+ug8sNzCEVwD14z(6(AglZevK736f2tAOKe`O+mG3~f*$SA;<j0HR8(lO-*uv${egVm%S(3-3ehYg zPA^$#!q{dBQ2EuzWHx_c7k1xbMW?Z>BMU#}HsJE5 zDcG2E0%z>Ff(~J3oaW}sDB^blZ%j);{&fwgl$eG7?x*l?^GS?fOVK_t9#1L;V)NCr zV7ViLyvY4Ys=BwrdKXz#Z5W1*niaSrc^$UxUWB$r5;)+unR~i&`66rVT>B5MmZ{<+ zhnX1oLIsa(t$~EQ;@G@S7R_TtF@qZye`^WgK^{SDeJ~13)(=3^{caFj{}RL>HbJEK zTkzfY95P%hVb#@#;H=4miBXyGL+J!0yPts_Vqd^m{v3Gb9|AsKKWJ!u3IA<%hu8Pk zflS(3xZ73$!ShF0qE0`HEf9yP3)Db=Qj7A6$Wf`M;`H}`DBbDTK?2i7seS$z_Q0Tw z9j*>$x`iiMUHN%%sSt;XbNmqBEet^-Ps5F z-S6RW?=^^@oep1|^PphpIT#8SK_q`QtQKkml`|di&#V_-RQ`pww%_pbKr!4M=Er$j zGI-cZ0*_cWK!Z9r-Y3t(%g0oZFYY-6oYccOlVxbrXM;@(20_To6n8G3=}$i0X1!vaPN$5Si5-_u1Sf)T)!f$^gM|3wmWk3 zK`6FF>vGQg48i_L2Mqpn0BdxM@RDH^>O}6s!r9w!bd3@g;sx9sk%2Ws+mTmA1J2J> zz%Wl|jEUKZ>GQ^+Lc#=V4Hw`7Ni)pQ9)e|=wwTeo6yv@0ar%ZPkZo1LRQXwW&{Yk0 zsy>H*dE)4&ErW|Q1#pFI30zS84SbV-!6&W{ETlSMMQ$^UmDIr4f!mN~(*)|KdEj|D z9bVO37~YPv9O`w_RaAA1WFTLEFt<*gc#F zZb{kjJfsA6q`iXv%TGb!<74Rb{Qdm+PR7@FP-V)Q(IJaX;{EEAN(nQAJSa!U?5 zik)C4HwV{A8sUF=I%u8W1OY8ZsAjzge_B|fZC*FLUSy6NDcASXX1M>X7`@X`b@!j|=^CY)c z^uzF9Kj2HaDN5bt;IAliG@8=``@qzo8=8G

61O=q~hr$_Ya=~ks4`k$)^h>a}Xx=euGLnWw4sbFfp1|-~8aeLvl@$KUl z98TJSSA%=8ox9JA%Ff}jl^w>GIm6M_4Yh~7ab?zhuxB3O)uJcJGYZGEqfzkVC*Z+> zbky&nIQ#w;mcMzArQ-FdDd0Rk?wo&v5X;j6rM{rSHAXt6iqCoibaF)#beOYoU@3p3BxbxPH&E*|ED9JeVZea48 zofwO=q0HjnS0_eH{1>?ZY1OUa_*@r3?2!b~@*X3C}t7!~g@W^LR}Cd*8YFA^!J0kRBGmR zS{!nR#`px&aTbX*druD4tb0#~bQ)-hNqha=2qY3D`6f7r(1uqr@^qAN?N` zoq0G`&l|=i`<6W-k}Oe5WIOW=MVqA*T4bwi6;i3RkdQU9r4U&Pr7Tg(JM)sgol+zu zDG`+xzD4yrzrWs#%Z2OAnVDyv`*Yu8N+k+0RTW-Mg=gB+^V`|8pS?E?RN7D1ZVjV9 z^CPLin>cD3l*BT7>2z&Q2CYBLYHB){=}RG&mm6pI{i?gP{#gUn4|qzO*7Z}1S1h+* zCdAbk7vZip`c8HG`e?A@Q@Stj9yPjEO?~;xsatg{&?rtV zyZNY9^(2X$@s3E|c}9#@)sc;LrDPw=I~?pxB)a_}M0;opQA#r*r!o<-gJuAzykR%D;@0lj=8 z0P(JKAocWe*juv&gd&6C*u^yHyIcW%b9&&2ycq5eGQj6m#jxLj9{Av10gFA-K-lvr zEFawroaf8IM^Xul(gh&Z@FQ~WX+*~^l_1A?Nhtn@50Wujiozf9AnBL~%y?q})7?^Q z#u@y2$EZ}C^B#6{26D?dRy?0LD(*^TtjU;gIorsz>Pe#Ec9uN5kxkS+Z;)+|YDq%$ zGh(pv9dVNRk2GK5qbXRNI(?O=o&9quM@*aEG+RQWXBg8*EkJEER?)QjO>}CWE8VW- zOOLQ@qxd_v*2<5f^$+4%1}v5O2c=WFk6Bc^{Sv*-GK@!0vVOOF)l{eM9-UTxO52U! zP}ip{gDEA-&1e?p3bH)rhoU~}&CfE{9kulB_$?Zqd!1gse1-a_W>YoK47%ZO3iaKe zM2BLdsmZr+>Na?odhH9K&xf|tR|*bP%X2L)eUIp?53Ek5xP*>*YtewUbE&+SJiVD9 zPWPXmNh92T5eL5yWSi7;60CHOWif6N7l$0O_D%wES#zA2Ay=~DfDswIrb4WPCOP5O zHJrMxAkIaD;Tpr1W2SSDDKh;ZU77c_rHtQZUgXiPj;w9%P)bDv8V|@tx7uou#p-S( z(e(qpd?OAj$5dg>TLkyzU16tU7^~5zgQ7$gd|UMf-nNKi&l+RA>w!3a!21UN@M1r9-uu!;7Mtlhd)BWkYDM)o7}Cr+OK7aN7G3*8jV|~% zn~tuUMR&~?q$V-{k!+ifWMSM(l3LJ6c4wB6P{CZH@iu|nKNUtgrd`Rf5RkVnrY9>V^-Z6XSTX)BKHX^RG70DDV;utT0d8z z=Q~;v6ULp!sMu2p0@M+&4+Adjw%dTMDl; z9sse3YACSFfUJpd$a3BVdrYn1aI7Y@=gUC--kGrV-Z0uP--={@SFj9tI^ub;8zm^N zLAMmO(M8tN(@;~uXkI(bSd31aE&bACBA#%e#=S0pGtZ%#bH4p6hu2+}+^I&ShG#pe zm^euk(h`Zq$R+Ysxq`$SG!nOSon+^=AyPCtK|Xm1(x@>Bnt5_IbzVN7UYxB(pQkRN z6Rn0+>xU(kn{7*{3^&utbYJ>>k00IV9zwr<2&Z!K3AFa|S<0bltY0F7zB*q(ueg@b z_=S~J%!o_xo_$28e!QVm5#Ol(F?sH_;}YD@ijx#B_0ikU9@3blwe(EWO)AqU80C zRhls%Pt(dI=+krjbVOvD^a_0>({G=X4_lgu=<_O4CY3`3ixSD!T_;F%(iU>%p%E!5 zmM4jppE$193OJ#$j%+=Dv8LoqfLZ(uX(rz{lBtobW$p)0GslzF(b){u)CuMnkGNMQWY7j zow@?IsLNozfFbBDtbu>!7vWP}H0-k91Gi7vfL`7*_^8ctyy>D)Qt}(ohp&;l#XVHG zrwH{;$06-=KIp*>OH_G&E>g*vU>@_fFegPq7-$q_3<}iEVrE^caa~}~Q7gX4*}i0) zGx%4I=uMguiEGZJ#q20ayPQaPYA=!8>?-0=-b6IopObmtKN4y%O|C5Ar)xdL=_AwG zRMcu7tuoZ0J6ade;C@s3vca6*(q2d799-$aw;uHU_9N8xc?dl)5=Za9jHd+;lc@RP z3zXT(dYF}2mMrHceY~=kBI!po!Mcw|aVDupmISx+2p?CacA8Ge_0c`MAJZZQ4sHE= zlR9`6(dDdv^iD|{t)ClDkIj#vPV9Oge>#ZPe+;0HEuCom6GvKKY(_hMP3XM7CA39T znYI%JT5CC*=A4~HU3_@x%k4jiZ0{&J*3?Os%&I5Xsw&8-U)cmjCy+jk5aPJnnb2oO z1cxh-y{AVwUoK~I>^E9*TvZBdM)Mz-?&sHH+LRKR&*`;Hj)X9ZwO2y8UD)WAuP%VgB3EsQ!*b4)*C}pl|2O81wzq^IQV9h3yt3>xC{>i ze~}#i%Cj1~mC4}^9mCM2$c0U=xe%Wl2cf_A!+k@0SmkC63NzHfgCh(5oqW)wH->UO zo}(3VRmi+F1O1Xeje4?nqXU;PlIc}KH*X80(#{&D_;3od;-~^MS$Ns>qV40FPa4YNw)iq64dvT+`T%3 z4lkTVzaN}U*X>uNhr*SqpVLx0`ofq7$eYn|B`2E1b)*7K{`9zM5KW$lrnSS-^xMIB z>KUCz{j>6Er&|$C-cw0S%{X+={>L=AyO(nNr|FAuK5h`35A^8z6ivWGbdY&M?OQoi z!KRW*Zz-h5Ch}609n|xh7q#J9vwo3{^c~x;JEyKmcOTKD zC$=ikxm#z^FUrzXWsflZx$!6I+47DInfDOALycs!Q3ZLi>k>KLnM6uT!pN16+ez33 zAdCJekabUfaq_=ZaqZD1COEfwRme;uAWrwUlX9I?WO`9DS-w1nj5OUOWx`El_Rk(N zyx|=YjhiI1*xpakDrst8E}R!1Y-Mlz+x1xW*rV`Ax7i$q#Go=1J-3+cm@N}A5{-REMS&GoG`$hCmxf$+rHQQX;CJdWx^hU1 z7VB!z$)TlmUXdsbW7T`VyP`Dm(oC9SIzpbUctHAA*O97?)x^~2GKtVlB+?$oiE+|a z;z1Ex`zn!^m7h7wrtWY;>^wOy8}n-JUN$#_E%MCxo3l)8&J#vOh98~SsEVSF0?KRJ zf?ll-MH!Zx0r7Lox0K{c4QULVfXS;CU%ZlGcn0*m~U z!0bjLybWrApCaGjX}%IxmAA(;1C+4Z$#3v?MkA~{UkJ;MliR9*(0bNImAdIas4>_EpB5hRgWh`tM^plIPxlq|9pHDGh}-+EP)+s21v-#=z9 z{?2CZM=3CEzc!j}o<3jmWq~heM>*v*%>BZV+@(a)gAqBT=Rv+-2qlH13B9N?;^uHTE^!kA?T5A+f!*1r$ZxvT)VbX27KD&wzJKv_>hWXT&&0;gkQ03-5kl?D^ zmF7NFyF`}?jL_#jN~r~2QQ?4 ziWXE-o8{1lBx!X2Y|7&+OlkIiWYxqkvcYAXq-eY%#iwqPjhfd9|)TY3pVrST%&E-~UA#b!<((RSCTBX~WkFGZ>z-0o9LQ z5b`q|p1(~6^Y2$7x}pUlW=(?59#t$fb2EP7ql)jdUw=?+h0E8k0q0;U+$al&9%*lw z>1PXOi3~LTrwtXt%24xy^>3~Di-Jdo&`;YY)IYNtec;JOJq@X--s%(zyX=Z;4z5IN z?Nm|2M?rL=sGYGcNo8D`7crj}E;dW5&9BL+*v+}PwTxqv|CO^xO^#f0Fd#~ej%3Tl zL!>`2k?d9~AP?r05ILzzB3Ir`5{<@5y3RDY(DsWQIw(Zn^C{Bi7)h$7-bt)S&8fy_ zZMyl$TB_&jNMZZ{m2{4v?J5VU0qzLXB@eIWhD{PgX*&!q3c7+GiCMpU2QAdZs7B%MEj ztho_R3cq-;db|}m<)}p7e)!HQt>AEWHMnv*lzM9#z2BMMIVs6767h_cLpAf*n-A^# ztBT5p%@Nk#f;^TUMT)E9k=p!B6#2FssUK=Yuh(}WH=CcRc^f~x`A-HC66ZsOkUr>^ zTY!nD6S&s59pDb$nEL8tw`{0uH_bIX}|j z_RJ`__`)A9XF0-)+ZLdhzXUW-&IiMdvM}ceKg9d~K%H-2qoLjg)RI++mN{l1gM)GC z_xHm{Kw>lMyKIJ3?uE{JvUuHV&&|dRjm>)+?wwjZuGS0b} zF^6pby@IrqxDx-IFmht@3`tm6M0$Rfldm({$kvumB)pG2?p~*Sr(e(@r%SY5x}5TF?4qZPLg~RUbLxD% ziRAahQTYr06g?`XT@i0d1xJ~-3TG2Nhl51)Lm`>1a-6&%uZf%B3|dg!MRH%%lS!GI zr2E5pqJB4&XoTz~CoXR!2W2!#yTLC`xkDo-_g4rf=Y=q*WU>35RnyK?C!iX>C;@?=?GJ!N>QJqzBx;sF`6kI2UQ8Jc9z zm0PE7p!VK$6vxD)u+e=ea_v^MZ-*%w^;JXf7X4xZgc_OsMf;h{&cDsf|BEtB+$YQ# zjXcKr)YQb8yJea)yIhrAP~Ak{SojlzLm|Y%FP%(_R}g>JSK4sBi<~Q&L5~FY67nFJ zw4S_5{yjTD9=ypW{)WnQ+V36l>MEr(R|HVGhb%uM&_eE=u%ITJLn-G?7d75dMo;a& zL`7m>Q0Ic{)P9L9H^}`Qy>VTFySqY@oA6MD`%$8XqQ}o@%Ahz`mF)v7{9Q)ZsXnB0 z4fAMjdO7jvjG@O<@@Y+hH_?u}KuWu`Om7*O(X=JENM_kDV(E4iiOwq^943KW$x^1> z{I!JUl@qnAvE-*$3{l>{i+nw5LM~=%k^8-Tr26O`&JpuHoS7msIKwYQOdr2gU>1r; zGPnP0VRo+KM+c9qp&d#Hjg4(bqus~QbVocA5y(MvKbD}vr(6`C_!9jQd5?y4|Do_C zF=+jx1QH7DdEU|xYLhHsh3zI-3qGKF|0FnPBtk=ZE;uc`4O_0gfaux$_^j<>e8(yP zi!EP_y%w?l+L3%bB?0CZNEr6%Qx^IHlwNO+f1L%JWuSGc$0V;Hm}G&mOdP>Ajg;H(M>}S z=!Fh_dTC85t!NbIUJZFhwY2Wi*=(Jce?pVX*EW-T(a4Fb(m$U&B``&+2kYpy9U8<& zy^qQcNa3tgjbvT8D)-Q-8PE`W1gebolFf2f)N4J<3}#o*=2^vNz7bcc?vhg0x7Y3CPTj6 zOj&UYGsSy_sT=*v#DOA8`v6ETYzs=e5QfyfT{(i;H1u5W^w?8`vh{3?lFS;80FD6ltcwABh48i>-lbg>DEh7sM(W z`nW{v5Vq&n$B!}v@#XecaB*%8khl2&t|{PuH5}Hq`9ap%ZBQO=15bI);YXVuoW8CO z)wyzTGh7f}k|{JR{2iKK_!RAMEJr{&R|tyT?ypxvCU-0;d5l$bSb&Dj-S4~ zZb+S6x6+m&CF-)VjcC}HqDGdJ{q!Ky>~m)Y9hs9#GuB&DMsg4NX?Fxdb7#P3Hvi@8 zm3rb|CWW`S_tDh`T10xZj^3U#MaR46b0_9Iai`zR!2LXx4Ev|PfWD_N zcjh-$oZS)1q|c2-tzJXabKe}U_ZBPs!?g&~A77%$cH-Pm`SE0o#DjxC84cQ}z%6+= zNbb+j2hPJx)MvAnt{505BUWd~q4FXkaVDI+yoyO$x)RyEw4HNd;tHobSDe!|&%ut^=Ujsec#`s z$6qH=!aD(2;VBK}FV$e*-^K7foApoaSqHaYJ3{IOFNkE{)s)F7h%P<{^EVg4#dTa* zf2$WZ*ofd`BFnMJisLx>sUdE+62SvQy}-&Euu-lEVk^(V38iS5FB1%F<-DP`(FtNvN5qOC z#McT3IM3k*_z%&1&p5Qje1Y?ik|93Ff&1EV4R@u(H08vGaSOu- z=>hLi`rW96@_D;*_a>T9qog8|{BQ;3k6%yc9Rcp|(kvRz@?%MnI<)*|KFC`e(mN+E z!MRiGX}jG!RAV^CY*%VVPQ9#O?r96DdDKo&&Mq<$Z%&@}KjK`Htl`+MapkDmDRPcy zSeq5_$}=f;3CvE-hs;mSndl9lCfX9a3K^;UqTz^BDDPhiO5K->(pOiY+U z<_@5-gTK&q4*^JYkOkR-xp4lnF1#Hw2ENs+p?$6c=-+jNM@@bp!+R3iWY56Eg_*E= z;u>7gzXuVG{V;z*6z{P!#?0kV?5A&xZI#7v!{WE#PVce(-)rzqm#q!{&;AfmnrOi!-?`EV2XWvdH4YIC7ZSQZ`$@Iz+Zf2hJ~5H&cyL|l>kNNZUoy4|0P zyvkD0i||uO4(~>-IhJhiSp$82#e+;oA2W9^M={}RXEQ}@H%;C9W^n`q960k9Q;u8j zBTmp&bMm~^jXfVdBihMLB>u}F38`yB$KWdyU-%9kQDglL`!7L=jUnX=%!itLi)i0# zFY0kSi&hX|vxGO$wWcen$L}JtPs5n{FL&kc2ri+9nxiyGd604=!?;T}Ow+$Drd(1K z#eF_!jJ|VP&TYST9@{Qyrq!K(WVa5238QrS-Cu?~t&_UzC+107k22p<<6B*jZ@7r5Oe==ePw#HLr(+SKC0u-v^$z1;Mq{2+-vrD){fL3 zHlS}lx6pz6S5RMY3VM=t8ok1M(e-*ubc8b>rDXAmljG=wuMkgKT)b6+oqN4Hi$X!v^g!;X)Wf)4~Q34;F&Pr-xL zv7mDz4XWoCfXd1$xb(ag_I&&Z-b*F%?E46(--y5)ViC@lmBiA2KEe)}`yiZH34#sz zVBeeuojI|f!gC5*N`v6YxHsH4aE8PEcJSMFB{XyyfZtn9n6IG%y8TjM^+^D3*Zf1m zUx(4u!q=#5Su;BRp$6q7mmp`+4CJ&a4wVM&K|{qhtPZdQogWcKo#~&LGqV#J!HCCZ zgHCH|cvdXr*qkllL@!-P`t#+8%@<*E=i5r=?2enTnDzfvZ;*hmN{Ns>R|uAkh|-z+ zM(MIeLR<$QHVmpNqzHFch5*<9+ib33Sr2trKgjJkp@kPY@zT55LU8zG1&ORt zU`*JYp)wvp+SQ?jt18(1iKvV8iq&5jV13YrgLkNnM`DeIVkPHgTsmwMY`4uV)=FDTR94$?WcAbf2FtcW#)oW;7JbA28x zy(0@R4vWDr-Wi|~`UAo5A>^Fef%xXOAcH@ZNcKb#3V4)`wjPT@eOLU@-oiCV_|OtG zbXEzu&U(+J6g^?;&-*h%GRchB`87=HcnRl@+FvF)^*rH?y2XhuPN`YaeupMiNOJd- z{Dou97pa@lXK1;ng*)yGP@xskwBoV^bKdPH8Cxt2nrC@w&n_)ovf&_ipL-8Y?wHLz z!TK?lm56XZpWMc+bD2lYW8TsoY=%){a|+$*K8q}#=Az2Yl}LBTJ(T(C z8Pb#PLl^x=P{*vlDEv1+w0w~Oi8N){c~u>Hofm_Hyb0W>vw)D58(^1@GYswafoAbQ z(ApFR{_~<>W_SX$eM^Jp@k>y#sTAT3Ya#1U2Y7G$0bG4~oM~f)_lCycy&J6X92a?f z*!@MjW{n~v!9Fym+>YM5)uJ;MH__y-JhWwg68iHy z95sV4!WY+~iks>vRYC&2aO6eO%VU_d`>&bTml7Dw2kOjEuR5}Ixg8=c@zg3TWhN5yzmsRjQo7A`9lp7&?ro!8)dm|_EPxxNq%nmbTqM? z{7!AQ&*yqIMnX2<1scQVSv3U|Lh>)xBYcak^L&HJkHk6nu+l#|X4FY%mmDJ7L_XC> zoi`#KBdX}wm0-q1e~S4&ZiK>0*C6lL2ho`xcAjUYA*JvFBxzBCROS$5bfgvSD(gk- zSAIqhc>Y7ei};}BqZnwjHC>#MGPKlb!lMK|ki2gKJ@;3GM#Xw)tK0^%F<$VvHxSOu z3xV%TqaaY?40Hyk!0Pc#*tnw*npamr&g(`fz0eKs-G0O5PX%n+yBc$g;&9Zj)i_Q> z5qFyZhCi|0puE2k_W!E{Ia&z*cQWCSY6=S#oB=1-C@5461%U&B5dXsyK89|C6CdoL zV%sYCyvh_BO7y^Hkrt%YDZ%00G7xPd4w{Gg;MbA~Wd30cZIq10CGspME-@?-8yiSm=5^9_^Z(K7 zVRP^{PGQt3yL8dO?O- z;d4cwk?wk+F*hvmvwfwg1L}zZ`?qgu-vh%t?4yy(plYv$VCQxpI;0!PJlXM?oS6Q~ zOe~pE9=3pmk4RdN7VEp?wAf7(p&=Lgq+`^!R zFA6>fCcr3v8bmi|f}d{zylW|i{8j?(T2G+%>L5sJ@nYWy70iFl7WYOa;sfKh*!q|X zu3g28m*PR#q3{GE+X(C{E(M)c1rVa11@&>MV0i5exTr)y{(mPSE;b0rTpu>j&l$wW zH^Dr2Yq0-h4o=Grpw@9A)U>L>A}b}(b(euMV`1<;$_J}o{6hO1htPJtS7?)C3u1h0 zP};ddR6B729sQGl{8pVt!`F79BXd2_(OgwDdaWGo=t;cW)$yH~_&!Eb{Ue$0uY9O| zpfJ>GWw3gz5s6S>pa0MGl((&cICht!c&-Jmas)d5Qu-UU=rA zLv;mYxtV|0aBprmp})U1bDt{=&>h8|F#0=#`&!~1{W>HK6PK6KElLQ2KYnMH^Y5k8 zbCXHmK~cP*i=gbI#?PTb(kYj2ph;1KX@hJn6g6dZpV z4|lp!VaxnXNIQ@VwLVv2u(S%&RU4qh%Os$i!EIkOCm-~QE zh8wg*I>DwxHc+|N3SxJd!s&WF0QMcQcAp0YYD(}iRThq36$LRFKH%&5i4-plp6u-@_r0ja48w3+jth7_-q=BSA>ex8A#ncnz(3|Y>$McdBV!ua<)I_~Ql5fu z)Jn6r4?y6oB_O?9i+f{94VPcn3XH$0Kt2m z8Nu}sD-e;kg-$gm$h3EbP@u}Bcjj)l}WiEzvK98BeBf}KYJ zkb_sjz_|kSFVw)oP0e7r`vtsL9AbSL)6la=0ta8w!9GgcaP{X@e1mrz7HQVO!d!{rMf=n<1~4<+zM0K;-FF7*{O-EvYPc-suQ7kS1hd2iiC5IPQs3; zV7QX$2N8naVEM=e_76D1Q+Zq1{n!fbi322f8-t7aGN|Ed!M{XRNRUv1qGQq^xk((l z!vtZa@IPe1@&|)U`cdEJP87)7hJUrEk&OTuewBeY(~fcY)XM;2swBDtzJWPe>2N$y*XHp)rRjyddIB0NTP zTaGHkviiiar8V4hyk8Dn-oE zBQXumn`9lNe&Q8se3OeLt&7ns^Vi5cu@c=Otw`q-KRo$s0#0jGAr9v43KTS1hTz_U`|RQPv905 zXA;I5>B&u9s1F*@Oe1Q zu0IKjcLzajyC3}b&kHszbcgz^Eg*l$9@O{RLhQ|zV8Dk#YmX5u5!MG0)kQF&yZ}rp zRA48o5xH%V2bUTdSi4aOhCOA$e}gn^2-E@>PgRh(YXVsx_@L`s3%b)^g{+VCppk%U zNVq%?O>JcN4?S(9ZoiP@)@^}Klt!SFC*+BG#{kjVCP*LUJs=IwZRlx11v)Xt++}HcM3D2yHMAcG|9!{5{oql{&S>-;_V$Kt(?p(rmK{Y8^ok; zLnxHZr3INv^tbsry6nLV`AUQ6!Vz(}w{k8BF3|=PKTDXaZUruLRzQ%RIXoRSgy}{T z$Y%GRj$w27J#GPOlvl&-IX18^cq457>jW2ET%fOI7X;5@IZ37c;G%a3#@$0;AU2#m zZ$-g*yI4?dO9UyG6u4n?4xR{QLZ`(=xTIVF*22XgV^9jWmQ}&p|65$1)WZ+ohj2>u z1x&ExF8cQ~ILq+h;CUiA5fpJ|`!Z}%>Vow>Q}F8w7u;%Oz-F^4VnGKHEC4)s@5|3@ zep)|FvGf1521EnWHWUF{-h&d3DIhI6o(y(jJ7!$cU7 ziUsxOr@{AfINYcWfyVuZK&Wp&EFIhpc{@E}UyKVBh;IRlu8k0>V+}Ept3WWs0>YFm zAW@3}cWG1bP+Ja@T^Q1fEI{X!6*T`_2~CmlbKOeHB{% z?g4s^EYN|6hnTNHNsO;R4kO%upVK*giG1rmPfAiA5OWVcdXz_+t_MN7pEsLKwe^s? z!wRJ6h9oUn@`tRcpC%EL!{ky#KABS%MV5G&liH`LoC(Q6#?kaM10GUH+IUDLREYz+8D<=x?#hJBrMK;zHu|g*OXLo`YkDZ?mt1iU6=W zJK+IWvRPg%(j6qwE-2f%9ZYxILCe`S@XTR3INT6~RlXBw7wZ*Nvi^!Xm8#LzO%IT0 z^-g4`I}7o?`O1Xf>10}MV>s8d5eeQLL;i~{AbdYY$oBbDq+b6IS$#^1ip-TIiy|Fq z#n=BheI@USl<`}#SN}da8W2zHzN{e&Zb_2HL*g9X;wI*tg%mn$vlyvAPC}-Zk5J@j zJCb!oLjusOrwn@kDIt<&L*%AY8Lr6Voh3?0HTKJFgVMQf7SE{J}Jw zllcyRrbZy^^*cz>?gqQ&4w#gF0$+LWgXzKsaP_zcC&alhX;%YfS-0WUf(oeTUWYon zYfx2L2q%^EA?Hvo#9C*=woMsOG4DJ)J)8z9QYmmuI0+I~CBWY$@!)nX2F?sc!%@d* zF!E)e*ROEM9SeiT$fIzm+!F$b12k9wIQ6Q6>x43_(UF0eb$`);Z_g0E{s@&Br=Udf zWoTBG3@TM>VdiD?a@HD16XkR(QnCFEdG6FoKAe9`0)sw~tDB}dSF0UqY=t4i=eCEe z$h%8OVLefADI%Lb1&}xkCDN|*gVUMmd^gVc4-_fM{Ip2z|RHem^vs3X@D@7Ryfi55JVN)K}_={*nN2g zE6n@gVqQNO46*vl(T`ARKL%Tu{s3S8-*DfI7n=_AVT;|u_)(<K7)_k-id|!EpfltQv5?g6DNwR;y2ccIDNkio{p8kt73%l z3r~K$S&bJ9-1r6l?mxiGW(>sQK0^HBcd$?CE!-dOg%i*X=KL>#yl;cn6Axj>SPRrX zZGfpGbr598h1J(<;HykE%oweNqrDa2!TJIZE8c*g53fS?WFb`kxD3gP`Jn2X2QsI# z!TM?j7_MYk>~**0=AR94{5Z-Uvr3w_hb|p4=m4UU$iT4?ptk@;au~Y?$ziIoskzZrHcKLpZcG4N}D8VEM!Ly2xF@Ri?&)};hiNY=yE_090??*n)WPavb9 z9qzZhfXVDG=(X;FUElhk#&-Y~dJVw|G7K+_#~^aSH<&J;0H2xD(Ef}En;?FiStN)J z+(a6JK@V;<0oYJd-cU0@(O|SIu9w7sa4jN<4 zEC%0yXo+7BZo+$20K`&E{hC?fpW+!!bQUWWfZ(8K=UbnwWO1}?s# zhItcJ@PvdSu2Yl8Wu8(vR8taH&KJWsKZxQaJrP_K$?_m7{J7{X4}Q0J8lFl|!aVd1 zrr7s#%xV}E42NKu(E#u*dIS82dcYvQ3x;@K!n&2uU||0fc+2)-_#ZTbT~s~XiY4%E zel^@vECu<4`7nvnp!-G)Y%dRjk1ktaRqIL^yU2P6?B~PQ#PxSFq)EpiZsG=H;+6F zOCz!iPms6v8blCNPWUoSj#o$o6Tf^ZTKwoJav%jL(WMKaAyLq^RfeGi6G-@H2M>4b zf<>E;!Pr0o@chdLp}|s^2;st_{KsGs-wpfcya%6spJ5&O3HJ*AfQG>gJOlILc{1$Y zDk+Tj{}aVhN5rv1pClGvD1-my$>H^73V242629Q7ia)ihW3G!fuCQh~D?NQ&FlLDV zZZ^i>$BpsQ0|+PhV*KHt752Wr8h2f{#iAcJVDTzPJT1Kq@A~14lWSb?0~0rV#?>8L z{M?BRjRNrQg{SbZ(sOu@?J2zEvp@d!ZYMsv#~p{+x#5a-7rbKB8F#mB#k_@%_+{k= zJg?XmkG)@w)!4gwKCQ%fUn;^EL`|^4Vq<*aw;?_(sgDJAEXLej+Sv81Ixdb?#Wm-Z zumEk1lX|g82tRW6Ed^x;EjYa91)uf!*9iaHg}@2$O5z~ z;3&%awHReQiePjfE#}x=yT@73s7@Z=I7SNBrV^gBxg>9CDjAR95>r`Ak}#M_Hn3~` z&Eg9rwF68y8y0B*KAlu&zB;mE4SH#<=3)cb<6l+yK01=^@;` z=s2!98Hy$R!!W6ez&XK@_*_^FZo3tSr}Go>VeKURD>DV(l}^Jc*{OJ)Qwn|;o`h#F zPsH;B&tS=WiP*LB99B0h#G%j5VdcC;Y;xobzOpP4kESN!WiBb$?M^DbG&2qN+)lx+ zdP!KRED>we#Nn1LF?d!{BtF~{fpbrW;ep~%d{^^0F19^{Jyr+cFcB}jdaEnmHRgnM zr#9f2U~Bwe)hfJc&kEcZi14piV?6Q90Q;9N!TslTu}_UA{(4mnhYl&>$xb<3(J%{# zqzmDf^8aAm>@&D~zJed|jWB3k0l#H);9Phdh>jfr8|(sI9;?ANPy@agiosCRYvj42 z7*+1}Lu12=XxMfG(`~Vl6Kc@QN%u4)b;Y5?@YQ+pzNeTBZp$RItJjb_3%TUZn^eM= zo5yluDa2!L2j*il6s&-`nM!!a234%#r;Y`Jwed%{#n?zp zA6pI^Vj+KHy!)>)PP>J0)?tie46LwH;syKrL#^=j)fhKkLO5o~7|)$+j0e6N;@1NDScaYV`dQle3I76Yxn31d zhA83UV+z>po*X`_BZGZ;q_Dv~aXeR6499;I!Y5V;;+YkExPS2syk^EfCz}+u6*JUd2$)L17w#arxGTL~x2F;oA7cE*X1ea~+ zgL|?u)CgI^8;>n8><|Q=$75imAq}z(@*(o|4OlRG8%Bc(+uN=O{g=&vIv+r7$P@Th z*$&C^FMvdJ!AMLGl+JhqQn~|B;Wz|s{=*=iGzJg6zd@e%B$%`Oyjuqk4p-&J3|A2Q zFA~9rzl&n*A%--ZWa!}PRku|?>>Zd=+LBnsg7xmRb%)CV5&X(o5T~-duSGWx-WoXt{oD!YV*Niy zYR2G->Igg*9D>xR18~yp4YVruKJXX*)1UJZwb6FjNwzG8qB;W1pReW=+kKq8&Z;tsvB%j z?@T!)a!!{Stv}5maz4v?g*Eo)6c&_U_Ki|)HXu)MVD`^Kkb4-m^RsP_9i~qQh;68>7ywWEQ z_r1yGvJBGx;0+nxIzZm{$Us4jHmr4=0g`1FKvZ2}>xVti-WvofUqr#urAc5kD;3Uk zq(kiNESOZ21buLdJN7kPK^CTJSe z!6mmw*i_jBPkFB1yY3l;7PW!<+E*Zd@(m=`b;7-uA7I;`e&EZGq0V>&tmltqyElun z?~^6i*MSpQlS2xuuhA6t!$J-Az%(5;`_XJRsM(4&P~5^sZHZ>rFWSOBDzjqmE6-ua zitDg}{5ijI@f3EjT7li?JAwTiD8Zt=D7(RIEUP*C3(l?>0weQJ;KB1hWBI?o;ny25 zbbJNN2iqWCDyjUba%3;OmoysuLLW0I@E*0KsFj;Vkz0c9{prWi~; zia@{=K=rR&csJ$}oVk(((^@hC3)11mxm0-8kqCN0(GXZ41ZPg{fr*-~a8`qXgM3g> zL{=NJwPazr*C0vW*h#8~)5%*2FLL1Z91_<%!mQXj{f5tiL9XkLDsA!FL;L^c(JHfA zYW(6lUHo`EJ(awi>5+CqGIy98a_pujM?0ytb|bYu8Ab2CUraaj%TdKD zPh@;&k=Xb|lKSL6vEK58G;JFtXX_NeuuKhtvNZtav9Lni2_(*WK~D1_@ZdR$>6Rhz zr0gWDI~@u-8vJY5Faq|gN5SjJSh)W(4hH`wz-yt?P+yY*%aT&T-8dau4rRhzpKNd% zlLvpb3Si5^B4|@B<^M-qgJH93m@Y4X1wpsqy>~5~dD#dFVhvo?2y0aSfY=z`rQ|)HeYjMPo%m%6%dOU7JDVA{jM>1(@?WlQ8(4|U3_J2|IxAK? zh4l-RW7~wrv-=qlHf#SVY=A-Nt>xdhm)^m#xR(%h?kUd?Jb>1KM))ID2j<6bLBmu5 z?DDLJ5|e8%VPz?p)fa-Uc>$Q%{4swlwfS6s`KwKyNs|rIA|Q zboI_5PE*ba7i%fff{Su^{n;q3-~WrAX#YSpymM%*NjTMMv!Y%XlenFC+L?%yZ_Gw# z8RA~+LAJ^^5%sH8#4z_O+2JS#Ib-=blYuzD&oTik!4imrK2&B6%mMcJAZ7=6g=zXQS^#fTJfdo47c9W!@58?n!nT_>l(?(~sk zXWtQHH&~8g?M@89b+sOt_oW>^KIDmoT0XH130(>rj=G%F| zpKjpP2cGr$kOuQ6p8+lDWO$qy53oB1vYjHqXKNTNcL;&xr9q%*e+Y!C{P_9bK5%jI zfqmUxV03RYd|kN(${jX=RlWzDYT5{Wr#FL)k~)`3En*YM^Nb;8+UOf*}2 zM(J&VEDjII(x1Nn(G3IVs9fVXT%#n9yW&OAAhVI)9ev5>FaMP64)LBT0?km%5dS-%!SOOiDl30(*^%`E(! zIS+2BTY%;&0zvZ^z*#Fx=-ILaLYCRVBA->ze%%?y`gp*%vMsR6b0@5C+6M)94#K;Z z<4~L#4A1zT`A`1{sEv<>Mt+YvjpyfI+opku#d&_WpAFghd9cKue;!s=K<9tFir1tT zT&$ZQ{v|)puYChUgP&n^-Wc|#8UOw_n8MDVY|I|IvW~xRB(g5K>)2c&V|Fe-&u>3A zp50RY4|Y|22Cd>ZKxVZ95pRM%Rer|QPGRtMIhd4Og~T^`aI7X9g3j?XNUbzT&pi#> z%oE{FVk{Uohr^w~P*|=K0=~`1p~uq?zOUH_J-NI!d?B*VX0GV9z- zuIigE(TQ&(J6*HMfM@}^e!rbO+f+^x&oq#ey*zJYY7WNy&Ot6k2lzB7Y%o@YS$5)Z z`906z*eb!)Bo+Q#qYgp*K62Me1IXwx0<18B*~@_cY_kF(;pHIizY6T`tc7L48~D#Z zZB}yFpwAwe}lrHTRR%|evai?YJQLRBNf89OxSfL7yflzh1Hc65D-bB z>q1vYkM7TfjOjV&)sVjC6RS=)cJ*x&IAtcH;YJCHd5 zj+5WQ1ii=5990hiJrvZJRl<^ltFY-F&vB(@f{sxd*vBP>$(CG z*_}n6YrY_}n`e;nC$qVU5|$*$?-X(GRHXa5I;rP6F+^yhGNN`EFm@6i_x(-Fqf^kW zYNcT5v}!@j7b`g7p$W1Y^YK%z5pLTqgD+Zah}q3^bmN0pbVz0ksZ|oD*TiQs@nL4f z##I*f)R)u0#WrO7s}7Q&aE933{l|>EG9A9>?Stw3{=V_;7jkj#P4Zl)lz2UDCM&+R zkrv-Kq$#nVr1AUva368-m@EwuGV=UAPz6^1)PT&Ty71ww5#(Mo<=GSlUT?F8T7~5> zdUy?dh*<|2(%xWS=mRgN9fF*V0kEJx1ir;aK)zWVe=kagyjN+UVVwg3=PyH_Ybk{5 zR0Hg*2jQu$(A3in61}4!<~M;Ivu*}Eq27fx`j*6=8gXF@FU(*=`T6aR&!bSR)C1|> zt#G5h9-_`v!yBd)wm-ZKTlrnhhMqJ4E*bho;vgqH0wyJf@Oh9x@MI1_-7X(kmFop5 zJNX^f1t*xdayi5#TEUzw0%o`7!}*tUfl<_hB`-8U#asoZy34~mqX{s&P8_VI{t(G; zeZ=6x8=l>JNhXQ65YgG?Bp^aSen0v`=ARRT)KmMwv3NRYA0B0lB~!?*%yweOEhTOI z{$3$cmgfddNpH|>CZJk`9)7!pL7gE3Zx9H z1=6*;g1*_a;qa7BsIqRvRhrVo*lIn7?F#1uMwQr|9*@uY%-ad&nbbDbhThb-MP4jU z9WHv(33qj<;HNMh6$__@uS{Xp>^YE^wFo>-uaNNRDdg<%M-sXS6R#=Jy0z407u=Ff_g&GX@2aw*sq zP&oRo34F$NK#TSu*d|D@2U@4I;}X}fDp`r_@R2p_`rhg6hb#&9yyPIXYIH!)%KISv zf`Uz8DSzI`hY_9wS{!^1=Ef&NylxECKMe)0B?xXX2jSRuA4s$HhVS)m@YBl??w_}X z;$};@?!bW6{<$z_sy=*sI~78ORl%8mFTW_0hLo-2Ag%2;DYEJ(n}s__d*wq?vH^)o zTM_YGe3@8wCz4bBQDjwXKe=%zg$M=u+joME{VaGKq$_yf z(1{=Iydd>Ms^HDRPJB_L0Mq@Aa3a65&6XHqw(>rQPY;sl+6$7br&u-dU15gP&N$;s z-sf>RHJw`YU1L0N>a$x6^I+(ZIf_PKB=O6HAozYMd3~P}X?J7tLf?(_{Z1g8k46$q z;Egf|3Q52v0jX$hCTr;n!m9KTy z==s*LK4%$NJamM5@AVLQ!y8J!?T2Z~{2ge1DCo|OhFC!o+~1H2uH~8Vr|2@wjO5P% zTD9=w##0#Q`4LQvMcE9V*Q;xFV7{CNgHYfKZ%q@NjYCPX}?oS1b ze7p?bV>99B-c(RdNCFw5Xqd$F-_{L*Fy-xjSZc-RKz^-<{8#+lW|bY>akYl{>*mlS zKL;W|=tJW(O~}($hP_XDSJw?mKIbC>Zxe@z-KGzuao0<79h%9GS^+sUrjSG`XOhBi z;pF3RJUQ6!LCj_v6QjkHnD8wOa}NlCn@BD>cGLnTx8*@9)n|aL<6hKLY?@s)jD%hyx1y}cXV)vtZ z!BZDAz9Yp}@cE^JVC3%%^sefF`vLL-v++%!&-0%8yT6d%+W{l4TC(xXRdTWBHcgym zk7t(L0=s7>be~HquS*&aC9zv^c6}JL-{Uj=@qLJ~dw=`J&Xi&zF`_}_LN*f9Hc#?q z?HjJWGAZL>-#dAbXj6fPys40s zsSoA8^Pshlg<}T%8EKp&KkIOXb)DYulFvLZN8`zkmOF02GC`f&V>Mu>86j0tRg%TYUjko;HReJ$;zz z%8QT_m7(C00$i9l9=>Xcz@n)m#P;L|^6TddQuJQ~dAo@tdxi7J(!w<2q#i>ad!Hg9 z!<)!QZ4c5qe;VnDDJ42By*JM03^1kVKhfyr$CxhRt?0i&0)*63abcSoO_R9=4jUXW zuJ#t~$S)?Nd6sPKHbAG`&m`?!5bL(G2}F*{3znsJfakQCDEwYQpfJ}~(9vgxRbBOh z*e4aZCpu9uUu>#iwUZ`|eK;POR7Js?eRAxcR8c|a%5Z8uIEWWEO%-&}lOWIY`TH-4 zvUoZdPVWiB@{JC73Jz1bjmqpS^?&G9@eGe2@}o_c#&eE+CUj-XG@@O2jQMQY$CR3w zkp4y+GSu%+UcNj@Vqy|W;K7Szfm$KC{(+LMMR!Ta%62mS#3xer=m*h#C=4(3q+!Tf z5pKn(!m-CXaPQtM5aIK-HR?8S^!f_;&a;~aik`5bYY!Z`a}2y@gu=S7{Cznv5uOjG z!Ptwr@XDnE3f|X&%(IsuF=GVYjpg@o8a8as^+@*G&=NLp;uN-8d<5<%@I5N=bue>P z1tdJo1>K@FIGd3OCQqW^YTGG*oMX`aYY#k+@dTOYu5jj~xR z(5nJ{d{!`q&*+=J5QeKKhKc%|Pb7PO2dT?!By0aeqE^U%{uyMEh3-itK;{%F{m+l6 zlr16E-Sdf;^Did;!*S-rdQCDp%#@a&{#q^U;7j+8e~yt0|KY$zWwzt-K`Jue0p}hF z#j~$+;f$XsYp;10#h;yo{o|(!@~nnXZgDuhQza_Mn=Qv$Rr9_D-di3QGL;tEP8G~Q zl_;pRpi*@k0S^+#^rC!RyK8NnIC zENfYphkepBShYeOwEwIpC?vK}FPDw$PzKh1k3#VXJF@HcO{#KjCbPpWj4|E&f_d>K zl*#(`huK#lLpE>LCLzz4kTE~Dk%Q+$Nb7_+@}VY^?2j!Y8P9JK`(-U;zD);dd;XEA z-uX?Yj2DAwUm0L8s)C?a8>}J>L4oHv>!dAV+gW?4OLv2VY2J``XfKSU2f~H@p|JCF zG&p}whS!;yFqCr@3?BMYA;AF-U>c{Tw&6PJy-}9!U{JtIQq#DmQ9=r z^TSmkt6&0Lza$DW%6~{>;YVVr{fbn6cuYR;tRY*Umy>^CS>$+P9J#(WgeZIO;N1dC zh}SY*V(lbFVy*r%(#NA11@r%yjH)PRp2;-k4!>{NzH%uMwu{CLeqW!Fl7(E34LI`s zza{&5-S~akBzpJ3{+IWG*&2fTNJ(C0{Dc_~HcE5nzt;;Gp2;B0;aMu4AVkR7B5tRs zDZ{#@FymJEGV7i`Vjf$5WaL~`$ftXHq|jgyk)E-W#PhDagOSNZ-TyLa`&B^#c_uM^ z@)M#p@_|g;I7D2Ogy4~;G)$>egs{n4kU2CHn&ge))bB-*GHC^Ww&!zMeV%a5^Z-~N z4T9vuVPIn#$MaUHpvh+kwt7{;=Bp1t_-#MP1)6ANH4|0<$L$9?FM-w|o#tD;@-YPfrjTaRuk5<#6P|BEBcp7}g19K=>I= zC~4xeLw}{AWBC|Zb$gIp{M17#Uq2zIT^mSwMkRUrG@lIgoF>!81d}TgJ|yUk4Qa12 zAfhKHk)%oQnQMW~OiA%UM*MvkBkW?sOq_LvyZcdy*#AC5rJhuBAH{;J{hnOs9cQwF z;0u4jRxBOU`2E-kfi8QOXYiO0q4@lmCUfF=Dc$TZ4&y4uqWcTyc{3+HyG3TY-naXV8m_{Yd^BNx^_s0L0iE z3cgu9MXjF?=xgWQtVuvRx~hl^2F~~6^8_g*%O#-GE*4zECxByl4$mZ%!Iq`-nVnz8 z(Iu@)ls%zKrE{)u5ut`$UdhtxU(Qb~Bx@6xFYi7xH?~WVX%A+TjJtN^uV5!>dlErr z&PgS+c_!U$2}ibT*AvzIZN#1T>PT+=OD@kD$Fs2Va4<+6OcSPq@r-#8;l2pwEw=~V zN>`Y;gU?G?9)P85PQb!D5&XG28FHuPfatagSkT=F-Mt?`@JWVsh$iga+7Q;;gRu6J zvTWhfUg$X92&_&8NQdM=1)sycI~>8!Vo!inzyZiA_69uW3Wr|uGq(MF=VtyqxOZI# zbO+U7?KpXOe|a42#lK`@L@!CzY$y9p-XW2N$AW#8IK=cy;h3a;{I6L|Q01DAhKF{u?$_?qYZsp4q7p;Fqr1o9_o$@6TK6Dv z7Rscbv@7xb2@c=v385$deaF|XC4$fzbAeWMs32Yc6CT-WBBhqtl)_H#Qr`(Kp<)5ox9_aE$@JeAo)1nkR<*TE(cZD-#2anW zDQZo8Gq;oLJtv8fe=^bAn@h^OD~a9NS`wk#LKZxIPc%#UGgGn{lzvix>jTPwJj48h z&)+AFSi(e}Bk7#$3ft_q!kP4gJR=kW%8z0pEblyg8Fv*99jb*p-JK9-E5%xk5`qP-&nrJX!hkhbXCKvOjH<5Cmz zBwUo-5z!^T{jA7gZX0QN8%$0+B@_Ad45GN9j6B-bK$?HGlIt_xkxgYki5QN9FByt( zZbS`MIm`q(Ni%3IT>?&ytD!G^JrwTS1v_|N-tI>fT+~j3*7a9Fq2v~1*7Ka-FG;p( zvnhM(cmTV7oGDxURFeJM`vy!5Zo!5Z1u&QQbR>vHL*~T*cqp_BZcDBQ8xcnsaaaOd zC!2wp+Dv%Zqy}$f6=6%2C@A>;B9jI>Nv6{?QZuQ66pbw-$-MtBUm=MkzYHQfu5KlY zOKpi`(KMpM{$tLsXk@g{gfp_SlNguc0~TNY>T?>W)^S^htGL^8P23dMh4eP>rb;k- zLWAq>(3slybh&6fITq5#Y}(sJByZVc_KXa;?Sf&_shzcG}x5g0`~3f`n7KtA^c7A%(# zT>GNIo(T+Nx_0pWg1dWA@u;#O`LH$nQ78}2N@n7tBx%8k{3x1G7zK+Yi}8ZhWP!ZX zI~plA2QI8kB?iW}_$mA&HCq@%cZTKDyKN`v=XNVfh85^%r-xkZ?lYWb^UrGaofFLE z1WgvIu{)Vso9oQk(BI6#3#vq+aUoGXM*sjQfCuyf)qB#F^m;Ir0)C(lXIKzo) z%OHNL85~+Y3x=L+!J_^0u!_&B`E375PSkrpiP&OnZz zXJsc?v-bj(1q#MJIMFwf>`V(~mNsj!Tf8I$XXbu^_{oXzz{OGU!^B2#`h~Dy`vNCH zR@qcRjNxxwqI3&=i>A@dqOXw4m1CjdHcdUHAt>B01T7N&AZzYJKbu=)NLnJM$(5nV zy#$Nz377G8O(F4bXr?t{syJyi--D^n_x=1Q%6odR&@J1O>7RS6X~KI;s=QB~M#&Fw z#^%Lbd%hO8Y~1|nq-}RBuB>-tqSEu28z%hBQcZ;n%L8dzvxao$1rcTaC{j2544Fy_ zh_hrVDRO>DuElhceeJ)=Pd@7&D5VG|IBgIfJE+{8yONvP9;uIpD9z|Tl zg2>wUYf11`bF%KSGD$5MVQ&0)iAf)@V=hHBT8NxqUp+6&knz_--|Z z`n0X4&9hEZ$0vnUC|(q+IAff7--UNuE90|wkEn`PA@Odxj8$TnEp!~p@TX1^UfyGc zSha&zXZyi=e<84P*ANUWsiULM<=BkHudpdrm(OO^Ap2$n$2U$Dh=Y@$rA1hWRP_e>9(jx(S4=^T2sK92efsN%pcq^%KUG^s0$dVdS2>*Y66=j9d@x9HG= z7NRr|tGU-lcW}qH-m88qao23>WOZiIym02t)LLfD3<;7mph*;yfYb%Ml3l+32?jNO5xc=k#{8O&mS^{XO?oERKYY@7eZDTL;XrC=$X12)DeyfHj~uLCL@Re`OS;(b2nU@_a(QVRxLKEKxG+O8YCn&k zk8d-l_m_CnCwAfV!K!d-`MaF<-Ke3;`Ks7Z7efm^WYNDX{m60o-!x39lB!H&Fuy3n zd^g|4sef??%^#D4(|g15TyF`TR-K6fG564h-_JZXD#DTn(gKHq$C#0R14Evu3w+ac z1e=N`2!!fRqttXo!O%0lS1qd*ujFT7(5GSy&3%FPX9MtH;{sgId$12}jzprtcV%Q= z#e&yws84|kHeX7oHi-x5(-)U%a*iMU{J5R2%@yM34$o=o<60W0Rz}rT&eHC%Aexo5 zn@WnTruSFq(SevTbdTB{F2d|6H~FtLH+RZ8^Q@_(7A}W^m~Ya^^v)V(?0huHU^`1D z209aieY?oJ>rtd9@eG;UQAlL<8VQ^JnB3ItCG-7+;5;h>2g_7IZ?zuC4O_sX)wa+| z*TE{`1CWsw0Y^I1AWf+hDnC4eo1J6WCw6*loQfCwc&Q%ybCM7{;Pwcd>`Oteo6mZ6 zhr^!W12DXZ|M`oykjUr1?`!CT;S^O!PnCh!&0}Ey?_RR)#$#f9yMfFYbCq!KQ^*`a zIPt6A$9q1Vh~;yJJer|Sb_@Mto_(TBqvHwYVbnK^Q{{Kfe!Z39cJv?OHrw3h5^Vo+ zet~+lplUT$&E88z-UZTXs}!2-Sw;(^Z_(!K&uNW`5EdD}qB9H+)6klHYN308x<#kc zpT?^AaB3%&cE5^33we)vMTVk$O07jI&KsU;t&G%+t{mEARdaDrQMim58 zexJt21rr3(yB!6&s3cf&ZLGlIM*zOA9Vt zn++MKETMO!6VJQt0yTwD*gESBEL>FtCmNd|2}U8bM4N59JrXv>!xg;U48uY}pZy#kcn1)xi+l zZn7W!mDZqOiY?{@>!Z4|CAL*Oqf$w-xH)_>PIDJW<%=VUH`O&*{<^Iy_+wxzUT zK_SiOeXAMG3H06~f4XRz8|7r?QnAm9RD9?&XW3EAosab8TAjG+k_K0cy*}E^(Wq3W zF*!gKM2q&BLNt07V(zOI=$p?aDbUT?ri=kB+VJeyVB_I4|mFLs%0J2S)? zUsIs7j?Sg41TNI~j6XGMilrO*UdXZGg|uF>jEd^Nq@$O=(e?X&(W<{AblOG<{HZw^ zPZ`N!p3`%BPTUeRz4cMe+!pVxUV}K>m!Hpvp!vG}cx-<>uGK9?SN?h2w4n&6|0%%< zn?7QeK?atV$_sAu9_7(=4Z#ITaY3$sHunCh#2-^DF+T4a{>Zt6-@jzyhUHOsJSzyD zhrF?};S@i=--*Q21Iz2Q(NM`4zdqMRs-}aof;srSKngee4O8PxNql4>f^Tg5Xx`)| zdNQttPQF)3d&(}+ZOUBCj+%d9rtBWbZu zpu!Ye@%?wZpD?C=!o>541`*o85TkA)56?a7|Xoapui05w6xfnomze5_;#-MI>!ZveC8$? zDATu244p6HLX|87=!|7CG}z)IRTRBO!@mot#j(5e*5!VBuvr-6-$>%h=RGqpw>AhJc159zZx$XET*60_t1$e2 z1==sahQ%(~XqYc1_&KN{m?2OX)Cv@7%EjHr*(jS9i|sBE zIAbUnJF-sTz_QKg?dpXEetcK+#MS6xZ;XfQbkO~a4(?T$g0nIw;21r59K9)lAA`PA z{ouD$uHrvBkZ_BV%u;&obSB+5CY}=8AnKX9j@E9nqz$#IRPy~WcYbR%S6jQ56IQCK zz8yy02a-F^hNHLrz#bNoPUWem*q<8xx_HIR7z6V&tkzk>HN*2r@*>!<$- zlALQm826Y%OO~UdjEQ^NU;E ztV)fVfqwb2p03FVpnESR(4#bqZogVauc_72nT_rA)bTFbzVSPKuaZwf{l5yJQ3M&RUivG{I(HfGu6qFr}6 z9*O7hoc2S!5YdA|p}+A$hPvQ+fQaDuzfnBj(}DTX5Ad5ahqd|Tm?xfx#+3JFKTAdo zi^8amFnrq`hQ13A;I?_YQEPY&Zm(H`HF^vhJT%4gHFL1YUllW_PeCtG!p9{tD4ivQ zx7Z;ndby8wAAd%*=he{vDobhIq6_rAO%zpaJ3|Z7Vr#CBtikLREa#is=FfZpwthvyq^YxEB_`TM6}9rr^~v6?Q#V02wn8SUuWLruemxJr62~L}MD! zUVVxLT5ltpmMT$P~I$i6hZn@uS;mtAl!dnJu)FiyQ zVhSFctcuf9W~0jnQ(P8pfunq9(GS)cV}1CpG~)w!=}{P793PIRHvD^j=4m`Dm4o{i zgm~|ij)4qw%Kb4_%K|UIt z$-=($GpLpmk4_m;*dY>yF`dWpON2j$9^Zs_4LtBM@A(z+wZz_0a~uwsj_!Y^;jLax zl+>Su*eQ+o>_qXx{ZaZ`@PXcJe@5*V)X`Sia_W|xMSn*{(GP`3=zU8!n(^F}ItD3H ztEGLMZFnZvVeZ7cwsWh$Uof$t88Xa;z(7W`^e#hm|1upLHOWHWN%pH?CpnrMNggQV z6Nf1`$phYHry%^Bm~E5dI}la){JjzUjP&|gx`ySdJTbG2b zyaF9JY)b1~T=X;o`k+n zHL>l)G~T5*9mg)UK!dFd@QwIVJZSHM3np#CyeL2XrxbwA%2D_wCHxlPd5!YFn{b@H0LQhJVV1)c zY``4!aZ5$vz+{{xABT2#WAIsdFb4J>$8*6xIB(@P^lflLfA{4WrMei`T%CtcXV1pX zQ+4s8-c*e8o{UmPGPt2u42}AKP`361{jvE4E#Gs8K5#ChBS9H7{%0(mQxQZbeDI*N zS}f?^gOh0cvVJbv@&-33vV)V_R9wAfV70lt!e5I*-%zIXZ60IP{)<`sSBL24+LFBw zwvvv%(}c{ZAd^}eh}gmp#N@0n93N4Dzk#~oQfvjvd7iLsMF@~ZsSspS1;3VbLi=M` zw$huh9u2ar;CLroKUT%(ic_HuLSVMPC;Xac1;PJx;nZS9P-BGQtyK>plN-sC@(N;b zHiaBpx07r*w1mW}>+<>M-^{+ELI&hcGLOuLEmFj`n>jr=S8dj{h3hJ;;M_(&aaCg{ z(%u~lXpQ$es+}1?&1f8LxRya9FPGA%E9$9mLo40g@_~+2|D<7S#c*k<3@T(!!r8~P za0;i3K|g2V%3Jez|CJTKGP1*k7o73vqHQ=MZWqq~a|{inLr^3=hKB>4;#O041e zj<-4ZZKMFl>y%?fg8*#<@1xfIH<&u|4I}p`3AS+(0-4@#n3wztZ#{X4%J*-gu*WrY z-&2UY6E5MEm+3ennu_mfB0hD9$BhTWa8X2jYX)bVpWm#+^o^`x_DCX=mh_Jq7SkcK z+LsYu;ZtNvQzlt7#Pc7qZA4|wcM`?-V!l7A0eiIw2*<1gt6f2$Kj{p}@mzf$pNkse zdojoIy_gf;OS3yvZ^UtT_I%}Z{8Z?z z$ays4`Fh%T<}kIH6i1_YPI~?C5<30lEh<{^n7S(V(DfF>bbYEYrp=JS3(86;tE7&5 zdUf#q@GQJEiSHl02H3yA7H8(H#!LwhwED6K{T2POnm^k|d4?hKcd`u;i8#OOEbiTu zj^mYcQSo~rI)19e`nxyrR43nyx$6~99W#JoA0z}@WqIH3s{!SoVys8 zZ3Z-cH6IJhW+9!bhs$rNVKFm_@7r>T_Zlh~yh`&9WYAu| z7gYrU>9ja^dZoskR`se<*UO#U661WnQ*tl& zBc3cY_aRG042WB|5NZ6cmicRXfEo8G&7z~nxyok!w`$8{{#??v0#4Inkb8MljrKlZ zsM!`b+ByY2#u%_I(wu)$>5p`CD<#Q$Li?3P71LVfarb8V?pE;S~OPGPgaC zey*7)7L$i9{A}mtvnn*2a~oIdH=~uxYi#;CfEt721vUlZg30ECxOdDOyjjafd#l>_C=h#G6N5eNkff?Dd?yikCAziIHxrPyP|{eh{RqzuH=RLOWg1< zti-R*i*epp7Ts#hFr1l#1JXJuRjST+H7cQntQ1PbiJ*$(Fio`Sq|r-T=(_nebTqky zUYwIox5&j&$Ns}~;ba%8l|^Xmn#oisc$llRtKu3?hj8BFoz>hkrn=zbeT%0*LYO@> zk&zh^B^86#HTM9nP7yRc%%S_c+PVNT?L*^2jTF?1PB_-XM_*D z082dyR-s@vJCn}=Z)w3DEYy1`_*tYe-6K&Qvkb;5hC99oXi;H$nm$? zB>h4#nR9Y6?;IOPChFZ{795LY7EkQ3u=5P9KJZACyF4q9lU&Skze_%GV)Bz|Jl})n zHs75(<{zc&5602bH5X{RL@^c2tf70xwa}YMT~uk|clzO?2)^7eg_pM|VVIQ$uG&2v z9Tv>N^cFKbc5(scx-G%UTO1JYxuMQ!FAQ6?7efYv(BFW+*PV^TgbDGu^h^p)4@koy z_YAad%0aLxz!l|u@7ewvIPPREeiSrg#-G<%B^baXDdK|hr^E#-@&<6I?lr2qw_yL_ zJNR-jMeFbi6pFoylW*nXs;+GG_r8F;_ov~}*d%;y7LUHV;rQ%E2>#IW$9;46V9Mz& z7~JTL(c&wxiLpgBW&u8UI3Ke=8Q^~n)A3oKD#l7G;>vzW9DivnHfapeUbDA!V`U4y z7JZY}OO;ZiLm4#UZ5(yjaGcI1u5<$5AHSnSo|ZKI$Y zVy-PX!5musimB+DMRuLvPIf9Lk&==;^2xB7Bs}gT4Z70M=V<^#hwVUXi!XG)i-Sql zMNpX83Jpue*q0$QSv46kmRPpJcD*9-myL(j$9!Q}!VZS`+(fk)@09u7M_xEIlbMqF z2`zKE)Ij}U> z%9SeT9i`*#5U!Yd5c^EF zV@29V+bo|(4QGXG^O6N{D0$f3mR$_EO6$3PeF zZt@#!0Z}Irc5;9Y`(TX-yYy5GOliIhNyf2oP~8U}2rq%ki_<{LQyj*u=pbwPxT>DX zd7>{JM)vvHkmBk|q}wryQPP%RZuBc#c)XY77H`ew_8%zcz8Zb!vh`Hx((fiT|FAn1 zxgJF0uEbDQBa=Ew7E_0+TXgNA$MlBhTe`M#h;CLMin#5o;JfeG zEVN&831{pqK%EsOe9uB9maZ0HfMz2)D?LH&|K8#jZU|q6O9)IBiwhbmhHz~S?^;cG zie+VY@#&-*l$N}XabL>NWP2h0w=EytJ1*kayiDv_o5s)ZQ?N%r0Z-13LWheXsJ{6) zCW!Ax-+NxT_o@e`j&VWB-S&9C+X|O>5*)`f9w~_iSf!$kZhuwr^aTa1yf1;@c8|ro zUPIJ%?mIg0<1tMxt)l^Fs;IHFqW2>Qdo#GR zCDNSEic0fuw>vBnvwIn(HM+!UfghQBD2m*FR!F`lG!vJgzvL-V2QNoUSmC%8BDA8Q zy)PFoT0Vs4c|vUJ7AN}s%pe8U7wKC*aS=wZ%`ljJ&?bS`XPKi3s4Mg@y>X_C)Es8lam!g;7d!JSJwzW62CPye~ctdu0q!QN;xF z`v`t}#y@@<_PCC5!GYx-*jVO;MXP)mxK%ayd(U0Ot55L(+l6KS@y~s#xZw11alzUN!)Ueq9s0#T zL(3aYSU9E@kLGjebhHu+CzW8KCcpO13<@1+Yfuy{cluDHebGhK_rXU^gH6@zij z!(({i*dE*>>50xW*5fPUi~~6<@WEAU{4)(O<&!aB)${k zyAwV_sQ)yB>VoiSRKSkb6%O)p!kgQ+&iCo~B>6Ow8;D_mE@G^;j zu1gmoz55=RuKxq$c+bfx(?9T~=N^PfUxK8K5n%9q3oJXd0D^X_fZ)|fvOn}Dxp4gg z32Qn@#vQjMYy1A6MXqObL-nc`@3ZAprn+#YyIVM|AT_EOt53E6IMTUCH&dk|fBND# z?;SpzNvFh=&^PwA^i|Ij`YgMf2JZSv=lF}Dc!4Co>QdmlW>m4(Lz}-*8)DC5WBhTC zpr(N}-p*T&>buwCeOnJ)sp*9eZ|*_m4E}NM4951Q;n;F94qpr;Ask9Wnavsa+3XU^ zO6B9R9fdeayaZQYt3=h^9I8CMjbA1;p@YvebSiy^-fM?3#!_5R;V34kEg!}WwYO6UBT-PVE&maLvbER#}*^EFHHfpT*@1lCU^03g4az z!%J&Vpi13AJa^s)XMgraONDjVHP#Uye_e)8Lagyhk2%V1oQEZJCPvh1W783x6| z!F(hObo+V7?xpYW{>4PruJ=1gOljaVomr52G8{?|Z2`?23}jAHf{fE|2qSluOc9A7 z+T%UR%_b94FzFw2GjcCeojZdIaNo+=gjaLPvWoQawFT5Fdp>ol+ePR7JV7@viKDS6 zbLfeUrBqew7Ipve;lzE=>?C3Uh7)xx)zCh zP)lrD+Q|Y*ar~*5#S|w6{Qfu&!wXfgZp~cu=q<){)8&}or-$I|b(oQE472XtNR2p* z4@)0M$j~Z)msx_ zU>gU&b5VF55DERx5M-(NQk?R}H$P43bWb~gEWK3d~uy$sl(Do?n&iUcCKt0o0V9zxQ zM*Ev_LtO81!uBxE=x7PoqCc7sez%POrM;iOaKeeVRS)E0mCDzTZ{&B}8KM38Nh2?I zFO`3k5WoiuoOqu%X8d{WWxNODrrj$k;m&^#=eq8=bAz5HTp{ zq0;LLp>k}VP;{0yF`T!VtmO_8b8kCxF#9&qVp^7#N8*X?1lHI7Kq1K*tRcP^yUAA> zajYFKhf&WJF=B!;>=|CS+++?8*DOM+`7$>Cx`-XS4*Jzw&~R!udSwnHLggfkZ=A)U zNj7+<>WD5qSEOCNi}-RMy!rbGOqU*-GNI5O_8i9QQOr)q!BRT`$ss~qc$k9FOX--t z^9!D`@!fZ`2IVSUc=$(xPCX<+@7i>ujMO5ysuW^=c~DQtg!INV#IK=fvrok8*$J?H z9ScE3G&=kuF#kjdu1W^L)%_6)zk0!W7{ih}Zeg#f18hXELATKwliH6VC-We7NSGkq zb1PiT)h|2eq8B5{tfaZ-(wxoi%=u64yy$#rAh^=-d6?bJySTx zaeFzHMo-RhE7R|uBh62hU(72eneygLud+DKk3X(O_^e&O`O&+iv~87t^UXI1Kl_Ov zejvG z#*0vXG6SInbI>ZW1a~7jm}IPi*~4|HirR{!M|R=C%tOd|bR35}ta0d`Ee_~Az`@Z4 zzPsjRBq&jiy$06mokJvJ zd283BDpnsO^f(yzFM`Ipxmawf3bT2W@H1!(z7EJ?#I2#&DCi>YMwMjC`V5k95J%>X zk0xPXUJ|h$XCm?I1zBbqLU!EQO9K5_9&m&r@mm%vy!^vSU@1`Ly0W4?)o=Xi!hif3rib~#ErDZ{SE#qIr*s7CsKN$+WC^@{`q6qun6XCsc1}@E- zi(g%faG{<p?B8D#ey$3U_b3~$&#^yGE;?Q=rZ-`g-a<%u~@ zeKC2BKmHj%W!F9u0X=M&?efWY;@XF?`ALL)f z>hh&d@_Ym1zjGn=+)Ob>Kgo$m+%tbMnZ+#>+*KFhE-sNJZVp$-%APz@sqIGgts07i zz*zE2`z>*q@r9h2UqW)d|B?g$I*5;@2;T8>(AO9XvrCh3^^+Q2g)P8OWliKtXrtk` z9`cTDgtdb)rq5*iWcNYrx^WzAEFwP zROX%}^*a9#YyF$BCA$jS!oK6^_(Is8%EtU5nW(IOkN(V5Y(AC($vp%MuO=enS{!CY zM4?$ag3Y5KM9uQUji27g>br+``Yy03aYD}K*lFlOm-m@1h=$8irXZ{7jV zYlaXjUyWTaR$z_JG8{$$xXK9*wcRJrCnXcVMdHGh?QMe1FJ5v_micgdl^Qv#KWe8}r45kJ~B9TjN6}zL_rwl(Q2~S^A#Xm;52BEYET@f)b_C z#Uv%Lj)ZHrlG%5}G4|;&OyNdg9rJ5^JElV8ff`y}HJ~B89M^kRz-jzypm8J4DC~ev z!fu!q971lN1@ot^ac%b%{EM-NI-D@I(gh}W@4-gI2VMn_G3jj(iUz_V!tAi$%{X`* zOT;`Yf~(RgsAx!qBdgc?GcptLIoYs1P>8oT%8$>&wIl# zD_H(iZ>#H?qctK7bqt<;lMC^1gqM>XNxt?4xGS&HHTRr&b_#pwG;j)H(_h; zT0EuN7>HPkt=98!WR4oPS51bKz7lGtjDm6KFzk))Ax(lNBAYh^nWf5*RM>^Rc56_y z@Hcrl?h;wGESCSQE)-s5{pRS1^F+I^f!viB>nuCkyl1U$Q=B)eZ*pkqTC7*yi3O?3xO7hhonPz7*_0mQ zTOtF+`SRFWq=<7)Q?R9M2COpXVbWYp%=TS@Ui&^~DX$JhRyluRl<{3Wfl{5ct z_7k@DU-Gxw{rG?S&b--=(|nBXHvVMmYCgDdCT}@<8EUE> zf*XabZupFD=wHMg@B@Q@q9 zR~a$>f9nx!C|M5s<@5O=C#Ul}neM@28K(VoV=>=d!}F=*cJW3}&+%>rw|RpyKYm-* zZT`Zgv;35GyZBsdZN9y7G1IA>$mgjH<@bzL_g%KkOUYfl{UGWFJGtPvJ$Y)6Qeu2^be9Y)CfXe7%^dByTgnk8D zC2QeZ*o3=$4-6-X(I^LL%D*2*zpD(T9+m^By3-DYzx5bBv<4v~exafL2Q2fx;jP^_ z{5oBX$195PRW=Wu6LVmCEdwhar@?Gci2l>@m^+!}eO`v*iFy#uUVMaA)!raiJYYL; z8>JF1C=++YwWIbZ47iGK@2zm)#4)Vjeh{y`O<*8tjL-1~C=Sqrmd7$QYs^O|+ZR`< zEWjnpvDo3h026$|$n)ll+*#=;F8;@FGW>HTH}t_vXkNU-OFtM%*QNv_VB>i0-;bsF zjUl1@mA#t$m~;{D`sWYaNRQ$CsmGH1*Z9f2m!p7bQt#kh&tB#)Sh?^w!!GeteRlGz z)dhT_6w`ZZkYIU#S$?NQI+xY{Or4Lef^mr6y8wx?YB*V|fgI80DE_Ahi?wSY@|o4y z9ovm({|>@J^B77ztPp2vgU+?~$iL&r;=fy{ue*)4qdc&v#~b-9mP;BHgzap-7cg!_ zOX(}LYZ1ikNJG6#I?O7vG3Q$zDt;EiJE9nGr+tHn+IL(s{E0(rf8lFuH6)W6XE&f7 zzR3gdP#a3q`iIe&d>I;aPlDQ}icpc^Jy1+*!F$HZT==^h&e4_77*&Dh>*Y9gz6`@= z{{U`(1IO&hBKa?ncgSSDm)@a11bAW_kKw(HvoR+W8Wr zTye0%2?`gkqicdKD*-%9z?X%s9 z%+>bhTAkv!ag`OE-q=p=2h*4uaZa80__~rmUwM$9_Vxnrwc;Rur)nkZf5-aKbSd+9 z%sRN^KP$N^_gHS4vkzy&vIM89>AgWXWqnJ(uM*r8i*gk}b&EXNoZl`}%(JG)$t; zVu*n)WWBFrn7$L1&vwNL-McV3>>}!pJEqw*clYnu_JH)-p z#H)#4@GH6q)xvKmF!=$KnPsRlEysn{ax8sNiItnGA+KGJgafVkp4*KLP9n55M}l_0 zm!b3b$kG+*l9U@MN}<&cGuIB-&uzxfGj)h*t3^R=H3mC>t4AhsW!NP{1^7bqInf(GGGs2L0I2h|Y{6Mz*BKU_F&a~V`ZJ8@PO`S3R zhXd5*?GQySKpy?3*8fXo z^-oShQ<`kLoFJG{s7VIH3^A1HU2KnhjN#d`2pX@T?V8Vc$a|wPYG3&BT-DcPpVJfa zasMDW%WAOtDre$o@j`USFNa6*8knrzj*+2zkZ|e<&M8_V)9WGzmF+NV&JCQN=#12f zZt$Auj_rkBY>)B9`A|RX9UF|D!@}_H<_pXze+55=)p;y_hxOw!aU&)d_FKPV$%j&` zdRmUmGnF`F_!~K^s^P9!gVf+!3}0J^M@r3To!$x6&;6LLB1+xlq-d9`EKReOrOTMV zx5Z41UMn0#*}fjuGr0}>RGaZhya5ly>M=yU9w%@gZ1h? z>3|KF?BKlSJj%wL#{M5iu*A?58Vj~zLZUJH-Y>$^Z7bk9ghP*sGK6oJW9PG%EMM%# zk1Sq<(T{rh`DyhqILvhB{nX&0;{@%j6yE+ds{`dzuhLzN^~P@e`}&sumZ5r{n$jll3%2=);&KPe4C3)uCJ%*9pr5b8?W;jJg^N+07gBVbmn=PyBSYuBl%h4YVsxmL2tCo( ziKv5*`$eaF`J-uvk3zACVVNWM^eBaOg&WzlQTat(&Z}xj(>sO;tWVL z4)3;yiQqoPVH6(?UuFmE(w}1ZtN?h7e2lB7AK>H;FWm2PhpD<7RHiyZ`{@nTO0xQ| z-1D$aI)STQX1FkN8-CqbhnH)nVQke*s88F7p7|AIa>n#!lkR!IG0ue;UJBy-m75SN zDn%QT)U@rC)V1eCv%bfD8?h)Wkm-n+6N^tTiSzkju6O(^?yyB7*Sa~MQ#jSic~6z( z3vVg&>pxHD9o8uG3RY75zUX#NPp5zjU!TZXr@!QswF9|jBC$ko6VnesAgaX-+4m3O zeJ87He@j%;?huipqNAEH5EjIn)w+_=ogUC$Q>3-;cLgb^+6aTavk-1G9jm<8p`dje zE*&t#T&0t6+;jn1Q*0o2>ITewozcqtxd&nHs3e}atn>g=e?7wLRRQQ-`V@u?Kk42c zg|k=V5b2+YF6P&Xvs`TYsV~q{`-+98KVha&1z*QMc;Q};5q}yn(9(o%ZP+6)gCqvD~NYO(|;`C>c2%Vta47Z^tpYHk7wCL$9VDJMR8Li+mNVFPCDZUlEMla%_Q!*RjmR1{N;o zaqa6#JpN#g%i~P(i(7;7b7nwCI-13w_sQ-UJ;o^>P7BOu!^g^&dtUB`*JTpeuqG6` zK|{5(8q-Ky`F2bk&g!3Ej^vO3E8%p%WpZQsnU34sZ0=Z48RJd$amEv5`GjpUyunpj zeoRguci8q9r;?Jz)ykxElhQJ|+pS-@A=^gsnFV(U|9v|a)uxfec-C+FY$zTLXR(Kt zA1acraXYQm@niRJ>f5RZ#b6)OY8HiA_h;Z#%^Ebjn<6#99Brbf5U0wnpEpbO6CKY zcH+kTPJAkC!?Bqym>5uxwlmdGx>$~B#^2y$o`;ga48$|v^Ozmb@+c7jr{j=&BMJej z;V5umF@Q@TvY9QrE_etlHpd@6_r$PW?l>jw22*<{?8v%~f@T}^yD|SKX+M&doIug( zEl8I!#emrp7h&_2z(zE1?h2dSg|1y{W*Z|yfmhnk%8(bdC1)K4cl**<3?6B z_RxCB%dk3Y(uVej9k^iD2_xeUSe&NU?5h_W=Xrz`T6*(+T zbvq=fP)~vys)*6o$3^JAUH#Bi??FsWC+2hGDzCZ@z;%SnbK+#|5jehbr!>3BT$GoJh@#6PzmNIA^vZ&v=r%VACME^I~a z*bc;;>crN%F05r-o2owy!J5esyiJHk>7`eR*V(-`-sO*49d>iiGZUQN-!|?tp z3^y)A>Gfj7t;A;?vA;wYAUHTjrZK24ie~Mc3K%Csd z>dP8@v8LGv$6`D%P;&>NA+9iCwLn2#*HJj-2&zswAobo+TzUC{advK@Xt4;dYoV{b z!k4Edt{;{cYZM@S!Ao-Evk5F2BjCLbGo0!*$k~%G2?y37-fq`F}pg(Xl zrxQGz6PmEvKD{ek>4GzyOUngL{ih+f*LE(KC#%TK+utnk+vg>)t6Gdx@hF<1LYH-$@Vysf$>tRdJ1P0Ie8fwh4zAGGRy{I=pQ(HcmW==2vDgChE;vB# z>S3gIUq`5wGnQDp!s@|Y)W>^3@fq9mWFO-9;wR8!8VRZ@&rseFifQ`Kk)09+4Ph)y zmnPuKdLi!5O@U?YJ1nfu#Fd4)&^%I%sF*TLXsm+y^*X%z(}c{&ZAe?&iTl-ESh%GN zpN4ip#;y|)T^+c2yc1KkyOG`1i<$U`jbfto;ahRK>itj}5+zEv9TBA$>jyC@t`FSG zZYT*lq3_p@XQ$h-=W;tP@9IE>co){SbwWzJ1Db|ycy^`<8#U{(e^fQB3(IkQ!Z)ZM z$wx!cC)hmyhy{bGxHvHxciob3m*szxRO7H}MKoL=J;$NjVK9k&hQ!Iis66J6<13g( z-=7Cibo4^*J9oC1x?*)G>%qRv4l!e_aY|zgIq9f_k6kh3_MOS}qvd$Ie$Wuhbtgma z;avLhAM4RRt6liwZyTh>G;s_6WHx36%=)*2n^r#3&yXO z=X6(VaM#O~IPGN`oUZs7&hpfNptU4T@bpx%K=jgMK|-hlxAwOg_eE2Rx6Sk*#rK=J zCe?PRsy7KIT$SR-rp%=aW==-+oJ}xu8c#zXO{PNo7?NbqbPr06NkF3|tghK%j`uC} ztGi<3OLsVb_rl+)zR0Y7gaT)OT#yRJ)f3O~*fR`YGM;0)Sv0nW#W8!AfZ*^XG^J2H zl25~`n;-EZ>k|s>@?jzN4L2OhaoVgJyLQ(j#(?GE%-V2kT?fX^?n1ss7i8<&vG{sB zdXw4_wy7O%9UZ7-<9{fmAHJJKsD+Ct)fI};?6QC8OCH3bo<5|1>Bf~w6;Te39}!zolw?qhu7>@c%N=UTu2@4Usq#pcR6Hke8+y4BX;x2!JbJO zIGFYx??0trEQ2?0wkASUHvz56aj4XfL9F?6=!8c=>`VwM$32BZi$D7O{BU@(FG6NL zfNL_VH{Nsy_ZnT`bip05idyhkWCQPGfhf6JJ~{$gZG557AOd3+)_&{{ytVPR?b17l&>S0eb_~? zOJqOQy{P-by~Dl;y;;B5(4I1KYW6+M{BR5&L3Y}AMV!g$>CTY17saX^ zTfS+n4|XZ7#PU5g&~tf5k_Q4XSKSu&RhrP=>W-AFt}xKNgN5!awo!e61B$HH;ea33 zssun(@hNT|3xRNJ1lGkp$3oc{X6NG2=)m&t9f{bo8i;?Df(_r^BXd;-%o}phxV#YA zpT03}Oa-=%ufem&bvW(Y1gZV4$kJ_xxgXo-=dkt8#(V74Rz!8QV&achT(xI*I;#WG zC%X}IwjWoV2eJ6}ATE}6BkXlA#^rTm`~41VUEGGtlC4OY--1=^TOilcila=Q^{r|r zG88%yoZJqz2W_a*ZGn|R119~hg^p7t({lcabcXu|oX>+wZVuit%=dM{dnj#qi+eg0 z*Eb5Gyeko>KgUD1I~F}VV-Ohg0_~5&acpNOikhCmK>jHb-2-v?eh^l*2eP<65O3;Z z;JxqzO7<4xA~Mm;dQ|mT-{Ni!4<&_h9Ph|t)N&s+?I$B2;w2eMudT@9-}l!Eq>~vB z>Ek1^+P{pvxfDbKFRvsaz4D|liS>C+auja27%$vVzgcL%d4bS1^pf!Q&vu~$(}C1V z&?Ps`n6_j}1+nUWOtK%x!_d7|FlImNbNX{A{a51)+3PH({4GOYb_@8(fKURJ+njBN zy!M4FnK&^}44`N`cd z4R6I5wx2gyb->@H4f?JvaFA+2@b+dbbZfy6CpPwj?eJv2-^YedT>IUDG1(mmdD;e7 z^=5orSI5S*8U^PnAR$+ZJEC9lb!i^1nr0&`B@-8>eT0s48t9i~@Mi&24I#|j-=NPX z9?|Pxfq%?u$gQH$8T}kJ3nG#KCk)}ML(%Xc1PS||VIzwT4vTEUt1rz z$2eZ?@F==^zPz^0`3PulvZgOioaL(0Pm+PvTDa%zB*wOkFqOhF2h- zxIM%7><}nT4~1TE7&zTXxKDop$+l=z#m8XO&R5{>$HRW*8#G!9(RK$&bxXnQ^fah$ z`GAvavv6u}Ho7$OFzat27Mv?ZNq+@CZmGdGU53|{GkoK48v^Y+;H1)tTCFZ{%s1D~ zYsYYvo(xvUFGHXV@S3T;!vweS41txqh zh3?j3Z2Vb(w1zLpm&u0DhcPRH=y@33h^8h$hX#Ylr<$0s4<3@0M1?lsd4VtZT) z(+o3?g^oiUcAb6&$FT{R`~C&&cf_OC%pVH*Y@NJOC#fT+XbaAp()`sQFjvH%=v`#` zzi;edzFUFjMf~79XD@>GJ5%hvG6HuFXA-?98D#j|XT(j~l~e|wB1a^2Nx{P@q{VJA znc1yE)Y?~)R-pyiJIs~LS76!~YchyTQU+=NB!}&Ndr)Aph~=MHzvD0kIz7XV)fALs zVjAl$y8Q!OEKKR-=E>US0W*n_8^g2R{BcM#o~`v4D7uz_$A%1-@rr}VvsldC76yGn&>l!Q)XgcBZsonob*v3|X8P(GA@n zy~vo_kH#(is5b7$(mj1xQQL#>nH{+On#KFE_1KVDgOcr4Fpe&RQOtK}tuH}IY5^|q z&BvPMU+`{J4i@-lp~oQu%cRp$68Rprk!cv2_7<;~ra)YZV!voIrYQsC*tzyLhPo)>3aFMpi8BxluRVfGyT`3WAU)fP@>V9E?DBnYCWE)GalP$-nL>m z_i@ZZJkwc^f6j63 z;rC%UUFlf}Yc{SSOzY-RIN_{f0nh5BwM=6_!!}&7M3= zf0T=-IiE32`7_i;GY;^!PZ&Dq6Uw5pVRVe;i5IXnqV*S>rno}g={W?4^Pu&o7wg@c zcpu4n_-$E2Yf^=H^Gbr=o@cAAy;6n54pE@bvwq=M_eX4&S&Au^N?7M5j~Sc$$&iu? zBC#%&u%bUQ-qV9TD||o__Ir|upl3vBUn((}S3wR&^^!SvfNV$ZI0yqi}E zWqAc!lsuJ0uCdh?5lYb6Swf~YzJ$hW)MKh|6Q7*g3;&Kh7-&65UxW+ZrT)RlqIg_l zc;}H9*{BW4LPtjytCwZxJMlACkNAS7;#>?B$;TIVJv>hr!FX;lmMeXObx$e7%vc;Z z?H97$tC)sTEs~4sFe9@GGmf?cvwHADsvpZQ4C0gv%fIRlq053rsHv6+{VO?$>6`k{ z__`b3Q62Dk*M>LqTHwrVd0liPGB!3r?0Fr+Hx*+XNyp(Ozmc-z54=hmG3IkKNPa6$ zjP5}C<8GW^)DIgC5$dNXN)2qpsDqanjh-P+BgMsO=xZ@rHCL1t+KN!GgaO?7(1R#+ zz}2S}2@jhvc0&W~8~#ExpcV(X8obl0!s^&6JQ1lxWkv-Ca?8+dTh3|%e_~AfPZT$o z;pCp5I8InD(xnFTb%UT&`vfl@h*Ifs3e@EjTTdw}R6h3}KmTkMy7KDrY26LtZ#s-l zlNn9Qr>M}Os%xlewJ%K>|BLU?oy=GM>?4`dQh4{Pj|}}$O~xw=N#czda=eo9WuAo) zN$DWsxIc=F{+3A09cqZB!vHx_E(wLx10=C-5`V3#f*(@hOI0VXp|TTHXd16T)>sUq z5&Dkg@`whUxLbvVX7~6_%_?;2)tB>V}qRC$>qp zF+0|bz;}(fbmI^1pZbl3EdL_$;5qgz{DOA7->`LMIUp9h-BWABB6PxYOdkrO2T^*P z;Q^P#Y4A}A8p!b9QH;Y^<|aiS1W8fbZ<6%SM+vGmZzx@?FGfd(4xzy;C$!?gKZN`A zN{aQ!0?8Yc9wT)$HDAYD0j7DdT%QfU$(&FWGmDbc0s|j9}`yh z!LwhIPWB%~2QMp9nVa6QaGpeOnQo)ecec>IXY8m?@=dC^x&w_&|KEtc&$;s+bf*HV ziDG>@+ox;r{a5vP#lt@E9`}T}y^%+O<3HiU4abRnunQ@=>O~xS9+SBh-b6~rg&f^` zm@IPc5Nc$Mgl)DTQMu~_>GOKL|3s$Ky4#$u+NXt7Uk^G+T2ZUHi~6v=u>R#u8f0Tf zCxmRFqHDL&$nTTrMWHv$-YC*s(b4pkpCr}%(}(1#1GuZ*1t-r|>^t6yjUp_DX7N{i zX)Cr^wIjN^9VrY)`+Sf2Vf(vKXVQ(P(S0zx-H+6)e>fK}LT`K-LJK#D(P276sdT9X zbySt26Il*!QHB(~eL#vfZ%Lt)=L_G1Anpbr`+%QkrHMD$@4da#YPzoZc##L{Fzp zrlb1hXhy^qT2LWJ>%3;rqeP6B85+>_8~P#hei?0L`v1D}Q!s;8u)3vJFv$5p)n5I= zB~3MIH8qKTY&D`2E`KLgbLENQo2@ukpewkZUP~NADN%p4iWqpkC3Wj-iDZ|)K>Fbp zXf0>lX~B1rpKe6Iu|8tkZPe&XqhCn<{()}U{|c9jEAYc`GQyU{P?O?i^vTnHj5ji% z&Tqu%%SqB65lQ;hNRrOKB1Okvm!cgHrRl@p!|1Cn z8T#Y4EET^lOQ$T6rLTv|(1&NG>69`_`t9>j+LJ3vn;)`q`_PS27UvJnsKez&m5AS0 z3`xgyEX;il--?|uS}!T(9R5*JykA7*KX;?`V&WnqQc@!SpW6tL6(aks4j#2QZ?19P z+{)VG^r_V=1tKQ@|8I+i{pZtjpB&jH%0BHsccb+~{`2sEK3{o!(z;Og{6F`R|M|!N zxc%>Mb7=qB{qxQoKkAl#;Aa3Fp3PAW8@!4Kpbf3Zb(1-g{;5J>p!?6DjeoXsDEy zmZo-8s_*^%e*XD=f4|4?9OrS4bMAdz*Xw$&+kD6DIb2+Pd|dyZ#{w=9F2@tjZijsj zu61$rc2xCrKWex|h|BK(dIZmM_V!?wQnmN-K78~fzx|1WPCkc^y4ZUkJbu#SguSl@ zzZaK+f-Ntz{gcd+j)x9!CtMDmIOw8( z*x8%ki*JpE*F0}8ehV)FRr4LZd{UC#+bfw(|L>Ri|L!kl5 zSKy?&6n5ApLar(tjuG)#sFQ;J;uM%WWMg+_HV(@apkQ?=l6ICL>Pi`|q?bW0wG0;; znD>wp2(KwdnRXF$kP9B0TuATBK_#7mp|zQaKbDLg#mTsTF%Ih<#$uUY3>KfCIIwF?ikr zF9&ww*1nD4owFU<&igS^E(z6l6XZ-=63(@aK`W{RA0kV;d7Gl>XFSfdR80frH6~4Y zct&b>Sxa@VVs)^8xV&VqJsU@6oZhf^l{K>K?ex0m>ep#!@XJy&kLxM-erHh>3$XQx&t2BiKj|HNOSHo6705bbTP-H8H zhI{Hz8rT9&i(TmXun*BEk05Hb2d3(M;J|YlsbeAdl^BMsGZA>I6os+9F<7%S4%N@% zpmj7Et;5NvQb}uS?`#>pO#NGXuU_|$T`qhhB34NHi zupc&leYkIb0g^mjSfA2?3-Q&ca;w8qt_$c|G>Pr$Je&v!D@1&(ImPd0;vUGF>@k=OZ}WgR-3Gm&~yw zyaxM^&gbY~=I5*tokrnlF3!zz9!~DANl?uke6Xy;SMzGPS#&_xtP8h=F5taGAHJFQ z1~WP--6cVTL>1tftgc7NPck@4zv33irHWA z*acMhb5M2-crc&f%}oxnHYLKpGaLc`u`yJfg$b8hyf-Mvw%`&hNw0(8v2qM4Hlpi$ z15CtP;e=+~z0(Q}=Vo|?wqmHS3Hou3(4MKrq-ZS`Tq{NH%W|mCErOciIXHLa1Dsns{)^R3JF5%1- zSnP@EX?Z21udM1ter*RP0I(D!8yo^6=C{xIp)QcV)5}>D67@rWJ)7e zd}~B%VhgIoTOhTq4F@kaqc*1nj>j5d`KtjEC(7~Mr4F**CHQSw4xt^jaB9oKc0~e_ zqHt^ph=)QYaB@y9j`3CDvT7^XMO9GP*$NN6S`;Va`qf7X-KpDV^!CM(_{NOStx`Q$9$JHu%MtiI7z7LEI9ET7#v}7lbQA5je0V46Aw~QM5Q35w8>Rxg-gqd8vq0Plx!DOx&8thQy;> za6K==1&;!-)JrjS_Z(jQs>1Gb72t$4VE?{)I2yM=rl%bTvpcZ*S3A@$cOd>;I|fWT zV7|Q-pH_8XnPV*`9a{0ttO_3`TA^rI34MoJn0y1;wc>I3Z!&g=a*%tj3^kUmkgTr6 zzo<)a?*SRzw7qL} zd%h&neK+3|<*fzS%J&W*3a=w(MhJ;cl0?e+KA|Z$w5&W1EunA}HU}drFbcl%;qV=g z1qqLYcyBDad}47&DH(^Ili)Nz1yO#fc+8)UT~V2+ypxF^vUyM`%*Gz6B793NK>U&l zSdW)ri&z~ThpW-2)`*V5X5?12V*buH_-3}lORyczN7|wKsR_Z_ZMdCMgEbjUZMas8 zi|&=^=WT`C?=t8pZ~(g$EHF%k<(>=pey9ZI=2ZwyuEVq0#YpFF#NCv0xFuPS@UvCe z$z2EIh+3R@T#Lg+wV-cRW7@L{0~^Y)&$$q@_7vk!cNX68<)O(r6+MlqC^LwI<@W?= z-H67Qff$4;L?M_H0jU+?a5@u;m_;uG4?4G?#01)G82ta z)nJUBkHT{C6}Z&5gqXjJCT5-ls>1U)VfVN=DZ^99o|6R#_fT&yUsV+mi;kJ!r$2K?~Eq9k_M04GZ72qvTO5Vl`Uv&!Y+V zvg?p&UIEp4)$nX6L>+@Y1HNZ7I57_u-&66WBOSLHeC)d=8RvWxkRKY4XJaw=&o&xo z_r>7)R_6R)hhv9V2o@QH<5bRRtjuOUn}i`}A{LgD;Yjzlbj=qi&VwT1`jUrKMN zK!p&LJ&@$+Pk&DUowc28SR_CSzEUJkbRO}DHzn7b709gcFtVq6AIaMIjNDIIze_KMxt4M^nTEi7_XMTr;X+GEEhob+ZcsJJ#5I#}oW2o@ z4Siwwaw7z9HlKkDD;)Mg!C3q>95oH$c%>NwrT?NaJQD-$zj5$6lYlev$7P}d9M||R_zeXY=c!z2Nb9l zWL{~51JiyBlbT?|;I)K3jo5aq751m<@NeEFNZVv%(?~XSpI^k<>2j?5Sp$lh+Z34o z5~0_Kxz_oJX4c0%hC6iCz_GXzcVjB?V|67KWt5{(sT6xQ6hZP$Cb-pduy8sNubK8f zAQJ=6uz1+c568Zbk>Fn#jGt*CaPvKl_Lft)apeTKcY9z~x)-!eT+wfH1mffXxLzE> zcz`W_8M$CdkPF5s;@DhHOqW%Ezm4;0~S z$acZaJCgK^|7hCi)GX}rJ^|LW8zm%NLkavoNLK%rWWP+<&Ng^(Q>Q8_ynCwPg^nZF z?e5CjzdEVv=G|r|CUidh*vb~SUc#P#?GWi5ucBJ*k5HT2ykN0r7G^8a^njBjExGR? zvU){0svoDZ?$I&~mD<<3L)}--auOjOo&%>3nMmAJ1O>NJ9EdB&hjo?McefJOdn++Ju?7L}OCj}N1I{Jn!);+B zlq=7n#W}7jfagY#dxzfG0byK~19(%jR`p+dBpu2-G9StDWH^48Hej zh3CmO*sX8F-Z}(@A0ANAK7hCX4x)wa z0=cpiI2gAD@&_p7l$jtuQv|L%zENA>M$^h~OX%-K19a!EP}=0SBQEV9LtuOk_MYh^ z4g;Tw%^erCl!p<^OaZd=3xm^=LP_elCD|5vo874Ri7lmX&fX_9%r1Ur#{TZ}o_(ch zi2WnUiUf}Zlj%J_$@uYk= zrRV&Prk#|&Q&w06ucan%5}|SK#TFbr>VLlcakF?tG1u#)Z7SRylOZvXjg5QKU?*9CkEe6+{bdCbhDy<> zT#MCd^~~>0NYidcN_#6_dbOeVUn@kz+OWrlncLqsVB&5Y%=zmP64!>EjvdgQCm=fCmt=}Va7hcbR9;s zsjibg>M<+u?@`uk$+?tHL?Ns62@mDo6TnI{8f8VBICW3IO44q=;>1q) zxSYD!XGppRS7JKSkS$@d0FmtrIDa3?adJaz$jzK3x~nAT>OQd)qU{BX*ghErBr<$2 zmiii^{;3q6i>$}f0!O^{wZwTTd#GKqfc8TtxL4SrSji31f-Wc$afipgLx_ET7z@68 zBGb?hLvPOFS@$U@GuY7DFcgXfai|)JKxbDHa=a1{f1cq0F3dbFat?pm$}qdG65mf% z!^^)8;%{oOMWO{~l`9b(*@lduQaEpDLDGh5T+?Yr)zVfZ-)n&3tsyu|m0LkvpfuCe1Whn@WjMiv=B{J1jSIZ@CXj>1T47;YCXBiwwOhybUY zcq~(+EoTDh>AHMqY07aXp2Wl1)`RBWXh0VP&ya3?71EnhOQ{&mr#3ttVht2obX)68 zvTDzWvYy`l%$j%kyH54&XRM+xvsod>zp-BIjO{*sXP6ZvFF={Z*HN#}ttAZ(e~9ml zJ}p>!oSu%3N3gpb=f89Luvs5Sf3j7hBmFALM7bWh(ke}k-V1~E+Gun>r||H#GSF zpki?wzWk^`>g^7AzpTZ=})B)4h78J`;GM}oq!lBqg9ByS#D1)?2kV>5wVgD&4-S7GNJ$tJ_TH;U4ajWlrXYe2|HXi z!?|iB=x5uI{MHB~^K1}2V~tp9ALs!`blp4zXBiJH8+V6zkU!pwox+mm5g0oZige8c zNN2~OZ+{m2Kc&O_K_O;Oo$B$Md8P$UIw-u4hZ?o^W;lut~ zyt>Vd;re#Smo}sR;VrxyFM}{C!_hCdko&$BO7A=HRI(ktXKS%&TRUzsb^Y0yR@5J_ zLu^3Y{8XX7QxNK#_?XY{319UreDp0NPd^n9qY^vpl8j-r z@dy^6Mn4hS!MgM}1X|^xc%3mrt#RekT_aaZ=i3T^OL!D!xr+GO=|;!h5T|1=hIa=v zCQ=K;bEv(a^C@TUd}=1Dnd&YWrJ9_ZsTU0el;OTYYFIvlGQE;Wxte5j7d42}1O0CF z0zoBAN{r&?2tQUXsHHEu@#!|M|3l>$gd#;S6uP5^tlnp8v~FuS>w{$s_zL$zGrxeC zrI!;;=J?NU{YVzw;=}m3ARZ;C!o^CCalO<~E~pOnxIRu?+6sNq9q<)3L&X&f46Ss) zw6F_K4|}3o(+!FS0qAk?XY`g>WbBE=(9RTWq7op0Ec{5wMMH8KEJBK5&2Y5eXDi{C zUk|P6YM2~q#sg;DH!fT5^PX3+|V%nlec zIKF(0(QT|+(fhC*pB^?rJgE}CKWo9Yp%VY4Rp5kg84ULpW2;aRs%Gavs3#E*FQ#Li zH>1bV@eqA-29-X+u#Wb`t9Jp|eBvn1iyp#*6ZQyQXb0X!wlFES#J4yroZ4Z4w2WtNU^kKfjt5qnAw>uxAxp0%bj)D zQ>%__+61l_^`LXX5Pz6@ao=e-)D$gn+uRnLw%fty>;V|QbA^h{G2BiFz>I?@G_!+| zG5;)@&c#87(Z)C%({bWMB0h#@L)M}Q*|UnV>^9@CW>!Gcvl6?rYO%ku5|zzOsP8F9 z#rjsXZKy$DeJetDb)aLs1r}Z-@F*%ndUy%c)b7DFunGQ5zsPEAM&z+7>=kIjORG|7 zwbz5HD8pUXD%^a14(}LjD65*scqWA~ZA%5)BLmx0BT;!h4y>5daPkjD>05Uk&N_)M z6&LWpG#TQ(zcGUMXyHl+h4W?_$i!x-yDOsRm?}7d z(wNlTfVMenV9;j-ZvJRoxSok8ZXHCeh)|2)q*FbYWr_EQE~OijOBdeErZ4RZgYx|j zNG^RvItPbY9+jJ@-iTcGltux&G1G(U^9iHqqxn>I!w4m(d7bJNm82h8T&1>m-=XsE z|S?n(~O+}AiW~IM;Nxt!RV$3BB)QTM1av+!9oyqLiLXI4s zNTZr>vMKy*C!1bnV*jyd2Im{0U3?9C#@EBXQwpy)t%X6RB6N0d!m2`btp3eH>lH2B zlQlxe4MT|7?0~}(Q^Y?r!?H{}ST*iryz}F*wQ|AvT_@rH+Z|hsLLn7@8aZ3z866-J zQM)qmfZ_aBCGgKEfikNK-S%a0%VgTkyA&}JO<3($1*hxHc*oNL@qxA2_NeSI#ppvJHf)eN_L^rj9iuM08tw+1hs6=Pv&1$5I3@rLnc z44!3Sqe2ce7H44QX%ZUsqp|UE3~q2M;3( zhRw3Z&=Awdznj`9tDwH*FTu7r~PB9I%Bu-?SQXaTcu+P$9ii`EmBs35ZPd>ye< zX(1&?MhIUwA?v0~Nb$k}s{X=l+BtNU?%S0Tx@Pq~^tO(0%C5|ccKMz|M~9VB;(yAt zE|o2##+gl?1PDqV%Ex9KCkSE$dE#U#@M zq%QF``T3)XbVk*Xi|>QUoJVzpl5QlTK3v%Fhzp7jB%vTGic0&H=x1vGwsLLAOlYBh zo<7D`Z%0X*IW+RDP<+h>5t2vXT6_>2DIdIqJ0!maBlN33j{J*(|J!IJ=qKaa@eKSr zmkpg`S$JevjF`nms9jfq0F!etSzU|fzlErAsE4gNleM54Ay(Leez!)1GG6bMUq#sM zP=xPSh7coA!|1m45YMfK;6KKT87~Ll>@0-2mZCb2(Gz)ZJLQ1vx8{&twF_K>n;~nq1$V1x219BibJ2P< zYbv8aR0Sv0q!1`7iz7}V2;R07-3jyI_)r8jhi{WvVt2^>2W(=XF-_jyh$A8*QRLu# zPwWo)hF+geSUqZ^76y*clbaH0vA{NMzGP<{e&fVa{7;B}-IY()OdleD)+SNnl8w~A z)x%VcXCrm++#5>SX)aA)nN1r#)}~$O%%Pta%%#`(yrqV-8mQ)(A) zMBddE5|#Tx^j{Y@7TxQDuXlcFA5l-F?=bsqmujPiOFEIU>l+^4^~9IfD8iYDC;M8a ziLeld>=Pa#?)-Pi=%5IKbOhn{OcMVpMUYV}i}xB*DCAW^o3RQW7OlgzJ6eoZOXE%Q z7OZ``3C`xbP$FW1YYeW`Y1jjyH^<<8!Wp+7gy2ZHH-;ZXQVT*8oQPbqwseTo_ppZjk=Pi+p<&v_ua=NFk!SH_WnMpBxf!W!LFK?IZs5JwAermWAy z``cZz`;6Xo#VO-BE}<yIaba_5)l2uNNVhoZS4JacJd#PRZmg#SN@lSi-q9yU&a#xwKr(cE zL^2L>?VS4Muj%Cjsq8VTpRG8*~+5cVM+6xz+kSsQh1 z@LhrBn^p1O+f{IJ*1(IMs`&9;hrvlYFlpCAAg=+w`xwJ_g$axljIrjFEeiE)a6aB0 zskAGi^8C@+>x*lKVYsp}6n7c?llCeOsk_sW6Q7LGj6A$f&w|9|V#pm~&}3Hy7H!Re zk4-fO4>FiwYaM1~G{BY7b?^Tf#)^x%4ByO!Wx#(3Z7s#y^a{iVl)|qx3#P?|s4YuG znN<#Y{NmAPoeH6=G2p(N0LS%_$QX^n27}WW_~!>sz7wPwZMLbw2DL7(Nc1xSx0gBm z{PZA~L*sI?I(F5m!FRq2Vz#WpEzbs#7^z3yUpjc`z<6J}`{4)`|93Hb*(O-gJfCFEnNJ4ht_D@w zj|hPZ@<_LYUisdV&X5+PR|IR&ru}^Mfv<~byB&Wib2Djrx1}~My-bt#=MJOqrfi_a zBDH8QYiauR?SGWHz9=pEnwPd(ph5c|5T`R2TG0zSO6ZS_hSNIIkK4_wpk5_N+>Z;8 z@8&yE?JD6D3N3kE*rz2}U$Pwa4)XBiS4MS(8or0CBV&#(z8LG_Ff;xi@7amCRj$mK zw?U1)FIo;c!+b5{3%(D)2SmY`I}+>k6JbNeAloJt@_Gr-m&gIK6S0TC2zIqum=Z39 zQcMNhJ4z8U^&eg_zTisD9Gty+6&pLt;U>>`f1~ASoyx_cT_yPQB^A%a7>`^$8D{4) zAn`gLCyu6p?G*!$?eQQ%q2N>pqip6F5*qyR?362f_q$@tjooPay9-Z53}ESFgtzxu z&@a#dExsO$7@fl^RUUayq+prOWCefmVc1g;gMm{dP=7XJ|4opJ%QM7Btd{gS6q0uj zKe6u)Cy^St7SeihF9}lIO=;Oou+pS16JA|HW`FLcmcQ<&9!QR{LnA`ab^c_xjBqZk zE5u8Wc6PB`PcT682%B;VouZofo>IvR1?j?cEjlQ94Sk%c9_Hn1=pJD$+U~djt$O@1 zHJUR`Jre4oT-`dUtk+$vpyau1`aDOsHO$*?CE)_S^O7M?A4|4JqcuH5{?_fC| z_g}?Hrlwy%m=5jZgOFWagzMkWL1elBQzndd_&6IS9x<43NJFnhB-ZA~q01^1*&R`^ zc<&FbpJ$;L;f}#fcMRX(gV_@YA=PSv-F>^E{z(UmX6fPDg|*m_wh^wb8N~`9e_v&{NnZN_{UDG7UjSoJ#v*E2P2I*gmF&4WF z37cfFrjF4Nc~nqZv=-l5HPPU|7JIq%7(GY}kCS&}agqs4>4WH7ya#(9xI-e<9Xl_d z!Ho=myxbXuyBVPftBM0P7y-2#Y52M>2034|p}8s*BHtL@{XV0+Zz;l%;UHu*GvG8g z6CEc9F{W@1ukV+@)wT%tG}EwPFc%h0@d#R+gB(Uru1`wFoyG{PEscTs9e+q^1*4ns z*?2ztKv~8ScNE;AJ7f;Y3`RK#wSPfzBWhjbXil+)< z5G-1Z*Ymg-A8;1lq`V_~i+>TNKX-}htrz4ePc`A4;E?li8swg<0$FnN5j*>=1ySkf zC)PZS)*q;igz6r$H@TN(^(v12;bkaGbOk>#yE8`ZYB!>Vi_58^+F+_GMTFK@<)Q1> z$k3;Dh|m)niuCIlCtBsSE*+g-PdEM6rM<0P=)pH@XfG2Hx_)38eQcD6mcAiEpPCG! z&J9#juWPo^d3WznD$4|kPGB&rM=+iJX_C!y+IE3(F>QY47=_AxedOb!UBvAB1NOZA zYsj>#8u@(a0ueQ-CMyMAk=&Q>yC>}klxX16%7n^N)dcNlVCCxIgpibN-56W3aE#rmA4$Ua zbjja&V&nq?{6=`S51 z59U;n5&Kc{wP~0{cTSR%+a8g>u0M!V>Sq#PEClITAq?JGjI8j*FpQIh=e^a~GO-4e z_ts%mz)H|J)v<1PEqFicqtRUtr&zYgnP&m^PZ#L?a)EE22Y#k9`d|K8aO`~XXeI(* zG(r(Ql!P;?;joELg>PychJIzk{&fLHc?z)bS3lHFGWqY4WE@{O087;(y!9`FpH3cp zb|&JYP6l2sjDpYgR5+N2<|x zN&4`WbeY~Jqp~kZz^69i*>Q&qi4>B{au*4^G>+JtXOQ{B5#*ulHS$0+ie#)(CdPYK zlE6>%@G^A^p`SUDtJywm|6@-&{ZOOhMy0SU`fI5Rn{ufo47ShXnom2bDbuO_f%N6A zhV-d>w`ggOAuablkXDl-Ad1R(Vj4PRzjQ`Rg*i?Aj&PeKkibCl64x*qSe*2C=b zMkp9*qFLA&_VP65thL1q*DfYEdl;=@$KV!m8atMFAyhLAPSXBxVe(?$GU4#@V01UG zC=6UnM`ukUcpUS<<5YxID~sT|KH-5*CFX&r#c3}+ZWKZ@4)eVA=w1KyO)@W?U--yC%`#jRWO;zTl$-Oxg!t@jb(2RWqDM~g%_?;!`AmXdj9>11WJD0z80rTeXy6eUIplRxwv z>ZYq0OI1#q`UVZ^yQ@1@w$qhblK6yzdOu|$vYhVPu$o>fx1AQ=nNA<&a-;t{@RL>= zJ4CnqOQZjF8qwR2%F`A+%jvyC{glbX6Y47K2xa%$jXJP!Emg8vmKt+k#3EC3sc6v! zBtt}ok}b>bt_>6;I=$)SboDY46S0ruX5 z1Y?kYHWO)|bCLQam(h$b!{BNh4vfb_@xMN-iOPi?li}ien2Gxvn7nL6G7Jqv@P#`b zk>Ae1c!u%gnEG;YNdR0e?Qy{95cD3~BFuU>!~3>i-)R$gZdC)+C{k-W(J@Q2vh{2{5FY2yCq2N~5HA+O3FkyQ^blc!Z9WC#tU)SW|0 zMU#o>`6}|@jyc)zIFkH4AVc;`?jV0hF0k#7yO8&S&)M2KA?$~Re_0u;<`a)?)2s;< zaaN9_DfJ<56E!xTM&+COQB@}ID4*_&RGyXueIr|zF0|C8Bi&Nz4?nHxGx5LasRnC0 zYfCDsr!Qr!lSVE4$f7!3O)0}KiY&3-PpsUBLS$au z6w6A{i@n137296Wf#h5tV1K4}5{hRT;p>YglR{?XdTtGQ8J9vP6ne-nhbA&{VTAZ6 zUnXZ4GX6ie2jaQIV6{3B?^NRvusa0m%$)8g z6^r>>GEr8Xjhdsm_?giMWz{&WI1z&>(u=U@Z0uT>$z)5?Aa*noCz#xzyyY1jh>ymK z4SvWmJd5?~nCx+?C$#QZqAu|O8oNz#ZI(F-^l0=~8Np?#3id~@M`f}C(n?q3+q4YU zomh&LA$|yX@?u5xY$RU(N7k?8!dihTl0!|AU-SsMQSpex=nRm@zi$({np(1WF-XX( zL?Vc=SO_9jXE`H>)q|hiUi+0o1(8*PR^{Mr^ zudV`*)7v3-p2nkP=D5M!i}X0;0DP8Nfj0HTVqQ1IzC4TRr>Ef78;xK4&f@j@1blfA ziCU&zH+{~4MOilfnf1a*JO*6P>mBzD1QKY8{ zV|+e06wdr5PL^C?^L!+i=YAwV_C6tv>JP|*o82UCZ$HtxTSFE~cMBL&3oJ6kh zB>TFP$iq9!h~gnEdMO7|Tto%pH;pqkHp4T#(N>HS~Dr~0{8j9&WQztrKYk{uRN+()= z+d0}q(3nmhR-}8E%FrAC>!Fruex_`Cov48i*_7;o$E>ZtR!WR3nu;xqk@G`#yqbh`hNvn5;z-Yg8M9#Q;Z@c%p>8R*F> z;q_d3h<-6d<%|~GRgCdl+Z0C@+hGQ#SlsK5sYyrVz4C{Xizg(qBN*@B4=;bm;r{Ok zST0D#66XFg-sBwUcQf~ac|^gZFamMhm#~o0?AMQ^FnA#aNqJ!~wVCscm%J#WFf*yEA>^>Jr`6@y>MQ zzB1b3?l$_-3I$qXu@t?k_dKO3^_Kd%!;11qEvM#0e_-*r?xf~T#pyiBokvBIhwNH| zT-G(M8|<1loGw>`xn!C5pYGcZONnJ6VV~sRM$YpIkXxUFiOqEjvc@cf3_l4bFYD{c zd5=P}aswd+Vol_>_As%`x=&I%pA$EYPvq7kZoJ$7hg81gLYLG6Je66DO@mAENKTr` zc&^60ce3bNunD}Jbx@ShN7L|jlp5ILZO#rjFLJ}uO2#i$I*CsPhjH|32+j$fM22cC zw8p~l-<%XYs7!^qa5gSH??YTC)9#61!5 z5HgmBpmFsmI`3ISHqrq;O$K;4ybDjew!z;?8-sroA#JY%@1`XPv5IPD;RK%!554x%I(I+Xld@g#W{SE4kq&yw3rcMjq+(8R8RL~iX2k3Qgm+J2E zwx=~8RnR4;x6`~@8no>6D%x}A26bgV7p=hWLtVEb)XtVg)S$c{Rh_tl^{RdY^(;A% z9WeTx6*wBpw(F2(UA+64U6gfG%jMA`l3o?T7S5%}+B+ZEKN~!VLkNpp5{@N~=R8T9 zaVcpmPA9*Aw2*xc<>YL6AMp{i*ILlGSn>*T!7q|`A9#|lIW-D5@JHRi@2IxA5>lUu4n>>s2^E|L+ zc?^@C3uS!WBzRv=0)0J)=|jCRX$?bRLkO;a?8CL4d9V>kLW)>2F24(d5sO);$P0oC zg0P9_2!`FfaCXufFUK8mz*Qfen%3A(jIe!;4vM}jV_b%X>Ag#^z+H;DuTl*A3Ph10 zA%xw60-(}=k~Ycj#9MNju(ij?#`8nudE+G#wyK-_+FMLYw$zYEyvbzMoO~ku)tx9L zM3D7v8svAAJNY2{nav*BM1~?V*o{9Vh?ULp?gMJG$@tL^Ix-RE?5154tUZUq*m0&h z)QxCiR;}_W>UGg`R!Ut9wYA5Ty8q+{Wi>rOonF3z-tb?D4EltP8!#Q5l8pK zHZNLTBeGcBt9ft&%@VuvoN&& zAF(?2kt|sGo@guHB}aMgkjn!jB#hNXd?p&mm|X?A>UfTP2n;7|l>{QN)roY8oFz|Z zDUofe+ey~NNp?uD647-$$Ud7thZvt))qQBqF#9V@pC#Cs$39xZW;NI^Wv}&INe##v zbl2QJNiEW?VaXQeQ_*8;l!yH&HNHHOI{8X~R^Kr}?e$wn|DZP0cYAH=&4z9CIO`~# zv|dp+V(C#je@+|STxCm_rEaFIqIai6r25`gY_^rhwbjOjnzOhw~K4m*rEpDZC?epHbEMbVi@<@9}FyfkXiBq~A!5tvCr_elUR+NgON%h`yDwI&BN+Ogogk(yF{O<3M*85Luy=$F&&bfO( z`*}X^@!gk*e+NfW#y%i-r8-H%yay!k-ylhyG(=#9Fs1KB=(tII|94x0T0ELd-`Gv2 zcg|{2y(klUs%IJ1@iCwa^Via^(U$c0s2kn6!j~z4lRpHhYbs=a+sbRdc@) z!QrQ*>Gc!xcg%A#rmdM=F(l-=?pewsrm{{Tmp{OLsQ6(fx1VtH#0r_*-=mZ$heJSYr>;gv~Y+;tP zCc+IFDR@`W3e|@kL4V(Gs8(o!^pBeO&vrhZd}V>jq!iO{+T*sRGg?@-=K%jfQ* zTG9?wZ_Xx~`p1|Cu3t_+Z`7i>%NEhoIlS&zqeS<|Dbhoqrc>-5B;y|bBtM3RiOAr4 zGFkT-Ilkcq+3b0PXnnp;63?}e^(7_5Ha&;DtVkrkWfMq+wJVu1x`TY0$C46ZD-upV zaz;9nh|Ks1&h7p;u0q(juCeGm_caq3GYcoK@L3OYwCl0p?d(aQd|k=x-6to&t=i1! zsZ?0AQV9CO1z=p@1esC85bTi(7b?bMx9AXfzcRx_dn^1Bw+#b*`Dnj>0DhA&W{a){ zpzR)tNz1q4q`1}i-*q!=KKB{+6pzQBUTH8<|0gUR-3W|h1I#)4hmliC1*P1{%$uEV zP+kcp(%pN7#Z{Jcgf?iwB8wSZ@`p7bcsyXdq%M`svDP|2+0G%X~Uer*e* z1ME(E_l7sE^yjlP30vv0h{aT?bRAt@r%y9yT2TuDev6$>c*K z^lh3rUD@)RINE(APYOSiw3CmBVrM6j|MY-tJy}aMv(A&GfYW5zo@}CKbePDLhmicF zH6&@zHuASvmasGCk@KB7oZPQTWX`BEmzD63^V)ULw1(W|0^0nUaE}5mWY#C<*?)4J zik>;-=DL_VsD#6qfFj0Uv=~OVszd77N07K@7x-(6V6oX9=r}$DjT1~z`Gy4wCza#F z+>Ln9&48UVY==x{InK*lfv(F;u){zH2WH)c->o9(!|VW;V~=6@$7C=)R01!ZiAV9rSuaP?gtj1jrZ-8Q~eGh^RhF0FY2r(3QKOj+@j%*p;l zLO=c{hUdlU!%PvXzUDV^$FF!p{LFtRjh`RRXJCS<%8}zVZzi9CE;vUQS6-)wd_1Y`k}Xu<^BV1cca9$19Yl|Q z=X24U4$?&`hw0M_dzv)jOr-^sOvhop+TP5S-rk=19)$t3-9!sT8dJJXX%mux7xbY?HP*1nS% zzn@M19x@`Ajtp|C;p0fiwgRt;R*bgUgOg*|A2C!;sx zjKw?g^o%Clrx=XF)#hwzMKFGIX+ojXJ8^v5X0*w+z^d}GICt4}RJ~FIeN$wxhmD3L z$LA0bvJkF4u7*{aEsW){6nOR^n#pn63>Id0P5XCGge|(=f;ST>W016vOA>Wp<{LzE zdQOi_J`L1xK|ZTEIbT_FuwszoG>u5z_}L_UjR%=|-iJI;4Ced6WKxw=MxI5L6N8-< zWW&?DWYgZe#QH@S`TXBEK2Q9PSU-GB)@Eo@Sy@@SU{HuwY}BPjrG|8=w+yekNVlyHr;%%eXyoN{w2tB<&Cx6LxM^QRa=QXjt_Lv;!^UN+X&BSDF1KHwOPehfANVQ@z z*{u>y{{09eQ#7nePrzm(d`gX&+?r3yoGx=;Ps@rFKe`#_SIIMbt+3{?gXm>jh$ux8>Su-pF)40B>2>zx9g z<3m!OHFGfHl_}<=FTopg%h7#{4fcJU&zkJF!L$11=o-BYzt1+qUovyiTJ0qaTvkAX z&=`=-`VQAhmcRnJD{w8Wo$;KQ3_96=&FnAxKwnT&Es38F+7q3)c|Dhyi0TVm#y%+~ zZsl!m(bcND;OE1fw4?$T__@J7*_}+*pzE>mLCfJh_#)>osgpw_bj}tY%-pSQ0 zB1=jrX*F#m;sZ@Y;PjTHI(CyAPW`0kjVKlG`b(_ez9O+fhO{$Zk$%}NMBOv>>5Sre zGyFW+-NE#hMHy{8h4eqTSj>ynl)C&p(v(zAq(9#qW`<_F^)BYY~~fqly%}#uCw4@uZ?6 zp43k9CucUTBd;$bam_LyZSJ4B0JpDP!tnsEbX5r#vCzddV)8g{Vt^j=Zg#Su<3|hg z@zap$u?l6ruI@E+kamQ`wByX@l#`%(c!1fHRs~gmm0-A~6DF>5hcj8Dux#KoterRs z7pL6?ogeeiG;ac~9&d{gv94HK>WjmAt(dkr0(B&+XsnlWlQT z$^;ZXumH7}bwHA$8s6$Z4SWg=R_D3H!R$`3saJ&mPF2F$#e>Y~q8MoLjAcrUF;x@f6K_i zE_FJoUWc~t`AoJ->r=avn)Kvspvs~fsH(pkz4(#$cz1-+XHSB8j|_jV^$*dNG5m9$ zR7VpxFQcbIjOboIt8-}SRk~qHF!d=7qdUZ+>5TT{bcexydPdWiHV?W`Yo!(R)Db>s z#LsHBD~+KS8W&T=XA7v*@C9*s z3VVylO67B8URpd^H5@{Ie(@rn`}dN69kYpv`eM>?`zN<)K#lCb5Y5#Wk8t}`E()4A zKj5;LHkwUilDVsy8O(!|8#wfmf`_NF1s5euAWeJJ^t#Of*u=8TOaF5aA0Prs2kydz zBrCWl^9!_dV&TBQ@mQKX02AID;)$m$ei~ejf4ggOSFJt9MJ{Hy9<@jHE4BE_(+d3z z7xT5?JfzPDV7CxoBPPedbAu6RKVt<;X553(0WlaHECicZOPO8X2SHc(wrNKcK>qd| z!BGbpkkH-2ZEB8XHgKmn_vLrZE(dmVKFZO8ITj<_LXQZp@%R+-NAWi&e;dhqZZ_%i z-b3;#JW2Lo2pPzTCtIf$lIV~k(xqNOvX?(1%%mG6La2dMWc85LWv2;~VnKXV#i-q% zE~0<+0%0G`r*aF;sZ{qE8u*W)D}L+Hu*nwm@;4W{RN0pfKJ%k~$|vYy!)TiLE{r<2 z1kv}0uhNRJ%QSk05v>iFNJYGg=|l0G)L`9FdeU(ZJ)ajumt}<0^F8)dsdEYSE1OTd z{O0mpy;(HHdM3Rvvw)me)S~KdXHv)5qoh%ol178yWL17Rx5{FaTYA%#2<0R7e%mp<0Qx|z6s03*1)|VKVfok z3gqd@<4|4=Y+9&|COe*kXPP-`dTQbA_bzx=Y9HDugko9CL+t8^MNfApc3^res)~2x zik+eO{M0_Y*|iOg`!q4dVhQfJ`2rjjwQ)&84G03{abP$N!kmABNc$St{^u6tJywH} z`eHcHDg{k_Dd1&V&)n*<0~L?|n1mxL;BlncjD35Zx!8ECuA_7rquMx~3*PBzGIicz zZjQoE&hq0&Zi~tnPB%q{?Ch9E)?8Xcs!rOFQ})h8x+k0LITB3n@6;y-<}M^_#dq#t z+A*&0@NeSAYr?s^j}ooknN;6io7zVd607?&>5D6~=-#-wH1^ee+PQEkEehOBBY3`t z$hjc;!FDfgUVD@V-@QRqJd0^S`2?C{p+MtwFVOAppHbW&MIVjbN%u7#qrI7_^x-?6 zeP9PPq+$~N^$6(1)!V4!{I%39`ZO_kHit^MTT(PGAxe_Jh{#XgZ*LJyrd^3Awk?Uo zI(j-0_KqXP4%f)q#8ae}Mv-E_W8{<4Rw)C5sC5J$cLSgcdgEYAa*HHQYQ9U*CJONf_?B|WV%mp0GHCAXv3(y_MNX!>2CNoJF2p$kxj zXg~UQX)1lfYj!bOe9tBnMYFYgsWHz^nsQx%?(&eKQc3l+PyY|~SK$NB7rV%k-l??U zoCke+U@N`w%!>9*SwZ{8N6|O&0^0n&fj${^pgAeWsfAb+O}tx96ckSo%ZxI@rPm1h zDl2#|RXd5>@r=tqJj#WCJVvGtnGntTspRb;EiyFJ$C>*IxEcN@xU7BITt$4nE_{cMlWvyGd%u5x`1Gx?@!ShpM48CdXF^Vj3ne z9pVklz@3EJMO%?+2rCg7o%vDclzoc(J`%!xxXE#l@RVCIX9iK2%de9hkWF`^Nod+% zZtbo|+*q&sM9!ds9I+@9^j^M13dY8e(0AqJ;7)$#Vsf0WneITv6&k4(aI|1O&#ZG> zPUF?B>C*vc`eyD|B44+RXn$eWL5>miN*$Kw>6U#80&Kl5+ieHkdBH zr%YbgDHCU3EuxnxM@%O0zKdE%s(+VfAMqKKHn)4cM7RC70$nkebHT?T|VV z@3_w$TK$oWGWX@qJZR&ZZe12Qe<|Uh3@F&behNB(7SCEx&&30~HCJL1fA;&fi(pCeT>Lk&8mjqsNCF0f z-}2AU>BmB)doyg(e9go<=7HLeG{$M(e)uMnY_`~C5!~~iB`}Uwg0F&L!MyE+3Gm*` zJ=orFR_s{K6>cjNtY37SdluuvEf^i(dQNt5XG+z{>}qAAX`n)^pPCY>sq4tWQ>#g2 z#1=A>IYmN3`?%!KYsrgzb)Hw_NS@plMawaI%;!l_B=21Z=~GOjqfI;;k7rheSl^?O z4{K$HZnp~Jk&2O!vN$z6x zGq0%D*YKL&949%Ny6YIZcH0e4CpP2s+8@v-oI`45CXsK$0&clwBFStEV^)(b&u}quA3kI~k83(6{;Neus{C3_1``Tro?|L%Gj`Hhja}GA# zMz~vE1HbIuVDpP!Fg$w-7UW5yn^+;3yQ!kA<8AoB8R9v~FR<6&5)Y-wV|kA&O3pV$ zzuF+&uN{U7vy(9P%^P%6Nk^IezU+mRbUb|P4PG=)Lg&X}7;6-aV}?xdK))*vev!xD zoA~`={RQ5a8sg_Yci^P1DrO{|h0sDt9I#G-lx0sL=cgNplWG{5Xb#^e6#;o+2I-b5 z;L$AwGe;fZ-CmCA)Ey6pfA}+!$uF31&r}#6p*2jKT&*C?YrKj3*HJ;A%z93*mvFOu z?s8g3r;#|`mpHQEDfeo36Uq2(M)N}L;ZM#1YPMr=f94l)x6jKz*x2VK?||^T<>-C=u>Md z?IBcu_xt$z%+wOzXMUR=4`-yKGqcHzkHYfcI$llmQ&PGg)H;x&i&lfek z^icrTZfD4$*l(1ZdxI)zcfso7w@lG(o)dQz=qlb%8QIfDevM5b9|zrt^3x6EQR8fK z>w_qf8h*l+Xo?XRsSM8U(sgcf)}X*HF`uiMr(x!8VTobp|5okjw|hxAg)1n|6bGv)|~h#(WaZuYaSc z8aDeIqDG)L<7=73Egp-+DI%S_Ru+X9cK*b*x8K5i0ihCMw<+DvKVLoN`p?gV>XScP z(^ZP5^z7&`t-PdCFEyi)8YvCX{9eg=qfH@v;%qandYg~e`p03E$VLZ7P{)!Xp zeZpBM`E$2N=N8O8I z_u(xNA$ko;*Z6~nLLYQBBtd${S1A8i1|yzwxV5wa+SY5}H`^DmU&$C*z8A0Qu*Pd^ zC*Zm{p15=mV%P03+?yAM*10KY@~t0r60`8-!$WLXK^A_m>_@2&DcI*3h9!T)P>=WS z%~bV7;s`tAX#%P;lvz!?t~8FqAt4LM2IX!}2Lyp6CZR zPV#lgZwILPSp?F%OrcLT4q}ZpKwf+kEDU|a><^j(q2BpSwptBy;MEvr)J2`ybF`pt z$M9ZGB%_Oarh15>5y<1f#vKYOK_o;{UK|7Hr+8}ZyihMPhM&)mR+)5Y1MA_Hug&inP{ z2>t6GO7r{mQC(hm$zKNuav**8kbSXRjna5Y>=jYQNo~zIy*pp4k+NhPzeR zU%QXu-GpG;cH|0mXb+`cJYR{v2Ix|n&YHdK#IlJGNw=O9Idg=7SI{Z&TuT_c(LF?Q z>ojsFV+zy_|75nS&NBYx8&Jc(9Uof|d~$LTCz@9Nhi<`Ix$QmApV|9B#6?Aa+79ot-!C;-cEONr@ zP~CbmzAM~?5(nmDT5Kqe3~fN!$avJ+6pK1t`AAoO#nHpXc+)nNU4DT7^!SS3x1Yg3 zgE5#_6OXU*?a;9`6fc|TxI7h%`tQ7IP`H)cY zPOR6N&NfY31lO4qI+o|Mip{!8tpbB-tjkfHm88N-t`9}I1tZi=$D#gd{V*MC^nv(g zL{iIA-FjVJp8wwDL=%G4s3tEY>$9=c(bxtrsl;KX6raO=x~%@KvtoS&@6Xh|UqT|? zvuWAnm-KP*HY$4R57bPZgT3AvxL}nko2nd((mNN@1>dBI^86cc_h>5E%fxYChBWBF zO(|-dCr9SnDUneVXL3zw7TLP#1*ahv$-PPO<@Cd53Pgnl1(y8X_vYLT(~ZH+%;;fJ z<|t8uRYofqnq>uL-MNfjZ7?*I3^A?TS*VSl0Um!TsK#uEX8XS|xGoJEROL`j?K0%I zYv97y?eOH_e4M=WJuum;vG}JNR*Jgg)74vXS9An2f4eYqe;igt{Kt;Gh(nFTU3e`g z9Cv(lMwf} z5N2?g+dtNav-|m++h?Os9FA-zq*0#a1u77K$FbDzfCg22nZSLkP66lB*J11w1+w04 zA(h<`i<#3^*_%xnxcB00oYnjXL{qlWMYCSgKPg!}r>2D14Y<&qHH!5Qr!TL+bt8?E z*RdEWyB7UzV(Gh(5^AkJiK@JRL!LSEdd-umjGLSqdo@&_-5hZRc31Ii1>#0$ZSA20 z4O;bs)3oZj;&_@~bPU5XB-uIjHK?C_3Vos)aet$<8IMMyZ!XTDoh^^J$jixG?&2^; zd4n}sleC1G*-3!)mak0Ay>W~}a~X4TizHMtrr@yC4C+q%fsHtShf1fxOC}7$5~|_; zjbwNj+5x}K3t`5rpFIDl8tmTvKUZ@N6!xlO(B@uvRG@)Axj*4wrWuY&9f#jMm!rUO z5=uup;Yxlz_U#S8Q)A5VZeTQqByLA(%QUQ5mxRqe`XYZh=6C4C7C(Xnq*l9G2*{txMO0Exskm(G-?l} zFT`ZbPFOTz-?Vg0$gjcaZjx+4>BqEKTJ`)Y{9L8a z{`X9ko#H3W9OL;O%76OFPoFsY`uenb%gBc`aLFZn-8_{QQqI8Kzv5_jRt;yoOsAs_ z^7SG;CuuZ4fT>Gc$PSyIM=hQ^B;^%GTuS z6>?+>=S<|>RhY$(RhU%=w{i(8ZGsRc%xpv93TE6^5$2{*A)}kTgXvf=460G*nC%57 zP&mJXN&VmfrYd~a#xNCr>rR2lU4=je2*FeDL2to9I5@u#6t*Nnc7!aBG*`nw`y^~O zz6k{pdiZ_*Yp5w-jIULM(LmM_rH|<1hh6>{cgqV)s$-FoKJ=cNfunDN+1KV7nEI^` z_shrOzqNZ%{+mC3Uo{IQXE>pyi!dtnF2-MfUxCozT#PGdfiJ0Qc*UX$YM;yCkMbmt zochm;ur|NPs9rOM%=!z=(3)S&58(|= z`SG)iw(~e<@{vW1>mCm?38^cBX4_RsysLzW!~o8~=M0;qdh!zy3uz)NA~CjPhB?{3BA8D!Umk;ex#nX?f$c`iy^ZR8#p0eUcwh-|+olz4Yu~BAgA=3^%NU$6PS5tq0-$@oex{NdUJ&4m3QALGZj6 zg!Wv4lp~`MGP50|J>;;l;R#sqJ>`4mGc;~D#2E<^=oPvU_x6p$1NYWqL8S&h4syb; zetP(7mKO%TMLaY!9M#XQ#@c6b;9SD#Y6fo~Y2KhvMg*@Z2>G9KN#__pOt| z{PD(UvswbJ!woSs_%pCglhFO-6Yy$~$JX}yVA%8*tdg(5pj9vA^qqr1djUMUlnAH2 zvftFj7)1g~4dEHc%h76sd&##T4YFEDy z=`YrtmG^QiU+agJHyp^Qa5rt!<{M%pm4Sv~vRGrJ307kHsN0>6T%9{ES!Icm z4JY6TYgFgS8Nfq(2CDg7gJ(-0vqts<^LdOiBzHY$G*t}XvZpkNpV|x~SJmNxauf*F zvcUA@!@KQk;FWC+EW5W6Rs>v!+Ofef?Z#JlaxWXcr2m6-ml9BGoQz3R??d$XIT&B^ z8lqUlhy%mm(76_;&mV{5Iy_OZ$QXydhhexN7<+eRqTt;}Y}F{jf87ymrfw0Y4Sqxw zl8I3P!DzZE95-(@L7ktTxYBGK`oCO1Fml^(eLCAeAJnad4~D;B}@Zq zUB=;~q;$M_`7O!KmZ`V+_mHl-u$eSX@xwIv<(P9~5vQ?xh$KE6Lnr0%Ok`(OTFvK^ z8}o{(yi;iX5pGAl$hmxaT5^jZRw@_desv|3!0 z3B@Zd2_{+Nsdugl1szh^m(M=QWbg(V`=i^vs^op8Q7octD9 zMM940laRns!R0?)@ayDG)ap>eU-wvee!9ZsW|If`a7moYojL*CZ0ng8;Vv#lF9^bp zZUE6A$;@Mw9e_6iAS-SaM7X(tS?xv$sof9H?yQCKu1FZYyctfWrb24qPB_GSF}&`b zfQApZ!D(JB{7icRm-pntoby7cr&|qjFXZuC_YJ82Gamn%+=u7S^ille09=tZMPEf> z{Oe$a3uejU4QY4m+@p?@&K?9Tdw zI#tD3|2z)+{wCnp@9S|4KiiqCqJ^5@-LUktEH2`GoYJR+aoiD86kYKVp6To3pI3LF zX|FO)%(w+H6BKZS?*p1v2;q%I+2FbJC4Y8f;pB;Xuybib>DvXLFIR6Y#^QV!5x zehkL+*nrzJKe&2%6O`R{fjze@z~Ih7I5BxAh{z-|>({J@UkbsXC*H}~KdE6P94260 zs4#~T9z>yLQ{6L5a|moxLPM9Gc;!0JIS~#YuS>{S~Lhw#}z9|(Sm0*q#e(}G16XGaWe^HemdYx$7npdjraSuhvME{h>W-= zF578_VZ*%VhtGCwkJ*6#KofucGsimB@%TE6?=^=NaFX)^#zEn)C9?M5+r6@2@GymVkBCf2}v^~mc4Df zzp9YTw+ch&{jO*#vhS| zixERisA2=>khD*5MQI@sPvGa%;*qfHP8qDuP61IJNnFvvK~!fS=naUXpZr5m)yjY- zo9nRa(lv+>ltYzIBfO2`b!fwTki`2g=Gwf0oURslymknde7XbQyvO2<;Fs{ne*!x9 z55V*WExf$oFIcM0#yfhVD7(rOyB^A-=Lp~}kU=$JJA~7daE|qM+~z(D#~1oxyJxaW?{X}cC5Oth9|@};%t2x99oT-F-8`JUzp;t zu_Ac=#cW&{Jqj1+YN6NU572Hh0rxKCq3Vs2DDQa}*3Tb;?2;BRQGX5lD{jEo)H@(| zPylaNUV;DSmBY)2jbMG@I=s7+3Q5-=f^V-V&RzT*&Yl%OV4@^ml}Lpl{VN-oq2 z$1rbo%Ru44Bp7WnBwu;%#~R-(GI6#Sd8fCUi2rpat6ynwN@s(}tH-&f8?JUR??;`W zdiywVRgDGHU>*3}xDN(LZ-c$qUAULSb1K;JxZe6FluVz9<;qj>WVaeBiArLzi3Y~^ ztDtJr+ zL30f=OuxJngD)?|>3f4Q#KRt2CWNEuBRh0klYm{*-LY;)8fI4PM=6CoJjHAGVLByP z(G-tGr)%-bKp~cyHsCUiX0(&OgQkaM*ufs&qn8)R7XNES8G#HtuJbk)9c;#02j17= ze-2CgYSCp+0zMEa;rUBp_`B{j=8gKJ)$%laVCRM&Me(>fYXdIW!Pf>`?a?DG2w(Lr z#jEpo;mdnw=&HLNr*TH8f65lu{hWg}JmSnpVGjmJ0wu}n;JUrnTl6pC!)E}PguWN8MSII@T2~_aM`d8^pEU= z?)}>EXlV?HhKz-|3eIpq^B&VKKV-TzJcO9uo660xb0Xn+mZYl8gG8*#CKFdopvz_( zk)Va@aET2xePm)w6q=rqD=VYO+P9Hp_I*FHy*ZRzcwx?MnzN3uVxfWr zQwrJ~PeHcA0@yP<9D;MtKqAjA985R?C-sDIjnp&veOm^PbSvV?tjTz#QwG2I&A>4S zrlHMjE#4`hiwTc(u#py-Pe6}(Ph5Nj4PH85t?g+-T&u7u@ zzc?%isKH}b3sB3u84q?|#Vhw7;P*4q?8?AyY@wm-C%0}~nlH_sk9~l{;@9w4T{F(R zT!2aZ{eK}V4%ZH!#a$vH_}(@X-SWNgHUB-~&z{y(;ZXJr+hGCPh177kA!&S*E zQCxg4PJF?804lw4mf2$L+T)53_L?GhVKcT5o8WqPD|CEegy*i9qr2%`jBho-&gVKf zj=$H|+|t6+${Gk)XQ1>&p80i39Up`$V&D-ORGHTc5cu&`#IV2fOY;GOIS^1=bQOCsT9V4WX1xh0a+vB!wKR4;jEu$E|D zTWV_Gs3ka?VN2R?HiEh+2A5apqQ-++nDA>cj;ox5V}7qhg#|3W zes6&aBP%iMs0Hflt-`d8%kYSmJ!ZVI!2a(psCmW$V+056Z(LowBQxyq+syqr3vHTHK}juo1QScPqRqkW82dgPZ^<9UY{g_OPe{Z^*RnC9 z{4@@4IEyuJ3eo>#0p3!o!k>pq(O?Y4rnBXESg{GYOP8=`aw|G>wKymJA?B@bK%uT~ zbWLx;)6@I;_i{VdUU`FIY#V-4{)+2rx^TwOFVxTH^||Xp?4fy|&~lL|n-}>T$Hz#q zQxwHmt!NoGy;GXK7%#`Rp3-9N)1}y{vyQSSvZUB_PK$NC$nS>$X*Q#o|33{ec44j* ztG4S8>dg~nrQ|>1&v!y>(w{!GzxWH41zq@c!B_lmX&Y|R=*Rd~?HFp>kL@Kbc>8fT z{_$!+nNJToy#D$Y1Tg$jX z$v0fB!E%yw_&jk-@gWPRnUkOIB}j>d40&Mkz|3jACFd(44o&4l%$KkTsPc9Pq0Wnt zTN(p7|9CdF-y8UlFM==i#ZZS`h}U9fVbF*L9{J8<@?B@_<@b7co(F1m+M)6{540=Y zhL4ZC;x^71mu=jHYZJUM{N^TXeHehxM(t2w9gP0%YfzJ)t>hQjqwJkn{O_GRPS20Q zmzDu&qnVDkR!3vQP!?)yN20);pToV3#d9Z05jJMvAzrWh)OZd*6;fGn3aIG7I#Ef zir0C)wB9r2rencWWCa&GR+7p4EkRUJ1!z#-(LyFMBU-ieKMG{TYi~!cmge_1hB440zRNITxDX-B8NpD1zUH$JnKVHGFIvjT-=kRZ$UydPO(<)+(wh~{Jj%|w;KbHBCbNHPoY<$!kFvfA-t3;%Q1-#yK-T_O z6l-xKoc;Uw1gj?CwZE*^57Mqk{+-U>OyF0%AyqtE%W zb{SjPR3TTEwX|lh#5u8DO^ex(mDX&@ulcO$M{{;dm;rmn#F*V?Gm{mdE<3M#3VZI$ zB-T4#g*_rH&&CqgtG*aRn_STTt>=E7ooA#<8oPqeZn8+fpIKF1_W%_LT{-2`8mkKb_~O8P|Ty`O#2&AOhU3P^HYh0@j0Jx-Vzh1mE==8w@7{W1hU+Grx!Rf6 zFyy8kQk>*B28x^S$Ky{1S3` z^-5y?vY(TRD&wRA`TPG>Jz4fNgxsGUOnP1{Ae&mO$O_XC!TibJxm9`Fn8$a;n7(5O zm!mYn(L5VAS4Bf}&;!sp@)`6^#Lyv90*61DqI|Y78ntiby>Htw=|?zb+zZ6O)yeo{ zT_Oe&o^KU=8i~$X6lp8OUim^iW?F?Msik;mA;pK$gW{wP7 z;3>v#uasiL#{9v_Yem_FPaiSvlMpNUj;{|V{zey{E^Mg&iW4K+P$Q%t*9^4d`i1@Y zX5tMzrSb^%7d7DA8T>vNgGt*9eaju+&b(C2zNPU#`|HLw(SFQ`IWr$P)XDnz@9 zXL+4D4==sX#(i;#=xdXV&o%_%0={0N@vc}L>Vk_8^ZLOAQ~sSPfrBUbd;jbpOg!}f z#`Q+QWWyZz7&8@OCocu_t7DkX;{Omdrc-k|@vNF5ev2&ea|I%pLGf1Y7Y$a9mITQH>5@c`g?=He3O%Okv-t+X_!U z7@@jY8M7Z+p?9Uw2T7lV+|3bq#K0LNGQ^l`1L44m-P1-nwG;vA<%Sd=jX z_y1jj)5p)iv$PTuwoJvL+twpLeHL!2j>cFgckJ66hrP>|Bizfx>x*|_Z)g@8rfkFo zYYt=Psd&uPti-~vDVXMX8dVkZaJe3jmKK$G`SumOT+O3)X%o(Qa39AQwc@HQb@8W$ zr?}N1L_B#}Ctl807vIWi#fQ!JG4pm4O0V$fTXzLr^eS@7-0_)x3?_e?i5H%4K+)zY_}zRZ zepQ@-Bj+u}y)83v>t#>O_%Q`D=DFaz+!;6}#~B}vcETcAF+LS)*)M}fpxe_)ILSnq zKYz5stj8*Nw2v_kAMgNfhrES?1C0>8DGz?^wu79se6X815!6Q{!Qz`AIg^5R?(5Cl zeCX+yB2k~4K9(QSC^mAiOoG9q1l!xJMJ5#(g!yMNQvcgSZi6u`*!x5{eV2fr57V*wl7A@$k?YvOeI~GU$n~S>N z_F&sqU;MH<6Ju8dVeTY@xav!=y13J$8Ik{@3l4Pvo#f6UKC?kRuo2L(GCdBf2XF;Huvuyr<)Z z`91cyTHrrqe;bQ$W>3N(=}3GeKL+gvS>icUYs`3Uh8p8cv1Q9JG;mVEVUvw8Ws4S$ zO;thf@9hvGBZJS@?E;s_$Drv?35-mLhp8pwAmQ)<@Vlf4=WeTm?wM%G&SA-tHaRzz z^?WkFHYA2U+Pj`jtVm*w)k*BpfVgfR3J$tkV9pQJJrpx zs#}CBs)u6ini&`sH3Hig&qHVZvFP}GDZbOXj@c?jcuJVv>s+nCi%sd6_N5l@W|iR_`*SFMeF;bQHeqP7ig?=D z>!`44yV&p3b#&zg#^T(YXr0o4hJU1Ju)PegQY|JWrDLf{C0_k2^lw@Val`R2Jma$; z7uhYtA8XTa?CXVC{4N&n#dzYXSy4DRW(qDG7KAtRL>N76DH@oJ!OxcSaj4G-9NW(c zE1ZU*xLJh#7Yhu>zqUAPgc+*d7>d;b!#RJc2^zN!MCo~bRQzR%(>fKqO;9_%3DTczfQ98lAbW5M@PCzHZi^qZzis9o%NvT;>c+6f zAWc54uP^)k%7vXBv5NUl6If5D2T<+g@$7QG5iAsR&W-H7sI%ISECM-NTcS$|A2n&% z-e;_S_Hs61VK1|8_2G*IF7=@kKe$1A81LVbEKy%Lj-~G|+3?|Y0~CzB0CjpNA-8Wm99QSz>X{qRU;Zk{%e?}J)0aWp z{S(-$-GXitMRYp(9Q=ms;g{WSVZc#=lM~Pj{d}xZC9^L+m?Xlp$*LIt7jS2dG9D^e zfOB|5Y<3cAq()0j-4})%tnBeoRUA6=^RQ}bHu|dVK>eeoILWO9&ss7(=_n_bNo~Zc zuSw!{5si3!vYgn+l;Pd95?p6qhU=RmQEhEDn(EHO#SL+2OLnMyJ`4>ltgy4$AGx^( zNRt=hf^b!Q@eJ@usPJslcED1zet2xc2t2Lz2h!clap|7-u>YDKDo=k7yA~^=e9TSg zIMWRp!>_<%i!OLD`3mS7--Lf-B#_v19)2mDf?*jK;LhAe@Tv(?GPsS2fGUdqU ztlmJ2LgaL5a*v4qz3`(s9)Co0J{|=e5`ZUkW6>s}l-1t6&h$b?vtf4^v(*)foZsC_ zXu2ye><>(!V#-l?U)Bo!_4;E?=Rh>n=;SopiulP>O<_nyIxI~+2D_(@MLFeKc%5|& z(p9uDchS8W_JyB|JF z8Gh7F!QPzTD)E$=&nX$_v=86JZO)BxvTJ&j2J(0F}Qd8Oo64d z6GNVPp;lKWHqjcPA9fID3AOdj_@kJ&IvYKH*W%!=YINhTU{acr*hr=xSHFrA2btI7 z4&c-2WN71__4v%^s#Q!SR;GK(^Sfb#KJ<^@1JZdHeeT~6& zkwWB>W)NQ26XBolftVubhDJSEfWD<>$Q_x4rAoT!d21X7EES&Hmo3oecVF~*qKz+{ zeuGV$3TDpu4oo`{gZM|M3)x>W{)y*IxKDO$%dZABP`WHNa>r z3hopT5>=rm%nZg|%ICY@zTift4Zw&H1#A*#KXW)C^Pzk&m>MauCpuNos-w=`?e=4B z-)z_u)thXsz`k%5)~9Vg0T>|6?_FN*#S_`b*pYt4{B2DaSnMNaPH!EVZsbbPUK-1u zTy;oD8qGAFn;Pc_llzuMgjUydJ6@A@1SHRz7(| z6l}3u0O6X++)%k7*#9;hG)}v~_#S^aw!#BcJww6ey*pfYO@v7cykOzi{cs>-C3IL+ zL$~Q}2pe@17MShw|9yAs??&} zz~5-6P$|spQZc499bfj&!>IXjg7?A#r>6zt$9<|8IB^!fTPcHQKiZ>g!aFFNV1s$l zw_uTy9yY(a3bBEjcp$PGA{+j}u`dPCfATZP>PUeEsqj0ScZ0pagt_P*1WGgagIkI> zOc5o(?CbNv(Iyo16WpP6+DcfO?E-~45%A&ZGRToh;C4toV0Ks({8x5|UvRRHyON|Q zut8<`h~QQ1@Ql+nN1uuz@|ia7i(QEWtM9`^(KpF|ilbR*@k}8O%j>qV)J+&nSs;ojd7E=E5b=L>>eJ3f!?+_cvl3f zH#?$aXdFg9nuoR*_h9DNKtaQI5S4bu;q*%t$hH?_nF)_`r}Y#6$~=SC>K3AqBxP3vY5c-$!x-L)0+xDS+qcFA;zjh4rv=koOuDARWY#o>s*!WJ0{*?OM z%m=$+_7c}R7dIOyd04d4G$hQ`*#&@AZnR_i9i zhieGUD~dq0V>Yb6T?1+9^Pxf12%&>_falQ9Fnw+wsEz&u9;++CZod(#2Hu9i94q|U z^%6#<0A4Nm1s#G`XWTtiOx?H!-~F`1ny^^h`8W#Y-U?b&{g0?FQ;rAQ-V{7)ec$6v6=Lqg8cBZ{exm&P`4mRg_q&Som)VX?#*e=24}zafZM-kxE@ z=4+DI<+b=&!AiX0v9{RbyAp{MMAH1~YDyO`#3#;cxVxok`?!c&R8hGbRJ8X|Pz%6~cp}A@d4%jV^)BNr5Q?fdqGI7Joyuldrb1`QA z9EGlxtMQ{IqV?c7l>IpqZ-2?e(!Ze?GQ0?_zvtrpw9{zOw~u&i#VL%RnjrSNdkR;4 zkP+*6pT?<`x%g7H2ybl;!-X4#^vP1J^Htx|I~kgg6_E1vlE&HR*0yGhXrcs z@Som6Xu~iV1lyrUeU89}83GqNzjE>-7f^dS2r|{y^GUM9z}=?}4C*yuqSR7iv8a?q z$fT2P#By$(*FwDXpqg~QIY`f}P^N%A+TyKlF&WD1ov2)>PVgs)c0BPvH6UgTz?}vawM`0X=UUpjA^g zbzjwx#@^mdcMsQKOUVRrqEZzG?LSCItz(HU#KEEa0kn95jx^uzvcTntrY&LiRNB*@ z!BthZx-oz~S02K_R1+@g%v`?y+Y!kitq@V7qZ4QNM1ec;wVc~!AH)q^BLj}IN4ctL z6JV3*0oTwH2s2*ygYnJ#Ak}}#L`LMgA}e(C9s1gh&63%@Xwq*bojW2w52?oqB0C{pn?J# z$>d-i4HEBgIDtk%EoAk~RGN~TN~_#nv${X7)IC6si@BmNR`RtGKQKKHBZE`PyLcIS z8$TkOR&(hqdvoc;hiPP6m592F`iuXbszD_Kp@-dk9+hsZiE92bx|lqa?yI-(!qw!h zRiZebXEWI<|4HoDn7-iN{ezS1W5k*IS8%ri=wa8xJ6i+dLy zMeX`u=%HPSd!J_F8J+zolkJZ&=Xc^E=75u5M&O!0Lhf~cUktD{M7a&quv|kO-S%_1 ze)JDm;xQO^zj*>b_vz!7hKtbjS^*?GyPFYXPrMosfB5%26bh{~q zrX8;bU2_}p=MVZ~cZ+_U%h_8SR2ooiY$d(-NgBQV(y$vHLCarlj_tBK4T7v$RSLwzGH-b(>Ty z|151Bye~<)Q&LqfCq@@Mf6nCGV&_82&O)yBU7$UIF zeY^I|6~}JK=d;bO$DNj>Y&9??LgN3%<%UKzE}+6rB?KaW?U|=+Fa9Gf%}a zhc=0)3db9M=>b{>#AC)AKin+TYL3D_^SaS2Y@zoMVq=R*am^6FK+uaioCSxE!a8#| z8wNa-!Sb<7!Rs2q7uDe~NV^0c=Dy^@=fuG3$;I3hbyxU1awhk9feuW&smM8WT;?`! zh_+vuW5liTc_`7IF-@Xp70x$h#qnY9JNT3|OSX$V*f~>8rm|X%Rn8UI^9E+5yKxsk zZ(tUfzPk)spOsn1#0kV7-Gj3J`r?Mj9K1Sp1YQm5g-s{@Y4}XRFWE1j>bWX*+sB{Q zSJI7aeX`^iQ%|_6wDqBtw7o8Y+Q+sTa`S`@AJzd*Ow$IutrE97_!{1e+D;*p!1@v>qo(-o_+ zN2LsMJG0>p)^ctGLRi1M=4{smLpaj#iz_m46`8qPat&{CISt+ST#Cgj?!?oI?zdPp4qr$t_?ia}9#FCxc$wTSy#K1ey;O&?)#Fcr~eGesw!+ z-eQT5hRfho9S#qkQ4#9SxtQi{ga#}KlhbT4NT~ns{1wNgBO3~PreHR@RCeICTIig@=|5o}cc0A=AjA?omT@G=R5O@~gy_;VZJ`iOn- zeUca|J-s0Rhb}BzVF3w7@41JoK5>eIA74m`;@Y(gMF~bfxM*h+==;Q!9oxEr{Zzlm zeYk%Bct0LCo2#&d=`CzlK{Ik1L&W;>j?j5|FRfCDrH1l)T4NVQMmM*k*lVCzy&@Wa z{`*17;a<{Q!B;&r^&K1Poj||at)zPdL-*6%#q{Kw3oX5(Bz<_oR(jxN14RzYr-Jn| z()FJWr1@ckXw&4YxbZ(ZaqOUs8Z?5vB|0J63I~3E{mLUw-Jygy=4V^!0>F0vk z)ICJXesUIMaQi$vV64mhwQSh5$A=_0K5yaYY&K-$kM(D>PTb&Qmha?4mzMGF*&2Mo zwotw%=7&RHcV+&MVj`!x_pXEM&kx*^wVxbz|1g4am-RUBtG=MCTg2^a$bxYeeFP8o zF}S?j0xkr!gHeY+d~FxjhH7E`S69bBo)vI-mKhp(-+~MU#N*{Ec-3bCTJ>|opl(R|kt<875$?+YKByIs#=Z=i;R3g{UCtZdY=MlSkiy>>=h@-c||Yg4ED@ zYX;o*`~jWo{b8s^JLF8TgtnK*py!MXJW@LV<{NUj$QEDN)jN>OsWpUJT?5gr`VU-Q z$bARu~<>)glM&Ph0sEv%&m2wL<>Ik3D?)-&_3TTm9Mmh#J>Ez~f z)aU72DrxO6?aC88m37eJC#PSn$8U>jdEyq&IE6TD{zd} zkbJDN^k0!Km27EXCh~hJSHXdL(ty2HN@8^?I&Am-XZ&2F$9&AoHGJgP7j>`q8*pWB zKZ@ST)^Q7ianx<}aL$BS;r-RISDXo`5fO<8bte zSPVRwg-hX;z}U@4g*#is{gv|ZZg&@cQ_8}Cq*#17IS$v%M;w*E3eOJJMWv3}cuw%! zkJYuq%xzD=?Y${>_7bRUQpbk@$Km3|ParoY3$6)SY~jrtA+qoS$T$j|kaLv~_rd{= z>ZQP@6J`)QW-;{UzT@0EL*Or!a3{A_bCM@NMIVL@<|s=^g3jxC8U0p%MbbkrO%Gm(1-Y|pnNtH&Sl0q=ntC8P4u2d8S8>c z>s~4IJEo4-!hCb!%SG6^Hkw{)D@s4vy`#;R=PBysK&f2icN$(cfn=0j$!wa8Q~<$A zjh9r?qj^nqX(UnKIj=!0^#>Z+CZW!de16h0MhOn>cfAbC3w<3Wy*#n(wN?_ydo7koq``GoD{*2}=V&%(inD6d>%w$go zugUtbZDX_fk5?}6cNV^sWNDP~&ppgUV|@a6_4?i1;!Wui{}rlW=T$56?XrUn{_&!o zdH#@|7|i89%7W-5IZ&uR2ZyBAu(|ySXkjGeAC(vMR(WuFmNEVn{5$D4>`{$ZMM@WR zB7eLvbVn499B>!U@7jr{j6%de+mmp-n!wx9j6#vXD3Ou%M7MNRv<(&Er0#1_yU++P zndiZdSqi9aw-r=pwgVq(4eUPxy4Dw(p6rL!A6IhT-vVIDg&a}jjB#*N+$n1MpbVqh z(6Vg7;?D6*6!6LWViTEOQEj9A^WeXEYi|PcY;- znce)|A7U0ZelK6WD+!lc{XoBpF0eeoX!Y49Ay2uIatw5(&M`95$;PfUN<5FwO_7uK z{jM)<_DP_#DQYw=I*En51>*BZ4LI$;Z=hOPz!HuaGuwi*yy2ltcEvD>J0$1DE)F`& z>h=(eldPlT>x$XPmW}8cyh!kni0IVQi!9o!gU={VVlN6!*!Cfpco~BRewns2e=7Eq zL@MDNKD?`txQue;8h$Bor&qUf!Aim0GkF8BNRV=o9-d&+pahG(_CoY5F+`jw1*=&A znv>7Kqljf7Z`T3x?)zY^;LA6%EP+h{`l#V?6XXg6?$E^l!2g{Sz8j;71$zQD(@HknnR=TfvW=!xD$v`TAJwB z^|@vN5GFythC0%h(44Hg>@A69F(g{514<1C;rf!nqcNa+U>p zno96i%Lh(IOE|7g7k-V4;1+9kax1qgaa(j|bCS=6lI?q&97f$x?`$Xn^SdBcx6X;%-zVyfjIcb;SJW>-_yk-s` z1=g~zH1~5Pm7LXQA%}GsbAPF~yIMCEl-wVjtY1zJHQP&8C=k$nK|S zZ||`ZgPm-ffjOsTDd>qpJlXq~#x%0kjUDxU#LLKLvE3W|1Wx`e_BUZZOKuy-hD^|7 zTWj@L<8KpY*nF2a2!6@`w=#^672V@?Jewr)EywvSi{6U9Ob~L%R@QRk`sPU#c4_^I;=em)ezLHkS4*QJZw*i{5Ce&=#`hJ{03%x=-_K?v4g)g(TnG~n`l zPEz^0nydOTpSRI$6Xj7kZM*{%2;}O&5Oov(V}IM8m!pUlwEPu6=r(+%=7dPk{;5P4zT_xCDhY-{s7LOY5TJhbNJy`bLU!2%i;FmPC;=vv9xO!U%>WmCR)e>uA zpSKW;F3Di2{Wv`Mv=J6J8DPXTp^iJKfQd!XP+##L4mo>)gMI`2u6KaRN@Wn5s0sh2 z$3dEGH)qwe2&5x3IkOHO@V3GYzGHhbe_EFK zRk9s?;Tcmlx6p`PO`O1bWp}dLVSo6wBU*S#*Cf`3& z3g4nkOt`NTypB+A?Gsx6RN$^^2GPulM4GT^AMI`wGR}VhnaCQG?Gqrw=q051#Ep(! z&1VtYN6`KEs^a^{xmUW?M_X^LZnmTJ_waKqikr+`N%1 z%AdiDGv09WZIKeW*ACEARV*?aych!CsB+tzlObqN0jKPE7J612fx($}fUY?L2fHsq zWIkkinBhwQKQPdM!&Q+}aMU_?yzO=lW9pYL&Wt6o9Lmy_sZ!xQqe zJ3_3#7KLpeKuTZA$t&^-&0HBtNAuRw%MoNf%M~pUJf6mMy*Z9Z6*`Ea_sb zkZ&Aa!g|+R(8S>337&^&47%8Yn`Kfk(0#F3%_;>O`?X@JpbLz5U5E2W z`lGg?IhNE;!r4DNffsyIMd8)Z`=2^4__q%l2mgWr6I{XN!Zi>-G=j`yWuTU;0w?nJ z!Oe5^Tpv4kSnU_WH5Ctlxm=sbPSVJEUtC@{(8`&M+ik#~a=TDtl$yZ%zl`K(jrqVI zbm`_b>{OU%!(dk5K8IcA<_bPYKlbnTLH2oIG?VRb!#?i_WFz~?un8BFcx(904A%0@ zH)#j+e>t2KSDRDif?{@kjwR_^52wDhBZ%FwrHf{h>D}fsl7g6t}HmO4jRGbdME_6$6VK)ys?h^jVE`H{X2hJ0C7zA!$v! z%ez-!7C9T_^MjmsbITtF@QVeXsin;hN$Vs#P+B@nH2>fRFq}GxTfX5i6qfgcg@f(@ zxXuK@6b%NtDFS;>7ZbYPLuSCr1gL+ff$ao8dZ+wUljiWU45=WB5YKNAP{ zaYUEoF}T0u9rP^F!L!fyfP^GDqHWsu${8dySD_Y2F&7S?Ai#6JTE!GVG->A zmndP*N-$xSFYkXRlRHpV!e8w^DXRE>pIQ}{ZZ z={#G*()HG}mtiq1F*1#9pfZ+pyqx99o@W2~-e)K8Hww9l=UGgBJA3dUpM88Yi`ks& zM`53ZT;1JAn4iOV8YN^Fd0EKP&fgpbkR`RdPNTP?rF8V+dWuL2rLs-CsP}skxju;{ zNuMZse!ZUV%GA*3)-kkIX)0Y3*HLThDKaqGLT495k?P=R;yh!i=5GLnoDCrH_4zc$ z1gJ}FPXFB=Mb|n$GXIA|$>EGKy~sPtruj58^@_f<%P5BJjz7vIl3Q$mS_K<0w44RZ zt7b`R>8we}&Cng3%FOITnbD8Ag3r^Q-5+AZx7l_-Nr@shqMRt zXa-#2{DYF)HRrgR;7<<8*LC2X$|unY??s>+l*ZM&r9gDZ2hP3w6#QGK1+v4MVZ8kc zm_Oz(@ScajTF(%3+wQ>r-f?KHr6>4i<_kW$095H&k3A{ZG3jR<#tfb-&$)&vX<8*qhNpUyeME&ZkiMKx)bjB#(u$6%U zt(C1zeZdAEY+#mg=h^A#b8LoA88f(*F5KVAtnPRm+wyoOvwGpl3>F!&ZGFbG)Ge_%5ml z+za;k&0M(g1%V$s5$5N8h5eUzgSny>j_!B{Glq`Dmo_5Y?mJbe?M`6+q&e8@Jzo6D zWezrtK8eo+9kt$X2b{ZlBzELHgWJuTc;Z_!R7$=df_f-jc#9j`l?g$2|B1%^ zTMMbRmmIDLP2L-U%Xl-n6CC(e@^`b8IGJ}hc%$!6Yh%Z}<3scH_DBH($h54*Ae~yJ8JI@{!G_YG6UNDqbUbAli$g$ zv}Ug-Z8h+vtse6!UB#K2`dd>3Od!h$RdU)gng$NAqKlm}G-iqlb%mRe_nC+6Rm*EO zZQ@sUs!t1R%)QB!Cf#K_YECkJ*Hf%T{sfcTyO*W?*ul=u*uc)ztYS|0N3nXF(QM78 zkGzes21|H3pLeuB&2J7=cF0wX<6EY9a;k3Q_~jFib9V-vk|>!Bgk=#5BDEHGFd37= zeU@ZF*_j5eSo?SE(v1WxH!OVWcIu^Ggip>}AWf3`5tY`djc3kl!3og9P-VM9S zVnbWlAgvx|C3?et-+IW-Y%r&q@v8LOqYquPwx*8m(X?QZ3dy=pp#DRxsm<7#KJ?6^ ze-XYk@szKL%umu!6O4|urQz#oueq@ zKricA-rXS|Y{c_FBbOSoIiFWK3} znZ1+Y=lUYVzFBW?*C!S-OOJ5-o|nVB1QV!odI+1`m&1vteR0W`CRnh381e%h@Ux@f z_x^Gc^@82;lHX{t?*?}~mUI#e`UrlJC-xX#ITQ@;K+x?S;43(#i8GWhu+Yi>(D`ZtU8q)hS214CEi9XfMB8^X@$llzIZgG>y z$I6eyCV^B@8A2W>R#16!EamQxqNm4_X^m4njXYUK?uJRDN?|+dtG!@X) zqF5T|n?Q-7o5H)b}G_G~9%J{4bhc+VY2AdEU^$ETx(+HgV!ETwcyEm$Y+R z&OMXZ)eMBq3ndP3iv7UnrZ(5$u^+-;^nsr2IvDG_2#){w4(#$ixH?Ya^_R~fMi_L`SHG>09|7qJ%~g4v~>04D30!r}(*W7*aRnU+gATM>4d?VZ%X ziqo#JQ-x2Ni}n*Cg!?_aA!H=Wj{V7=yL@78)q4y?ElI-u!og?e$3oHx3kPaPgsgiBb$4JXSa?Y zXVV4W_w{wVSpDe?R-YKm7M_V^?U%+fUe}uy+kNJ_d&nM*iR6o&wb@g%W0Ltd-}3J! zs&Q+>&+}J13%O+Z2tG_x8=50tN=nZyf$z1xoRK&kMqX3|A0g8=(aRftalLR)(Eq>k zGe^7bu{heo2`@f8iks>^QGcDK_>TkW0DInox}1iwhWS|jMZ>jhV3FDT5v z2FJek7y38p(BZq5lm1%*n+-Z8x@LM{vve&V28X#5lN$N*i}IZNi#L246-(~qsk3v} zffPrnl7E;oZA~6aUo}k0&%%NpS_3tn6#6((LQdfi zS1LQ{M79qFy#}tNlUW-{bI&RY$&aJ*tVkNOBbiQxCQzVJKAHEuK(ZO()U3UQZZw~# zC(>+6j7_5T1<6z|zl|oZiX*aJP7l2{QfPAkohl5Zi|d@I)Y^^YpWBeem8q1eWl#US z&1uL&BTBQgAT5(&Ers?-1CnXt@+x?j}zWK~Xvf{i5qbnoyp z&j)ba>TJGp^m*>?7E8YFlo_0v=^?rPClrPbD&uqnE=0waQQ*3@70TKYARt{C-&nl_ z6CgF9}ad>%67w8UF5wyAq zz$ZV1y-P*{9n6O_hswAsTf-oF=rW0x>=4j6X3WQT)pB-9`}yuYLJl_E<)=pU)@jV@ z%NAY==gUWoWY30u;AbqK$+B`sutm4Fv0~GuEUI)DEBO%3ROT149qUS&ZR}C@bfJXl zl{K>*k7j24?moM3*3C3CKQV{ye{A`yzEtX`NaH`qlbLXy&D4$PzLEufaCe}OQ>Ibk zhtYJ#buRttbfSO~AYqSlaHN*SeRR>WYUIyZ20NFA{bT1-)wx$b-dib^OhX^`rR!$TpkY9sn56}pHIX6H}jx5 zuLmBkybAt%%<%1!5%_t+H2eak_}bDFW4@b+=MC`0pMOg5`TD6?KV&3kZ#BbN3$FuE`BKGcbWC&1{U{YGS^xp>r{-G zXI2j1bYL>;>)ykU%L`^hY;2guvN-mgU(NI>kDc3=&X)hKV9^^7vDP@Ay@|faa?)=x z(Y{AaW9E0(E$r)h4S%sef0amXyc)&d)1|;Ix}=k4O#cnlr_-l}IzD(Lr9E??&v%?? z+dOx=!5y2GD;6bNeEuoZg zQ^}@w9;LpXL5{!?E*(M7GB_&wqepx5j40uhF1@SKrH(^tlo6*)df$Gr@`b-xq1{h5 zC%KbVyli8Y*RC+{Sq)6hvY5Sjbb{Ts%3?d$9b#|f*0ZfAlG&__9J?R7mSwek=NpeA z`!X?^H=LrwmRve$|6pz#FYz$v?k6Sjw|9T!=AGWeS8Lfp*NbV~<)=F!e3CBYv`C;& z;d)s2_#ezIz6Cqp55dyL(YWP|3s$w3Vy~+oKKg7ZKJnHE+xwRZ-ZvK%j_ej9>& zX5NOWXJs&FY$!y?OWi=k*-E zDQL6Fe!KW?Lyo=P`-u;I8^Zo*a7;QWncZQ*EO$;JGnt*mp8h()_`DL9eDNZSoOzw; zOnSf)Ejw6P*AMp1}4T$l}MpXm0n%drRZ6@LVw1HYL5&gV{4AOn?{iQCpti&{lyG-B4eQuW&&@(;iCz-jER3N61JdaD(o{P1ESqvf z0&CwVmb%}>(=x}4LgxJevdqb#DXHm{JTuWlQEJ6A4WR)6PXhsb!1_IW9G(IR!eTuCGo}NB%Ks&mR{3 zR)#jKbu<6D&zQ=-HfH2{lih7O&K5r8*;_83UGzM{L?zMeX-pQYOrOg-+66yhuo~kd zr?bBGHGIfXeb%YfBRSk6!xC(}>a?P7^Bcw4oKeFw{@Xi6=nUT^(%tU{+f(0h4(4T` znCuRF$GroW@wKog)ezr`EYVrX33Z1SW9!WToKQ7TJgz$cwg~TGz8_h2ung5X{$bA+^&yilY7}uphnfS7XpOrGc_s~|5VG8+55=#2`BBV?vVA*0gTaQ2Kpk8iiCkP;8zv zy?HX5#G}0F#aCBy|G0|o>jY4!WHXH(zMfP@#nZmHP1N6XH?5?2I=MZAX6Wptl;~_) zd$pe0CnnOfA4zoSobW#A0GU6_qNRd9NTiZTp4NLP`RX=$IxC5~bs}h1&vqIivyu!y zgwXGXd35i)4@D}>r8QsXka>-m&JJ;=7oSGZx(DMZVU;BX1Q zx;nBSMe6jS8xDP_ZPj;Xp8cIA&brTR*LN`guZ?V*$vyT`{|M{Jl`_3^sZ3qHoXy@f zhizx^%qH869kg4(?5i&EGi^PXLPsnAOW&9s*g8eDYW8CGD_WfkHuvJ)6W?)KBM)&h z)3-v7=5#35Y=?(xr$AXc5I48{-`SUkQ}u@JYBD4$QYj=FP>K?<*D_R!sE~>#WQ-&! zMaVqQL*{v&XJWtimXro*Fr;}NH2akXX*loiI^Xx5?>pzev-hXU518hK{3)lSqCSZSK}8ZRgIA}^cG;B8_(1LfJ$zPaqn)_P{N?g3+FZpe6D z^kw9osyJ>6A2<=To>2dCIw$|J1&MTAq1C=Xipb5n!~I}lPTHa_aEGq!B(6;h$o<2i zWX|4g@yWU2j|23h%D3sm%{3-B0M>(F|tA#UuPI%hZ~D5Jri`D--^`-_ru6_JKPqz zL$%8hM`HX?QhEYI8sV^dbqc`)F}TkQMYKsWjvqgRCsk=E<~t7&?+BKSE(kbf zK(i+e<3?xjp*jL~o$+XS<_{nJXn3#k#fGjx=+m6=Wu-Ut-K`;;<_Oi6-SBtVfrZ=F z!C}8KdZktIeJKZV-O8BxUKxUVOW`6Qhw1#{c=S^ce$4_nMd$uUTEEFMs_ovpx>o))9MxkfmSTckM&UR#-;lt(clnaF%}^k_x+C7GYPChsWLHR0rpf8oJ0U$m z7qVlUD7#z*<)IviK3;)CYgMpuk|M@_F2nfDxwvtJ{y%Sp@Wz1;`rHYyIr)<$-J^Y7 zahueAd_b1ETqIK#-Y2uBHjraB7s$5ZSYkG_k<5{_CdYiz$w`g1wBTW&qd^JB~J!c_jF&Adnpe@4$JY_;cx|tt{=_F%l}%Xg_}5Ll-wlN^e_3!y0Ef6z0#v1;H8cmzaS>m8qz|D~!B`k3{ax zXEHnTG4U4|B?VfyNVM8360X}wT)z#H#68sG*SeFqdq)td6RA`~6iO0Ct`osep`^TT z1-X4ko>U(ag6;8*L@>pgtUBw#UA0NNZP+ndYjiw{Q|4RAxEh^d&Ylop2g60!52DI! z|21FM;h&A{sgyppb95uCnCiz`$uDQ8{19Q!t`lURsHZW?5!KAZ{wR)nN{QB<=6P-S z z8eS1yxkq&UkBn^xEHnrg<*qO zFkEC3;V~GBM7I=tag9aP>kK&5S>T5SV4rf89G8=8D(ooYH zg`U7fOqC5r_lr1G)du39S0VW2;|WPaKP(w`Lh=)uKbYHLWRD$GV(DkGw7|JR9V}DQ z$Dg#7nDApYEKbNnqXUlUI{t!dBd-b=)vEhEoY z-6e94z2w%m59G+hM}$@QN)Au@Om4}E!fJ;oME}mh(Z{nO?zI4Xid|kE?qM|xjA7BBs8dJQuV}~D8DUS2Q4Z%fD5SH-6S8q=UwS~a(ULekW zi^J#H!BDSG#GjEU)Oe-gOiVT!jkAz=r5mdBnh*Yxg0Zic5qd5U2V@Iz@na5t2qr-| zJp(eP(TL%uVdjM>cwUHy@mL5>wnSjbL~p#E?T;zS9&pg`#MxigSi05)F199k^1=*F z4%*O}LξRSDtV`Eq)}~yH-4gH5{MbwTTW>UAO%m@g9W}y4`JEpp ze_b4RzH|piZ(b*vI7t(qA{b!k89|^96(|{^!cdi91xeA`M*J698 zHl!(6Zo#u{`1IHr$+Sjmbon^6y^o?M${#1ay|5`N0_)y|VB!6EjEzO1Ss|V3Xrj?Q zEfb53ld#)37a4yFVDdK)0*kMpY~2}jil-o2<_bQDAoR`x9C;#*X|p77{LmcCwx0yAt%A6c|AQDS@x!-rl(asb z43+$wMD^1{qF-f2ly1e7Z?|ueI0+fpS#Y4$8j5tLHZkxmAU46`iO{!u1IYV6K*!9d)Nk=y9=yb;T##-dfisG6GrZU5C zQyIBQi(xh|8Htxeai~}e&%33e(Rq`sn~_MK#n_P#MfZt8Z7;dnF&WD(zLM7M{E(#B zOZ1YW6Dr^n)PdVBEy|zTi4#M{ zI22)zD_xct-{*{sZc z+M~6w?S&e&|5*mX!6mq#LHUGQ(%Ah=2(7l$Agwz_rYldv=%zO$;N~xq%2txj<{a{1 z)<4{#De>gm<$7ZJa1XKDwVQE&`j+#L$3+r$5ajw{!t`c$GfG*5+*9fSm~~=dJ6~@) z)zH%3#oTtzQ~p-wCc|Zp3V&zBW-k_XhIicO3aN?I*X-KARZ(){sh(&FM(jsGE9yxN;{BoF7>bU!8;z?*dE`&%__EV!XXu2D$Ua z_?6iMeW7%C#biL}?NywARES-fB{+Sk5W!)o_8@?2cdnTzQi0uUD##-W3si0x~N`(D0J{F7gjn|uKz#My`hbwqKkWKI)x zV_y4Q9qQld?#b@e8)S+X&xX-CBg~Z(BAw1N$nmL5@KUB5{xVdPw6K7UiZ*9=Mb2So z4y&?Ja|GF4JH*+!d4HHK9@1?55-nEt{u*JNqD@fPL>%Or~w?#^Bo}FgYSj`2GnQjw!Y0Yd^^tW za+<>SFYj)jbkiPPvqhRVOaYJK%#aodI#q~ zbNNEVELaTTm&M>vvxWHtZ(N-2 z1A~Y#j6R@q{q|V28AU>2VIqb;M?+kKVs(LW*nBkyFOH>R&H6$F6_jB8y+VwAyo!>X z3>cozz_gb);NV|LwY4?yk*k8t+B}>-S&kD%>3CjUg70ycP8wTWyiQObF^e~)FCMz#3z?_pdgS@%@pgXd`IV`Z!X&xW zFeV$Ln5Q2G7?E@VTF0fmNXCrW=mk^Q%N7D`U*{0x{56{CdRoJ@YNs$;-%ww%0X}Ze zm_F5>$T2t8CE(k^S-j(a2wWx9k>Y-b#F)3?XMiy8&Xr3@C@v;uHgs5{53Ugk8HFZfUK+@^_2T`9KwAqskD^3+c~va5`Ea#+il~*=&sA z!v;`m-i47f7RXaMgfb5YDAE`!|A{AEUxQ#F5{NZVA|NOqjm+j`R9=loc2_3k15;7E zIv@Yg9R9vT34E5PL#(_U)~0!QWLpKTvo*9|c_q|>Z$QB%1KS*O!7IOkq+jLO<6jLn zy&MK+Ik+}fj4tt1Os9SFvNxpR^6_k1?@vKb2i5$l#31!eEE-;gqMgPVc@vMr;za-~ zsQ*TNrYoc`T4KUl8#KQ)#jH!FFt*aeicRa_qOXnV2dPF=Ukxs6mxJr2Oz|Ko6!K)z za$F2gbLK(NbQ)&d5yLs*esbpI06AdBB_fl561(fsulif7a^13%s}TMqKG zN2X0?KMiqM+3Azn^CtZ4$%J=I6ZHpuXL_AcYpP*}N)wnrVvc0W-fU9HC(7o$w&7ep zX@}2k(VFjMV%dtg0rt$iX69IWD^944LdEI`e1k)Y=%Z-jdu)u*5+CVvy-l_{4v?UC zVi?{ej0(&7*di;2g`%>UVJ?Mz@078RqYPI8HORNEh5lU@tGBGfnlwEOtDA!TW`cZI z8+dtI;?HAO?D05&AJYPnu73Iux$^JgKCdgD2SM{&aNk{vm#q!hqgsnUjzx%{QjIj4 z^G|qDffY;hQDIa9r}y-@C-ZQ-I2CGNbxJ&v$5C?I_$*O+HFZK)3eeMwquf^gRO^8h2fZeTzIILv?n|0S|04&*lS@$8k%#-MD!~Zm!fs(T)B}oO zs9uZhZVfmmREv0vK3pBmMVoU04qFf5$NoCljy6LtupXwz%V3pPhuzVIc$Q7afL{?B zCCV^1tpGLy1?aexfzz?M_=hhEysatlq1gbG=#3Ry0wL4og2ecvu$@Bt^Zljc zV9FlQegW9cPr0F@TQU4%Gionw#6m>_%;{W7p%euKRrZ5E8(KN3M^>ObNY!I z=5JJl#zI59Mi}ypQt@ZvSyGn7Wkzh18Q)iOBxsc`vpbM_P?e^$wT?juncIpkgI7dr z(k0G72*cD&&)_b;p2vO1@n8(?Lm7iznasK+x2czC4|DC-eD>C%9!5p$HuFw6lj=Bw znN>q>Oto|tw{~qhce%U{v;0^$XVsUNWaMiLt&Id>>dQ2CZdwLAUsjjtxw(Ycy-Q{u zgcD}poEDNGm5RQ!Fi6@P!qh_%b=%kAOp-L7majylx)Qcr)k9UtYBX(Nu=Dp?G<`9k z9H~vP@!5>I@mn$T<2JmxY60<86fgC4#q8O3aQu1{US5ab>llcZuv55nDGI)GLm{ys zh343akmSq7vhA6e7({)urWBy;T`2Ct5n^Qwp9m1YD!tiwsy zA=HHxpsJ_{xq^fE%HN1xH(F3~z7bAx6}UC6kshlA!It%SA5;oA$tv_8DTR)13Hq9f zaEY5sYreTCt4)L9!&nS$O@>WH7&InDLt4ZapJ@JH)8&SI$y2z)y5hB(BX|$?BKbrcKo*`kZ=cB%>3tW4v>Na(s3 z;b+EP66qt!8WiYihE6R))Y=pry1$(5ZEt0_x#nQ!6;0l~lxe)F>t54->*8$dmKU7( zUu%d!-x6lZS5-#Afx|eJg)`Bf(aiYCYs{K^QMC6&1Y?)0!*sT)Fyg*S%q|%@=UovlP_g9ov@!^YB)dXezUtdXuqiZ4p)k%+>rvaih)it;z8yMgYRhR`Hez(NU zr~46F>WFLVuJBxW3bJ}f==bf9>Q6pcMX#;OurNGaoD9y=I0!jV+=b@l3zG8?wWb(m z;U$Q8RtoLT$JQuNR@AbL+8K71*{+NKN)>QHYU z^%ywcjmQ0|p{S2D>lzJ#=$)spl!7Q zer_tT{j7?eO7c*peA74Ar(ymPA$&OYf%4cVV%_UvqNSKa@MJf1XuhhV8NqSvTFH)T zZ{Q^U48T}XAnsfqXL3vgwWS9u*r9#X;6LLI1gn>!^rI7-?J|eGosrTmb3c}Oyd#}C zrkTT>?9F0UC)YFnd+#uBPt`N#KG{rlRW@^2DV@popay-xjqOGwbJ%4*POQThB{)49 zLbReVvX@t~!3~1itC#;~mWTymcXj|)U?WFumMR-EKZ3)~j)1evZfG3NC5N*M$+Dh_ zcy9KQD4Y<)hNcGDeQ_H;k>&NcOskc`*AZKU#LemuOA7b#puZ^!r*uxE-5sk@l^{H=Qm@1 zVkP7qn$c8VipbJNkoYPXB-da_ycT_~zeygn&ZNL(eA|>ne$qXGfQ?O^nY^UJnuX-F4~}X zOBkD2pssD#BCefUvX6ZFw&2Jy=3n_ac!$$b?p+rFt%~* z0QAMqBltWQhF%-MCN1UIN>e?J%}spLIEQWPO<9D{;e79!IiYO z-;B?nTJY{&BYv;#hs4|>I+ltsQL!I$OY7kl+>8_cO_&*42~qPVY+F-`7O^ITy{SOv z(kAc@RN%mP4Mqb>VeL|i*lh*K@6ACvUkU>3QehPy0aiN(W!*9=4!3^G+Q8V1SQl2B@027W4nn#hi;ua2;NT^@+5m@qH=c zZ!6;4d23v2YRB^SR_y%M!d+pN$Th5V#nFp_s8aDq;;U!yrFDrRPAhwTyOMSy&x7?_ zvjkO%U)nD-cNlk#Z=B-5{mgnfQO5d47*nUrGR|W`OpE9-#%p2-6ZcYw(a;KKwvLH1 zPQo@!@}w`Ew=eH7TJ?Y1`4gAI-qV9E->al;v9gstJQ$A%&*xaT%pZ|4e+}Wv#-`*B&izShbs}o zmLa^N1j^U5VL6b8uZ4-w>`Z}RU=$jk#l!h&2v#Lheu(60q)7*%Z@M2YpYVd#iKCF2 z=Zcm=cW4+pAn~X@RC9MiK6XEj4jAC{1Y?-$QO)P5K5XolV{$ozHiK~L6%~Y@NNEyT zy^bUXm6D(_6?RIQAG>J{TNB$*fvo3{MUnSaes$mgRgR+TDSkzHV z=ZyxKd~AgFhYD=ww7@H*3VTmC!;a4DF&7#T`|Jj?ObZaYrvPgmu0v0w7J7*dNE@od z3DI(tPHDj2qC(7j-iX3SB~TD=Kxcq2F9>*H!%5+onxtH!n@)9&E}c>e_o~ZFeJH_0NaQQ#D%CvPLh*9M7Ar za6j4%<{`FtCua%QtBzPVasVE*4jX;X5hlBw@vP$r4*fU*b7^1Pi#&+~eUSRh0*+ae?Go`D55SarMQf8H1DA6@{~$s?r(ta(-M5pXo6H} zA-3;szz%VW*+|#p?BfR93!*uE({(5o6yQR47Cy#agGE>^zPxJ2OgeVb&s8DoZ4)+7 z94O;e1D2^Z;SPTjVm39wC#Vs}uh*k#v<|-?RO6phl{oBNhzErwXlhQ!YN2fK*Ce9( zXBs97$3t#@BJ7JIapYYT;zh#IUl)R?;Q(-({h--@8mWHA@kh)P68r~|NaNaP8}?I- zhw{qzQ(aN9J087Rhnurl2yZfml)V@lUw&c2%){A7dkR^>hRdvkSs)wlVuQ1L@5Ad< zIu71xBA!{FiF=M6&aDn2Re=KJjMXn%r%btHvo3O1L~h~! ze*T$T^z;Td>4`asw+JMaP75-|dC~SE`#} zJz=!h6Xo+yBUiu=Z`%T3_LA}QJ8it5^E|Fv3wt$JMX3;Xnzu{zGq?0hIANq zQLLu05DD8VQAN-HYu6#tpECKYFAq@(|REM(^<;oSNNyeo*tM*Cod9tlNZlg1cfnEAM$!78zcaFQtnlTeBX%vYgw6w7yi&7+%|{RXIb{h} z>=?Fjtk86P8cCJzX7ktmV8!(;S#47pWGIZmIbMusF~=VL+Vj~HG0-SK42B)s-z*@PYQSeG^SSY$26n;iZFY9=yxU2DmPb^c%v z_FQDw(B7kuj_-%^a}VTf+>P$}9uVJWkC<3nc>7pj)e%SB&9s7ajWay9JD~h8)ja69 z!E(b<>`Oa|S-w8dw+X<^1Ue={La}&7Ff^V=gZv4H$nqqLDaYX&9pCLY&*0qFT+FI1 z#NlaN1R9gQ%`uf?3g8r)N@g9}~TeO5K&j!XlXGc7P2Y{EwIW>^R`f&6a3 zv&cF~YgD6PO$p>}%JKJp4s?PGsIDg+g{w1hzBmzQbyARcAr8|MC93U!Rcr zqx(qsC+cIdZV4%Vx|!_g6(S*vbcxu?sbo&<7V^$WfoSgwAcM}kNsaasGN*AC=jMBD z!Z+Llz3D}8SEgQ}VgjtCJk>CK*}&slv75I{a6WG*6M)_2Zy52D_N6b-@y`h8~Eor2)xVEnQVz@L7qcN+-Cpj-s5r-kDge_1P> zF_9^$G#CAjiFSNE^`%0*X{)VXA#%|fW34#VI|e+1kNhhnTh&J_eASKJ$$ z6Z~Q5c>>x~{a_Yy3Ma#YaOq4S0z-mumd^RHxuFo{i^N`fzW%1iVhgRs8&Muj^U_4v zsb}IzNGev$&cpHIY&<$j@s*4+#7b14m)2)Ck5uBcP9@~_RzXp)9O1L8A^Rr}yqX#a zB<3RGcokUlG7NYX!2U`H!XpwOtdjtzLLQDRFNVd0Mrc`;!o{!(RW7yA@~S{3(*)U( zI>bC}#*XMF3@vO%_U|UhWHh3l-W>{Rp)_8O`Cls0V_1k^wZ#}8&W4n9E>bAx=ZQcj zl7r9C`ra9QtWJXB>O@?zO2G59IJlILaluHW=SS!*J)ZKfGgu z@J-JZq5@7>5k?(nQ|FQAZqq4V(2EZ-@9-`o8>h>aC>+SOePv{S{i*r@C! z%;1hUTKiNsGD@#k67RZM@SN&QJk&$Tp}Yg6x0v#&-e{9&hfk4D^VX9Y6nhU+I6y}3 z1d|yv4iespS!nT5C6AL0n04#~&N)Vr5v|N*Qy(tS4%1wzojoazy`cD(So>x}I^i8S zdA$%;n~u!1*(7-NT~c<|5wq#%Zt4xfno@tttq8-7@L7gO5WjMB?Z(6FSq%*=Wu(X~J2S_Q2Q)M9616>9gDqJ24y1$2v%ew+uV z#R+hzh{lpnpxlj0NM)8nYI`G^kClO;+oO4v&>d`s)bvK!&^doIJ+5DK3(j3`h8ed7 zC%-pCBfSBZhSe~PtN{~H0?%J%Snweqm7@hzQ)6qTpM;<0A$FqM5WgXZtGv92{Dc;~XJnHJ zGopxo*D3OLLm-ix;zHW0?8xTf!^HX*?Je`$i{#dYlUVC4(z=F&S@U{{`alVHz;+vI zMJk}WY>aS=>sbCLlG-WWCah&%C``OBqToRXox{s<$t;xe&it`T)eGWu9-pCl8re$z zXdm{%VTuLZ@;D7!ixBiB20^JM6n?Vdxb!I&zcS+BNqJJ=RFjc?i1MgTq(Le#13IC( zxN$Zc?KB6ELq5KJD8n4b5@8C|7y;cC_ zyH!{bS&YwawK)8&2DuTm#@AX;HFk|ybgUk+^!V;tb=YH6ix$laXs@Zn0@EU-#usBI z$wBU;Je1teLU&gVb_r%8Ei3~)v=%(FJ{>P#P*3lx=@@p+gy4;IEYr$Ht#dA(`IKVP zKq*ct*FZkL5vG?rQU3ih-pqUqaqC}r{X&9wxDBLw@yhfAc^yQIXChupj!mf`-7BnMH*pU3o_$2>*K`wKiE~7&sgi7M%Ond^ zDo9LH2l2Xhk#gxDQID9}ILzCRpxtF8Gk|*9yb$9FnbG5&^WycH46PNN#>bX=f%siE1FGbNk}5y4!@#AIPxTOJPS7RS%7&v zi}9tb5{k6O|7d`Vkn6$NsThkuTOO8;a*@Q(gLpL$A)_78ecy!~oQHzVHSAZofzSmv z@u#m3^WyvPP^lkp+6GX+aR^agZsBCjb!4~PK#f2TnhSdHjDBASCtpEZXE)UnUdE!u z7a>NvaI3BpTDG0o$?L-X@159WdjXzT&Y`FJJWj0Y!nl4n%u>3r>kwa_U%pI|=T)gRxnsCFvPO-+RN@?r(rr>wubK8X2zBeUB zUse-85dqGYB|)r_W(qrMnT#{p4!od}5Z=V|vb^U9C>EPdF_SqWyh*WBc~{?lr)w!6 z@43?yUb@ytjPa;%p->g|xG2XfnHDT@YeVtz4&0RKq8iz5JTUFTRlf7MWpNJC%P!!h z!g-90bfN!jCvM*EgkC}ye9|srYvU!Tm3PB>*A-|CUq+$cHR^?P6>*I>aPZT098m59 z6MGvOYJ;>s*H3wNeF*0E;nC;~^vu11jqcarKer1?gI!Q(Ixshqhs^iD4IUR0I(WEn zJq9^(QqWL}#m zn4o!V0VMw9f^Qb(e_YGJ`u!RBZJQ4LkI7hMa0Ywmdah>{kM#vnSTG?9s)i9rUJ(Y5 z4IyA{gHS~I9_qE;_~PM-_N8tpTJ8b|?ftmxY63=cJx;KzkZv&>+jh=`+d5&CNGM=U zmnp8(x`%L$2HMRw9CoRpi74a37E#6Au#@9*R3g960d-O(O-B8>I*vttuFYnU7R zq?vCmbG0&<$?di4>$rhRZ@FqRk)-0&Tkf?B&D^*S#rFHWK+P>T<};ZkRm`IzTb4N! z#>y@~-hN?e0Z!e%4?Yf!@tcQ9xY#^&TkS)b_-6QyYCx!T3nWs^a7a=Cd!mGC&0Gxk zUeCt0sjH|j;d+eS-G*4peQ?ZhhQSUu^yeK%V30TZc>!Q7gTVJW1a`-$$JML|@Li@_ zn7$}<<;3H9OFRt6Q(){>Sq^dWe5- ibU%TAw-JB-{mI8Sfj&%(rtts!`y_h*_x1n#wf_t6b=$<+*&$>KjNlJ-{$;yiTKd+%;N@BrrAxpv&Lnki^ zP6#%PiHvfz8!R^O|9GWJgd{8zt}+ZEB?PeBJUWrEI7gYf4wOmE8!JR7kQWlNEGjBABq4lRbiBNK+@jF9&_(v)Aqn!a zvU*;zatX2WUalqH8DFFPD!b>SFxD-z$9M?q%J64K*0uc2QpQM&2+o;1cjZT=^lc;6`U*E7m^m@Qo(M~Zd7IFA58(`zeJ%5nFt(>`o zOKNZCLQC7Ys8i3mqc>l2?ka6uZt6L1_8sCT;4(KOuZg>TrIB0V+`u&z@8d3Zt>=!8 zi{m2qn{rnFKC_-3z3h8kI}7v8V|$AAMGJlMSntS(tZK_oc1`jN`z3A0rHIFI-jy4; zlO6jwzsd$q^s~Q&AuM^Z=5IQr~Bf#@VsZG1*7tG2sY&{ z!PaF_c-DmaG4T!)hk1*6y7(VCu;&Fu^W_|R>{lT8_@Sly{7iU64Y!bdcXrXx? zlaRM>9=an7AnGW<@})V@3Mc0JCiJu%{puuQp_0nnEg?Vv^93vl#6&m!a&}62utK#l5SJusCOlaTlgQ z_klL_Z;wH$++ZlH{-dMquW0{;o76ZtkFMO)q>;n#@Gas4XvF;8Y~N^8Zts#T&j0yt z?&|t0+?|A1oNnbO?vuwCZi2yCZoAbkZbVcSmnwCF#E@DG;-}8r$2eZ$5SD5D1%go|*Dx2mg&vjpR)%&K^g7r}OAv+ijYt@`hwC|D)5F2cho%SiCCJ!e9L< z=v!`yb+I!L6*w2at;4bPa5S<9FTsPyNeJ;vh0%%(jN6ckR?j@NKhHOn9c1AUdZQd1W~`bGZQCUkdQ!{9O1n7~rYK zEG$E{-!uUziS4pC5)iyqlMxd6Y(-<0p`EHN()3PyxT-ere9lF_zQ#&4ZAlMx7G=|S_lx`*mE(MGPBq_U zc7`1?e#-h4YFR_m5s}lw4V2PwfM#^+imnHZ<6QMkIg6qxoV4m-&SYc(3m<673^^rU z8&~Pa&-vK!T_3mSY2&_{5$K||dEwXYm;V%|*1m3zXy$_;H}-SKvh7f$H=;BY`F zbhTqqd>{@twglo<&rp4HXunESIyYrB<*bA4o&xgZ<#qih`gPwhBu~ipYj+_mu9QICX&uf|DdeW?#oD=TSYW*geFY)tF!aSJ@zt2FX$yTP10@v;Ej?K`hV!)xRxKvU+#qQ`p@tv zmu!Ce_%7;yEJXhrYmA(r5BoA(Y?ND#cwIltF%LoU%L@FM*M_IIdl4v6hKz4TD2XnC z!^c95S(^>tV`r$(MP?A#FhA&en?VF8TxoH`)(BUO zQ8&YX``qzgzZ24K+G6DGSqL_p3Gw&N5XD&nHG23`VTE3`>6r9w66R)qp(BHwQ5_@9 z@n};_NYTI$_tnU4E`Tn)v0=U^3g+e@cvKp;8=7FRgDX@i2l9K;VJ9~Q6)q8|T$PID z?@|yO5eUuU2^je`8NSmJ@meJk(ORKs@r}ZpEs@wA;0pVd3vj{T7gr43aZ+y@qP%7! zVeecRPjtuBxgul)*}>P>3B{g{Q2#IkhxJ{B`S->%VU1)(&Be8Lcf7wBh+mS~IPm=l z27X9J`|JR`xt@n6$u^WHKS1>NcL-l6(_A)qRI}@7m*&f_eVQEu44a?MeuB@@5%4hk zOZ}PyvFK14d}|EgTX~nB`@Sdsq&m#kXV6W#cK-I=-~5@~6|^JpAD__qgpZ$=NiSs7 zg*Ew>JhktTxuOAXPAEg}d3l^a{Ete)Bk=XyW3Xhy=5IrNo9)_XHD^hTYSxUBX-+x) z4(grvaUih`Nx~j{5)^iRQx;V4s8!#EGZ1YgUPN?4vxYY*+@9pg+ik$ z0_RK;QJ|QNOi?@@R|cSRO$r*`rQ+v;NUS(wjEb}A$QhT50d_9WMK-jp00q!@hA+K?19&Yv2cz}z}287kR&f`yx;@< z7qhY8k1J+%xMJHHSA2Tofn|2?xVz99o)M;K(wdI5mJ6Y@tQbjElkuc&2%fhs#J53L z@#91XHXrl^sc43U~n@>@rlP~}KKTp~&?474@Ah+X6 zA|o>iT73NhEp|1aX%$9%lE+H^aZL_QJDAEhH<|LY{~6PGjdmK$_4A`Q4HL<3cA~5< zUsN@C(!l+*`2S8dlEL&`tcx7jT$#<9YnxcJ=WzArIyu>9b=Pm0!yZCi^D3xjAzpkN zjBC@3Fl%2iB<_X4X15(&4w@mu#2L?o8eeTHtOH@6;D#&IqFo_ZIUBR0=U_#N7d~8C zf>A%?F>Xr?R_$;{#pg)OkzWC=))ZX$9fPgyxtJwYfKxlO(Xu!PyBdUfxKNI&mFr+r zmxh{wWq9|r0wwG6kh;Vl6P|6xuZ_ohtavX{5umi8HHFH zUX2}ZuR=$+8MP7?7}B+gC3J{$g>6rn#glH!K?f#anTrCuu=NBh>n#^O+1o&oA16}M znzxNNR(@*=^{b(hkrMn0IlD#^wTYxP>IA*7+bFv7r-At=%d*Q0Cg95BGnSpPk6F6D z1ov*kW~N_i4yFEPT>W?z14FA3c|+)@d?v;V&o z8Xu`Z`;Rg%UDd}}B~uLTu*9WRmQW)LX!@C9j)FD*%dyA45PO8?%*NryK+O2%5W*-5BtG2HyK-Yr9jzuIWBC7!@ud-XdhUBu_w}@8XAVPdO0YvTaPEr zY1m^s9p<|;G4APRd^nSi;5p9Nt5bjuVZPVy_QQ5BU*yGRp!a$p-p%_$$`_3>TyH*P zuGtB5dW0%(js;g~iM3ifxV+;7*^X4j{3UuAu}uSZgMU(AqawrYGMsp}0SI`fNk&-T)&ac`*N(M|f)eVZ11J3~Ds$0<4MEVb=yqHhz4w(Y8= z4FN?oZk92*eVI&=w<741%|Tk^dYA1vHO0%1T*vFp*XCc0w_;Cy4zteBWz5U( zo2C4*ccROyeYn6oo!p9H;4~?b+i^9I$={H{q3wU!>bdS*$>M%CC-h;H)IupXys4Zi z>|4gndvEfmKSv7>xPqCdOS1({eAAS>{p^gV2RHQGKW5b=gTSL%tZqyqXR8M8S-=Z! zrj#$Y^8FjpvhO%au0Bg;vA5`xMk_74@tS_^_)WG?f70C4 zdh&Yljn1S?ApX_>1eJG?;Z8-okspdV2c+;T{3l(m9f`^(dRXMAiVGd@X}YyGp6n1| z+K#apk#~evblKr>w?CfN7~_cXXZpN62qTAN;$@I8OvMD;6<3UNx6^R*t|8tfMqpoL zzOd)gFzUcq#Qq6IzfkMe*d#+^g$p8TW3kpe191_n@ixK_i76o{SB}Paw+QUqZH1Lr z12DMAADKHmk*9BjQ_sBNX6AdC`NXO0+s<`AsAqLX*61Gh7QL1#nCJV7pYc17-WE*3q~D#i(OZiGQg`w_ zwijvo>^i>hKP{Tu_=42m8>8S&KDF=r%J=Z&NvPfN60?wDITiZ{ znV@*KH9qD}$LTl*U0+M6iy7m_e-;Anw?=x14dUwDP%uy6@GjnXIe#*I?g+g6kw5s@ z0Ho+xBjR-g!VRMFCm{sig}Ny1unO-CGY}y3TZPD37%lKM`FTR$77xSr;IU|kN<+k@ zJghzwfjtgJczQhzCauLtShyHgYGD}it^}Ub3ed26I!2vLg|GA`+!((es^yk=Ws-|+ zM(d$pumQ@07vrFJF6IOlV|r{MEMLSyTr36A)3acDISrk|yij&F9^Vz?G0|xWhLqc5 z>0$woG)19DF9iy2I$C7Y|l?J;bY5B1$BrVj#lWan;EYnLopIdA2S=UyU{0b6+UG&zbaZKKP> z#$(}_5(<0ZL$^-bc7-u+!QZ_s=XSrHfuub;1)sK%0_7OVr*@g4vhp00k=3Hv2Ygj^&DWd!5#{F z4ApJU_*Ci%^9~Pei3`FFJ_@UpBH{abIx;sbLB^YSd{s+8MxQ6Xl%`?6@Vp1jOvW$e zIE$#AsafChn&=)9W^+PoIiRRj)H zn2f?GUzGSPfyeDQ=(|Uv;rvvL9qfm+YF|`HxgcFy8@aRGkT=>Lz2mH7`=SA3YEkL5m=1i+ zVbVtond9^6w58&4Q)s0jGk&DS7GI8|()U_S^^GB$V0pc14YQ%!>c;HnoNOlcY6-QS z-w&~;4@74ZH*@JG_qfoo+gwvKPiE^I;W2bQ7UtDc#n)5p?7i)lKZhs6_s%UeUR;Z> z@mJ|UVI>8B2|~=Ondokrf@^Zh@VYOH^J_G){)i4@&yL6G{&8@5HWHCj^&z>}1R<@H zvBJp?$y)9>uk4PNA6oE{aKR!0vu=9liyukTFn(JEE_lX4-g*g6#rVMa-&(x&%*Bq5 zWGt4M1N(7nu(W0aKHNygEEjE*FU-Lq=`zHarr?Q+EjmXRqIX@HP}?Km^dTHY7uRFQ z%v6}$2>kG20M`9U!?VD}DEa-KZYs?}xyfRr$j(8Zr5b))*}`P77ZkJ@CWI(pTl^T@ zzom&ozMRQh}E?mvM zPKFukT$Q$n)03RQwYsIVriuxi|1J?{H(s5q)O*1;*t>A4?$O*}aXD`L&vi7zI)lry zddjVNeV2=F%H(>xEAi}=RP&mQ8o1nxZT!_+#nqXw=H7fUCC~GD815#Gjnj^aCfx90 z2U5<`Y`GRH+H{nTEX<*dU-L-r=tinaJwd-;9H8EnH8dc4Gu@ItF6fw`HNM9wNBs!} zhW;l1^>ZUPNaCg90F-o}rJT0W5M7#p&Eqt&GhP9qF$^YEUMTXngV%C( zRHV;Am-%8uJ#dD6^Lsj!9fmGJdmF0|fF=Q_$xqBE>oxS{Ll_wk-rcNg0@PNT@+VU9@QQ$JM2gAcuvh zKclL~M4z9dA>ZDhc@Em;qde8Dgx;7jdk55eZdYW~JkC03i}&mVYojWV@(K0xZX zXwZ~Hq7;GCO~15HY*=DWW%PDnSO~0XNzR>sBD|KkN&*9Jflo54!XdrLLIII{n z5mv&u!YkD9d6yQRtuun^d3}UTn*ay3v6!@L5G8XX7371`nz)S&&- z22DR)@N19UXVPti`fdW%N+UQK8Y6BW z2RV5Q=xm${Sr=9QRnlqx^?Oy+ehEjxwGj5&rJcLXWUYR*a@?rwSR4#AXx979nqPdq z45txhsD6Hk*G#tObwUksXImq^`!=0qa<9;%l4&TcN+-ilWMSfezUgMx091FiHf8>{ zweZ}QMdv(BU_AW_71uh@AGbytx_c5%%(CE5TsX|1=rMu$iOcwC!8JSXHE34mW00l6 zaqfn)R^0q{PA)Q(jkbt{$_Z7WE;R9beAIbroPuAY%rTMYuxO|;Ot%=ps%IKB_Ss;G zpyR0Pnqxt|Fh&(?gzdG)vS1^We4l~Y`+PCG$QNcG4Dnvv7vbX;Vc)+H=zh0COmiZ_ zBG5y21&#dqC_%>&@YqfPj}P?GL(9AZ>}@Q^q)TgIP%slc zlk(9#s1#}nQMjG91iKu|An`B{k?}JzS~C%i3yYyXCmRpyG*HvO3S-t3!a5}xJp~LI z=MykUG7G0qEkTd;bPTvKU#Jt|2opH`(qYVc(NJ7FoQaRQ&wZJuYE?JmC}-gRS2TocLFYZ`=H9{k@jcrWbv64*GNhkB z*?*BN%y#g4lGjAn@?Y>{(MiVbzM`(3mF!Y-3OhS=89g#gVk;f1*t!w3L~2gYX_IUx zZ*l38$l^#PU-jZ5jZ*l>e?PUKTzy@Uczz%1cStuUKWm2bG;#KK#8Gb49965IzW2GX zdur_KxMeshrGyaM?fesmnx@dSiMTaS6A!OyAV+o}-0cQp_`JbLI--t0k!m>Ctqkj4 zB_uu`4X*-CNE}l|<}*W7kqw;J+rURo6-8IAp_%N0?oV!*bx#}0PXpkn7Y+$QcWV}S z_=ln42pW-vYwF8koyKtXVKQW`3o%~sW1P> zeld3Ehaxtq1nz>S&f5z*Lh6PB7Cg2`YP3HR zd+neuDGkXh<4}Ld0QtklAdNds^#catp|Kn^sE3x#-%Tv(H=W+{hqf2Alkv3!bR?vK zoHiaIKk2O$qPCsl!gi9T!$B&o+(N;k3R-csf-FjJ&=#Fv^ejvT-t$LT2HoAwD+&MS zTGvJ#OFTv+DV=jn`pgyf9pT=63gl|bEvY6{7n52;Xixof-mCV2=zvBsld{NV;;ExJ zuEU8tTPn+mm5ygg@mie9ln5^Lg$Gxy*vXn^d2sW>Be;Eu+FZ$YP1g8HmP>GV;y&&j z#qCeYWu-o)Y;;wvXu{D~yk2)G$u;Ssw#|Z`qy%wJjz_t77e90R#%6GB^Ny4Ip^ebK zH4evXck&KxGM2fTDsVFWMf}n0v_EPy-I3Zv4(=7SIsOoBd$5z-epb`V2is`vnj_Ru z-9QPd`^XmeXwSgkbY8MgzzMr)ctsC6RLkPaBRK^1HPh2Qn#kU0fXeSq6A)Yb|54!`Cx@*Jc_pZK|kgPg?Ghb%HACO?23T%k6DPnoP*o7CD1*)2$n*h zUSnPWH=}fP>Db}f@KEfJ&PJqbB4)%&VPSCyzU@!J9OZCi+G#>kVm{XA#NgVS`LLpK zxLj`u>sMY_ymBVQ4h_P-HD*Y>ZHiA1CZgO|8u1E72>fJ-b7mSaRguRNJ$Z;pOJnqg z9!eM@j*o}Lp!8W1*(X2J{t*u;-Q*&zc<_l_&JMy1Ou(s4PQ3Tx4SexfMT|(?1kF#{ zc$xKpjf*Vj*kymtOUsjU{~}_&2PV;_TRN6s1pTC~pp}x>Y^C5bUDEn=m^nUC;N17* zvONXzH0gaN^L`+~h2`C2d-Hzq+7)-1U4K6s$h!wjuo2H~3aOq|a2g6BpHlpP(1Bcbz=MlrA!`tNf? zO}u`y7%_8`!3(&|Z=EDYO-RI++-$7r42AYtTdaPP2C<9+glr1NSN#R>N-99Ab2?5P znt?@;k;qpUxb(18tWz9{2mh90$B#^SXT{+&n}|VIBN6JGj4<19yy!H*kE@>G1kZV! zgg=_sjX~&7N1XSvM~l5FWEz#Q?2!e`LM-vMSq}$H)$q31i&6(5ce@gfSt{bQ zf->${$fMw6A1!%vkDf&SqU}r>Xa5+&Jarep?1?nLKJYDF|G6F?PlZF{d=?!q9>e~v zizH*Y`<9chN>ThhcfM%wK9;fZV^igd9dz^ib$VWLj9zZfWCg>znQhh(_Uc|Zbua7} z)y%uXs<5BM9e1YpulKNN$1bzj{eML7uSb!>)4{AKxr==-&169;$Ef|nHF_qwi^eN_ zZ+hmuhkdSa=P&Ea(Q)pDv}V*8rZhQ|RwzdxwRSz;d%ULLQ=LuU-tXf-8W_QK zuQJ9@_)d!t+#~1uKNNpP9;06>qqTY{c6?L9r_3>q(25?axbEhdllrII7Sv~YiLlX1FByg`&3f|R! z>ErQt^m^$@(wujX*4~gtj?@GorGgKav=Om$71Vqz0!R7w^l8y#HSR>>JytK}P3id-)IM zXJIS~e)XNb(~{v5Uk9@@ZteWsEE_iS<1aS+L=_Y7t)`s@uXvNg{i4$`ci6Z`YFxwp zNtD=Th{5YzF)6tKcjop{%i*C|Hb>x9qoUw2yn@_E+lpEaY~Y_LOn{fJ3j(tPD52j; z;Ae+(aiTI3Q3#YrG$RGw|ehI$&&UUgi zvxb+m;49ymis-^K)VNI#Gtz8u`>!^>w1WPg83eQTo_B$ zIYHdRL5bW531^NId%|}6%JZ#fhjONy%sAP=KIV1%Fne=y0(X3I0w?Dk$<5T);5PT~5&D5-N_djZS?m|E?)lAE3yIVwL&7#~!gq+RohS+C*-edN3EhVmw9t4!}P5 zRrps?h__zr$)?qWY7TVLsf~T4)Vk7Q$I)~)B;>Pbz<(Viv$Kel2T!JSA6d$F&LHEg zOlnjurS2j5w13k^a@ETw$3vC0Y0d#kJ9Lm%9K1>|bU%`P)@xF@Rz`#FKP2wsA6nWk zfyLq{Xqeq_=*-r{!MlT?Y;>B|6lh~toi%i4>!G*y3`z7(MNhC7Y{polK&69z2)=Rn z;24<7xM8)X8Ol7Dq3e1oCWU&UNzhnHCIh!y*TTis4XR`2WAKPnOo@)dNCR0+_g#RD z!gz!|@Wq)lX=HzP$Hcdb@SnaL_CA+`-9$4?u@SOH;tctrlCV892K-eWlnzxvnD|?o zxNMKHtG~BhsFmcd33Y8bfuM;1s+VCTZ*S@Bq!+WV?y|6x>gZzYk zIQ8IEY&;!F_4B&;mb_w`->D1o6hnB=bYhfFMZYpo7rPqEUav%B*K?XQ=rJ>VqslGo zmE)ScOIYu>g(Py_%N9soWL7W5n6{IzWy|JGOtfJr7dus!OM140ty7WXP8Sa4v{bgR zYl{>uuQCZXRQWPX4BE#=UkIiX$tBF{l05h0sR|dT`iLFueohg4Rw6>Q276>OQT8p9 za%}C%Vd6ik9;GW}3`%I`!pD4`R=Ci=jPd+YKc)Wyy%Ik{GuwYstoCb~nEjqcU;adX zHT`5X`5n!c7RT9}(pa%b0;iu1L&T5KDE~SF7V__?R^USW6m&3T>NwoxBvD`@g4cCN zEZ=C3%31Pgzv>3%8;kJB!X3}drJ;G&7Z2!`x&s!X^ru%Bj%Y8-Sd#uEIW9 zihusq{|vTOEubS(uwhZ3P0bj z$9}r!Duaq2;&^8Ji4rGDVTQtI@@o=mVdonXSCz!*iWBtj5>Fv2U+HbG5w89&C(l$5 zy0E#46n5z2t?L<@CpKTS=ARPF{P%_W^u;qqnrO?kzpI@sT3@rEuK(6YZKU zfmJ>OFi}w)YtAbnFj?>dvee;>*Hn`{44QeP@iR>ayB_?Z(W^`_u-YEWJWY{UAq&M* zE|?Y-iY2REk=i1OVdH(VW4XY~PWr>dRTA$iL$R+r5v77(n zIl^aJ60(mjg}RJ0#Jb#}b1(|CSI+~j{zk_I|G=+k32q;D!Q7(vRJqp{(ktemYM`J` zy!wyiO7w8J)&$3|YU6XvBl;oK{!R~N==J^~#aq|t`(hb9_aBIH34bUf=z+jty6A&l z2L-6!pz||cQ2MCTWO(l+vBD-Io!1m&nNQo|mr_Sm1qBuKQqoij8YrE?r{2k=zyB0b zUX)4cmpzzTV;k$ero#uwx{ERw`cu%uOrq^I{L8#uBI8OImWhdMceW$lmpsnCCf{a( ze=^w06Xh&se-G0L(&yMqEl%mL33sA*jZN=vAtT3RPRhmGtLlGV{cpsT5nxzIy?Uk@qySmS834}u@t zLhkzqN*v>Y31h?2?dl29?r&s~KNnNVqYxovByY6{`r7NIkT6+^>`n)G3pq7CwI#^U z3dF%_W?0)Z6(_d1L-Umg9la-L!BitOY1jkRT2N}+LQ56Zpk`|*_(-alyK*;eh?d6c z<4Vx$5XYv=Wu$EXmgc;DLzb7WQHFg9mF~YskAFU;4%Z9R-M*RJZtkafYqry+zP0J|ZZO2~X%NM0nL&!PN1&zV%z=W0jr8AvW@+OH~ zGFwGIua~fwr+4yVC!dPc>K^btFDj@tjcD1HVB&pvK4;-PKCkp7?RuKYHeb5WTAVMi zp*sh1hdC=wuRM-hnUTa@eYKa{(z${=IV7Ijp2%@eZ3c3!u@{)$r~6E|Ad@9qoTA|_ zk-WmM20l6=m>yVy3?5eq8QV7g$n8#1qC_3vKevSGmA8pz$0U-uSSPvNU(Wm04`SKP zr$vW@Gs(ade9zWg{`xa@%7{HoD;_41{kM3US(-svL+a?gK`~WsuOSWN{Zuq_D|Mf_ zNc}F)h^gGCB85`&D7j7_!e7&is<$-xWEqJyi(%(JCG-xEhJ@-adhV@?`{|RRe_s{* zH*KYaU@aU>wMV4?WE@Fspyw||=;z&WcEMDv-fxEHUxAn&v>03dnM3Zmg^*8(hFeNJ ztTqVv?(J+W6~#i-G9T|hLI??vyz5O`LU=&rEGe}BDU%8 zKK7}%gN^2fa-Xb*am#MzaE=$1xc22kxkd6F%(-e06FV8oyj;uJyzDtlQmUHV+E0mQ z?EfjcGJbedmSq+hAD&M8whyIr+sC}0?R{Dht<0xf?%^9h){@zD38elQNp-uwo=WT9 z!GA1%Ov`7krg8f7NMcM5)sJ{j-|p8^zu---aX3o5ZgkSr`1ADR&wbi+`58HOU8jq7 zKWKO802mklBMq?!^x?M*W_78ebd?5*Vqa4A41M_4nWLq}5Pm26D7(c5UxXa~$ZNI` z+w+z3&E3GA4#z*v8$Z}DY7xzcvv(|N#O7dm!(1#=i-&b*w2-qkM|GV60@A~w8xf29 zpM#)yeJ*-z643BC5G@malffe&aI==;;UhQfp8T7n(;N{eoZC?BvcZ3^g?U{v8O0@* zm@s`JjIH0(%(a>rhtUvEl}BvwQ^F8gT&a_SuVNqlJbIh-26vIIP`k|k-K5P9uju*X zdV21$leP;v>ySsEDP-P8x}6k5Nv*l$5b__TzVg#k6#1J7+A&LU3FoF!Rah`ku!6ZEv5a{`bc^5PCCTjJ!rk(9Yksv}K|KTH@r$Gw3frwJ)E%r9aZa0V8OUW;XwBqB>P6{-D;u zxn$EGLveoP)Zy|~@C**pA@iNo?Nm=yCEes#b&D3dJ|^QO-L&cRZJMj~mkg360e4wA z7(OF4Wd&qqYhuswF^GBij{b|$hwcbVysH!RAGzODu-69nZu-F?$^lwWevwq0JH~#E zLh*241RDINrWJuOEsn?11M^TMWOhyqSYX1pCFoT#5OTNUu*+qxp!J1f*+PGG3Hj>T zCzrr--E3Ss`kFcpx?+ugFr04K!t~u^S|?z-%quf6zJC%*Cf}#g85*$MtB0+3N1%1` z4XS-K40bbyVbJ$Jg2x5Q826J3iux$4q=POOHIhqpE4f&nr-=WWXwmvx6w_8o0qF%a z_4_6o(%McVjtVMOhYbyS6-m`-54|;u3b_P71T=JI&Is++xG) zCAdHBLpc}ii=1Ko5bnrSNiJHul|>FZ%gi?jqWJJ~rj{JU2ALF4)BtZ5ot@7%Ix4fq zO*XV*p-Yo{@3vF?!?V2X!5*5a!i0w9lwtWn(0anoV$3Bp5K{-`DKnMT`6d3&z{oHAUnjf zU~q!35*hHC7Vq~)%`jm;-?-t)>3JBvHWV&pbMfl+SmY#*z@K-Hxbt}qet+@8!vze! zQeLRepN>nDZqs-*8~Ba!M9HtoI5(!AVr|C?`jHXV$f!ZFY!5B?DhnSmHFOM?!;x!; zDBDm1u@?RG`N(~`9K40jf4D)>Lgv}!b0b+t6j9UO9kl$^E&&5pFf{oDAfAaA%<0;7bI{Ev_)4VVpeo1o|Y*G9Bks#{KDr{@o6ABL_Y zDfEl}Sb4L1Mp8_DxH-x7DX;~1^V#`(i7ZUTgH}G?$c9egS@83djQ9S-mRiVi-i12c zr1f^(=tMEA=4e|kdh`U&EJv1$&HBg6Z5vtAR-V=9Y-Fbnc+>k$%UI3&Jk~gK5WD4T zK`F0fSWcTSTU{W|#@DVV=i$mMJlmW-AAi#_!lsh6OY?b))7pHOx-9(~euG}$)}f%V zAN)q%ma3MVr1FTBlv)`=(}!i#FvioPncHZ2_HIhl-$ho(O6YQVBds5QhidDu(}c#& zl=b;8iAD5NlC%VdnjfN$5_z=U6}-Z+@)&w`FCDq2ij-<2?07N;f3uFzf`vw~8|sC9 ziv&-#rU!9_C=T08>J4$7l<41>ibPfYNf4~ zg%q5p^Sxe_VlPDpr;ecnr*jlA?I%C^nk|3RSez^)+Q~iWIKMn{t;nb1gr(!etJI{K zW4ZI}9nqVlu_DiBn<>*|g=o+o8#Z~qCF@AYrVfRrEUj)U%l=!k$Oo}}kYtyqGS zs#M~F%D!=8Cx&sSo5i{1GoP>#^559H+8Xws-c~mMVg&2Fm`TyCW=v)a#|&RA7ge9$ zOkE3&M9tRiqV8qomM*f_>3IJI%Vo-`qRb0CZy$Z1-naeZdrVFEOPyc&;=&7b^w>yx zR4+|8N6)2I!g#8-Bof;fM*bCI;7S1dkBzMDs@SdiIPen!m-RgSUx7q-AJFM_Zc{DUa4w33>X>^GVgH68_CT^`H z+u`C67jiovgCEj#%N-;)@g6zM3@tuj+>E+-ezQD3vbgJ@_<&3+x z=|M@grOxF)qLWu6MZ>#mDAnq^sMKQ_6ZM6$ZgM5VuzZ#hzL(uxew^*vBh9&IzGRy> z^s|^rN?gk=aVw`9Mb0r^jI+>r$qbaFxsj(E*g4nztSC62^_IC&nPUWd;1Qh=o#prYrsZ98KQ0sz_CHGnoWc)5X_^$U<^2z5RNg6s|v|s;v9u z(X@j$w?3km9^%N^BL;hg;*8^CU=ZuO|yDeU#sF zMnbhYo}D~L(+sVUy=yjV<(;rTeKDpicE^{RDX{JMN1?+$Qe}=7%I$^oA5UU%xYrRz zrfz85VkYFfj?!H`n2jH+@4;eRP zk$v%7$`$k)&xq?}_Pm%vIxdoN+AY#{ZlWuR+vw1Q9i-&2nVg*xs9w;t)6LY-!vTxxIk8g(Y(rq5({y!O8!Jm11){_ zk2l|Sg12Zlr1mqX$Zd}&jh!}?tUTw__q=S%(afT<<}DPoZ3l&Ytt9)tCaT+Xi(2iYhPiaOuYvJb-b^o=AI4#3S%->5=%D`n~`A+BO7{<@99SFNoya?eN% zna)8<$p}+7G?0~$S&d3|#YLg7-xB6`oN!*s;)51m1%4py@Fz5Dx6s2Kys6%@K0L^5<#D980WMaKH((aq!u&gNB6Zf)g``eByAt%&4J3FU4>}(4NG0N?=s@ zMw+bAPf^FDaM1q+oiIH}i{HN_%h9h$d)X~2%;f2k{xKR-Rz-P%*<>*zg?z7vQQ`NQ z^jj~4Mk-$+R3wx7 zj4u3l!!oaU5X0VmB5TKHdVl!5Xsp*V_R}kd4fA)S-Fw2BZCEyQSzW{))VyV?SGThr z7f&)BrSEJ+%22Ba{T?QN=oHg=QO)?w*UTnm9V<4-W=?nin5NrcVq$Md@rZA3JHOi4?}mj7|iF@P}R=i2zp=+XWI$5^R|Xgj~*vv z@9eSTu?70?w2(Bjz^bn=wMp`U!2fZ=~~M1|$29GeVlu1fAFg zCK;~yvd|K_f)=?TlEd^>4)79~+wIQ>1s_`iWrh)6tR1RI4&(O zpu3G9sdU{ZT4>us|I=%|GV>VO>=gEIVijd_xx{?4=}KcPC7<-CqcLGLaH~1RUsb1s z76Y1`wwv}19Y8NX&gVlq_V9nqgmW{d8vK=jnW8V^*-Z&i^@4t&D4L&Y#JX2K6X|N4 zCWX-rqEqK)uxceuCZn;D2E|yi^0;cY*lsI3D*K*KG}^*^H#aajh3o8U$`~$G@(nw0 z+{adLRN*SujIuh`uEJFv`pqouUb9~pG`W>NH<Ohn2e_o=5ADVos^>*qv zIKuxaT*RyEN|DL_U8J;pA`Si@6kT^zjsG8SQqkUsmKN-sp&8WPDaCB% z)LF+ZDHb1IKy_An?8O3L;)}w0pLCiE>ILod2m!Bs2h33Lyj*=JV%5KW*n$Edw)fj6 z<}IwV?qg*ZCR0U&Z|731;59rJ?TZ>H+-t7%`()y0~RCweqDNn7Y-P0oJQg1$W z{5e9O>@&&tZwOsm7)s&{ADX_@g{m_r()x+`7OctDhh@AN!`9RY)Yo$n8~VeS75mO$GT&0! zD)eHh{|OwDSuU;Gzmf93zovI`<5_)LCNp>+z*hTDVWar6J~)UTltPgc=Ur4X9!GlN{_h0+3VE1H+|fFFBNk*4k4PFEdH@Xwz17k!f7 zDcLQRM6-vl674bW;G`o%xHmZmY44>xuJg z51co1Ks&qF;Hc3A=O>hcb_?=tqpHE&p&Xw6Nq`CSLurZ8GN@F@1owa1P}WmQt87i6 zaMOIKUU7)q@iUJ$ub9J~n){4X9gr^i6O>FhXQxWmG`YDw-g1OL;k1JcU&>Ho!)<;- z)=0|K2&F~GXOQFP5OQ9>ij;ffsA+T#U7d1*E+z|{?wIu?FO)(H{_CRW7v7NdVj^o(X7@as>ID98-iL84%U|fl&KA?oOQTpwT+Jw^j&2% zL%{kPhJB;TlMiWr-FX^&_6jwrFlxP2P1kZ8Xjj-V+T)Z)N4TA|Hmrqm->fH-3QNjw zok%q$?E>~~Me|qY^2eX`Be&DNl>Xo*e^Tj&sC7k$q|{H2wU6B)v6)-JMXQhH*qoQN zZId#WDnAeg_zr=FkX7_HQ{*@4U9N#J?987w_5;BLVP*kZI2w!cvoHw|A6-|}r> zjK~VCX50aV!YS}Xg1B@YF3IR6{VWl4OTy1_ZI-7c5qD%+ z%D+1j&7zB<)m6>>qu)QsyT1}GT6%zAmugN%XP=Py@rm?M%A8KSY#?KW7P_jwlje19 zrcVvWC^D>`R+(2*Nvnj0<<`-9*-8pq-AV^0eWAU&Zz)cul#;i;q=kX9?3a8m=~yL` z&5bTPfASBh1hkUnqrH@uGl0#lw`1Fb11&A3CC*N>*qt2_ABiv-_5k1hN5d@I|uM$p%QTVl_wDwNo*j%?cgqn@try+^g9DyieF3MqA$lFRJ#bdaUf zkezz;AUTaTc4gDjvx#(jj|ZJfoK3lhfg-I(Qf1Uk(p)G*U5)uX#TD>vVR4l1kidUc z$q>ESAS*eknnZQgdqt_s?r|e``*0{=JJ%ZPxqOqQue+pn1x?+SGFRvaT8YW7zC8{w{OC$au4BXm;g zk8rUM?#&dTlJ$G|Y~BL@^3Q{nof6;mv(OcQT7q%?BaWg^LHvUS?60YUDF^wH;%3c}?`4D~cnf}VL>=38Js`PD1BTdK# z`M!i*vUg-nE@d<&+mf9hDQ4%ihOm=AZjk56kEB`no!X`E(e+>_3Usd_Jz<|}^be=S zihKAO8w2R8#&U{`b|;g4J$$!Ge_HBsgZGq3_+`FwB+?D#eJ2kSiQGJ@zm}QM%pE&K z>N{$=+u^6V@_3F)?qB69jXYrLTn*^4T~4{%ts(fs0dNb=0*7ajWH~4mI!da*y6Pk- zrVgdZf#op#_j7o<;}ytT8*(;X?GT$K?DPAT(L2r$Z4YQ;)^P*edfFA+)kcdoY}`;Q zZwTIQ*1}U6e%SV5Al4cD1h0whaJo&4d$INv=&pPY;+N%M+^bKMlTN|JG6^)*q=Jgp zGIBJ^ggAi*usdS|H)aSuyQc=Y2-wohy1QKF7#GUfTf!xEUEj)+ zM+;l;lKHc4dg}a{qLpq?Tb-c8))%vBn=M(rxRiFpIIx+=!dd(DN$j&OUq=MP9DvGQYTimO($bvMhnzLbbu+(=O0ZkAcGDiIlo>EsSp|fhF6I z!Z7y{bn(GSaR1#3??*g?)uBH`9~y7K^7d8Y07oP&3Y8!H~JPNV7XJEnN z7-%R?q%(Q3@c8mB@EWNNKRSiqmo5hj@3_N=lc%{__38A=!;_1vKgHekSXDhFXQIII zZI`6~$f}-FQpC%|+LG1EuYBb+IT{pYKw17qG;aGc^0!Q*v}x-prL>8U8j?eolfKaP z8ULuwHH%s5!9f zg8yXB#XmHBTq2!xdQ88r^^kt#JE}R?L|!+pkjj~Jls>(Jh8H%|nTkUcvpbGv{Y;{# zQ5}?jV-xuZdtmd009p_(^fjBD=(EpBzM;PsJsqgTvOW)_1zL%cD^64S%s1+cGeZ8+ zk@Z~4l)oZ7$6iV;{VUq|yoFbF4SQ^8mw5bsv}cfI+#sadx?%uT)+Yv_p_JN zw+cS6VzPW5zzQ>Wu(qr+_EV~v$;-}V5_JV~>J~x%n7;yNc??$Vy-o?!_cCL%jm$63 zmSyBc(chOXFna$ZL8~*AxmX?|qrHJpIAAb0CwVwucdbI=nYO@f!Sadxz@B1$_MScb zuxTGSZ=>aO*kccUxg|@gD=u>3Y6X1nT|L^r+>BgPmB@(u&MQZk@tg1G@%~HYNv%tT ze-qUsdY!OEbo8ki<vn;pJtT zIO?euK2)^FVD~BFo%e0f$yyuN?$*S)gM;wBn=)$b`~z~s-oeA{8{CI=Qs~jx171cK zVTg)4<*X@(|LzlXYh{8~#|BDRw*w9)0$*e@Ep!ID%;azwr6eWFj%L-;XIWXN~+0p6;xi0``J%U2aC3XTU;a*Wd_!i{dgX^VWOa0$0d)}4dI)f7<04D=EBic zhsgNvF!sHxm7X=+g}yES(W*B>UtYYCReZ`}Pfp*UEi;b7wr9WLblP}qh;zZ9fC0SB zrA4fK$UOGsrYch^V3Gy*obYjY3YpmGv&el-bkeGrOTF*SZ?LH4mq{Px(|;f0wZClT zZ=LFp+@3j>o3}R!jt+jsmpD$NcHcC<%U=VQZj_eri#GEkV%G6tc1AqZ+3*=tjH-VR zs&Xp}Gv;TX_Ter#eBqdEJ7-mXfVXjIQ?s0peVgDAGKFEyX zo~6MerQ7g%?Pc&zk)wgx1iSaYhNnLs!;mC1-tNvfz)U>R3|$1+qeh@cFYk;Pr`4$VrpH$YavvdcF>#U2cM7Wf~0sYfkmoQo&y@2M!oU zLUCLnpFKeY^=I9o->-Yz_uu>Zggs9_V*dK~Vy-ss8tGW-vq#%gDCw>vCTl;HEYVbD%8%xgl|V zz{f?tD(6*lLwKoKJl~ueBXR$@keipfn3MgfNa013xPJ+%AiG%}#?95CuMu{zJtP?l zK8%J1ul-3oHx?=yFTu~ON~lTHCGKMptjWFs@^>2`G~^g>;&K=Mtd+)J#!BdOMgdl8 zN#jqw{@7_T1Xpep^w@QV*u7~O=FA5CK7E$>n=FU*V~629krCQ1kHFgG!MJlrKkST< z!tGaNA>T(4BS%W(ed&A91jYQ~lt!3(@g}q`D}*%}dh{!(9PF#=V3X`B2$%DxPQ9^E zJ|Gp$Rh(dHjw;Dj%fq>&>foO_hwE+XPwebE?%JGSZo!`dNmOwi|KHWc{FQ`re0RPS z;q7$ZGvY1ZPj(1Zz#v-2#qp>55^2hxYRdY$owAELQ9pU1uM2-eY17qMr-d=wH&TO* zaL{EJrVL^$qy_AE+6JaHD}&kB4`vprJrFZ@KeZ|?V)H`MSn#s|CQfjJl!6otY)rx@ z_m7}P>~V~plLTXN3!mIBD-FPD%GRmV1 zJMPo7gEvU-$Q3deeVdNAo+W(0hcy2jBHNKasnax%TKfG*p+^#EjEf@klaHY|4Kw<6 z#Gmw6YcZDu6S8b=kZi6n<~0R9Q^bW#{))M*XuHSf>X_}_bZfqyME_zKH}mU3ZqJ26 zYL=@PbGDFM zUf(II9p5ZM&Eqt%`Vd!U9moZ#jFnuR^Mj_Imy@LRe0E#%c{|@Ur5{^kt;2^$R7)Dn zOzEeg&|ei!C*Ht{Y(h7Z+hawRx@!}4bS|SLL9ft^KS@K-YuVlsc#oB5>EqX1WYtwq zpI_ae@5&EIaZnx|UVD{v-wH-y;l5>iyrAPd)LCxr2qy1r$$GB-q$Eede-pQjS=#Sq z&HZzjfy{sG-1`U?CzHs$bkf;-5b_`1v@wsO-Awmz8NX}d8s=SlmA%7emiHxw4OYzn zKH)fSGdhALZ)=e2yo$$msbav`(X4a*115=h#G+ibu-W6(`H#Abne@Fw>|p6S=6B73 z23{M)wc3Y3ZiXJz4mnGPb;H@gzHnB!V=;Tv?*&CK{YS@l*3qH%WNKDeL-J}T>C=vM z+A>m+gm@RxmJ8?jq0e{oewL3VfyamQi{~qId0Q$ti_^K>vyHd;V=HfR7V;zE>_%;9 zb5Wu!2YcZ>PlKT9)v!LmlyYJ+;P25kxY$+)Ggv=?{>y`qF&Duy{31AucJNBUEwKCh zFW7lT9{U@&asJXj;ruIU^wU$vGn(^J=b{Qmnkr$#17lRxoh4TNWsJ*0mC$>fD%Rhe zhcC+2aecfLK3e}1o<6$AU5JoFmj4s_itoT=+id>i*9$Oz*hRP;l?U>LinP}C5(Hms zg>mE4Vcb6x8ZlxOn2*YUqX9Osf3Xs^SZPD;UOT9jY2@Zyy3MC}R1|8w_Vwg~68bTHGN8N79_OmgRjs#Jq2YvBm@wzFvC^)7$WfHN1Vm zCJh+PsyC?MRpBLnyasQsXvT%oow(J<9v^m&U|S5|v9^G(?8<8)o9iI}=;x8O&0Vrx zQ!Z-U7>fag#klUwL+r4*j}f;MQSZrpXn!Qsqdlt3I@6xg_l~9Un`d%)E>j`1^bWVH z`vbqtMvAWHXjG3v5wwMG;^K_=a#f=5T$jQ{F2lhV*3DlGVV-kfph3N8rR`FP_sE6O zv>Ptm+AP`mAP?Mk-Go)*%W&XEg+%3V8$9bTh4wyDXnyUMP)y14T!< z)$cTMtD*&lh1nohrGiZR2&~)Zi0&J`uq|yf4jtuzx12=i=irYKla`AgaQ^sRz0BWi8Xd5SqYw`*c@Wg%zI%c7^rFZgVISJ>B2N%mck z!naTRpfbJ<(weFz)-Nx?f2y~j=zJa|=q!}9AIk>5^B}xPTndN2*NH;z&WDv2i^1lq zy^!m2g_~Z{$t8Pba|abxa*s_%L27(uwfCTZeB-58yma*q?$P8a5YXqx9j9@mf5M4u z2B|T{sATdz`VcI;6Y>7bd$@1;LtM@k;~tM#9Colo^lI)s>S{Sio@W4tIi#=w?O)ik z{qNX;wh`>re-2p0cVbOJGcI4#ffjuqaqA61t3F=Lsv`T5kcV{i!XMBYS|SqkFj2wib);U~N?cMp!9FbsO9_mQv4HrlP3Lch;xLC>aU z*igRuV6<>`k2cBPDMC=4TvxZw^^M%;0zDr@~vR1hD*o`m$d<*vTQmEN1 zgX8aA;YN?s!J-~ZoHf!G4>u&ik;;*HSk@V*myO4#wW&Dd)M$LE=#H+D3ox#HwYc@u ze4Om#j$PMAqledK^ybIoEDvY=e0U_P7A6XuzYWemW{!6oweeByW$xHuS$rU@w@v6R zh_P_z7ruD_QQcDb?dByY+c1ou(N+QTEG~nQ!7lK39K)Y*jRS+)Okjfmp7j4J5l=RO zyg}wLMDmt%ch%xt+c$Cfk*`F@X19oTkJpCVJz>1Fb${|1Ka@A@+zIC1&7e3{2k=u8 zEid0r!EgVPVw@qUpV)(S#-Ff$yso$|P(hrq{U&M;IfTje5-=Ht+?jXJxgp=qfFv)I zaT643p6}>a^Ez0}f<`-IP}xVUeb#~Z?!Un0Ge6_-3}cj6Tg2+63~I_I>DL_9Jj7&P z?1InRdby{wyr|{ZISTmTQhl`hFK3bp*lART)?Z68DSHmufBKKCWinV?NhS;anZ%wL ze4$@5Cj2k`B7V^O7IMEofGsKbM$g-}QrfU48v6S-)yQ9`GcxV8SHL4`i;L-6av7zZ zlx4;%4-tM@PD9H#(eGF4Y=O|%UGJ49Q!Rapbm+%c)+rGec|ww(;L5jI|D=G1bpFS& z3nD&TmGgDIO?zVh;~o~h=-Bu*CFQG0Ha%Ap|1g4>b=R0JzmLKE1e*<)jgbc>2P}w z)r*dY?B>I3e$u#PPk!;95{as;JUQKzXJ5Z)(!2pO^rmeC75!3Y^4k)r#A_MZ&d4L@ z200chIz!R}PSfy5HMH!&Q?mc>2Dxh8rcHuJe`#GNjeq}@T<#2D(XIEX>_sX6z0s6E zUH65w1XYFZtxV?UoxzsmFQ;#IbMf5FGK|_=h8IqQ;4AvYyDq;C(F|iWQvHl2S6*P};AUJQS)a<4x;;g>nNlap}`KT%5xu5epK14b2!+E>|eo=Y!@l)ja6 zeGS?mo~8qRPKSM_zrxJ z-^?EtdcDga=sWF#v*f^UZRf!O?1$`%Mvp62IH+8+UUMq4u=g& z=K4Zpuz0UD-WqoqTpZH)6NhiX?1A^7svsBo4cN+Co;d*nCK2o%mIWzT3O6>*$3i|oHN^9$ls>0+@gU79B+`PViB{+zqZSx=LLOLBw3 zC4C@V+~Wn|zqdgA%B^r}V--gacf-NA)u1);j8NOfaohDSz&-U&=up1_b2A5uc2BN@ z28TcJJy{Cls*A2z}SWV-pBF6%qV^75rWC%Oxn7>fD&=k2z@ds1WFvyXzHW5Hw3f-{kF2@*4Qnh^ zk1w zJIVR*Cswf8rpDZEK+R{~l^ILTL_H@zd}VVPehAs8L7I82^0Q^lJu|F{zb;>+S!2Pj z&$A`(x-tCO$ySv6(uBr$caWtuqaqtYhibo%d|r6cse@8StpcL-u)xdY-6maGF9_~h}2CgYJMs-hf3|{UJ3;c~RxxWRz z@^!=7Bas**I|6S_8-@q10mmjqi3fb=Fs;lGo71gO(!2t#4!GenISaH%G{WYMGa)X+ z9Nm{1oX|>!Kapd(kCRuz&^uwEwbd9*cI0zC1*P1*?J?Yli9@)kCHuRdyQ!9atr!3ehTkuZbM!-E!e0H2N=r!}h;@p{d{GTfei5pOJ{*X;gP3$LD)PEOcoIi-}$%Nu3jp(UW zi4|+BxYuq4Ea%5#7Jov>+o64MB;q-~O#O%+w?ARklRJ2RZZi6$S+T5JjV$qB5gVQ# zMlTimi}Jb(!LU>pgZBnwir*6mU1S5PxAS?UZ$zJ0rqICM3#iL~99hrf$m-=jx@CHV zT!t5r=9C)x^dy^Z2S-r(@+EYB*bD0ALa8KR2)(zlq0n(X6y>K(V>E~Ip{*x)D(|Mp z`?m8pBaU&O3omk~-EY&rGZ~z6PJh_>wU2WS671i<_lf@qE)~h`rbdFYgV+55j**Shxw=*Utp^opUJb z@&b6ndxKrPG@LX%Ol|9aa^W#D5V30=xA5N`+B>C|!!-rml<*wk-0z_g#;5tC0ebv> zX?5yK>YX@(Nu_8zyCwWDle1TaVCkk0az5a)9ptbQSz) zW9eRC2+f+7Lf5l-QkZ^}*O#^iYJ3EHbQa*UDSDVRuMo~Vu6Cne1y#GxbuPH6#(kkTe=@U(SvQ-lu#>&Dz)WhR{-0;9LcPwAG6%K(jF4=?l zrQZx(w=oklCQrhx_r>U)jGTH>s)Kekq6~bSqCeCqUf3Wwz%C(+b!`<1Y4EML| zLtW%8?n;)nM5S>deGfRsbsQ-L&ttlH)pQMxtGx@Fo?j*BK4p^JgLYo!m=7DM>&%K~ zXM$SBJ-q(zE`D8e7JDz|Af4-A0EG3O!d-E%O$_6Yp4y0}%o-H^0I0|TB*VNb{cPX5Ag z_#$nAmNAz2-19vrXQP8kFAeeRQWFeP_5zu7k&uTu0eh1JQF+E$IMCsZ6BbUvzQRyE zS-B7S<8x3y!V3e6XW{yvG2+*ivoOeU0@iEI#mJ<6=s7nGTa~8ZNrQ>#cH|sf^AbMy z)(d5iy5YkbFZeCg$uP|b)5dF~-J)k)K#nEyUFP`W&@U)93+C2lOXDI%4YZ5zg|NK0 zoNvh^SS0-%o+(wp!X>JpH>MIcCNXH;p9iIjBB84?9a6nl!0q`SFs3{mo~WmT+}!ox zeXN^1`Su*&Zaf*XvYNntwGuvlI|0`zOM&OQqqOY!byAEf z!-np|SEYT>*Ln_qRjffhjdmFNb|$ZU+w1FI%V_I)@K`S;L)^7xHT@X26L2NU&Z9aF~`rX;3Dl zANdb{y*U8h{i4BY6@L-z2siVl|wvR#8v!Iz@9&P7+oS@Pbdei ziqBzV<|}9js)hxV2E&QPm2jo73XG=Z!Wqr~z&s%XQZ7Zn_b3k5T!@4<>H(lU#2VhS z3tZwFf4**u4Fr5k0jtTo;7h;9aO+YhZ`0XCm$&by(>^=kgRcx(Nn7*Y=|A95SQ7r( z-^x9o<--0ry0gLOw}67pQ<2+)>2%a;0F&JHV!z+jfq+3{wxj~-QxW#;&B22K2Y9}z zf*lcZz_-W*vYlhhS&n8uCQ|QWVi(z(TJ4jp?8Y`K9jVHW46|mLyXz?;WDD?Dt!UuQ zv!p3~n5^5Dw>5lyFdtk827t}|Kq|ev71FxPpk{Lk+&|??!3PdQi^+8urSl&8 z`I+MCuYch1G*whP?TgpClf^N?zPM|=8fpvAm~@*7uH5?$6wh3TlZi)R_*ExT>@5MA z-m{Q-NjU4C`_sT%vtYu)g+P9`u)(j0f(tgmw`0k$ZP5en4;E6ItuoB1_JQWuJDkk2 zbPCJv$Mv5-nCl2v;*ULFP3vSlc*Sdz`F)o@@V`oYDPWtxO|KI%-GAbfeiQWW)tcw*vPNBdLfucCvez*)v$MAA?R5hhEIv*kQckj;>bn)JxGfVC>Kn;8CuNNn6a5?gc2xn@CwwO+~yPrWokRmDI!^=%Z^b$VlHln*W_Itu;|#kk<_G(6om z52szp!coyP@!Z`hIPyUVW?`iG&czU{Iy4nOPMe9Nhh^dFqw}z&b{gjOh*4!w0j!jn zjEd2dFg0crn#`F6GA)A6&&LSu)O7L3`}16A=16>3A>hVtKjF&jI#Fwy6b|^Rj{0`* zAvGk63$3^Xp|b733HraoJN};&Qwlp}D?w-Qeh_^Mg@I?YVd;qscy~GiK0n<9Iqi#J z>aIybZ8Q@Uzox>%#&Rg=It<0VY_1~e7k9YL&bY}PPU5XI~4E{ zpA>{1CY)APjAM-#beMd4A2jV!0F@#y`XFSTIa}nilp!U&o_+!jDHi4i%0=V*5mPWd zznwCcO~y4Fv9Ekqdq}P!9*#Ie1)l2h`U+0J$T3;5Z~f%FkUeyNZV#$NliP{RDg& zQ4Z?YQ=tA$1)K?Of#?4OhSbp>Ca!3K1rEI+d*undyS$k@G4BuD_&gZz>G#JOk{;2V zFagI|?1;09N8vd^6Vd!fA9cwLHw9bZ>dHjnp1Pw>-ef%6F%6gZU5CS8reTNPH1us+ zgs1be5re|;&$@8bjEupt%fiIfk{EoD5RP)bVOUd>jpm9WxWjB3+8>#Q=BC$S1M|b( zT2l~4i_vjqBCL+Kz#Hmjc=3%MzQCK@Jy&PcpX!L|*QD{if46AO+UNlq_gS1s}-n6QCV<0EAH#Fiptu{j5Q z<0(-6b_Q`7@A$YyzW83d5?beIaypvLWc*_bn;P51k9jx+FLp=a1S=Q(FI^F%?KYE% z?>06s;~gpX^zpN;c5%KB``~SQ8eLJ_L8AgkK-V@O948gRFBQCExm!B;)I$n5@k$E( z_e2@IwvUE>TYf{+Y&n?pLki|bBmwQc0`0zwAnD;$n7$?fK7XtR*Bkk8Omq(NQW$){ zo(1&>ZooX9cVPJY6@2_O5xhJ;K>ZA5Y&`l4v}WJoS{#+Jd5=AYOdgEVp2b{)FduJg zo)=!<k{3ZE*wVS(mx$oMxMM|}vzc7sqHTK^87Y0XF7%^_&n8-nsP z_hILk#rSmGVze`l#Nv`!V*7ECI3s&8N^e|(!Snawji3-LU%Ci?viX>G;sYq?hhp@} zP`vPR8h-e83}n~1;q@O*IDDMo+paNy-51B>@fYKXye{VWn8A3_zyS*aRB-IK zJKUtMpYTLU8Tb7C1X7(7VC##QQ26@;eBE#zBBisy^==J}cP@vhg@<8s&3V}1k^~o< zeBjvlC2&0WGC2N80@KPqPTE@z^bY+6m*-=kdY?L+6#OU;-uK7;DnhSi)<|Zf=I~R6 z_3_BC0j*0ycKq2bq?OPILF&7>XqkU}$jdhrGI1L-yqZdP1}Nb~H)r%}{tS*|H$l<% z?=+7%mINy_^2S)jt2Bv#K(tyv{&H*pHcpd$9VCDdrB!!lAv{c-ecO z&=VG7j%Ox%|4hc&uhMYm{-t7*oHQ(KO-AS68FVr3`n&{e?#TX|Hd{sOW${z9a{NB_6! zAk^Cy!4K(yD00b$^nXd5j+}vLR(L5aeer<*{UV$WSF2D=hY?(CBVL$UqNH$57^{<4T7Xp zNXNvDZP=Gg!`IopAk?o-Jb$4riGEu6aV1rzm?TPu z%h6J>vMU7o3tZ>A{C$v`n+wkle24z?0F&|`!UTm1XghNo#w=67H+}6eK=cxlPrd}3 z4~Jl)K3#AH3S4jq@{PF@9+r=a;X6=`AiuCyj9Q+hoo?P6f{i zb@^DA(Ri~(6Y_T&;6?$D9r(fukFRTlqtQO-P&Nns-Undy%6|~?#vj)XSd7kB7o*y^ zOw80-f>v`91g%a6THp5(@5;|W+Z74e^>GO%8)l+#q=sfeYC>=oJ{f8g(2wSq62wD$6!*22OjyOf+yA2aemoGs5#6P2Y=MStt&Qi zrUK^oK}H*Qto;ki?JeN(+Fr=ZlEvZOov_C+AD&pf1or4TWIcQegI6iy!iBd%_htng zReJ;hnTRpI-@#^dF6=(H4_to<_t$hGc+ULA$%Q6^#&%PjqA;18o0Ce~n=bNmp6TK5 zgO?@Q1$Ssf!gDg*pa7enjG%QvBk6f_GF1k+F`eNmLhf`U7+mjx@?Q@@>fcq!eleW) zZPp@@HY?0mJw(`*=YXgEE;+@JiG*+O>>hXLikIh{9!Zu*n+z|tiG6SZY= zj&cKd=g6bmxs}+Eq>h>%3OGS)1U5xS;~^O}?ECN@O83d)sncfIZ7+>Yv!qbBOyInI zc*uAA4V}vrvC2RSZFgBg``UgOwA=v4UQ@uQgGOzTyci!FiacyNR%b7h@QvX zF!_rZ-<&pwlaF;!_m>X3jxxk^CqKaqbjN`)Ubsre1HVP9;=)-|@xQgVD_;o=zUui7k9X0>vu17 zndOdoQ$ItXrXiA%F50;2;l!<`;AiBH7u-eYQ>lPHq3xmppNC>6=Y}blv{25|oXeV_ zh;jdhpw*K8LY-j^kw^Z)Y-vS||NIk@a;xEnRv*-@l0t616drkHjx*lK;D+A!&}XcU zgG{5*?AQqWAW;xzFsoxz*eYD|SRQXZYJi@48T|A&94|dN2##s3ZpXrUxzSQWOhA$o z7zuvXPrFU%Rp=hr*icq|;_*n{)3Ag0crD9jSy=KT|6PNsh=<_ydo!d8bAek&^`~ti za;#H9nhKPnc;yW}lCyUrVfpqG{EtsN_dYrgNjQ{I@2$HLBy-l{y&j z_!R=B1RkZ_5RFFYqE~J-1hg39yTxvJH*`35{_EvBj@cpwO-HvP5e^c+;#~I{V^+uv zoboIf7rPdLlE4_YHIKmIzc@TR#|1}xnT?~}X5p-X{uual3hw+8f`b%e@t9O3sx_zK zzmN*_2F zF`0v#U0iU`P{8QpRszmrj~o9UhnXXS(X+rGuU|d8 z(QMH=418ybE(iX?wXX`8v~wwDiF#r6*b{KM`!0le`C-?NIM{ROn&j2HLM~d)4DZw$ zL+;{S{_c~beD~EznCI|JQqOPVkJbOcNnQlHap`bYcQ9A*^)qkbCe5ar zgt9La4$uusqdj?oUs&oo-+Z8f$`c!?zj?HPRVP5|l!tVsxgX0t9YJ#f+TmlsZ+`8- zsWhm~jq80e7|$##fV*keq1@3KXX_QAd%)KkP2E|Fpx z*D&!4`Pt&2Elb5^hkeA~%_oY3=URzl_C3UzC#=M0r6-E{tv=$_F;8byG`)YBr>vD1Zr$F(n zO;%!qF|)*9*UuH#j+i3W(itS~HuMmG7W_Y^+maY3K(+&g}UyLvik6=sA>KnESOBjK!}5qd9j!$D)s@JR1> z?y#VrkM5p^s=Oe`mOxLye3hMf6zH;bw}Zs$Oeo++6nDiEqV!tQbo zN9^zBIEOoKW5*nXc-;uBJ|v4C)Bul1%46TdmH4zn4etvW&^Cc9Ezw(v#cJx9t@$1T z9Aq%k-3-6pm&UBBf3RmjAFS1=1}gmt@rsJLrScz)thNHDzy0u~Xb39LQpCw;9JwX; zH1U9pD?VC243};HCF)2}#GKu3s2wH7gm^Q^v(`aV!AD)-YKY>9&+u@dI~LydLaR+4 z7K7KJi zunI@r)~UF1s2V!$cgM6GFO2ecN9Avy;InYvAB)t%Umtbw#C;2}PIE^uUWBjrDdNVR zpG9`>hT{0?t{Ajg3qLIauI7m%YMe5_SE>ClV5vYH>r3IN2nFome#5~{RiG?r5BGx< z9{nbb3VP;PnJ0sHZwr0yYc(`4j7FQR5x8KeA_nWLqyO)fxG`TI4K*9VFGvmtSA^r0 zKL;UW)=jsN>OWkNH{g?a8z_3AK+lSdse1n&7=8Od^%Z$%{_?f^{MUDapLc;RZ|m3u ztIb-WyQ~*F`mVyb!z#4#ts6V6n?lo`*ip;h;Z*!y4toDb(Ruh|^}cc39+i|?Az6`4 zi0524WrPyZqD3KE+DqHsvQkM#GD=?~`&>6EBuQmdN@=O6v?wI~&hH=K<$0WQ?sHwA z&-?B78Zqe+rew9%1PIS0HgWar4})-2&$D6ns8buWD0+z0l;eeh!<17-?;1p9sgvhd3<@V=o1 z*Hk8;;EfDnx&+8}*VD{05g8(3Yfg@M&mq6{k1-dfNs_3Lg5i!t|0aq&}lHH?D zmSk#^@xf-O%y1%JybYx9=W3EzJ^_EFSCc!TzNA3PpVT_Wko711$zbUzl4>4Ja_XEc z{dBo={?=2(OW2Rxw2mR21%AY6(iZYEcojLC4#Dq}D_BL2A?~5tSaYho1M|dRqpPqu z%o@?e%f1HqJR%s2MR-&&YATwf+=XlZe#1hwZ?L@O6tG!C`0qmo9W20o={E>58HL1}qi$y50{uNyLd=I~_Fs1y}FYuhADl7<^j7JXbK${OwSlOTw@Y2=h z%{m&!>`qDssVkGojG`;h>z58*&u)Qq%>+cj4={=qB6-gx$oq+pV5m2TJpOG;H1-=4 zYrRLHw~PoN<SEv*I(PT1m=mPm=6GIjjOOP{j zcai*?yGZeRU(&q$5k%yL5wF&8^7Eu8Q82y(U42`L;^dQLp4ECXt?DMk{j??jbuK4e z20U^}s~tQ~7!xrQGa{}vhnVqy1e>xcgcu8R&ZVCa;qVuhSKNaA1sTwMtbm&lPbR$e z$?#k*n0eGOVqqm(1ZKvaY=4zM9;%*%sn#k`8qkYBJ51=w*{!G*_6FW~EkflEHEK9y zM6ar5)|4fdM~jKbSs*!o`sEXv&smsUE#z2MXERK6N&+_GR1-wo*BNTGRlEa1QcDDV9Q z^9nz}*@R_q_up60&ypa0ZlfS)(Z#Ih@)TvSjfg*Y&Qa(uV174DBbLYP$%>rCWKx?F zD6Y~ZkHt;MV3q}${oo4NK3zes@83c^MSRJ`y-!eg!IS)U^dax{x%nLTy#`G9kt3#l z#36V$;d|v_nFV`@ow6TE)AlEeHga=6AzyNun+ft|d6LU3K7qm?A2Q!?3+ec_g7|9Y z!A}nhVjgHhX8CH82s35)`Ft?}?wxX0{xq`2zKE&*qd=CuF(P zt|#^t?zB3B{L1&R^~WdhOuq-0cf^6jaSCG#N$$SPXx6(7RJw>2XDOsARNc+Itc}Z#g%Xg+oAbM z3&=R>1AkXLq`8l99qJx9_brKm^l?ZXRv_9hMaZ$=eLR~Eexj^uLMktqk^P4sGNV&x zk!MbGiJ`w1nK&KG57VbTEeV!WPN_rMNC7s3hHwxsUyeD?Ke4Wn zE@WF)1qcXEV(xD3L|PI-cRp!l{kOW11r2-1m4ER>@zqJPepNC1L@kZZYSN*bi6h6h z6^1jv|3YrK96iQ4dCTVdAt@Cjo=ezGtV2) zm5>c4b$Os)JdKFPrUPE*xKE8GaQvYXiT|3;Zq}&8KK?9r&n$6bs_c(K>uhO$vOM*j z^%u0A#DJf%$MAQ$v~FPmA{FPY|YyruOb znsm)s5&BWLpMHKSP^;KoK(k(-K{-81x=ux(-aOTa4$C>tY=$OYSz3nOBbg|6Oo638 zys$OP8hzd_!Z|OhFsL;H_cSD-jtS?*@#pRzy`fmg=Zya>=Ao>KE0!mHWdqxmup)j8 zD=B>t6|V}jsncdLJ#im+k94DuShX|p$;)9|zW_WM$i*86x*2ofaLCOHhVHro97r+% z#ZOzn`}7*f___uYm&8Ea>QZi2SOkInMwq!Y8#K*Zf$9DPAHJ#)zSapiFFKvfljipJ zs-c!{%UsC0=aS@f1~*I2P$w$J1E81H3YCj5L*8;z^bReA9KRBHQX2zT4c$@fz$!5M zvKihB>%oKOJlvfg4EwS}A@lqbM%gS61)qw-)7oWl>2)WQav%cF_mA@$R;V-S<)ZAM z-+pviZqG{JbYxdJe`asYazz<4Equ#y07Wi`;_3ZP_-j!Nx5rP!Whz;CCZ-Dak1xWy zQjRhF%LmVWR%DZz3=Ffnf%7G_QK^b!NLDxEH;$*gizi9*)wsUq-U7~(C|J93W?RqrHPPo_=hT`=y- zbDrpeNciDzNr!6}(5Ug3OhiI38RGVl!>kR<_j-);Mka#yI+=6V(CwBJQ7ofd2 z5t_V$;eCKU9NxKxozzJgJu_XXf4c%k7VU+dU1rdJdK-wWih+jF-B46<0c!SC!@}_j z@T{BxH||z}kM2hp7=8suErp?sdnej!K9l@o#7Uri4ij!JOlW{UnOdYz&WL?yIyQ3m zd?QU#;yst_YuE%|FIjPCId|g8ov&Oh{)56vPQ-M&8<)-TA;wcvNNl4!QChx`$TzMe z+P{uku1{P^=$?gy3p|p%DJi7qm@l!s;YuDoa3WK@{sa5t%emQ_JK251ibx;W0%DKm zk_RDk$*XO%h=J%BGkDQ}1mx%wNiAU#(Ur#>5t~Fbe#{`r<^SNFnlQ-Segn-f`{1KW zCA?j%1l_?^;C;Ue{syIix#b?{JRJkuz#q1^F`&rq1HYo5GsIZxKtza-iMcI4i=686`JTzt}WneqA9h6Sf{a3{kD5#tWz z$VGADdZ7ijXe@=;lqxo1_!py-9S9!3q9N!=6bKaCLHVBDurxm!EPVoClzUbv+1G;e z&<$wU7KP7uDj>o84cw7<25E(#nE+XC=Ws%b)VJ`F{3Kt-V{{aPo@)}H{ajA&&KvGL zDog&8R3XQ1suBUyrI5RmPyH$XZWAjU7mf z#9HENcGOb;)>@(>?La)PY#>)dE|AmPxc$C^3z_WTNb>kgA;Q*!#Q$DQ1om(lZ$n2& zXiy`^Mdy(DH)Kf9_(vvrSc{Y%&?0I(Mj`dYMy8nSf7jPb5#fSiaER(>`edHN{n4U zE`twMEpRAg5(!=5NM`l?Wt1;u<495)UTrO4imUU`zOk4UNwXuGeJ{xp=kvr^k&ncB zmVr>uLGt-!56q$)*w6g`aEqcQwTgWOOd=(I!BwOuriQF~&XQUF`S5Slow6F+=ws(< z9Q!Fi)_g*;Um}YPs+N)+OS;H7m(?u^i~(sIGrE*-85QF81K)dNaCh8bO;uJW+&OTN z+)0opsb@;ypu{CcQ)UwuREg9)On<<9V&vgXuO!r|oMJpv?=pq&1R+-A7V}D$2d$Sw z;KuY|m^ybFbFwrD6ywWbq1-hHaifU#gvrgS`y=)mE>q*4B6J=M3(Qj zBBDdwu8*73PJOwKW9(Xy9pz3Wes>H>dB1|ZsIVjl`%H;xP%MOZS(0a`c_ix8OmfIX z2#nrMCsp+_WE$7g@v7Oy*agcHy^qqw??XH6S!q;L(%%Uxdw;;~GaRRb_`w1OzZS~Wdw+G^IwnZ;{U=x>@zGX&B(xbsBvy1Ez?;;1Z zOG$WiCQ0|AWKc?gY}2U5@=u)C6xPtR+N&Vk!;<*48q!Of$s7I(a(1kk@t4V={)R7T zvvmYbh&=}{zrQB4iUx_|udifg;3GmyLWuM`Et(*5m%cuqPdzlYVi}()uVz~wbaVT| zRcf2a@Q!D2EY1LSk7u)Dyjm>p3&tK3Zw#Nobu|2WIJ7ej$8EFFb5Azc*Qv$Lx*P}f zz7LKZ@I}?fZ*Xp@KmMszz_U%d*!1idTKP}Gb;XM84#Rxba?39q*?NW@UVVjm^t6_- z7;eL5ry?2eI8iX?_L`5l?rHrMJ~+0~5$yDRU|4cB4or9e?^_U@w#@~(I}+q^d^3m` zeuq?RjH8D%Wug+aD4f>~qs5RY&XnRSJkjNbDwRwD8j9@8vjPy1_d?5e3q6MrM~f#W~9 zDr4d4t@wTBJDjWLhrw;!Jj65wMW@!`(CaKT<Ci1-&dc5eV@bWa?uwwD}`P_K#?nU~rbY89VE>f%ZLbx_~ffl`dUmH&^aA7^J zYhgrzDklGajrT9Hi0Z)@zAhCH5L+y@;6vAe{WT7mBCxJQg-NvcVRmcZU=pg+m@k0} zkgVei?}m4Sjkgo?BO@5XgRX&EcQ#Bd7h^^)l)((smmv153GOZaRIPFA7PQ6)61PzS zlBgliRL{86#a}KXfgy6mREWSGMitDbZqoT(iZ0bhnUz}fPN8sZ^Ab`!Q&PM3c|+3z{dR2o5Eogz?gTI#e>g#{zH2QVJKE0-4!x+VnlppKpDG%*K)6sQyKtic4>#dk!6;JJ!ym>c`#je)3~BR8#`TbNF!J;Opw&fvaK1j2BF_?LWqI zx&oxAOF-EZdr+Se0Siei1Rt$tkefm8`cVaLoBo3YTUxexC8!o2=Z z^|04&41yi`Nzyr%=f%y!6_qBD(i&;v)Gr2^D`m;p1r_qjUz6-t*#sVwrlVnv55_(9U?6{;&md*CHj69%2=1Itrq*x8|F=RI>ePK$b@1INF*K-}bshi+< zl@>SSRV9vnvLyed2pCIClf#e2Ne1_O--S?KS{fgjGG`oiy}Jbg^O`LF<9h%HGi%XLrY3nRF5;@8YwLfECxUPi6 z)R1bYSNs??UkVi!yIJe$>(T9lJS{FiOe;({_TdQw`s=SSGI3ULLXXF>_v`52Jl$HI z&!o0rLb&$%!74gDZ3Z z2*62SFY+jY;}ssg&1)Fu+-fQD?TWYRS|~Z z%{`A-=gcH}=jIZT*S=)Bxe`%b$93tAjEU4qKg)UTMnt7ajx@}gO(KtQ^PlCK#Pi%t za%Qs}dGDzQ$#)cp&V|dej5hPV3 zSu2Y?08JM5?~MiROD60k!+ju^6bxajEuil~a*fX!q85I_c1KYTg^qsvqm&UDdru<%6NN=kLP??YD%JkfnOPhE>JoMGO# z_KD-j?=FZ%i>G7LxG-&Z62rB7^J-3*TC(NsW1L?qh5cq-#~U}6V*c>AW4HMZMyK)} zbIj#8<1sG}|GQej+%fNBPQIQEg?;U?EKd$DCe4M{T&L}uo{Qy?;e6QCIS*#9l?BI+ zNAUdOY=8ru%zHzY@mI;mC8VbY{ea=3n!hmy-w8=4P<1cybdt=zvpivxtH=pf4 z(pd94R1CGIh|=KSa+n*(kGF#Yu`)!8_ND}({+gX=IB^kAhYL}K!D2k`bscm2>+qgv zFD~V4#c9qQcPRQE{?&@dW5wTba_JNrRNH~|iG{5FCvEn2#y6~p3ZWB?F?5l3G}WK7 z6YU1qkyG!B$tr~s!r{M3;iL&>r;A+m*+pe2d)*Enzx>Orm`H||t{GHcb!M&gU&UI# zdERs|SA`tf`HgI+Z^_(;ugIwMS90!>8Oge3NqfCTYGbF1)rM%q&=|WC5E1ERn&&KH zDtRpIlgl8+8{0^W5Y?Iuq~;W=+W=oC|tGKpC>E?w3L19MMSun z!1TUFBtb2l49nP)n@h|HUz#45cMgW=%Lb$#wFqm@@%D{wG7?ULWZ6M}l4QN$(kBvfD?O<2HSw#PkQ19SEVR1VBjP z68@Cz#G&Q_Uet#`^80BaIk^8RiH67I-aH&mUsccBC?C>qeMcz8tzsSr^v5^Z^&q^{}m|nU@f; z70PyBg8jp-KfcJ>;yLH4|J>) zB%@QWGp)O2NnWBBN$k`o*KdTux?~+PCv85lO|U02FT+V!>q4?afk$rNTt(bQc3AeL zt|Cc&7NqvbB68$gIQhQDp47(9C#;nYxjh^TFNO8VN~%e^Q)S56{;Nzmm$O{3><=8f z*99iKY0biqeB{q|K62ae86+w+)YyABfb+5kAU|;t){Ml|2*oACo6vNCi~i7gwveZI z(hFu?*#hmB2JmNYJu`dXFta7?0%P!MALG?)0<%g>s+0|2_i|_kBLFzQ?CD9NURqeC8}w8y!A6Wad}n-Dijhh z+!#tH^nXy7wV&wc&AN16oe}Bz`kt6Owv)O~wWM|3U6S7@NscQnq6>~bp;MMVqbfoX z^uj7#_PL}Vo!y*47iAuz3Uf@+tzC){UFr`-sxv`cMGc}ca4Ht(H@xKSO+QwVo0Sf zf@S4Ru;{%B?>5J6naf3JuCRRuHFbN{qUTb*CI@oFCK^H869Aqm&^+4 z)I&yWJ$TK(2&Ehc>?hZ6t6I?m_IXjT_>?+McRT>KWswl@$p9kdCgauDGr&aH05;Bi z$Xq%9klogn&I~t{GDf4bm@b+5Y;5e?8kZjltbBAC+bMdVH~4H4`pt~Oh3;3e@B3ji z+Q@O!2X0~RR6eQ~@&;d3`q2j~w$jqYbE)ZI2}at@gc+)SoDV>r+1hG?a^(lB44%wW-+b$FQGQ)LX22J6?UWWa>?{EsO8>F-;!_dFJ+^3|rR(aZCxLlK+SI{YV2Z$14vzeqCYqk9jeN z4|~;^@B4+X6Pl`D?G)u5c8FwyVnnFf_t|W$P<4$}p*ps_k*0FOD{OSav`SrT9I z@*93?y}B1C)=JR_k=nHClotJHIf}Y+JLrz~c-p9Uj>eg!QvF{6R9k8rO*apsxi2E= z3aLlbF8w*p3QMLKQo!;j9Ha|>-=#K^?etjS4gdaKAwo6TWO6|;d44mLtSBcC zDZhr=bwtucQ*Y|BAq~wAazTGb1*$Y2O;i=}Xf{(N{P6b=o$tf5124ch-b z47F9axo%}S$JCHUNm>9$7280hq7kAsudv&%K8AnN0wm$CIB8oa3UhYz6N9%RBq&LN zlu2(RYK?Qqj489paeapDDD$w)69-Z?S(%J&p2MkWwh{X<1rjPNN|Y`5N#a8>Sg$jQ zd=}&<3&%R(%$zIimMhKhSG^5RrRPJFw+wn_RKUu44e-?_5N_qK!YU_jzwQ$X&F+T4 zvys6;Z4tP3b{d2zt!FkY6vv;1zKo5J8zWp1TVpzSiygncfhFl>>>2k7_AYyd)$9Mv zmOISCYo$u~!E+m%l^u-p#cQxNF&_DYbb0IY&SAvkkGQdZI<+%ZqaP1Wr*a?UXe{?$ z9jGTn1>YQ}UoB&(wuAybl{o~4&(CvSRc}fvBdMp{8k)jA2YUNMiDhdr3Fp{jBbsJJ z##@pFGjD1;bt8>ZlBDAqH*4-3pHJ@ThvA2XGwIdWEodr#jk(R|%&xSrWk)_nRY>zHTD_*w^p(&sm0F zzs=&df3Xmop~|wK*D%S|Bg`|oSInc>EY2DIl=<@05^nF}y4o&z?C_4A5dS^}lrDzB zAq5S5s2>T7`yRl+-v&4xDu7ZmYeCE518m{z0a<$mwvqV(7aO_zcJB;wW|j*ys7jL= zGZl&Cb|a$y=>#!)zkpm?Y);tAD@anz220t!D@f3M3zF(?P9~L|Boa|v?&7-wu?^=K z3#zW5$8nd|Ye|tpdv30LO^2N)`Vt1BKS1L<7M!&OadBt^R}akB*}lVQB=xqUmtDG7(xu1M&BZ7|_2}2lk)J;G}~j3SX^iU0> zFW<$CqY^l;`Kt3qS5^E1>Qd6;xm?cS& zRW^U%P4XwW8QjU3kMohbaPH?Vs)tj7Q*r-w&ch*A3j=#%p(y4Aet(t#_rfp2!_Nk= z;M-wrt2Txw61$ z{aALL)g2yTPjyd5`Q*9y_NqJv{mx+zE}@L!0)AwL&T)=#&YPt1jal=KV|xx>Wdoa? z*jr{37RO98YqFA5*!vDr%zIrAaQeT?Lr$6AINyzjpS8m@$AftAG{>~ReS!u^oug5? z_i%;b75G~87rf_LfMLD}J}y{K&2rjloL&iC7@R;~Y;dP%y0792fmrH)-~zqz!ix_1 z+`>Fn10N4G?V@&7>B2cz?ky@vrk8e5w`w?2|OVN-A*yx@>cA887XS?%!wu5 z2{kxx3i6wY(*@cxC~tC=^?qoK9)h3If6FRNa^xYYJdHN>FY!iJFphmcj&IU4@#5)r ztbbj>x#G$&pyD>_a@XINflqjq_ZvM8d(f2g52QBz!I2FH)bEHfwa@QHvoKBirO<$? z25Zr_aan4~YSULT^Qm5<0+oK$fc>w$C>^^%bB@JQS^w)e@535e8J9p)QcLKX@kjJ? z{{~vwD2xYhEQFS)ADO_mr>tp%4mA|aqsQ_?X}-TVJ$~GPj>r4srq8=+xLp`6ZCFBo z%&WsSA!=05%ABqnQl@8*j$&_s1RXdgOC@TB>8NuYiZ=bj3%cJiWlt)unR1)`zy{)` ze-Y?rvK^!P)KMf$5ob>DV}kQZN;xw{|Qg2VM-maRQ*G>i{^DE|0pna zwX3nmkun!QFz{Gk7>YDb;D&x#_}-ojmHo%S?MM*r5IPK3tcoC>T!sk;U0hR92!|JT z!?W@?@K{5D`JMRye#rNLYr+`3kFq0YjD<)l*Z1n#sX;D8EwuEX%z2ULiW7;of+RoD zj)Yw2uK9!>5S#P>j_j6Tq&!~3dbj6r=GAqGSfGdUquBrkMbK(~7&Lwbah}klkW-!n z)3v3cYv>dz5)rVBv;epHWlX;GD(q@nzz8qTW2E@LRd<9f!7y^Q=A=bf_32%?Y`UWk z&YAL?Z9U14_rzyo`8#Di^KA!ChDfY*J%V>?8`uDuRQ%EL4IlsHqxy+)XwCVChnLIH zb2?M0`rT1XxTZ|YvKG+tb!s$Ks2*K6FQxqxVRY5c-BfS#b`;~T`N(gZ>AuZj^xTtN zTDMV${@3uF)j$84x$$~ClNvh*_4(8(h9=WG6EE6%+?tB7k*5W>jZxQfDg98pmFf-| z)1a-KudhLy2KZUh_l*iP#I+SyW_-o9!o!$l`WUk`4UmzziLMJ+44S|ZCCYlLfKbW1kxtn&@V{`xh> zCme8lRy*_h_(BfnEe)ry1z;D)RL>|*hoI##;Fi7(Z-j-zC7Wva?pFwBx@Y0(Po)rc zv=55q-$6v5CL_AP7rMDTRhY8`$NzOBf2?GQ<1s}tu!ToV)fZWsrdW_|u}b8syeyH< zaV6PY#&XAn-(2tH6+Ea^Ws)|$2fG7rVS8;ER9u{mOMhMk>lDtbIVS{a*Kb1!UJUFs zxd_|4^kDYX-FWW(G*Gs+gi~W}OiUod`Ka<4&+byjc2645F-R9Trv9{;MxA*_Zu_t& zeMNEP8nQaj#_E|0q375ztJJ#)4fFP4PW2k>j80^O*974C^(-p>@(kpAQ02ZKuj1ej(twem}WGsCc^F_?@z=j_H|Jp##d&r>Vw&A&t`s9 z`sjwVS{DNY82l>gjlhIYL0+(#= z#N7Hj*sFXF`9FwLaaSX%WT-^X?F_>KOBuRxxh+-6;G9XnjPOi|EVavCK)2mpP8CvO zsnKjh`s8FJDq#w{Yr`TuIQEG1PdZSM4T<#4J`eh~*^%t4^GBZ5YB*`_J9VUFuBNUQ$7$5*w*_|5AR z_Q7}jFQ6L*%*#>wTO+o)u;}Pfh3_^sVQJoFjQJCe+rK5?)ni@g%O1shCG+skkR#`R z>&1eIdAP(egKgxz1FAZsIA?x4oBsP2uWi$#nzju>^ni6-jq<5-M$gBPiI97X{Yf&6 zfR8wsaO_gia|dzx14%9iZVWj~oI%jK1N=0!fp9yT49z`of$RV6UAqff9_fS7c1;*D z>HtAKCph@Z0L1lW;H%YM>^-9ZVN<3++PT?`@%DFIE`AX+I#$YT)-0?Mekw?<$J%RF z^>K>E7oF??j^e8qlhJuRlU-js52e?=!VZt6*pkj&@14ic20Jm!BoTiGaJhG>JbbpM z2^(cA@sSXVt6LkQFy4LO&0vm*V`vwrY{$j;R8C8k*4njX2y=!LhTCInttR4{Ea|k$MU{ z(3Rv7I(@gqwDYG?-v1+JiP}*En;5!A-JL#la-ouFM!otAam!0fdVHlV4IZ6E6KAJl zVzWBkK4wDmZ%?Ijh0f!x5zfyTI*BfJ9>?*6Vfe>$01uc9;F9_poSe-4pQq*HUV&?P z*yk*MeH)1H0uQ3~Wp5n6>xPvjTX5qH9W=i6j2#>oMZWSx3^9a5`QizG?bJb*DaC3XX;?Tp7KZG0fx(O|*lH965--Z2 zUZV<#9cEvfuF^h;dFFKE{~r zDCvVKQujdddNJg0=dp4e|7WnE5;|6J`H#Oo_@ZGCBo0M`X`w7UaV^1EBQ+Ra;sxtk zQyI1HOPJ=qg)!tfxPPA<<#`u^VoObItmTuW7h3sm=eR{=1(8-b5B3=6@=kO?t5RaJc&O3 zF^HO4=TTpFDm^;Gly=3b(}MFU7+$YHGnUv<<-?YA#^tN1!ey+sSS_cYxJ;I|Lk!(G zbule`{~ishPNG>~8Vb7oL+yWy=_?10zZkrdR%b2eZ~zS5E`qp6)tcVk>O}dsY0>9$ zRj4jtKy{v3(ZEwP>3^ly(T%(I;`Yi=4{>2yWt@ns#J*sK+yFPnszJBt-B>D^i4ljg z@j8DLx)^N6rc-_>!g01X-!{Xa()PG4S{|DOKd|ExJ*>;NFnl*TpS@kQl-IgqxLQ^I z1S)YJ!a9I||)hNZ%tIi&B0IJs9TKyoH8 zATjs_ZgV`G7PoTnsAkyRv#a6y2!+9*Q267y6(?2s!2RI}=q!>0mz&pdijx-9P4)wa ztV%}X!&&Zc_h7uVelmR@|MF6^PU6_1C)JCmtl>T2%VM{NN8zFTCu~*y2bPG;Ky@ug zY%DQF)s4=0>We?l%k#x=)1tAsG#g*=GVqYaE)=>{g`*l@@W{w#-1I6D!{-Roo7vK| zGFyli+`5iSjc3sT&QJWq-GDkg=ektuv}pS5Wz=2RhK}S?>=Oc-X6H(yPpqU<%){y8 zulDq+*#yqf<{Ty^m6(?-L3h+T(j610X-cFM4M|)^4J{aYY^(-t(tzf_u%j!a)#&la zLe$@+LzU_ny5xmCHEOtw{Z@Qb;>2|N=Py5vs*J(~HD5SC<`X<&+$sTZT;}IgB7k(m&1>Dv{5zm3U1UC#;q?r*>b^aY)xDy znx@ZU39)AQubirxtA7K_e@$h^0u7*B@;#Gru?|0(7c*Z6Y{A_^6=XxsVzoDby;(Y> z8()NQfnnC@!Feb=RR-SB27;b*2y}h~lddr+j+Y}BUMN}qJ1Iv}9&)_K10NtPO_@;H zRuDeIai;#92ZhYx0eW z8O+Kj#TXbAUz42f#=SR6u}3%tD=#324Q=1fX14I7u~sh5nL7_BoaHfZkp()ie)whm z2DF~O7mcfevHpGp9`#ShpA9#VdX?b)^&z<8bsbvo`ie{K207PlG+J5nQ{S7@Xl4QD z!CrTn^K#9g=Zb-TG|{2|ysu+ZyDE+OVMp_23DxSV!TU+(G>+eyYQ#Iyd6}o_+w>(g zx_&CXR8Wlr6)&*dO@-b%>`GH^AElG+9BFWmGu_c=LGyEN;j1nS8rNYf0&}TO|HtsGy$~@pW_zkbnb#*b`w0Voq&HC`m zj&>Y9g=phhg4?EWJNo`GOpiN?bwyinp^pRpu=GXAX-&AebOjz$6UCtdBV0WG5U+d? zM3*{sRzs$q&6>mU@ly@iV~^^Xr!s$dz1A;LAgqVC^zu|#^&^kb)~rRTu)9p8QUKg3 z+6~Pe%djQX57s4wLWyo1cpdo&LZVsFZCeBUdxzk6`!vgUT-R~`lq#_2&jPQqkI-Vw z?OW|bVZdf9v>sTFhkovcYk2|CnbpE{&8Jv3KZkMIHyL)=yyA`EOYEqa#Hh|~Vm{B; zVnug8LeCF%Y>2HoE951LhCd(Sl_pJ0J}QQ~Eh|wirwPSJHet||CHTn0AFrhy#yE>` zEa|+2Et^WvyOg4x)*~EueUG2Z-=fl35k5Hm7B|=m(I4@A^bebg?p*@(n5#5h_L`4Q z_sPKhF|rhiIX$&Ti#|D*jX9`FqgUBb<76Jy464VZatkWc;!I;}?P>P?WAxD}SK6Jg zLYH~>;OgI>@W>1ex?(rRkUWMS>4+9vu;0w;0bsyahpFv%ZI~aZN0oIGu zpsnU5%#p~(nV-*bIi3jIni!6s=B-DUlM7Jy%1W%4tU={X257LTgk8JkCp-T^IhF^U zV{_h4<>_tN%c?DI#NU5AYrG1Fnc2-D%R ziqY`UNE2V}ItO8oOF8-y3z~xxMCr;K*sl5y5~JnG9P_D`T_5F0%dvkT74#OW=T0T- z{-Yqiy%ZjvJqzn~weXEnG_=J>g5)(3xccxiPBFQ`jHxKWOm7S3Lg{@xZ4}F#xi`eL z(dRX~T}{|h=*}K|eb?fk<#|?mZ3Qme_Jdxc$HyJq8ehGj0*Wlxw5AfOPCe%?%!u*6*)aI^xT0DTRpF&V4{X2Hg6Qf)k z8oQ2NM8A{Msc+GI>bG$Yt^JjT#n+Um?=3mFZqTHXB#r6aj}cB?U+ec6RB=Jv4jq#fwpvXyk`DCbzMxrM*aTTsPVM=Hym`_rq+ zIcAS8J$cQNN|4z!>_QPvZ4jnuKcwisSRs1lWd_1sK3bwIKo9Td=6UYfm}m4DUnal6 z5ywW%lgPz)ImOs8IRibnokwlcbjH1MjjswsIN2_!}6^ z%VitO#;fP%RI;TV_fg#93_JB*3PWG_@Lnc7z)<%hp0UI@^Q*v%>G_<6@2p#x{2c+X zO=Sy6#kt^uhf&a*cNG>om4Zow5K&+D4zvvU$aEVi^7JOZrHAfxveAlQ=9CgNJwBV-aS>|I1$uPQtsEym(xI2%J5s@T3%YxD13qiEr0KaUsm^8x%E$Sv z!e+QoxyQ!z$*&3A)We<8$9Z&8r7L}yxSncQS<~l_SJP2GTRM^Y2>%pY(XdsnbPwfv zzGXL2qfM8#brU*ZuS{WE8Cp)7L~kk1pr+1~>Fte$c$M>pWfh3f_ZQ#d=V=8f!R3;s z7xrP-11?WJ^)e=kmY_cO?#G{c8Mo*Z<1!B5AF#?F4Py4=jHCv9&U&Csxd^7tQpC+J z4H!G)Cu^r_%5KTx!>X~zc!{rseIk5>sbU&={sFD{U@(^VuI)ARtVNOeCC2&SXCGx^ z?pnhW8-Q(AVMu%YpmBW~WbDg=4QfB(3cQ3FYX!*TGtxvX@CS)lD^1?#aCyJnS6mit z2%gN(0)ts;@HlT9thx}21!EQY7S#4&d0XEd zV=7`Q*i+A*qR4taTw-I!9=$ijD(`Q^%>7FEut*pMU#!K+HjQ}8=Lo9%`{D7|A?PX*O&3ZY54so*{za!UinyCOyG5CkK6`!D8=mk_uZbMz4 z4y0V)L*5`B!>umkUE3U-+IkUJZ?C}}3K6*KAg8XBUdQn;@t4cb#G zY45G2G|=z;dHlZL@9+El{r>wMkL%v^xaYXXIrqHZukjqlM%vrS`hJ0M{hCR_d{&3K zz&P>sH#KO#R}b_5wxF}_4a~cIErr!CW%*4uxw;E!*Vk>-v1cFYxK_~@)lD?Csfl!g zn8(n)s}y=`Gl`1aBW|(?)O^Ko^4AfXW;qxSCC9>SKn899_L7FL0(#~dV9zWy{E$9Q z)`Ds1%XNk0R5J|1b^07)fiII4HWzEODHg`ys+TuZ@B7ASDW9uJp*jWd{uXZ7> zx;Z1fAQ&1J?l7u;OHt#Tux{02+>WxuyW&UGIK~jA6XwEdgC441+@V`~O1KuQ4a3{x z@b&#w(#sf$N9`lfHfn$r!q1cWpP#gu`svw$hb*>lppTEQ(-A|)A9NLx<+R%j_p*&% z4cbVXv&tyOxrcHK)^K4$ z(?Q?p>&0f_ZjS?vVxr$0H}0{dH;WayQ3rB3Iyj#jS)5Pzb(V5sFN&D9Sqis6>@17X zZg2}0F`XE#;k&hJ}$i_?8}hVv3l<2qA{xw{cdxZ#s?sm9-(d;Kzp zGpruX$);P=nHIUGW3KlaBi+sl#g_EY*r!W`m%glM+z}~Fk`9+i&+oPHu2QVf|Hed` z@%aOZ8?2yw6${#=lTMunI*BhRrw0{z6#uG{>L<3-yeDjaf89o%53i9yM+5bnJfv3) z`!F5g~L113G@7Z)AEVd5Zv{`xk2VQq1#X0E_zUUYz&i| zI?!VL{$ZV}FqLCoydtvbx%ZSFJ(fc2FLC(G^^s)ZZEACPO(UHjQ&laC;UB!FzJDj^ zz0+=r)i^}swtc13v&!gK&`SCenoUUi7}-Gl>@?^7e#5c)M840eOq9$e-$@WUYDcUB~+`8TM z+@kL+hcEog{ipvWTK=0`kobx7&adYTswz1v5hw0LBIAfP+HrL#q|V_}IoqM}+_GEk z0@2t2Dsva3wsa91a?plEB!|GkatQS~{1aX|kwi()<~@ zG_)axO8=CR(U=~37I=_48uyU?ntFP$=M_Cae3RtE9+NA}g|a^}9I&8|uuL45-cnfY z{+P7HWwHK-Dl)FB;_1Z?G@R4HnXSfXh|@>LwmuTJn&Zj?FG!eMPFT|nF#TeEeg{jv=uz`3>#PeCRJN~d|qONVZJ)J|KJs6 zx+&rFBt;Z!NI}~89{IB4}hqkS8>u?k*{{w^8Gbmvp-R7;Wy}LB4lt zXzTfR)LfoN1J@RkL-ShF&w5A!7L~%{x{}5&>+#faR18D&Yv{tQFT$e-m4r*=?vf-r zh3ai`r2gH8-l$~KU21IfS=}S>Rc#h{U*19YtGu|vh+;0}?Q$+sYai1(xX;biljoJ0 zM{>sYEPh}g!*+1;{MCC*duVty_oj3?w?4O+`@YAATX1^^3BL#h*&V%t>itg}?^mYL zlC^W`xU?L7(0nE=Z0Mj5S>uHQ?{49iMKu)pOcaJolu7(~c;lGAyM#fP9+JkKwbbeF zMTw7csMq2>$(0|V(``HHPWdsq`}Y~0JbHmP&17DNwvVad^%dHo+DmRK(x`Unr!CF* zXn7ICg}qnAn5C?D=JSGHJeq=;i7amm*N5cAKeTj>8J5EneQZ8I_Kf9Rvz+lKGZYRr z^Wpufm+V}8v2Ah`{;9d+2dnFjei4lyZzFK|?HnBOHNs>4U<4)jqt<0Mrnbz)*c2BC zMsi3GIz=Uy^^tnb5^iHPv1Q3tI+3e@mq+!G;jMtLRg8;kD}lV)5jgux6g$&ONK5@4 zO@8s3)q*e4vX{klY3m&-HGe|ewOUzjvzdB^R8!jA-E@CssuzS673n0ivqjLqXc+h8bD}V!|2b)XjHflpw*`5Ji#U^kT4DKy z7lMGt?LsrP3Mwlk+Wg9&>i?Y*Ce{Q9HU2fw#P1oLM&liBt4U0s9tHpsJ9b z+ij)@tDC~$)Q5sX?SsO=uZ7%hpId_INvo-D?sFRSJW;sVRgP9>2z3Unp-10WknP4yT6*aSnJzA((I<9OqiHqm4&O>ESDq(rRyQbb@1P|K z#S|oUjigq;Vg20qWYAemliWp7zicF=Tm~WZ;$HfjrhrJsiHbX`fJ3!gX|CKvP?;67 z<)^`YUp)z1I2dem#@_og5dX^%L*4u#`z8q8Ba8qi8;EtS#&?Yfa9ixL^}Z9N#a1A7 z%t8bl{7Ms3T@mIHg1wXHVbk>wbi&yJb-o^m@i4`@U*E~_wl+${46(0B3v*l_QgFc- zjH?`l3;lmd_Q@4eUMYpWvO}=bdw`zv_vr7Xm-JM%i#*G&(tKZ*3k0`N;Hw6*sU>pW z_KK`N=1_{sO1gfkj8qbSQ^QSB^0CYlPCS@KiC!b%$){0Epfl%QaGPr$qAMIW&07$3 z*_*CB&LriCD&d$Jy98M!_T1Aq)49=-*0l22F|I_RgY(&%&RL!==jsOcaMhL!x81GD z4-?Mj+dpdZI*)Yt$MxSi`DJBXqHP+NJ*0zs^_=-!J-4C@P1Cry4Ey!`VWps`YL~Er z%cKRo4~hNt6imH4Nf;N{!I@_{a^qKKP$T4_e2L){KV=Kg>4?(ioqy?2U!+xGSp+d0YUbIiG!WBI zO^G9s5H}iUMIVq!lopHa*!k|)#*p$K6yIV7jd~AkJkPw04|LP7CG#*mI2ihy-Ei~X zH+nmHA#y^(@hQs%RrPk5sT6^PYw^hUvq5&GHHNH9f%k$qBwusHp|1<@VM;v4o?M3e ztRJUw%?A_^1C^0p*bpNQZpS&Yh)nK%fQirpll^qw~Edq>}R2?YGC zr?>Z;XvoiQdQdtIk-25$6Xi`UTu!Rs)u;;>OIjAd|#Q?xWTf?sQ8Xw`j5+clRn$fZSMk zO?4JvPbWFIi4oSX__&383J2g|l3M7!|l){1Qiy)^S z3q3h6xQvv9M1l(%JC=cIo#N(X7HhfAhODkF3UUQlvtA6{`4iZ+)`H>7(Xg|7OLrJv z^enH6cE3?renk{}-VTDhp9o}CKakPlf3$>(F;Q9ouy> z={;GHVgEn6mO2?rMT+SDt>?lr##1}Ea60xr7s0Scr%6=vC|$nL%RGbM(POy}^gf}R zBEE^>)X5Jt>Z}Mp%SdAEIx$SUI0{GSt0I%d@xwgcQ^;>NhVIcq%Puv@dWu7&l*1Qs zTU_p#jo{e95NmV7tnAaF9%^&zbC#1N5d$~A9AYF=rUM=6txtLP4dOk*N)h7 zejUy&N=L$0)~`G7f(MQ%Sk@eYD{lNs6 zxTTF%rl!!|I1%4F8Aiu%7y>4%Lh>xb3>xZXF~)_`yL&gOC}(FY<`v&2P@-kL-EC z9p0-ToGU8NA7>iNp+Em{hnwoSpIwu9?|@kTS3nql(pHHdDI3O*^NHn8kC?=}=^y3l zhxf63em0+0$+VvphY5G+J?6q)=JJDG@@Apn}`u^W*Ft6jlkJwX@bX0Y+mmMN;Abs`TG=~<#;=t_$~ zot%WgizCpyC=?eXQ*mW_7<|%HFl+u2 z$TY@d`xJk;Y*vNOQ#*{ku@GS=Y*2Av6gJlhFmr|>;v*-aVXY_@i|8WORtEv=#v#^Z zFxG64g?x|%o)7;HH5uy|tBO zlgsqKp8n$cTTD62?VZAa!-3qM1b$ z|1WONk4i4{$4*)l^F>%`G$6Pc!*qjpkL5*f>C)t#`Vd*=2*2TZjQ`L_v**ZTw`?$y z?uVg@%IQsknc#PPu~1ZX66RJ6!VkNR{6bzLK7s z1V#qSVo8?_7OotJLU$eP&eK6`q8Q%%n}|n8W?`)phwnwBQTDm;$c9+co zyB5JuArG5wq+&wzTqOF0p}Rj9H!_lOHfK28)gw?5or!+q7_9j<8HdM*!z4Blm)saH z$Wa$-m%8DCqc?6z`M@Px6{(H3tPWs}pmVd3@pJ@ga*QC=XpHso+F0v67Wcx(!c}i1 zQmmz*(J~S$NyG6%WfWqs4aVo${q%kR9pX3iFuckr$U0BMP{Uf`8S_s~C9Q8s&c6W1 zf`YNCDuW7Ft8(XlhtnIY3&z1A64YekER^is%gv%cO=p;vMf2z@Br@kX@tf1Ruh*V) zXP?P)*B=Lymh3OVLgtOBKIQ-?a^Hn6jIQC{Ex*Y5Y!Trk-iOk@#$nuork9+!P6qet z?J=73>l!__ts+OoM@?BD_Hhvl<_ZrUAIkJfVLaJy1$7)$=8QgNkheq#oFq1(b?!SF zDI+56IlWt$oHY%a*`u*1_9rQgxkEcI|0NFxS@bciW!^h^WCx5y%^WG%YL0|njRJl# z-&>g;ZLE7>jBaORY$+TGkw1pGGtmle88-O-NEIuieXzaR8~GRAut0MvMh_3c)!HP~ z>V@I=4{a1_MdOQWCT_e~fr&oD5g?O->&>}1bUF;q$#YSgnufi>d6*Qj2&*%fpw2oE z+s3D%^Se2u+?FHj%6jCUT?H}reROYIf+q`;(7YxX{$>-fY`PC7U08{HS6|rdQOEcL zM$l)yfL0}YSe6e(ujy=5k2J)~kyCKPMgoUK^x+?@hvwPi5#=k5V(-D2ylfDrbbX}~ zhCO|@Uj!-z64(~nO*$$MD6XoNdf$JduhWNN7t0;)`dSMUZ1RM0Bju6kSb>A{v>4k5Je^1@+hI{hbQZ{-Z;k*N+>*Z=wlk66n zs(OvOB{$M(a}{IJ8+sI5%yIb&u6&NWFTb|4lowH9#}lrF%LNsv>Vs==9K{5t3MTKn5|jGMJpKgqU4= zIDAtdU(OB!->!}R9kZ}6e-5e!$zfD5!&%78N2`GVv5(bo+tL@I`y-)n$_JuDCZH@l z7*dmy@%Ql(4AqsuRo6Jo{jwgi*O#O0wK?MVq+;0qJoxSmU|8*CkWVYZPPr_c@|cI( z8=)AKm4#r#Xk2#qK`Y03VNG^4%DNWd$y3&^`Cx^J$-WSnSYmA>i~C=xqyLjGPDQEW zf>jH3rVU2-6;_k-`9U7Sy_9q2FLl=bqRkiX(#q%isd~*x+NXMilGV47#=2chkE4oC zS0AM5mKD_ZosH`yWwiFw1i$)wA4e44bW0k0F!uCjY46iPOmeFyV=AF*3U;UY1 z*MF3GH7w?R`m;d-liqBQo;Np6i zN1k{ne`2L8A8PxQ>$u^<_dg2Z-*sv7mA-16nzuAxaBMEmUsvG|UCHKp6pJ~X>&FBp zqECf-lb6%1Fl}7=J%@7Y{P|O+b^HUz&-|;MY5bQTb(Hs`3`d0OP+q-9SQGTrD1H7I z>{R+iLrbsF9J4KCe6E7-eztwYpgw}(#t+C|;D+vw|PR^Mu?r>Pb-c9<%h*dHGkd7wlQc9s24lwM7Fv8t6&~tSF>BoKl*I*a zPvo75hTl~$cz5y9KlH*&lHk z6S2n237S(Kp>xX=bxRdsyWJGwtIVN%aw_7z6!57{1qQ_`_^mtw1yZBYv5?_To{dA( zzG3M7IT#QA$zaA?C3FN&!1UQuaDC&ZrU!xhg_r$C;H`TE%1RVD1*yAy0jFT{;N4Mv zN3$q*&bS4dtC%l$NHs3IIU;LCHN|cm#I%eqP`=rA;h3?lf+Is8GtSvl>S}itxc6-5 z>^0YM%0-c6)3b)@jPKxFjx7>2?0QPkFJ20#Gk>dTuA7AweQi{ESB$b__mkgfC#=3* z4WCPcnj?QS!&YC6^SgJHznr3Aa{KumK0~O;MQw~gO}0F&K2;0*(vCJQjMG6htHpLR z-h@i_V07J=MeVDhtlp-G=LZzWqDSu7j&ejK5 ztR0{?4+~ffKDtvA+qW{^mC6uUDEi{@M+@{yhoEj+CLHg_LcWK?!h|HaugQn|ptUgh zri8kA>rr~I5EtI8f~K!MZoSIE_2OcjD+{2$H0tPQhKQORM`Rx_*{&HR<)rs0{} zG$<$YXjhwq8Ok%!v0#d@VgdHLeZo5(k7-_>n4 zi;YA}F3+#=lr{+u?Bu;ALb;l8p?F=YfGZ;BgzJw?7M|3YiGZtfu=gL2}dG#=x3CE?ZlNTzM>gVw(pn2??c&MqA-if-r-%|o8 z4kRHw%N`?(bD*`T7_wW#k$EQ!!q`$!RW{@N7$aOY2A^^^!tZ4o+|G`{zwuG5XOInn zViFGNSwMEqGMJmBqcSxXB~?~14Df-6T^KqWmf&EDHY$^y;Mp^e#hVtW3LcBBcyoLo zVuQeOhS;%g0ogel!jT)mfZ!R#^~hb&wB;yv`>r>=XQb+6E>o*UYzSH|Hd4!|8E8r`%!1 zcYo^TCI`qNt-VTUA~Kds7`;d+clQcCj%*R;o&PB~Z*xHKZv8#FG3%fpJ*`JDu)RUp zapnr$ZU_|qDI3ixtYG8CVL5nA{>|B6w&LHu&F98Hn~c8a+RX)ZeDi|iZE(#0E700- zogcdSAm6H}E_nPo6+0d&q9njUI9+P6VBRAo+|Sa0VDS`m93GFLqOpiC*TnY@76Y>X zTQFo2Zk4IwefLz{OVUMDof=}K4bd27gZDqpF%Y7Fk-N;1uH=Nb3_rV*o!|HJOYmo2 z7<6-ev8vn}O@mfprENM)`j?{etp(POPQp2-Y^>ZJiNkFFsZ&`EWZgz=_#Oj?y#oC9 z&xYc`4kbwrFE#m>)`P(vV#mguM>$X?(i_Mi~Slx7Y)-JQQ)J zzyeR)-B7fPL(tYCNcyOXlcfy9+oX)rkTawbC5cmk;;6LwNbeGumY&KtlEN>t+x>_- zPt?%|`_pu6!U<9}-$IkdRFT@|!~gM^DI&6r_7wQizTPaN4M)kn@eIxEI!omOPi~sw zK7oGqAY@F>Mb>0Py5^Cw;p}Wf1GlM*SVX)-`>3){jVgND=l{6rI}LWUC|x<@ox$I zo*H9{ZOFm-ObL|99TBKJ@!;&l&(b;TGt}T%N7WBAY0p$YdN8(v@~504$*4m#Tgb2) z`?oRtem&X!ZlQ&S3?t|9m==EdO>acMks-sx{oy~7nvFQhZ%QJu`5Zl(pn`P<>^eEB z;$N;TJg4w5JnDvFPc87`rXt*jcwl%y5SAD^V7AIf$`21hnOGXC-2L##NE_CFQc!S) zd0?Io!N#kh@Z42`B@KD#>@~rtsAQNuDMnm<7M7EsKG4Fx1YZGy6 zMmEeJ6NV0R#>I9E)VNGT zw$n^FD>JR`C(78Rua3`->KN#mgxyW!pf+9x?=zQd=l+{{<=odS(*2hgYz zM8*Y~+!-@{E}rpT!~?H39TT0(z4|uc7b#5X69Sg!KZZG;w%0N>Arsh7Jb zkwmQJuhJTWN}BLF07=RYsMWs%a9pzgon|vs13#%bx*8FNrLae zY*e!I(^%vIA=Bd;K4=rlg-Q5f!umQ&*?3$~jCn(nvHvrx(>3P7aYG62X@p|>uI1Pl zQ-p7)@*w%n0&2%oVL4Ea0EYrZSqm`kdlqgsY=lDWCVZ|8!rTDVjMr`is{C#X!~vlWqBUG zXDzY6*9r=7z~rme_|as92|p|`rYabt0@vfp)-9;@RYm0Fv3Pbo4>O-XK>U{-IQs84 z?YJ{r@V?EOCR7hcMWX`NK4|5_W`E&pxnsQf!GGKxhuH|S6mMR>+6s%>=F<(AVp{Z9 z113vvQ^*-<5*b!0Yxo!w1FSkM>89~Xx8Yf#FQ1-?eFZWv`Y=Q|6`P(63wi~S6T#-E{01HDIULq?L z8%9~-U49UnTBC7VCKhX+yJ6XdRJ>#JyJ2H8TIWZj$2}h#_(FITWn<*AAdHIJgr4ym zv3DQ~U48=G-?kCqqbl&(KLsx$EMeiB$Lc;sIDaA#ak~N$?NErUU8zuHvBeL=P#Biy zpwKK8t|QfP*)|ZW;@NCS|gf~d=8W&r{q1pG9%EM2R zd0sC?xc;GGy|2kj?k0T@zeUqVv{Kop6BIe-EDhb;MBDV5$tmCO0#x2u72%bM!z_&F& z;}0(b&kN%CYxlCb0A3Q!G{Du|b>`RIX1MI5dz&=2N^=%}OF6HpE4eL&?Lzyf5nO#a z>)i%PasJAOo6@f{Poq#5exb$ycWkR9y1r&}7o_5NMqS{Y+@ABTQx-5^=GTH>$KD!y z?Jwb0MI7OjQv_U4)&yZd+)-iNs_)#;Uo&`(nH}8AmVI7QI-Q2b_nUG0Oa?5^vHFchD!OiO zM5SjI0=ok7$1okCp?SDcydE_HQP{UU4#Te}p&>8{WA}N%dVe_7-z`J)KW}7eaA3ey z?CbGE>M3_zX5;U$nO10ZGDTJGY?u}sA*##>Gn53l5oU-F2N)hnY&0S~#~{vp8mdk( z{fC5AR4=&EG;75+YX7|o>Jr)bDwzp2g?!kK-vJG`E7+FRj0M|`@FIobD_@B5r)EFl zr0Y72Ik!oO=@`si52kDfpH0RpYCeaHq-2mXloWAIxa{CXt7MR#x1ja!BXEGN?wuv+dI7RUD|!!IL_ zjknR*8?hdHM(1PD#B^vT20_{^3)=DpxM7=$hb+dbYstXnhRt}tl4)d(vuF4zR@0L% zg7YyiWPkAm1r{PMDHmb~=0o0|X>zre;pd<&=)UfQGd8)H&{~fBCyF6%l!lmujWB*u zik?%2xO*TAg1zgpi|KED8Igk)ohV3MPR4z!cx*l$4zbftSh6$^{07q>Qq7q%ip}?;x#oE(T(g0tZ1bgw**NOeL~=*Gh0Sl=7;erBb?r8E zzIdqMMnu1GbYCZJI6sxXJ((l?otPke)S5@HzpoOKxS?>z;pt?V(n+<_KZVD`6$JG< z*0kn|C-%v@Qb3=x(E4c;**(j`Un|+>#>HH-T_V#d9k1B@e28@Ot+H>>G3>(MZI@v+ zeJOMbAWU)Ol(~UF*z43z92xmn+1wz&qe7g8%(R1 z%jTnbh}}0IKgGQ;bCEZ;<~w8SQ-8G0S&!%ub+G@Ngs*%2@L+BZ$|KtGs`nl=d*4H> zNV56mxAD#OBj+_cbu4I>@z86|+xZB4vqSK)G{jHM!m znrF~J*d5{e+opI3x}f<(mqYVPDdlF*P|4;szu!Zp`W_^hK3&nZ9Q4iigNAt$wy&(i za^Lkx)L#r8br%@1`}TN&Cp^cy!*HS_Y}U`kc5_>Nd0`9ni57Ti!=W$R8MAD?aV_2* zPd!-u>zfna*m>e()gov)JL1IeFeJ|n!<8?~ka0T{$7JFV*PjS+*;p3S`eEg>M64K_ z0xwp>EgCSul&~~}-p)d0g9CKm&V;RBDuRr%AobA&H=npedms;IPA8x{YCN`y&w{bH zJ8D1Lp*5?UyxD#C;=Uy;r%|#fFh-=ZJ(M&ZkbBJmW0EcKec@cJ6Lm)LD??-o z93e8v333nTLUZhFeW)n2@L!n_Uy z*Dite5L4Ks>LS9%9DA=MKzpet?B6X#uUZB2r?jKkbU%i2C2&pKfDZ?Yfb01Xcge#1 z+o`BHxDMm_NbJiDM!pM+=UATpvveW8qz2%bwl`ApWW zr)|PfvknXG=UozdoLNuz@BW~Gcf+VodW!I7gcQ{!F<;=VOsDPHM4^W7U}`Jwp*4M3 zRCfBNQ0YXCuy;s_&}qzBZkxzsuG_Gdvy3_@=Uwvz-&R$QD^iz!=q&S=3-fFS!trZKnPRa5wI?PXGU=!*-AI>>IBdrT!`rAV9O4M{|QdVrfDfq__i9NM!}fv6M?87!HAr~#<6=g_+`aG zlBT0@+(fh~szGMzFr>cxM`>^0Q0(<~I?v~j>;yH^-*88GY?3tHIl7li_MFXkywBvv zx$on}$}aKr@ih<4&-@YQ4>4=zS^nwaD&F}{C0}~Ao}UyTDVZ+&ov-!SnDuXL5=l`3y|$)}h2>38?? zA#&^avAu@;q@X?Aq-052s?aISkY}C}W;vv)(M}>?U(=S_e^fm%6sy$LkeD?Q3(ICe zdkTjGj4%1-q&LP*48d^G2vms&!z4c*aTY14PEE(rr77rQbKlK>xp?zA7vEcoa5lJ@ z^~j51IiVC%+X}GHx&*sd6rqvDa7)ZH(Cc0VjTjbpG-jb*H6Pj&^KjygH_Cf;@ix4L ze$JbU>hlgb$y9^yk9NWDyS_N~+6!&wo(M4u!q`vC@kt>RmKh;1gp{^1zQ`amXk zmO)q?0P~p)%Wvd`Ko+x2GH`{yk2Sh@Yphc?V&4JQ%h1)u?N90$@nRh8^af*e#{hM? zc2i}^6N-oG*)%=m;NmrVx#d=$xr)Pv{MG1a{;1(5 zKC!EspQK*Tx9o15_62EXL34?X1#!rS>j+0DG2+iu?M zX*z#py$!1|^>g)|Ja?`8li-zZq(Eia24Uk0bCP(MLJDKIk^aa=I<@mDIg5Uv3Ih?Q zi!vC?m$KSsl?LV&>*9=v5x%MzWBhk340>;cORHR9^Unnlnw~75_eHtV68x=Mh7Cp` z2zwEZ$eH2jJQsqecS51RAP~|i_-|qRN&WYnGw|PU5fM@LaB&zU`QK}C_VeHN|M#~47vLoqH2?qr diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_none_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt b/research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_none_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt deleted file mode 100644 index 5b295d789717d854d872fb57f46441170e8d20aa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 65259 zcmZ^Kc{EjT6gDztEXokcP)LPHhWqYYBn>j9eujinij)RL8k8YKrZR-gmCQqk`|gtl z86%Mfg$5K#qFMFbZ+-uK!}`v>XWezrde2#RzxSN|KKpr|ZEHD$pNB_SnCJi5X7Nbz z>^SVP*X!7Bon1Tpc4!~m=VM_e!Q=dYY-jj9{Cv5iwB3&SdHEa>aXY+w=TR@8U2cB6 z4;=A5>~>5~o<`Or+iLn50GiE7(g3JELDtJ`>;d+GnV+x~yUmf)HHe}t{0 zYhfn;e+B)&&4AZ2X~ORkLNX_ViK&+p(QutZq+J$~N7I>Pqn0>$t$IRK6>kxNsY3GY zB_%E)Pf2y)d(tlQjg;&Y28YihWXrPwqNewW1eFVdnWqXUZ3K{fVFqXCuwcAE8V)U* z0WDKfkm+IqRnuOuAoVo7^*joHZ(4)vRbxTAmFtua^p2fe1cjGVMbVNPaon}DYuL}$we}buCUiQ%nLAGa& zAUmLzpS^tPD7f(r!;`}wVaI>pLG%m{`+@B*h|hQe9?CheU%Uqv&-(@PnLO-SE5F0y zB_HAJk3kT5H3Ixc`Pl;bg6uS3LH3nMUN-yZC%7-(1#(6VaBAv7@8}D#=^KW0&IoYU zzksW=YvJkJBv{)M3V)YGgO~0V=nE=>hYH1T{%AUM9t(wzZI19f$^_2Hn?SmX4eU`o z1`kzF!<)}u5Fla#$(ggk;Q$|0R7%6o5fk|3xCZJT6ChH`V7yxpc3OQRzoXueZSzLS zsOU`isPdgeMSLKud!G=AJS1rag=A{Fmi(0OAzr7&LAf%EOiF4HQ(GBwig6-uf1D*6 zJfS3^dJdUlf{Bw{3VCdiLQ3YSl67YWOx~)?vHWc=kc)zb&>Y`K-dB|o+x{G)G{1xV zUNlL}^X5SJRyD|4uoB|q72##P1Q>^DNlRJZJWSY&Uk8 zy9>M5bPKznz?glTt;hbgW(7NOgs`ciDcf?65xYQBo-H=s2F+UX?4!;`?D`d^>>_Dk zdlju<2kzEm|2=Qa4pZ5}-dO3vzQ5Croh9$guDNN!4lG~J?iNvD=jy4jZEacX*c%(z zRfg8=gu&J9t{JNAvrA^MRRo41Yu+UEwu`WzXe+RH+bFP0C&bv@H^;!6p8>z>TuA?! z3+iEY@VcZAj#>}H<(`M|V8)KidIp94`XM?faa9ohrp;oYx?kkl!_w!S07 zHVBYnznMJ=2YKrt>qHKeD>s1cXHoW+)hpO5L(JLrY6k4;v-8-4A3nn!_zXwnCD>ut z7PDvMtY(YOSjX;P!e;N>tHrjrm1L`rh_FvJNwUrF&tnJNP+<@5na7sq{{+#II^cBj zBk+#NvyWt|u!rx?V;>Zn#g<+p$}Y2%WG~RwVk@e%**5&^*c-2`W)D;@W?xX1U{~$= z49{mwfU4a*_Uvs2?7RET*{a)DuzyX7ve&F>fM>2bpej}eb&+4-RF)L`!#x?c2P44V zFVGI0ig+l0?G7Vqejps31D{)-LXhx#m@C}^nqMM8?vNtn$qbXagIbVxHUhqtWWvm~ z32?5;0UXYV0_*NoB4rvzSb=5a(IH9LIrM?pZ>l3J`j3+GYF;wNxoxbzQOabS$PzNE zR2<%!g>r-vGHGB)IyL{jn{)GRX^XuNLTxA{kc!J)GZlzC)%INah zgPgGR^Tex24!$Mnz+9^*WW#-FI-&Q47O%QOWs3vI6A>p6X^950wVu%U;X2t+El*od z=22H$2ae4gKDZVX4$_)M;CD9}IHK|(Jt)nwXe_5&D`#*nv>1ZUu_|aY>;j{ZY?!Q= z1vylcvu`SuvyN3vHs5uJ-X%F8X;B2593nx;Rvt9fvRHB8H%1j%9_ z_~-eIjNXn!zKG|f|CJAz8J>hnPgBso@{e@MWRcs77s&J`If(xn0dAH>@V7J@{x-Nn z^PeIT?3Bfs;iSna)Sm^SV%hL@OBeV*tAf+XhOlK)h|}*}M)zi^b9{8=!D}cP&es)z zW>`3UP~iiOaW{^~l52G8jspEPGoR?q@q}WGhTrEL!FAbjZmv$JA=wl3W2ge%9Q>H5 zR%^p9B{|3(j3Ui#101SVMrZzRr7BSYv}fu%>)`r}bv%uOoWXNPX?kEgHP|kI>C?g( zu(+Il?(CN0HR=;kHXSU(zt zA8wW4vOVD_d1yH;=ih><)!~>LAAz0C`|;tMRhWES3E#{Rz<&SN)HGrbI{l7Ar<_=< z=Qv?N=v%7US;%=)J6+eS+eDLe5^&;9Gt$^99Pe9?8drNbr40r|E#f7|@2oA_JVpGS zHh|{-i2s5dv3h1MfhoYNjMah^**8Nh$!=T-y3xA+>alBETuus<*Wl8 z*3|BX1D4K5l$-M!=dVLNWMzkG(8IC!Hz4js&p6JL>#?n&3L9@WW6R7s z4WB9EsKXm7<>HKgp2Q-LWIUFLdZM@D8@l8!KgwAu;jH*o=+E{+wQCXBd@dX}XWC<_ z{z}@jF&yXYD8;$^qw&rGE41C4PRH*HuuN5@Nr06q(LO568r_;r=QwRckrkzw9bSQ3 zwxyt%j|I){KY`*}dHAIx51p4?#Hx0G+@@fUd(Q)womh+|tvNW+*NRq~+VD&4b^O8O zfNkDGbd}?CYRa<{16aej?&DlW&{m9zxx_)QzdCrOghQ*!jWGOrBbqA9F;@4qnc3P3 zjEv0-yn8DE#kDnXS^aT@M_(}C-;lvT2j*$TYUZzn026m86<=7K!4DNL@Y7N@W0$pu zG5xrQx%zP}bA-qMK+A(M=R!P3t?i#7cpY?!i@Lc7uayT1^d=a zV4s%@TIM$&}M{O);y+s~ar$Ffwk^AKPncukosHwT8?HDp{)e!(BD$1!7u25LG5 z;HTslxK&z#F-_5Csx;*oj%Xv^$v47__8e;bUKf`R)}hTVF{VOAh7pnpS*12<7pLx{DmX}$C6&FLc3`B6Umd(XoR^ce)l7ex= zE3o7%hg?#YL5t=CC^z7bM$(&6woeIb_KwrccO2@elSEq`LveNRElht~i_45-(DdG1 z)JsyJjb48^BDRb1v|K+foGHtE_7-KvI*QTKd6FLc-N@;z`aw-Cs&Mm9IYxPwKC}0h zBGbD5AqMfU!l!<<^qrG2n$8+P*9-&ZOSS_Oq6AFar7xHly&HoMsN-!;6`qd7Le#aEOuuziY6C0PhUF}NkG z0$)DSXVm#BF;eFOt_{z^GxMI}>wHmW`0XObO2mZORcyv&kMl5}x%X@U=ROu>2{3!w zjG3=<-I>RdyBLE(L*^+~Va958Oo`l$o8rIVt!ctM$#P(3M;S2t&iCVeBV#-rQA?wT z*!Zol4WU_)+0dcK?BK?2_M-~f(LdD#Ej-XaF8RgF;xwph&?p8zCc25kPI+qaRwfgk%vUPL~ zi<0W?jqq4&5j;5MOL!$0l7`Hy#Aiqhc+MHaUMqV@lko&IUtLz4LN`65*F)VG1l6g0 zkpY#i^Ke5a7L=xiK}%I12mX{}nQ{_-x)Dlw$^fFflR+wKHH60|&`0g1Xm#cTYL`}H z!JJY0xkm`3CjwxPvjVKk6T>4(FY#jiZ060WL9A$92j2 z_&P|Ck3*^Gz!OAHAWLlJR%8Ma1KRXEro|L0-ksj)(&4j9bvA{og z9{e`Q!hb6Y>zL>+s=2+J&Ph;X|^BTddGBJ3!{0YgH=_Aq68svCLCG9@41z)?I#PX53 zDCM&Vg#DYSnco`ZabJsq59Z)0y#lIzU_RA&Yt9+jc!CvcY>AG_$tbomhPxIm#DkW3 zCYOxGz$bMcsPZzb7bOumtE?MK#Y!lSwGbH7?SsRED>jz$-Gn@p09x-s(PYptq z&(qiis-O}Z1Clfe)Z@G$jx7t}$&@g)Uc|Ibj}%7S0IS?_aGl);5f?+?KSdd$YW15Y zH3?&=zqZNU018X~&SaY_^@FpxD8O|iT(vb9YaZEQ%fbM%N+S;}cDKUb&U5hc#B<}; z@enloavSG1C7|u2NsdRV9Slw;f})%@#0k~YQ@o|f)6I?Nz* zajnBi+$(C27Hcc%;`qfx#jKwMwfB%!E9;1%Lm9DH%*BV@;aI;n7sa;tqU%W? zqP*V>^RiB1*79@cwq-M3_$q}xvmer&?Rj+7={59EK?q(fEkV}POk}?&DAs4hxnM9& zPA->#dIt~I{d1A{HmnoNHeW@#?FqE|;W|ipod<0L^RV}x!$;DOrK*XTb{ZwPi0W*>UCh#joV!;Uwc4Y{qySn-*m zENjoja98#YxQ~jn*BHyQon*d2?NUqlq_Z9#R@*_9*Kyc4k_DS~P|!b80`sp}!>~>@ z=gI*q_?2A(Q!W&=`?Ek)ClD^}w*%cv>p;}O68`jl1q(xYw*8biyJqPfcw3|iiD_q8 zn+MLan%B$*Z&x1n!(WToV=Q6zPH}HY{dJYf<$7R)`Y63CKMU|&3pCvP2t8@nVeG{T zvT521eaEk%<**-WCTOrc7x;m6V;;onuZOAV1j;#j6_kCH8gbhl9${muN~45|8~=c8=2PQQT11>~?E9<#J8 zRN>mcrI0A7LypV(=9PqB@ZlC%fJq8Q+zRe ziA?twz{lI;kX76Y`HRIs%k&#Hsy%=nXS=Cij}2^N|ApRYMRq#RBsli{B19#U^YV%? zEtvm|Y%a)xqo;ntk_}?)k?4NlJ*EbVj0{ZNm4)LHrcfPy4v_B}ythsPx61i25qO_l zGR=n<(J3G~`x`)9Q@fg;n=tryq={(+h-=?o%8kB_4OF)O)n>5Im@8BO$ACG zKV;3gFN=P*7x2ULY;MhT4Ex8YIbMsyD1TW#z1aANE^}CeZW|8cVaaotV`71CrPau` z0)N~$nujH25%@b$0AJ^PGd|yuLmavb$j>Guk9*q5-H{ojdb*vy3E7X!R$s)|zV=vb zlSg)C&cO!>c4%R1j_p#s_(0Hw^2x?=HjZ4b+w|WcOI?|TWLGfSnQq5Gr*hgTokKj! zx4@G`E=MboL39PSVV*%1Iyy(9twA0K5(2>GeKm;w4h1#!qb!?&pU5EvbgN0xdxF612NYlZrZS>{D5}LPg3F}@)6Pb~>68M z?rcV$`UL!V%m*J9wNw7hyd?i$JJ~-@$@&95M7m*$)YKm(h5^TD-Pcw4i}yG#h*iR& zE>Sq}u7JuVNMTp%KN>S{I}Q81vDV*U6-o7RBu(Qt$k2a6_|kPNj>|8@$2vyz!14() z+`k`G>i5E$`(?yLX&dtC-a=icIGj5^n_g4)fGm;^A<-TnEU+%k>0Lzp)|YdpJri+%ZVFoFL?O@AW?XZviK>5hF+Rzk!tyewbS&Tz zzFbCWGhZreG-bPSSZ5<`=h=(}0a0kvlY%Fs5|O!pobJ`dL|5%GYmu=uUhOQy`d^;W-fL*M#)hz1)2807VmjVVX|`E`4^McIvppi>WjydJeGH@+j?a zOvBiKcKo$24JA&1pF-_-_7I%*_?Tl1BwZv&4y%h^;1)^L1*w*KMb6lK*JOR%w*DRY)Jc76XY1 zCDhmxfCu?kqu(-r`lIGJxxQwK^w4e+pD|3nWJ$xsf;v(cHO_e)ql9c~hxrQQG{n^e z8l2|Sft#&#PsR7%65JB=iabrJi?FLa&QMsBWOgeP+s(;B-m(&4oW z&SW^l+h_j7llK%J{@Q{U2Xe9bZW`@oRzTI-ICyYH8#YTc(Z0SqEUukGsrz?ux10oe zDnBP7M_eJrQ3yU>P(zCk!#MN#JVsK3kC}ONKR#EJCr0OIz-!0*oEO(x(fQaM=9Gpq z^PKW9K1r#lvrPyqKVPO2&1}rC|A2X`Mp1L`b99!dK+5bv>8K&Pa$Ov~@1u&JZ%<;& zI#uK=jiX2R4{~XnJ$U(IIX<<1jv5xDc;0LT*B)AhOPVj!$}NKU;BG2yhe>GK5rEQ{%vk>^n5Q-PZdMb zdIxJ_rtsU7T0DHXk=AGFK;*qxIHbQ4p7EzqXwOA^OUe%kSAenMXCpjN!<< zaoQ+lk7Djz{^#o$r?kC>SoTVTQo}I0y7&ii)KdfJ&{1N0H>WP3ToC_Q0Ln>FT0ega zM9g;J)NZ~?b?U;X*plKp{eRA6RzfOydd-)(WZfg`aiaLB)ddT;0UnP&LkspYq$}74 z{9Fhu^B-fWCCA|3>6e)CrUrSFMrp|G+2AO92FhlN!-cs*=vLo>?$$Gyg;QPF^I z=Np=+DnWVDO6i(N2i&ow2uBRN@TB@MRyJ~Z!Aw0|xGJ2McTZBj6N&iB{2v}ar_8** zBg5>!+lhia{51aq52UR6%sO}L6xIg{F$?CYFfXJhv6#;gkJJqj(S_?_IwFN+Y3X5q ze;3Len8CF4cVL{Q5C%y|fO~New0X}0_ux?~d!`28s=vZ*zA;$!?=#EHfk4(=8wfta zkV30q8a74n#|sw>Fcih5i|&!tpAQp3xisQ0xQ$GuT&_!5agLfV&!d4#TRHPR?IHF8 zhnD9PF5cqDeIwWF#*dAXJ^89&Gv_9HEGpJwo4Sf&YMc)%* zSat3b`RaEZ>Z=w*g4Q%Wv7`qljis0$C&qC1Z#6u`Ck&GHzOX$lm26y;iQl8XW2GD) zWAA($E6!?iG3PQ?Z$li>F2Bj?PmIR$$4Mx3J{*-J*C2Z;l!~4RAik&H5_V%T>-pSR zJRV!XIu`bpX#MpgnEId+m^z2^*h!dQ9gY4cYB}Qp@#K`aA?rU)O%xQX$6_fy zW_AB}9R8Sr;r6LyobiRL$Aq9gZXqUY9>W(zijn-U2Y>VZqF)^}VcEVw(EKn)L>t9$ z$nh?kCVj(mRt)YlX{BpxmH}^13>+C+3=_ilsQcU!^iSzR>vL9<%i(dVt=%=W+5j)2uHJ~=pIuM+_m>TT|ICa^>4JIOJpS;JGT>mKiE!V z-$xSD1t&;oav7c8d=L4L^E0;hda-GbA;Lf{@ju}NjsufK(pm`3%NkL>{~HE;ti${R z1N5nv5~yEKfj#klpkv;}S#6$*!j}E$;GBicj}$n~M-oB8po{AZhzHfFU@G--Kk6P2 z#SgaA^laK~Qa5!0UL7fg>tg|6EP%GN8G8%vIdSo;Bk0tR0=V_FF?Y=+oaHUnx0!6 zin3Ar(KaBM_9VrFhH@7u4JHB@D{%r%ve3$}9|!iQVpD4iCz0g`8xN#_6u%NIx;;Sm zzpKO3ZQpRDwh@KJh47l_6dCyE1gk#XB3aFbcy!@QOpoAa1Z?i%rLCp3)8ROAuZ1vmn787Oh+B9> zxQm`JNao`F67;&2jj0a77<6_kN-W!m-EJ;;(r^O?mAB&7)^5~e9%2(0qdpjHqUPxZ ztS@_0Sw~!^sovjqOqCL1whh0=e~D&zU~f9nQnZESiBXa{dj>}E-o^6l&$uG{Ca&c? zpz0sBfuF0zHTH!==;w8;jW5FR99P>YZ-~IbLUF?PuLykRPJr#}awv)0#roo*fWMnr zSo6e{7HBL0xv>&h`lbgGdapxDRvg)QMu0?xTC?mL2U(B%>xt%KEm$mL1RE^!NvW>_ z5|TTkFWRzvEV8go6$wy9{i6JdI`A{>}1Ns+Ap(JquTtDni?f6-E z*Fgb~NqVycZk0oq#sn0O6~P4oNz(Q(0#~~>@8V8w{AMN}$gu$%cqW}r6dTO&ALlh5)e&O49=GGbwO=&MD~&Z;pU-l7 z&`94d;LdMwJKlEc#x+H)c!j&}yUlSyw(tfljogB5vBCK7XEwgOSb_$g$+%nfF`Yhs z3yZ>A(Re5h6$0DnweMWLomUc~reBkWp)qpv<9vAZFppf$tDr(JW6`Cf4U=copr;vcIv?ERpKt6QHFes^!!Rw53q&i5%pZ(_>qcZk;ukaz11diak% z46?X>4&4Y?*^tJn6gr2AH(IbO0Xg3AnBc0hV8%o zNdMS;$Vn9kCBd^~lXqX;nSd&e^gqqoc|?}fufGInz=2<)m*B#$7es7BGu_zij(_hM z;B$eYy3IbmUJ%{N6A}>^T{Jq=3w>R&+G~i7`JpD6LXKZ|!m;)|383o!5u% zTDTLtUR=ktBMoZtz{6ZU;oAMBIJAE|s=@{QYLSWFZZVj9^a;HJb?E2S zh7t|2SfpA`EpJ{UUu5Tl5dB0fZU}t}LYw?TzT5iWdD|f0l0kqE8z)?LcuR0)-ZNpu1NG9o^zdBHw9(Z=4dy z8Yr=j1V`eCOEd0%nt}bi$<$ZX0K#;V0CKE=NmioYx>E3mNE@mLCE?YUJ@m`nH6USq z9p13cgGN>&Yh>AZ41Cp%CJtFx8O={)bwYse!ENwc6AD2#*J$;r5bT!8za)M zomLKo%y+k8R!=Y_;tb0AmWj)}o}l&e^LXH1H0#ZxDA->{)$&6Lo@r>oyv;j_kyJbV zxp^nfNs7R;0XuPjh!I_N>?~cxznlsh=uy|nQhM1|j;r4jjIv*dmY;I#@=wR0|LN=K zw#ga$GzK{xEuvs#u?FOV1tII@0>C0Jwz}^C+pb2D9LYi2@a;dG@{qv~^JO8PU0}lV zm&I8!_?YGVCz~`Xer-FLOY`D!}2eo6q( zZ6fd_RSFHfqVVFs^*E8uNA`_Jfw)gwK@_ja=i!}}dk7GZKepbX z8;Y}uN1_VcUr|N|7b;<7hY=bpn&R#UbFuJ-J>9+3k9_>Ei|p~2F==ab#{5Eh8?{GEk&|VP? zBb(uHZ$9YRxxfP9L)5AD8osoDf*JgwsMWiOeC5V*q^TN|XZpaEi|^}F?R;^~f@{cG z=z%)vOUTg+380vF2Yf@KKz<`HdG%=n?pqm!v6)i1>g`Ohlf4M$Jn7J|LI-l&-AF*s zek#x3N;^GW=#c*?>2keIbYeb|_Mize%Oi^<9PDIa5Q`JWpKJ11P6ix$WN@vg16I@z zQmgK8B5>LP=;!0`xs%YK$1%+W4D9 z8L7hZ+AVO7=KyH>?1FvHwlHPA4{pjtk+1xvth=)UN%;#+aIQ#);9EE0N^}E^S9!yq zw6!#!-v|pI_R*PnAIRC{3izZu1PhBR;riw0L`QW2hNg#M?FT))bYc~0NWK8aRNLTj zc>#EYr<0|>_u!9JjcCv1LCe<@T0OoK>Pm0GuFIRjTSXYnE?vX?3(dH*G6mb$4$-8O z{-kY&DAW#|CmEAkxPIAsJTbfps|)1tLiA#KD=Cv~KP(0*E7FL?Ngi~emx5(y6tbD(T>r+^V4`a2E56_cpeIoGZrauV?cu%|4E~Be#GZy7s!;ZZ(v95MA z*tOpP0WLRlZ^DE+)?UF6s*M=@ZZG!!NGHzA3!$g24Zc0N0IXnR@|B~97v6_q&7cx) zsONGSKPtIe%^>9Nsen%Q2eO{;6}^1h2%F+pQE!iZ;EWCM;^$4UYDkB9DvROO_-XRg z?FQ@ga1?nI=MBpj+5-R8-Js3m3$j*QVK`I;RF3{3?$YY zhb-iOjR#)i6wv=f;9Z6pq%6vXgngOdu~i2`LZ|AE&wfVDEO@DeUo0qZSAgL6;!vZw z4A!_h!QMnKXp?e=jqeHks@)ErhmMnm$1$wpGIt{Xnahz&U4-=t4?r|EAD+3nLeYOa z=#E|%>RZ2~0fw*0o#A4bX*vMD{H4%p-%56}=cBf4C~n`Ui|5_V$PS2s+)oeTVN(h4 z%iSawMz&(LJH>`27m@wpF=wIuai~bGf|sj~fNE?Vo!Nd7*YCZB_vJ$|^h`3nJkUnc zJPqOG!7;K_XoNZ+TY%9o)bY;p57Z^ks4i#wIJvfUB`7vc68?rt>Zt#f^ro+bo)zO{ zXRCEx72gM{+`b5>PcJ}d`al&5M#!H#hVWOoow(K{(~aDk!cqGc4uxMt`EW`_z8?nh zgH^DLTMu+x>fu~|pMbS~6f3j-L#K!ea>u3=20lInNsSm7y2mCWjXHQeF%+Ah&c~3H z7BbXc0-nVKAR=1=^SXM;HSf1HU^Bt1EDuUJuAsR*A7*@Q0AX$(UB|xw{%zP!CUTNl z2S50essvXU|4TsPkTZPn_X5M8j?m)23^I+x!M#lZ#&^VmZyO&C;CVrNe^1x#k=BK< z+nI1bhU<;3HiPt~EC~3O0wD_{VQl;{X-jUVW3xQ)X}ty}OwWY%rgf|{>$bDBJ#ww=Ni;%%I_OB|s1M+Q9Y+z#TV*En;uB2exF#gc#+TzKRF6+NO4Lm5}Vxjz{G z7#(2sZ;ipA)R)+zQpWXNrPKYv8$fwmHl%dh!;GYNv?+|64_;D?;*Z3!PpNe7*kiIG ziVbfgf01PMHhSZk5)P~DVVa2$8U#P)I7~GWtEXxZXV6Ua!v0YI=q6&Ipa$e}3(0Tk z@l0{5__G1}ocX@~huu$5#NK-fmLroJPZcMWCGlVr1?We7J3hHl5!BMKf|> z!KMvh8FPgejF+Nw`%AQoi@{*ky{z2cU^ubm3RFq2fUiYf)IK`~trsJ%-X6ilRaZH) zw`_+H(@gN>afG;C_c$k~Poj!#9u9VizYr$m?Iokar9_O16BcFLcE_tGdmR1o+iy~3i+#drZ zT`>6aHtfnhiK8Ad*f1Q71~vDPSEvJ79UR=P9Em#&q)@NTj81NMr_MSXa7$xG)dP5`S%9g0-HS(?>@Xe{aM!?<@aT;c~*#1QN_bld~e6-VF6ryV*z-)jDyk?GidS|rm~Wkv0-yH9@-a#D{t?l`EPh&%Q8pU zb80^5=ts~?>N9bpk1j4ZSHO?=E=!?r8+m$hIj3Lc2u7OfaWvhxk#|;FtPYPN z>a#@!Z^|vl)5nBSX8Ig`TwQ3snfP(b4# zCRQ2#XbWKCl@5IPnupn7T8mc;i|M(;vmyGX0epNqpR5wHLm9c3c=Lk*)4yr}&wsN= zukK9NBexFnQcsw}v-u=Nxj zvxIwS-WQB}t+^hgypx#ezYV|DxZr0Mf0P=E#6Kz#=rF2>+Z_7wPx%nGYbM~?Sqju! zQ47+Hd|`HpG|0vAfmZz{Fj@be@ELxl6Uoge{bermV@!sLzfz7*Ji9T?uM<7I>QUOX z3dLh@p>}r*^18o5H=8bGhw(H19zx9b+Mjrid#;*I6dGSQ#OT?8=7o8<`nCk)c1D4z zG@r`^ovlGlbS*G!mycg*JC-d0sYtzI2Dq8swq&`4P0XSpkLi zI^$w#7xaC!3=2(>>i_vbWFBjQLUb-!WYR{J5)yG*tPMAZmtshP3U0I!1@qBe@MI2$ zxErm*THgt*N|a&_n)G7W(fR1O#nU=-$fH)H(p59HY+#GKp0 z&pa>eLbElmut}p8-Ggr7+ov~iiCI0K;qS!c?ryaCQI01iWf`Z>bD3Wq+`9A31YNg~ z+obTz5(;eippzv951jYI#nbcQ2iKdg7*~aY@?y*_JyAxbCJ7DRm0;!9JdDn{gca8! z@j`Db($~3|m|lbPLW=R^ST9ZuzQ!`aHngbEz_pJzVD*((6y%0z@`Y14*75~Y=geh} zg@4A|tCH|@#tF)^-ITCaOmU9CKaRimenBZ$NoMi!G5qKjgq2gloCH}OSQ_6*W@>z- z>eDT_-&K?;E_;V-=NVzm;!ZN3{{XaCOG9AuTs&M+i02(z@I+rC=KH>&S9GGt3TahX zvGEQ0SVU=In+DFZ*oy6Tb|}>1M;LsPkY_=TC;CH(-t_*H2qsjI9$Pbm_ot+TNc)?T3}A_TvH$ zCucT2dL*4DXphlq_hhf!06=>pm{iRPQ;ytAX{5UTaD^E~rVox^(@QH+1aYTwG`-h__C;(Bwu!W)(FM_4YMncFhE3Rjnn}7aNGypb0VN-A=az z3F6UrGAJtajxO(6MkON@z(02r^ozeGQ4fyN_^f02y{Z6DzKX;opJe*skUN|a%!2!Y zreJXTDz)SKoS$BQj%%Nv$L5>ateJm(;UhO5n@SOaNxc;4k01on~swo$TV2OsJFkwoPb6tPdo42NGB z;wg5ouF~^GmiW{T{IA$q}p8o_G8PyH)1^uG2ZatT>xB} z?Ey|v29U(ngtYtS=dmc0V>I-+3au?n~|Sqw%c|H0^adCvQ`AJjqb zG|lD_`7Dnz5X0}n&_ia!G+p6+&-X~ucx6) zX$uTmr?RAN)>47|WcqibF-%}Er zFV}+pQ0 zD)k}sFY^Y+w$rd;`!MJ)e-9luqG9Zf9?{>+p&slu&Yv^M&?cA!&j<5BK3uK}->I()OX0s61}|T*z9m!W2$=o`>A! z-e4uan)F`3!6|sMlHRJ*=BSlc5VsqffxqHB#F*{l_KdzKecT=$(c1HH&BYEHHOqf;H;}q@I4WBa{oVFo(9g`c^s<0(U`bcKXRreK^)SF7` zQU8^5u2Tg%uGK=>tP$9eSOV!(9h~={qqg#k@KDiB+BfGh8FJ19!Q5V0nArrkh1js- z$8+jFBO3p%w8MxqN@Q3(85YHLLdK>XD0(GCUYM*zwez7Uxp4*ZNtM)T99;xk{yPmb zHMW54jTV;LT}yiQR1SS#V?$kL7!so)Rp@o`0F^u=IHh4pW>7;Y&-MV#LN(~#U`Wyx zY^XJpO|Lz)p--bbS+@>tfyx`FVM1sT=&z=ANoQ8zG4D`J4qJ(bZwZjwJ~^OwuM?ik zO@>-jBIP%1@$Hi+yl2)+bw$`9@UICrZ0&^{mkcO*)kRcqR?;DPHB4R1aNr23Ccf;!=_swh~J(` zGP!pd7$tIhZ9Q~>N%H`QL!t29D;y-|`+?dEJ4mbH`r!F~QoW~!m?+rES-;2=63#w_ znwNc0DsmJuQmw$Grxu2+b0E$9I(h76hfB;G@uFfi4n0et5h9;C%%~!3w&e)RA^Z@_ zlWz{a~d>G5$K(FsqprE8l8W!}D#fMr*L`4xtW1l@fhw*lphl*L~JgU5_xdw=c$=%pknjlu9RS?vn@i*^seK z8Z@OAQsWc6n4G)>%b&=j`M*$lZ=(T8`qN4NZa+*82USxUx}WU84sv_>N>V)@LirTr zF}!XGp6BPqt@SFD>5&3AIqp4}_JI6nkw#}G1>x~|#pt&`30Txh6WBMgnOi&iEH$A`4eC(o zz?}~&2`UMG)c!yqirJ@O#3Os$=y{#9T)%}x`#dL?r!`5Su@YU}a)(Y&@ZpU?LA)H; zO6MyJq2HE98q+a{t}r~rN`?^@TC1~OSp4Ed8zoSwu_~NYZo-*`w#ZtYPv&pTf!dvQ zFtyzp)TK<|o;cS-KF<#vZz#~eA&7rx$uW%r^31`wQZzc7fR^9=&}J)FLl8fXd}3+X zvZoIDW_6$h*LTsfr4d(by^FU5s<7TQ1-GvEz=1Gs?@qW9E`SKMemH=4M1`5rFJExi z?L^#QbC2e)FgJ?-5Jul|HHieSUR_bftt-MmVh5M!=`D7lZ>#M> zkdc{DN<$I%bDl_<4Ma&%p}mcElD$dUE27LYqVjysMM;H{lC<smgD`W@Tw#}PZSCsY?c2frXw!sKxFy>e7{ ze~mL&iWI=N{wg`_H~ODs>+9oeZ#DyqFs6sZp*RqP63Z z_^2IeQGqHwb*z}4Nm+q6V!3Op;fId(HfYoRh{h=U5nfLNnXa{vY5S#$nt{1^W7=y} z`BjeP1#)&aCB+mLQd4~oO!MN;Z0Ie{(y@GA3rm1$E8pQQQhEqk@GdKRj^e@2}!Em$R?}|8W zkt-^OC15${BBeoK9i(c^9;-p3t-G?cPt+5xh-HD9q5%wVG=xn~SD|C;PUxO`o$6=(hd+vSSou#&*|Vj+ zxK^eLSN*z%2Zhp5R<96WSvPqblTg1vQM*d-=aXg zYV402=Ofys3$a<}6j-;-)7V)D9-v(R8jN52l#X7Ti>Xbexb~4aJELPJtNTiv75H=u zb&t)!Z?%j0KP=Pvwnm(07BhfnI^@`g0>bR7jwlS?6waSm`hm!ftC48gjksm@GrW^B zjPbM@ANShgw)HC1e&t$Ha6gK%z9Nbvqi3+hmdo(fCZS55Czb`sVV74ceVVL*57LV< zAyWah4PJ3`NjZF4&V4;Q38yVc!Vt4FXdgNSjU3|`>)lpFaZrw4@VCW`>}pI27{MEF}e2>75XJV;!Q!$N&2ABzEW~#k(y%7|8rE;E!jqD{=u^Fv zP0QC||ERX(nwo6t@MH%FT)YZ%6%3(ifdj{*U}5~(Maa9qo&Vb$vDQGIy_lxJzT)Or zeS!XHleQjj>+;Y{bPqkHm!5w>Vc;kkxefjLz2+IUTl-&Xwxtk9U{S9y@Mc&^3TVb>eK<*avh9ip8X} z1zav|0kK#lN(NFS(0%uFw0$VS{@po($J^YoW7kzi;pKErzqmwXHZQ{GD=Kjz`vGrV zD#d@#mZN}U2!FHdB~q`bPJB5B=cO!1jFk$;iiv@!W^0LguU^tbWjk8b!l&oo`=Q=s zN=vlu=;*YUwDq0^O0Nz?&6pr`dgq9{_O{a6d75PQzl$XJLNH(WvL629d^79XK46eT zC8kpyOstI|B3wpf`lM85L619DasEQ56;s)g#^*RmS{7mJ-lkjTtXS}djko8V@$DL&)p1j3nSGn_4-E$bN z|HL9cB9D9TyM@Cle7u!#55rb9V|h!dx%=v+Mw2>I6OM;jW&NN-9OdiWQFu1uhj8-wxNhFhpUbQw1*wb2>n z86;cI5{4HNh#or2jHm|Uxfw69GPD>qg1V?isxX{$_l9x(WgxmWh>o5ON5SSBcSnKw+rym10|Fb$)Pbx*O{4{a!BMaE26(Rgt~{> zl7u}u#CuCRljo8{g^noU!j}us>&ahQ?pw*X6yVH}TC<^G{2W=Ce4mb}2B7T4o7lKI z9EG{R-yW?CVYAPJJ1Y!Z>^@SLwZ-WDs~HEra^IJ8oJm|jV12PA-20SC;Kuok7uXJoW~0c^ksXgMR5bJ z&wYcgOC7MqdLpb`{S=P*e}-!!{;-6mfNGgI&-b20 zJwb5e>2)~Xn+M0-qQLAN4^p#Vlgow)BsaF7%=kD5c0AC5j1gUUH(~;=Mh8J~bp&|y zM8N3;UC66ZBo~eCY1@)>{8LfFVEFYioZWyRCzTH7lH#!Vnk$`IGXou7=FuNU8Kn2% zF_;`%4X2duK+xV*&?0`H-rnbjagyekr`N~C+zNuVUR9irGZC1F=ZKf2B6igrK(}cM zIM)2s@O~)gToF70iAj!7@gB40O|_MO;CALJzPzr9T-(>HAer`adqgC;lI+goC2bV?pw%nx9Z zqawEO&JxLmiLghi3XI+afvi*q12yLON!kz1wQA_w*T%pVpYZx&HEhv41`lP^iR_vJ zs%|+07pOT=Gchq398Cko1qc`2FT;|>BCvJjEI--Jj@JHEB-y>oU}1Cw+$@WLlG+3C zRLTUx&o6^-nv38c=WBdp@tNod#uJQsO&kJwaCl1;gf7T~R=?}uYZ3%|dqm*L?v=Q) z{UqM;-9QaSgCI6qi1$NDf+w4k0=6>?K~zniH?2gP_bIIaq)+BEt2b)mAAt&bu=OO+ zfO?oxH2|-laJeGQMrin$3~M>RxiRNEHl27H2Hg_i{qGETGVv;e$X|y@?Q&?^!ARxRh<-Ief*h#VQJKjqO^n4X=$%W%y9-L1xf;M&|S?BHS+uu{q~pC_4jAos9wBlzotNB8CgV zX28mzv!HoN3@RV4AvuTZnBM&!#J*k`q*h2`(4cE?k4LK+vw2h+() zTs8Z*2C8lPL4QR#7@JRkD}EJpm7EOj>j`G^7Nx+V*&|@hYljOrszJuf3SQjTCW*5Q z_*?Z-Nv2#d+;o2nE7$#m(`TBY=1CxY3>YT?^Z6ul>vC`lPKCF^Pr+qL14x}MhwF!~ zLsDWUtW?Q_8odM%+J74U)Nr{$;tY|U$&jkh2%oIJK#b%d92>8LdU_JP#qZEr6EyK0 zcU|8<%ZKq7(!67fNe;S_6EkXTy> zk?KRRzKstr_f3JpmK<6%Ujn7)6f(zKufUl%Ef7~e0tXUqL(FXgBF>Y!8FCiCr#p>7ydsD(NIj(~Jw9r^0#O9Y~3!iErE7=LmDTps1YAGJ96Ar}Ta1A?JC zH5i%{yx`;)E!ZX2L(*1>K-%x^@VfFE+zGFTuUqOM=KNKN+bIw4hF0KH<!kvQw4+NwX??5c&|D`CO(na43T-EoZMR5j8tHJPjpDkBDSw=)mh z3hBqAb8ynX0UBPK%nVLl0Oz`nKtBO!(3?fJO&_O2e=Tv?LV~N~PBGIoSHZzMDX{qA z6|kGB0{NmZsKtOk4shP!;=l$*!7mWjuc`#yXBi;r*Ginu_R_`{Q#^EPjJnlHkb*!H z*q9Usa&x?4;QIuye!tDYZR(9uZil> z3}_o+Awwe&cGcWt9v!#D)9C?N^ZW&UWvmLXwr9Y@#VN2>ZxuM~^#tuipU~E1}zE5Kkod@XOy>oCq6jHy12bf=^jO;#=NFwg3l2aXt%n7$- zX1AOr6LPsC@8EDp>3r$eaGagwhsj)w6caMFJxXq9mo zlPzW8$*U2V*C5Ihxt|L6j&K~f=U=(^zPq5ab^-rHY6rU9(Py6@U(R-4c!$3KpQp{~ zqk%%B^s<8^rY$Z(iM}2@B>NRJm=>I_o`iq(1M$<309?K@6vg}e&}`XWG|;riMDa+h zi@%FohB+N}Qa$dw5sEF5f|&opmtNTSns(Nl#w4dYRKGodrwX29y;C4QdU}tZnH|79 zpH{-eXf{&~yBlaebuwGTF-T5$>_rDx0VbsWClQ+FN2V!>pez*Q<$o_Rk`&{Ps5N-G zYcBOjSxr0|CJ+h7Ogd$U1G*jw!3~39c>0Sy`b+(w9~NJv)4kj1R#hK-ncYV1O{1y5 z;#XRvWsj-%!tlIT2+lvb8THh&=r6&EWJTT@(pfu~t`l92Eencq)0!4clq$wwhegmu z)0f0>8JCC*5hkN(4>m0B;c{b>*)1tIQ0VV-dfxpeW2+F#+z_sz_FK5j$M5H8RWpEE zo9b{#?lc;yzNL+qeCazXh&mI)FzZVlr^miUQg#=UH6t;w*A6e;+JpN8{Bh3CP`tG- z5Cy9Ouv|C^*Bx!al)ocfpZN)=n3iC)j3c_SKd33`r-9pSG0Wl|KDxS`&8^mF3z9pq z?E4~qC8wiYR`>?<*8ssI2aa z3$GqR(f@9s$GS1Dzm#Kj29()j&NJAN+z;4u{xJ?2)!`AHm-vOe#F6a}abg%nrNnob zBO$<64$fd_Zxdr54ZcQ?>^QWwJb(f{Vfdi=8OD!_vG`GrHU3Y69edr3Z}$43Sc){7 zofAj7AO5IKCbNe+#8%EbuahF6Bbi(@Ru}}Z}LNhPt7>gC&8ZoEywz% zi?QPCo?%aA7_JjKfb-nruvhUlR&nRFV`v6D|JMXIpy?eR{mRGMLl3b}^d;W#eTk}$ zb$D~vW84}30i(2Mumwsg>^OHeHVejZaOeiM8XrdQZ?2ePaupvqO0sh~{!90Y&8(S; zI9sk%LOpM1f{H{Bl+wc6;4Kd;uJ+SG}x!Bt^4*6}ljm|Vf+=3o?=7mao2uAubB>v)v&D@sqjgk!Ud zuzPK^v-!-w0n=A`)`tROPAMxkmVZ3MFh>4SnP}}tqeq5A>xd*S~&hc~%GLJ&H zj$jlzaRr4^KchmSFe@TAk@cKai|g!Mu}bO%eQt4$rsS`{v;JLpcN(ywV@9m?=m55_ z9jK^2_y7{uPU0!c$AjC~J8)Q7m3Qlh7*A#P3Rw2X5B)8L*e%v0X#8mwdR-TVxuyF- z-N^>_{|Y7s^V(>V#~yx(@r6A#|{wp*>U`Z#}ARShmy(V ztB07Cck`+9G9~Q%J0CY_{-vSw;m*?B3et`3*`H&x!1gT3xm_^Ngf{?U5KEIvF1n)D}tm`XSE_rXW{EYThNl%w(F zJ}1mQnMIc^n?o9XuaekvgA8kSjn0f+f!9u%;nOG~yyKlp=bwJb_qt_3Z9|VzIjxD* zx8($*yL<+Dwp*EB^IHw4KRt)0+YjTi;CEDDMhNNFw*e_;Kimuvg*CmiaD?;5r0)}A z-+%JMXP5PXFDcG5QBvip=HG!OX>UO6xe@Q8)-ql&=YK8rJ4-itCgEe<-KbzE2^ZFI z>{C4vUToDkbjp^3@Z5QjnkY+r%=;Ks#VO#D9SgR*3czUB6?k&m4gt-VXmTf88zCai*V^;q~Woez1dIMv~W0lYM}B{h$N_}5ylFt3t?p`|kj zejY9d-(P%iYX}70b7O{EZ%d+^Oat9-cb_zKInHT5jnFf*0Cv8a1ELZq=;sVUocrM@ zJ<0jg4;*y>-6Js|Ula<{f31Y$KhG2AcdbnI27S_!{*(N)aRY6I%OLS11h&lZfb`EH z5FU34>ejdc?)pibB=pIjtF272nGe|-v=SPhhQdG5OR#>m158xTAR)6((crg&81UGW zMinZ+5#<7?;$BZ%P87_0TTS*`)X@eDNsPZXi8;y-1cN0MJPpc0G&~5HcfzpLDV{0G zisDOcwjsjP3}CB5Ja~!cLrDH*7(ZYPOS4{(zpL&NAz>wGeQE(-7N_BFl?$xHZO}04 z3*XJIfL2xk3@FCJN<9(Soz>5T-j*gSo@+t4NhvH@Hx6H|M0ix`1DN|rLivZFLN*@%@ShQ6t<~@Co7FpK$ z%>1S5Fs&dC658&jSxIyddXn0kY0ge~SU~*e7+#bk*<3*Jm z8~HzwavFvgaxcK_$x#sb@sGr?lVJ7SK(LwG4-KUXyb}{OcxK|Fy!`&{@Nt0#e$@9t z-9--kxwBsY7jEUfRb0;titdA%#-HFus0DA;;4+@_>I}F&Sr#>%-*8#bG7P)b$oRWc zIC*FSuX+Agcyu)j@}vd{9j@X((-xw;FD4VesEg1$u>^jWT?5BZe~1{g0f|yuXfSsN z;bR-YZO8=9sffW8&zZ2oVh0qrrg9$s5_p@P3Cp+aglQ%(Nz=2rq}cKjS>ncJ)?5;S zvNf=7unryvaeZ7PmyzqcNj0}v(opU`AN!jRzKZQ|ZeJM$1nz;)!*vy9f4Cf<^HY9W zb1QkK=m|ZyGJsc*0#8^k2s-zSWK`IZOxuGbTvia|OZUP%&1fjP;RA>Jw?Xf1AJ8<9 z2IbVfAdw>k-?9%7zs8M(&Tk?gA0CGUj^(|?BomD1dx5d(TO#fLguj1lE;lcEThVZH z4-|)#!LR%6@P*^RwpR;7S)>IukuIUg^CkOzGl1Du2UiX}fZyhc&{eMs$4X;Ji?1qS zrZyAio(q_s%LK&_#Skc!21BoQz(P6`Ubo;`hxTxp#kX+Rnc=k@mg8MLJRMBt)Z>MTJoefLefB7)>s+kwF!&rP z3Aty+$lnqfy888L4D6}LMNLiU*q?@1WG~RuHx4ofGx{pv>^1sno*4S*h~U8c3cB64 zi{g)ubj)5AgIOtDHZ6|oWgcNLCGmq**~jkvKx2W5N$snh;bWYQx= znA@WVrcH0i=@t>rHyMG7_l{tqwk4fdF$bof^9POeJ+MwJlqi}@p?v2)ln+{fFE>fk zprj|{vp6>+3R(=>(Uqh$*^XwrjZ*&$oag#nGM(t$W~k-wM0nnhnb!M%_>Ha47}Lg` z#Ny_F;lq0=RN=lb(w0%~b9^(kIa*0ZdltckEe!k^Y9LY%W>MGI3(@$@K3vhlaVQ;w ziJ{bP@J#Us)hY$hP_v|yJ{(3}fe5TTHU$e@-;#J!Juv>P2th*UiMV1gb&1eHFEd*t zvhrA?XGZsI7a=e993sTlo5^d7r@OC9;p-fD)&n>mu?p;@d~l{BkdKZ@5z4`SUWd0en>ImZX}L-~d*lsJ8j^RivS zPra$Q%{meNb#7wj$9y~#8G;5^jj_{;o4vo+!|4YDF#cmXX8SxtCHG47SrLxP-E(k6 zDwr;DHK5n`^-;0LB2)?(MbAGS*jW^S7h65)=JRr-anJ3FzJF3Ekz0TlO53q@btNjd zc;He0H`L0omH$C&lpnjLh9=xUhMMOSG30F;s@VGA`%q2HTiZ*2c1hz!>VpxBWl&`1 z0R0G>xO~44ezZ$NXT>C}=sbo2y!*7cV~Brf`8)o;the-OmM2PcjE55wJ5a`=5MR!b z##OYkV#H9M*j)Cc-)02h$nH)&HsvQOeJ;W;0iWpIuLkr3k*pZxy3S91}#{L>UQ_aG<2tT}@ zupDiQ6>y8oA#C9I5AU0$S*h4rY{%d$?9evDn4ti^S%DAp{^AtW)fHu<|GKg^R(n{B zsngipJ&XADio>vZq6+U=dle)kv%rDMc+39{f%%6xzRyfamU*>@U1Gn3eQ;>SRsISaUa^pBB5))a=iT?P;iosRNJFwC{4Rc&-P-Wp`^dNU| zHiIbAP=-am4d}PO4nqg?FlJLA+R_8~pu!)6ppfgyUt?VMFgE`9fU4~`F!J^Rj@iob zDJKfygxzM?u(Aaw*@>~8Ap&fT#C0yiDuk+q)2Z#xT>83yAHFDS$8o7XoF@JO*BZy+ z21^ZW%q!zqW%X25L<>c26Y+X`3Ho>5!pxX>tmI}T?}e4|LaZTfwU5F5ItY!KB#X6;ufUZx<@P`EA?)SGbYxQF^o>haPE*$6M_zpB{ zG~~EA{+OpTk?qe`W~J6Evk5B)@lWC|Jp53EuJiO|u85A&&#IHz;ORTq+@3vbjHVNA0Lv-yYfE>_~^ZDVxHM<;4J=`qbyx56^hizv*xD+3SSEG(& zF5caD9_=)i<39~q^qZ-RQv;5pMnWup-WZEN!p`I4vJl*V(w}?2IX~3Ii>UU?3Uhxw zq!%-ts7(GJ`lzXr^F;i^XE!FW)+I&wVd`_*cK#(f+OGxQ+@;BjwWU~d%Yb#CY|7f7 z`GlEUZ;;PRr}CVOmhf6?p2MccgIvbRl&5lX9*@_)9o~M5$F=)qS>;F}_Mqwj-DKql zIT6+1_UsB2zI#rT0%l++$EY=GiN%t6TF4KqBCFJu!G6*k!q#R~JPx}@dp>ki=0rRd zHc6&;L$*`B=}%}<*Bkn^(~C}wIdb>Oy8@=8J-_1lqEwm~qlWW>HsY@{dKmhxnkMYc zFx2tPApIL^Np#{A66drAzn5ik^S%h&9n?hE9?KwR^6GF+=p9jWkfED?d0^h=I6NG_ z8{1#zQr~0&vc~uk`ThMWahDo2ESl0st>(|hFdGqEa^HvkZtP*Y3T~1^^8?8N*GT3> zauE6U;RexbZD%-GA6+0Tf>#S?qf%ZU^$L4q_Z-dgUgP4(Khn_cl~^Iay=qFe9aYj=8i{cHMnUIYCd>_QJ{B~x$BBr5Z` zn+mN4#eVN;bcsFHUoQqz9fAD8DM>|8hVEOz~|~9 z_23J!Z_diHmZ#!ziO&wuc{`8yMahikG%^VN`2xIol8!ttXCq!?U?vC@8{xjuuNXb+ z0V*ALBw-udKxgt)9@lV#gKren`#)uj&P>Lro8M@f(n3aYZX{gadJ~-fUV>L)x^Oor zgSfy6l5OyWd}p7N>&X)!`9dbScQ=wqG99FR`aJk-V+3)f>X0H@OTt%6Gjxw4%`5ch z8_w_{8Glq@-L?a;YKAwon6H9pbxrDFx)}KhQ_yFTAW?F10J-5fNHg(*v*u5TYttLr z%;jvSh1bwc=LAS|d2Xfwmp;3~g>CDg zC{PDNZ>@ut=dN&jx&}=B5k|B%>?>Rna`|r`m@p>W1>qv+X~;`qLX zc4TQ__?fq~edZG~Pt6NXImf{pBL{dJH<`GLO+ksQMRtx#vof7cL;E9x*i)(Cb!(gIia4&rMONvwWnlA8}Ez{y;02Tt)5 zQv2{E@mrojLg(wk<)lk+g5$sFmqq|9t;zhh{zjWMCSzCjGbULp3X(Vfg1he|cp@fk z&``34@SoP;@=ss6&r~Dydzb<1!;N?g^c;D!a|C!{++HE)^zFQt%4R&P{RMFQ^Aen& zJczW6kLzw_5uxW_Kz6GvZ`%fOUdzXPIC!>z2+5^U-e52_*%M8&xz{n_dJFK^@Il_k z7p9G?a2lBe*krGO6Sh2<@O3jJpU{SOBUK2wv=%=3c|)XaC|Eo@2T6Z<@QF5%ndX)x zQ05YORVfWj_i@lKxB^38a=>dK48$!=$dT$&+N-{iwq|sYJ54db@3{+)r88lM5(C3O z-|%<-cbm4IdB#_K^pK2u?F5CKU=WK70~u9YSbZUfyt1$$Rj*EwUFO0ZV`2~ZUyOq7 zg1!&{+d)*y7aZP2fuZvrxb<5Y8aup+Oo16WB6FRj$J)ZvrZBMA4+e>fogkg|m`pTk z=I`n$rx*3#@?BmVg0Fif49DLE_qrH(Ue-ZaVozgMm(q^=Ye>b%Fj(1|11+Dfz$d5U zoR?P`q(m=~KY5nKEux-eAK-x%IR|b_L!qM18`S;Qg6AGpFc#4Uc@+l;l;MHY6Fo5G zW~-k2RN;-SFC3c1hp&rTAZ_VQ7}Sg=a??WS8tWt)>`+LwZs&uwusH9qn=DU<{{=qf zW|KXhe4M5`$jx(gv68!fl3UDpyF<70>a!;EK4eVhsReH1{hP$&{XP}~SNy}UVN{x3 zB_zNu9EzgaIl0hr;0KI_eSlkK2jNCeIbY=7O03n=#aBlw`12G`!^peqP~>$PQjh4t zWxPo$pN5h+j@P%YWQ@o>R{$R_7uj{8lL$7;fw-U@Y~nH@BcgVIcV<9SzaKe0DTcY4 zqDfA?`bN$j*$VRmF2St(X&~m|4H{hnp`w8jJt16!}*~N5YQe>N;nkClouULlmJW4<*oqPNG_A| z!4yut7K9UBrf?OCr-VV&()DoZDd(#i zUO;)LAJfpoWBhZ$i(&L%3OuX51tyiDP=A0=PMAefTZv|Bbn_SU>ZLa{>7>Kbkr-&3 zu@zoz{YGN1sFQ<^(TvJUKcc-^64Zt4V4Bzg&Yxrl4WjZO|Fwg(NB5F9y$ayEahxdH z^pW0zP%=g2Cb@A?A4Y>OL&o>(@ci{D7%9BPFUwqt6HHd(9kFtL@rgswp85ealzwo} zd@gtiMbg<51lX-1(rmIt7(SmB0#2>K`)#|8H@kQ;Z~qxVo}`r>Z~mV7yoL|6L4BYL z4gEH-n?qKymyg}Tn+tWxWf?12b4nF9<4n3RJ{Diz=*0~Z-Ix#I_%O1DKdc`~M!ras zRfj_8_M?gz^F#{^Z}n2^^rti~Fo5n@Lb(iI1?~H?h@KToWcClPB-3MCn6wmSdKw;6 zEmcw6d#QyQ7FP3>d(+AAxH8115}2ee0M0Y6Qh{9?apk&2DCL%VC;a3}xN|2GmNUnp z;sr}i%)3XG3$<{i=_jh8{>sqir64%;*}%Oi){vel3Ts<8@rOgw=#T!pRO7w^t?gV$ z?oZDnTNBiYXyau%u0EIa*m5;c&3tksM3KH;eup;dXHZ8w2Y$Hb6e!TQh6)W^XigCX z8Rn3JldunL-- za#ex#*;mN+$ou@OIadVc*NRd)5*0|( zGaIR3NgoYk=A+OVMYOXIrf;^$5)YY3(zf#vA9jSJh+7vPEb7Jj(pYq?nnfc;)ZoH8 zE0FTgB?3XW@J1GQ9Zzjw&G&bqdcbU0e`Y>Ut7jvR$r9wbEb52fr3Sn;ylK1)>nO73 z>^JQCw~qZe#gJVrlZ>ackCJX@b71#K!_!GzR`E>&N_X^OLShe=9ph@A-`nYlLJ@LO z;q9Hkm^xZou>!9Kn`6XNCDa>JM3cKBxMpl2+UROw@PSS$Vq;DnHWl-K)U2mDK@)Jt zh&8^vcovseIilXAak|hah5!BXQPN?SMCgF&-GibIn80bP(ud>GJoOL#;qODN`sLy4 z%V&f!-Ap%rK8A+xIRCGWCobc>rJ?JAIePCJ2|F7?rp~kDJ1_i2v;Hw?X0M0yKYyjB z2~%i^@?s|YL_2@mP634ESNt>244Jo0;#BIzH=6Q64;_{hOq?}JC*5Ah-#aUqOw!CH zWmzkj`p`al;JznnaQ*-BE$(=4!X|3Q&Hv6e%EQ({A2Pl2FP#t z;*6>(sJ>hgL&Fr&E6EH;F04R7PAjMuRl8&1Jc*dxcBZYhah&|!gLk*~W1@Wm=6!Rd z>(!)z+L^=Sm-|TR$z*&iX~>GStYh7n?|8!|l4Nt&d0Djq@49?He3{b?4$ez?7bkH# z-hp&tUGN`@4ieU?U!A>^0Q0cLNF3}D{+rrn_VDti)y9VK3kzMq@ zoCCAI_6|SP`7`x2F~-kt*W%SG4P5bJ3GV7&go97FVREn?!bd5z4@#gGU-YTY+zk3f zR~v_UoKJc@5vRQhLe~ik(IsLh?Juum$}9db4NWfeWmX7A*45#z2_=X#v{6UyEi~Hw7ew|Z>=uDgFR7raKdaPdDr-wHmk{p(A!KaFP(O`86xkAIoiqD<2Q$? zc=bOk8fYVP`>3QG^?m%44))BUP5*I7kq@Rc&1)+DmEVM__Lg|Zhoe4rf0ur0m+;fI2BQrRoYyGNGtWJbGS>%0LlKemh~*gcJR zR&N5x+f)4celAOni?O4oy1y4&>!A%8iX5j@}jOs7O!YldQtYa10s@>=SySh<=Ob7{1aV$Kp20ytKt)$Bi@d)M!#*7aeB5M{obk1Ki9jK z{_>PX?X~A|$NMC#mkmV6w@UcMLX2w28k1HPQ=n9gb0c!+xrB=ty> z++^71cbKfw5JCMfX{h7O&8$sA@t2ke$}KxmQO-X@T6VNE!)AA>!ZtmO(YME(MjaHB zAEfN{IKGy?01-=_z_*sYNkeD-s@P&KK=%5_^4~2UpwV3mvE_z63LenMJzwt7wWV*F zeFwZrMM!Xk;7$?T_%swX)|cTJO+$xcBIwk2h!`Faf)5w;$c0aes64+2Z#mba+_x~S z_~Ar*=bDh}$Tj5B!HJY-ql7Cm15xi^5-M_9u(hxpX1(4(8)qr=3%}{nLtMTJb8OJ6 z))C8GRd8R|WOTRaqp}LJnEgx|ohpiGqjOuu!rz}6p2$6Z_Cpgi$<0Td*TuLZ$rewJ zE@G-T8iB;vDd>rp2D=3VbgHcw8~JH2tG9%r>Gg5qK4UuXTJ$pB^p4Lk_w^{O+5)^E zd2&1>V`JEIKO7IO7GW=*e1*2FrZN&8```=L+aKQ;2wx%|65+Z>R3vB*HWqJ2=0PCc z`M{N!hF6j*Pam==uD3!eYa<O2cyx{-M%%}T$W0vcV{T-m6=IpDI1kG}Z!N6y{KmR(ldNTO2Na5CpO5rx;wY38`2C#~_Zv)5r4Z zVV$2OL)8w>2ns>ml4KgT*#{Tgibsjrn{n4eAG$6ujJyrvJp3`-o|J=1^xcgsU zlrZ^1oB1{T>h|ND0h80|euk6ISw{Rqp9msU(L;>PPm)>N8~FNi1JsyA!p=v+6nTy;vA&bn~Ug*_-j?g?RLw*AP6VgfnV+@Ssa>6m5QR!OTPeb z9<@hhi9^ivQbFht5(i^p8!{x3L!F#0@sOw|8ulCDEHyuRZLR^KX~pEI;7ZbXYb&># zM;s*w6mT+^<;dnY^WQjGRoL9qq8+z3(9F1oip4q?h>>X}+0LE2g+U-a<+Pcb8ScS5 zzaP<}M~}!O<3N~g$?1RF_rnqXUqh9xuP{|!gk5?n98*G!;Zn35@5(0NX`UU0=(q{I zPqU4AGsaYSJhw9tQf`9LJKo?V<_gA~?j|B_aS&ca!F}#ESU$WQjwY}MCpi|>5sPA~ zu<nH$_-4Vh6%I<={hi0NLdx4;_a$!ll|daDBol5*w7jH?X##OLIl( zxxxixd7S~o#O(o5Ee#N!)5q}q|Io?S+`fPm4RliYY{*yN2eP-*Ac}*_zklz-q&zml zu=(dPX!AEZ<4HSFQ}qBZ&vU@&tHDK~EGF(-3!T?1i>>;_^tbO4QaDu&)Sp_z`xqrq z|7t+K$~sUU2GfzTCCvSEkBOS^AaQ@5PJZfolGK7CBD!rVG-S?)I0s4C?RJ99jw+(F zML*L1C)*8fO00lb#cLd+o`QUJ92l+tM5bK7!ev9>;JoD3IKTQdG~Q9+?H)1Wnbc0; zjfnn+?FK7&7E9!KduAVm9Zkz|@mh{ydN%@FDnvnPWhAf{Zot;H3Gmu&5t!u0GH0$w z($WAiTC(gTDfZBYMoRWWXMsYtIl$fYvWA%x^A+zdm~(7j==rSrSQ`+i7Z!L3Ijz)AW&>0SbXj#X#C-6S6egJ|P-Wz5g)r>FM15eL<1*!-drEVVDgy@q6R z*l9AdahAAaLlpgYQysi5;vwZg1~`y?VCtAdF3Z``)3vYY)pRu)ko}BQbh<+Ny--+k z))^G~YDln$7Jv7qKm0Y>P9&g26^{L0&gJ{X;8Jcg5p?MzS#5K`(oY{cy|iIjPzRwB z;&jjA2-@&FmHDc&7=%U>pilJ%IINBY!Lwo@sTPUskve=dOCL+`9E1bhbrw0if@hHM z3x)th>d=5x&n@yIq^Qn}I zDxQn)qz#YrnPsapiG6<_krD7DE0iaaS3ksv>A56Q_~JWBoA{Ury#f9>{~oo92&WtN z^9V4@;WRl2N7A`Hy&H~^vH3`=&MV-6MgnrM3f zm+=*Fc*bFJS(>|VdW+!;`fwc5Iwo?KG)!B9c=GhA`BeUmjtCMwAO|Y1m%v%ES~7KqBxP(Lke5SC;KO8jP}Gkgiq9tU?{oEE zz@^uGog7W_;-@K;sQH48)jBw^JeWw@6jASHLo|LUjdrgdGXWcXphKnvs;ol6d2BCn zW1cG6D3=E%J4@*VYlQ~Hh^yLx-EYd@+#yd)6%k*Jc$U%rn zUj{PXYe?IlaGI(2fGYjr@w+cSBDeIvk_6pEqIOA~_)QcfqK~}DCE+~spZgWkdLxG^ z<#p0s%T%#v1JcRAW|J;<4%jEIgYhY<@W^Nr8GBKJRWF9ntvUkZe!U_q1O}m8SC*&j z*aDujGGX=61}OQU17^~p7*e{F-EHT}cFQeejZUsdsY#*qc?#t-5fAA785hvhrwP}6 z{EFvPns8z`$5~sx2nVJrV7~i8oVmyd)o>BooEN}|f2A~V>>sUp?vAe~WOBaIP?Wqj zOy6iP=XU7y5~+d>Bt4~^Hm-@{c7GJ(doH(|b3K%jN#i6vc^_1(u7Sehf6Rir=TWKm zK1xrD#uM2|{Px8jATpc-c`4fby9Pe~~AIBP!oD#zfID?WJJUVwUxZi640IWU#WcI5kC zF$xFx<+T6BdeR%$ zOUlxglUL7&=}(C;+*Fu>x~DyG7RUe7(!E9Zr%k|~^^37*u`!Aq)4?C#4jig# zfRD8qkS^2$`M+lKTGIzW`S5F!IUa!n!$Vm9rv&TfY$B0I)nHt3Jw&9=0c}fezYvFq z{%^4=@>hSPzb;62@um^^*d(HE{+$@{ACZ^A%lTfXs;S7o zFq*gBnAE;p4$Dp*gyrw_!Sb{R(cMYur56f#QHDzYS4m^opdk*eE2f%+?D%Ht!5M86z(Z(a&!z1(weHiLeut0nf$OF(a;JRJNLMy}aS z;@_y3rzYFq@U7OWl1e2ruxY*k(bnsL)N;F!I5wPTA&0Ex$>8?ET1JU`?jt&iLBt>w zCjZ_~bmlAKU$y=CzN(En*#05qIe~C~Z48X1tp|3!4T<2Qw~;#(ux=?*t!+oilz;kg z*!&=*Fv~a|$!Zc+6HcFxJ)j}8mhtnJJR-|HzLWF&lF0B%aq`Sun3Vgu6YaWOvh>mw z4vLz`Jjm{(&;3@)E2^1n$@_4q5r!PKWRtP<<=`3VL4a2pd=9c0X_Bw22{p zrP>01$>kc-tm_O1pM}ESY*#oY_l&&Uu1V)=yrTW5Hq+6Ixn#k*ec+Lm0R_78V01$r z{7s^1e}N^QeItasEK^9|_RC;vUj?ccqT!;E3sKbL_OK)mN%-Lk>zt!t z?Zf?WLE{FwExLeF@pxKxZ^BQeS-FeEyx$1ICl5pYpQYfxJDE)Ty993iI1CLV8=>oI z7xB70fzUgTSDZ*QV464N6A9b>;G+`-sf-(ZmntTb;lb3=Um0^fIGykPcCs0wVe#B5 z=zMbt-h?KTy`0}WD$o+0pIxLC!d!j0AfDUJmI3OU_d!MMb)xadmgdiGrL${Q=%-0d z#OskOJWUP-%b!lreX5oO{nq5KHT%ks&fP-VZ>z$y|50?N@l?HC7)ItHl~U$JQlW_q z`&mayN%==nG@~L^B+VI$%reiBp-hoT!gR_L14D%@6}&T-unEb zAe?B45jI`O1?AwTmkgRKk_vMsP}oy&4#vqCLtad)slw}*)RC0a5~qU%JjcV>!?WN~ zgB%PzEhh`}_mS;v=M<3kfovALMS9iPu5VHVJr%N^E;?XF+=n%w<;-fB7_SAzjfUi> zStTtuQ^NnwJ)|Gc<#Bc8)4-bN56)e_pv3PZ1+%}>14{xC_btNw1-rP3aywu{L^ixM z_kgGGy3DLCb#Z^F0}h$}qE*fBxPbd?FJ|oyzSGTM^zUqPL`{Ykcw15b+BC+9K1?hH z8t^OB0q*@$0a$c`Bpz0Q{VN?n?ynjsiyt5!L-BlRt!30HUXC_Z>k(0&DKyx+!%aIK z@Njs|84UfVPnsO?#zS3fdiTKW_tu@D@*x{U)@_Fje@E^~(;^J|6oBjUzR(AT{e(XA zg{S8JFmO-{9#<4|ZWRydU)D2S(1Y9^fJJc<)8>59MH$?<{y;G#;vE9U@1elFB!}pl+VECZ6o~7CL+m z{In^Io0tk}g#%;~JLmuN>B6T*mMFgHD8MyMo|du&uV}LnuX@UNSmzA9!$Zovaq){l zFCm(B!6XH#7dml3LX?XT_JB3lbD(iWC?uV^MT#76(wTGIap49>yt(5f{TRHRblEo$ z-&@DYI`O-8Rdg*C3{9uIWdiH0c6Iang34&BiYgwFlw+CoP1L%SM>4lm5LIteBHR~D zll08c+QAQBzOzQX$;FgSh;64Sa>YDPyRq-v|9MR zIr~WBhXFEQH-;psEaCg@k0F;&4Ui&{LnPzPb-tnZdn(G8!=R24x}!~v{%%KdZh-_8 z3!+Hy@gRC3))A#DQc-??5C*ztQ4K#)ST3=a@%2YZv}XyeAB;uM1*Mo(yB1~W9`1*d zD9qU;10rJ`$aKA8dgY)sY6k7cpEpc!m-b1jGu4=sc3vbcUS`B-(-yiUNe-tUVDIyjEWD%4v-SYqOY85DJ^nqUryBC!zLe*A zZu&;u0~|cQU3ATj%p-@2sM8 zIVC(WIS}0@Wa9UWp=hvC1ApxnqZ>t*5;Db-Y!}%~ef)?_yLrcOI)RR$!^3J?_ac z;$Y<>SR{M|3eHW2k*$MN)L%x>sy1D4OP6BH@L%%ryF9Nd(UA9I{(G3D_X;`#=kqG% zWq6;rpCK-c@$-ctp{l=57f6I}LFFhuX|!4mWsfIA@bpH^(1~y4w!d*{dap%j)_}}Y7e)?Q1BCd3mJIl+VSioVf&kEFS zU5gu{$D*^6HT`D!{z_(yI!zJ!NtJf0(MF}QwTm-X&|5Rd;t`2;%;{x~^Y3$bgXYjn z2A8-k+E(Op;CcR!&?(5j5sL>}+06b*I{LCa?`fO0#C*yt@?_{C*B&|xh4U-W>0S$N zSQ3G^Hf*OW+k}aY#~p6m3KzPX>S59DFkCk`1NX`Y)?NjSE1?R#W<$o9etmDiLdXpj2!8hVAsC+_*5hP&w#(-8(rO2wU272b zM4IwEmF0QDmZ!+(e_wEE?|*`VUQ>bGlMFN*zlR8mEQf}Dlc3Lc0}VfshGz^1u|487 zI?YdEjT!;ta1X!P(oy*X|P zJ$0gtpYLKz{WOKK@#t#QPC16h!Z+eS`#-e*dK!Os^e6RV>*bz4cl;XDO zskkod5A}3AL5@_b!0s*WWSq1E6+X5X8*Zj!|9L-rzWOb7e97Ym5{igzNd!^-Xv<&T z^qs~W;BbbeF`hc{h5p$vj;5*4<2tkZ8Q?$&Z#4JtB{B^;DHkbv%>667OB-W3^SnNv z@tx*-uHe6v4kxM83dxAie9rmQJGvmj51WKDP?zoTLPXZmKR)-#M^#mbNMp~ud4DL! z_ITavN^$WFC(H>vQoDO=I{De{OOl1M_)RPS(oD|{xMBJ+RHAm+Y$=SNY;C9ymh){^ zTGHXK&uN3YF7}n^W8!B?9LZP5a-51&RF>mNy9ow$-=<;j9+*V;36V1kw$k>MDR?IK zHTFJY4#!hzm@#G(_5Pv=H_k4HE%SYdnO!EjDVqtRzWgV6nDqs1Z^RI=kmvQ4oAT6H zk3;y7C{J~)15a-CEZ%2pC3u`LfO&gu1&fLn3)Xm5VJhE{Tu)mGy>ZiE^C<-?ACZiK zci*D(s{#BN7LD;$*ZE0VaYTEP5~&rBr2PSEm{~a!^Nziv)w5b@fkG&CVqBx3_mql^ z)u&>aY21I}JQ6bgAve&bPAA{HNh?_Xyu9fNopzYdxBieth8gc-^fw2sZ9?$rc^=(# z-X68m4KOD?yhd}eIVdcR1x?8V5cQye++aLxt1MkKReM9#%m>Z>J0uG7Pu4*3AzQd# zD+S*}m-AOI$)QKAYw6jC8niWLHfbMMMourBLG~#n(66G}gqD_)ya9dEeo%w1JXS-; z2ISByPc8V4-O{jZf*styxCTC)5rv&6N6q#XzM@`BX0U5=l@?Ui6QzIqL8SE*^c5@u zU)Jbb`_BMp|FdV`#k16@SqPk$5qRz}6*RY;A(5ta{5`#$RKidi&oD-z=ZF@4k@=V_ zma`zbr_(v_{5f>@vpRb80;T5vouucV-levi+4t@CEUfp|K&x@#H1Rl_uSLfYOPL1# zMulk9=pSJB`nOnIn2Z546zNQFEjCMC0gnn7knt<3(6)B5AYhrTV9e?P)W}kTOQy4U z+KLW5)lgC1;SzD)s|}lY7sC?r4JI9U2FC2Y zTz~KaU(wMVdpzdjpO|WXsmxKR7b=3Rn@ON@Xd%2|e(eoQqseyhcVxTIPtx%~1$evO z5{1YYr0aJ^WVe7mzu-7vH z?#&S=mdW?&cGWmqS;&wN^P@njrV3=2XTcZAxsYi6g&%vog|@lqQIWZ4No@9izz@pktX86o+`i{+A{&;W;H9 zF}>VjTE}hhiy$GncJNm3C}VO&!5bx8@O{siu6t%tE0Y$QXY+;cX*CyS{5%c9l9yrX z^GNtrhr}cP6jk`wL6@ZplaD?DP;Gez4nQLK>1=?W(Ge2gt4&f`6S%B}q2x{H1UOjf z0LR99fyDzhYd)e3c?JFCAal!@w5Y>4 znd|xd%X;&1>j`r_m0iVu{(3h!PJ0HIbiYB4TQTfe6HAMq2nm!IC<>mpMB%A95n$H| zyb$|MJm25Br$W~~Xu{?agm&G<8Z^FaUXI!NC& z8&pN7!S7ouVZbF2$f`(~xG;dt5qaQl)k>^~tx1AsJn?Iv4AI#KA=Eqvwo4a-pL{gj z*H}i#=?dDoX&udz=_fB^65vyRErf5$1Bs6uq)+SR+r?d>WLXni{P(z4yfHc0LpI^;qA?M^0i|HNvduosewGkt_gss z^^s6h8wgqNSHj_5O$gGO4SSoN;B)~G23W>S`-Kb)Com7r1@<@pt%G}NouIR?984y} z5s4>ZRHZ$Q*55B711yhcYaq)L*eda)#6H2rANl0V5k6K%en7>bd6=(K3QI+;c)#Me z^28IwdFh9RdE1pX@Y1d=;w81@g0Zs&R;*{P-qh<@-{VNW>Gp!I!dTvMuOFn!$-?}@qD5J)Wgw7}(?7fe_4hu!Zi;D_7{ z>d`v~yM9Tcm$M|9JZn9?jZA@?)~stc_a6D--bL@z>6rfbCSB_Lmzxnf9uBFvfMnDv za4ir8(|yahB7OlsccWij$&?u4s-y#|v$q3(!YVkw$pEb7Y$5%}HhA!jagO~XiCa@p z-RX_z_?w0aS2te_Y7|$2#;`Lqa^qlct`IpkzlruM>Y(P69vb$jm8@axS4E=~_!#H} zQ}e~hs-cH^7xehG2=Q(%NbX(4 zb$VwrIXtY&2uhy|_-s zzLO!By_O;gKa1IGV^H!kIdfXzooy|O-8#G3H+!YNCR|Va$y(CiJU_O;g800rlS$0h~@QC8Rd!KWis$Vlg|Kv5O5~GD~ zQJM7a#xr%xMJu>d^(M6!ChVsh0=jA2j5{>p##TxpleV-Z(^)%T(fFaO)O%cFo!+u1 zB+YCp9Q{;J_(e1D*8uCiG-RMv`UDhx+(8cf%>moUt6<|346fte)BBNP0%tQN!J@WQ z^f>7Zi*L^4-O{(>eH;G~jPKuP&zyO@QM0kUJ^kgRaqk?A>j?Nt5wMT{*L5`60ThtOM(UXOm#Hvg?uYHSO2d9e~7$i<_}q;!0=l{sVo zgriw}0G3}##yS_)%lWn+)2t0~YoQWa$r_@pY7oAPEyfbfYv>Yx2{*NdV1rZ_9blYG!+Juto|Mxk2rf_J>sgK$EdK5;&|m5-kV+I6M6CWz zPcIGP^mL@@LOFMQTJiy}ubV13@N=r5Wx0r8=q>X@?&5InH)nion~i%OH{xu@-&w6! zgH?L0m-=i*yfp0os`W`^SHX}fSL`y9-^mSVQS6Fgh-8S5WD!@GLrj9=`9 zo$-@!%O_D3O<0BsUmoIpmOF|J5ElG*u>hT2Me&E;BpN0tp+&wPc%-NwTLo{?<7h8F zw@tzSHnHBETP>Z(yp|xNi_=*jFF(5y=dZntHLj@`6TTMb_GsW&772R$J09idnxQA# z2OP^?i;K!qQF!iUe0Q)ChX>QKL}><2V}7Z&o3(VRxen&2reMpnUKCpK7T+&?jupo| zu|lti&NG@w9V(gQ;a~yIjuaMz*~$nG&v}G(+E!RyFN#4YCgFyYUg-Y&0;Z)tLw&~o z9Wr@>MTTXV92tUDrn~Xiwou%4rv%;fTkz4WJGhH|ZrW0a{&Q;3ZPqQ6HoA`Pld{n@ z#2FtXad=>92zIX(5jdTmDzIREyu@uEP`u3z4d+VGM?a5qMD`yI+bk}~n&m9W8*&#I zyc#F)zmmd_^7#x$O*DBn=LBFq>nek1*kcy>aFGhGU~JkdpOa|0r5e8-zK1E9^%(Xn3ax}R zuxL#f71lGRG3jsUD%StgUoeU*5}89*=OoGn9;8ACRoL9_TAlmi$@nL$6t$UWdHb?@ zymN3L?rZL%aj(1h;_;vPlh!uUDT8~_tvem9%Ca%TDhSJW&Ok%OA!>L@5nG=Gp)c#< zPU(C{ix#rprd$vT7iOdL<8=J4;ESmV*J!QaJAaq_Q+~_RE}D_%hs|FF*p=9i(SzqP zD{K;;**H+=vQ~|7PKT&p>`7c{`~sKG7{!wxF5=f0Z|QdpQ>xe;POCs2H!O(8Pu}%t zIlT>Uc~)ce>XVqXd?9}Lse&dl7Ra58z|BGDQADL2#ipj?x!x?CSe%IMv;f`h&*Gf; z5S;$d2pf*5qQ-{ZIQ;E9>ajkw%fX3)>Q9|G<;OCVjST1eY(CEET#>>8kQV$g+%6c2 zb{D8>%L{&t(c+INe})BfG>^adps;X`-4m#R-_k>9L40i+c@x{9W%_&V4K@GIsJZ8WFFI_jnGtGTZ>OYQiaJ zdW;0@)`06qY!A4)hZGh|Vp%KeE_--mlj{n4MqLdGR)oM)^__5(hLa@~lTg#g16L;N zqlDoky6#^qN&jO8V@Bq};-+h4n&CQnHvb#--z0&usaZ5^+K}0pQD^2JXyUH839$io z3pewID_L#TVWumYLI2E_#ABgfsinRHJ@mYuJf1fff}1%AczK6>JvNE1yv3d~@?K~W zHHkTRBgjGG4oleIv#wYTTK2JQ*y}wQDiwn#jHPhj-X4-RY6Qee9eND?i6!$TEHpAe zQe%%X+f-2D-*UPpMw0y1V43uD-dws3bG!6S!q0xw@MP-%Ez2CBKGJ-8Dnp9#5oPd? zM*&qYcvRdw*oCp&rMnjqA7@Xd9BL!K_U?yx@hp(OoeQSvez0U?7dh~zlOMCQm>On3 zulty@ljYSegF;h3aI6p2Ybpsx@|M#-#TE2%=m}y~oeNU2EpXYQ2_p2;;iJR?h(4D< z&b-kg_Rg#~In5R1xID1`Spj2JvcZXQH`L=4prA|!>}8h1gi(8lO!9?YChYSZT@6DV z`+Ri@VaJ~eaM>6SG5VQAu1|u#xYo#*UpGWt8_q)4k72NI6yl{Gp)lU|4)>W{#_gNC zQ0b2n{@$7k@jH!pzdWsZf0w?27y9oYX2U8T-`kS6Xy*ekNwntg*%O4S%XRSki!E?_ zREp=)Kb_ZbP?hJdJP7e>p>X4oDAe8hM-C1$$KIFw@ImV{Y)a^Zyxw{!yIuqrjEi9x zcNtvxxge7h2f@laK;!&wP~VsXeQWN)fIs^ z0lsSv!tR!jY))1WUEbfwt2klmbX|&jW3dcI{qKRb(;FzB)(>wgLSSf~3Ncu(NXsq^ za)Z9p*$gcO#7*kJ^jsAb>cqnFS1MqtI6zXSiNkl!0p_usXU~s%@ZC@e;@gWrNURbr z2sgk4y))qO!X92~i^IWhuZZ9l4_gGl>XK)P5me33m5tF2PO#@Ylz*lts$F_wRqq;LuqlCa}V7|YOB z!!ub5!nKTfyWyAF=Du+_Bzv0%+cuJskyDVMb`QE42RKzz4H~ok=$b8}$V=NxcWuih zaj{O2JU$UF*0KEKD|6Ue6iAL$KIAe-4N2CRQ8KlAJBXGf0`x|JjQBpdrVt6S$XwIM zwu9@UQKHkgfGEdwb7OLjk-tvnu+N-%yT>L%%@s!|Kb1!w`T5W6$aNepSUX@gb$uv!4e%j>R}Cj`gu$Ll3Ff;>;p#O`@`vLb z$(Ax>IF*_TOM6QoYf>`&RI-56^X?P<`a0sZNgbNUu7Hr1!w|{lN};-&Va&u}s6LSc z3eBbP&ovoZ+$F&^>N6*DNQpFS%>>8&m!RtCUzoK_l6Noi8FYUc4-+=+z@(rIY-n!Hm>a{1=fP+VVuJLx&$vo#B1$Sr1B!cs z-UmCd{1Hl?dv(**OZTAkXaIKX)yBkbMZQJ*SRz||j8nfkhK2`l$w-{>T?Lp@coMxO zR#N}zo+||6-7h)$rvr4Ug)jEL zi^X$cn{ZN2J~c{HBSTm7h)U%jSc%m*oov-Y#jW+BZI`PYHob;}nKHL~i3aact>5wOc|0fL_#U^14<0KyV zEh=!!3dXU2jG^U^3@<}<8c(Oa2DaTvfr9_W@RGJR!JD(v{CyKXpzO6pf`BO?=(8BW zPi-bRuJIf_?%zRe#J1y0xn_K%BqCUoB_W6(H^exZIXEA~F?lE!C-BeUl}&|M&1K^) z??jx!x-?75?jg_dGp>kuiOc)Sae}!kS~Eub<{?|u{ZWJ~sz0C?^P$W&7{OzH6{tC8 zISvm|x-Pz$s@`6KSGd=>_p`EK{*vGL{tvtU8Fy%$uPB%8Vnwyqd*kJOFHr1=uwZmX zCtmW4LG2Tg82pxW2mHSC2z+wTpO8v=YngQqi%`0F(B_(Wj~G+()Evab^sz3hG2>_P%K_;{_J4 z@{Jm|@S=2W{&GxVvk~LsVSG1E zLf~xj0eS63_{G8wV}*6GI>;4^a?6<`^(B4@Ve^5XcX6Pn9IIv~qJDE0=ExUf?VB@L zRuhX`K89n`t23x@Xb8V%O9&!Hgasd_Hlr%th9QFw=#7?hbZ?vqI$jw-F~&z#W7l(3 z<^xVM6Xy#enqb@HF+AlPDG>231pMZ`hRJGq;NDw7i&SL=(-t`jlDrv*b=!D>nrlAZ z)nGi`rrnq>RE}e-f1*f-vS6XMrhsl|o?4SR5P?U}6 z!N7bW!GY%!1VPhf1bp2t)+bHDE)P$}osGf<-!@zmAtNXsP!<@zkQ0m#>c9rWU_2%? z8LP=y+@8(eedWakPfJwQke%Lu+Ye8#cj zXYu5{IXEx4lIp*1qruM@zw6U^OiO7&4O5o?xKfJTPymkcT!HNtp7_4B5%uSJVK(c^ zm|F#){nAo2nsOUQtZ(B#=TdCvnB!}(gPOdrpc5a=MW>Hvael{Vylx^Rcv|=!BLbsQ z$D@+HAO7Y)5}knW*EitfE@eT(0~LX~in!p{l#^(;SPs)aD`MF23Ff@-K;cd~f$>9S z!Q*ln!EEN!dfOa@PM%(P+CK$XcRj)BY&Ia=KS9v+M@aA}y9YC#^6}u+X6#+}5Dzy# z#NXpu@a#V39d3JuS7CxcQe9KfRi`ZA8~?(`E6b7d+Ku1Hc2qLXN0rm#1v{!71(&QH z1rx{13M5l1==q&_aM|`XWb6)sZ+oMmN#+YYo>m8HsLr=;c#hL5j0F*?Mgm>tu<~Jf z8->$@)Wdj`W_GSe-?|DsNnhi)reVAj`UqErWZ+0?C@NN*L?_8e^klQ19}9Ql!)blYTO$$EkiNv}6LixhhvJSy8TcUl5w=eqLAA@T(WSovueGno;-K&JXxclf@nkh#+t`CEON<12 zP8bUuZ$HQGCDZx!nYBQ^jKC7FXz2Mb3x-Ohc^4{%**k+0yqUTid&NdDKkFG@*e{EB z;|`GXtH;69PFYwre+!xIA4yk)YvI%_#^^BRCyiax$uH|M=iY@UaARr{IeBMIf_|B# zZfzAY&)LTHn3YkRk7^jjI;5{lQ#n(;+2By`4QH}|IT$9Ay@7vd#@Ll;dr_7#oBcWQ zfyJP*?lhbvx!}-04Ggw-&nd(@H!4CwZ&5wWipz!FRo$c_;VoUT+6w0n z{-C*8GQ|3=1@yj(f`{4xFyS)W|Nop)6Sv%jp1il2s_1%hZSS6ulQmS+)|*)&9^fI|Xuo067%pEH!!!1er(>s1>e6+qPAuc z3(09j=X5MLdM%T?WX9vZ#N6W_bpB19^^9?gpBDZVgwrvM=&~(Y7S2h?gL?5nLSD*>FCY9oTVVLjZTKZq2!)x>uzExss#;$Wcm0`g`O!Y+fewT3PT`>UEfU5Ur-0nY z68O6A0&|3(fLECca91RltVwSsQz!ZWIne}er(c2k-<#0?*AH6%9VBCh^Z2`aL`mR6 z0$np|VEd|Jn18tw&Q8gNJtwLeGeZsqa>X=^c_~sSQt)5-39=WoK&$6Mkg7MLesQ9B z;;b8;;n+;fr2}AFc@bRIJp&Up2n^?^k=95N^6J83;(vwl6Q=k=M`8+GUJ?ok9hM*` z#r_pdW%of}5GaX(@v_C_SD-L)?@K2lRRlg<&j!3#1e=ff!zzzv@;q`oJ(MYm_gJ-aye3T^G zoAUVwOAnG~^Y(*J@=Xvbe+756n?Y)cH^{S2LrBjF@=I9(q}k54TDBZErIkR0Ec20c zM1q|+^OIzR!QDtd81q&a?tgtnEDlS+=}TK-kx?N$R=fj`%vxYoO+IMsRE57bW|(Y$ z1nnI4spP&?D9xR~i!xQ=8I_%f-N(w{)fyGv+|!D@!+E7JHu56pZK{jjX;I4Y z8lO)b1HE8Fa}My_6Tv@tD*U*$kMxH(apl)6iCe7DP7YHu!t$|k^iOAm-Vl>n;~ zbK$55^XP=hu)fD)l9kZPsc-Zprz562SJPo+olCt>iKP`2A) zz0NsJAbxfb(w0=i?QkJr7))A_tBB(k9_NfAh;txY_M=fpntAmfl z&vR~_=V4*JB2P_Mh4*-08O*IY4GHRUyv|}V-YuhWu$W{=r|=J9q^>zOiAjKmbQo-j zD}=-TC2)LuEcl1=z{<0m*gK?=%ddt=evCSpKRe;{ zV=XXxu$o-bl%$8){xsR-G{2xUhwK;M2*-aM0BxITVB?-{CURXFGa`23nGQ8P&mAN2 ze(7+0eghaS3IoNgH=K^BHBPk*#%=em(ImUYp#I_t?5w;B&H;NN&@G>=bth)@6&BmD-KAR}echc$s)bfU zE)dWOIL0-{gyO5v4oqfexjiK>X%a}mrRV@?4p;z{g`u=+Wi&3BSB@8YL(s9Rk*3xA z6CDRd5Oj=?e=i06luv)?!kv0J=d%V1+)AjzffDZai9!-?vzjdNh@vt=tBLxTBJ!|0 zm#Z67Kx6l6;MLoDSp4)aJ^bJr|M2<|GG)3V$nNqdl7@_rcP#{$ij-r^&S*5R52epp zmn!pS0El0fgp`O^^yG;OT>hv73z(Pv+TMfQqY47)ax1}oFqZ_b>!+P3lhLWQ3fJCd z`|Vw6ba=@Fvf<`*Pz%o|E!+5XcV+-?eV&4^?%Oe6f+wx7;E}`uBho(WN8S0Vopg6$ z2`!GiPS1NjqkVodcwcuuj`fhk+1dwb# z(t{E+L@u`P%KBiWw8}Lb47`~AVMO0paQM(_~xpMYYyQPlq^bf}Ku9PNSTump` zbV5j8ENI_9q4t#OGqx;suz<--A!Krr^1* z5@?xOza7nCs_BPc)f4SzO%WB!)U_~!8obW*#6-Ph0I=EP9sc7(8 z@u~Sa`nbUv`v2v@twvM$cG(PsuH}Qwc|TY`q(&tI5VPc`2)u4g5!|gUN8^M9yjYOI+ zqWek6uD^7xMGIoe_-y^&G>b`Bblpa2(MN@Bum$;V01z?>fCvZ3(bqM+eHhV z0<^iTxZ7m7Jec@=y+Kbfe$7!U=BW$|#NG}QT)DTBj=QNv?Q{jyr#co7@dQ+HnKr%T zSV>J4O;IH&0CnaC?VVIv909!GV?TvkVK+ifV8n}%;c zR$e$;fjI1ez$|;&n`ov;cS_%)C$uFnn!80Wt`f&z^T%M}vqrj4MU8e&ok@~PyUC7$7817V z9N+hWJ*voOVu<4r9O^hswcW0lY-y2i?3E`;HbqR%*N1&2sGM%$g2Ey2x zvvtgV_<)nitI0>OtECO)Hy7icooDG75mOl4lmY(NT|oG|72UPS!mDq+a7lk{@C5%H?H$DF&~Bv||kHD9@hbc?hT z<8cd#Q}s#Otg3{rFK44qr7*sT)S^#LOaPHF%$>q=r8iYGsFr^q=Km-~)jceCbZa^N zx7!6Kdu2fNkO>T|I7`bGGDh<6HZ%}AisA1kkp{XCLcfNAOPMrW7E7lN8zUI|_7dJ^ zbMUC!#f@__25dVH3 zkSn23vOEOr>s`U;?;OSlWqr`w`e1PL3-RO)lI4YRP&#Y_%@GNp`|J|jS#beAp4bd( zTe3LKEo`6N`J4ZE|1_w2Pz&XQ4xtd+~)W8pgM949(>mZ6^9#jW{(YVVr&T3YcIr^OTR zvB3NEmejQm6X)&vAo^DpO62FjV&iSFWP1o44-Nz2{eIy4g!xO17J>ey?^G|}1S3=) z^E)lwpyOL7EMD*oPD$?vlIjnSroV>qa&O@M>1Z%%SxCZ6YU#eTM*i>zmW38dfj>Xa zLjHs*80e@5kMJT`I}!^6{(;cXa_wGo*2B4WTiBKB1y#-|prCjSbhi&c`A{DmwLb$l zC$2is?bh5`g{ufCdcs(A$LwgkV z*}j4}Umw_f=LDTr=!x|Kx>z8UPgHEr!=ZC8;A&3=0D2RBo+|PTcHtjUB|JJ>!M)qE z2y(R};jE1(#BW$crn;Tu$Ney&vtBIZKe4VLt$Wsh*P}=nD%u6xZoMUqa=V~NDGJ`3 zt^xCd6=cfl+5Ey|rc@-cn7`R@2?^TZ2|6r4f1`U5NXM3PYa^y$$|o;uYf!=a-*%Ht zvXvm`^Ab9zod>n9JaYHi42<}@3za4W)0Q7TaKZEyl>Gb-20jSRo;tAO*#N(;rHJDGWuI>4Ik zn`(1rv+TPVrqrb10(Td*`Y^~@lqA6pn|rXrCkwjX?c}Q0Szyl2P?UCEfVwFsY8nk^ z!t8y4Y?i_p3tv;Y%!!t?Kq`%{HaDZ*%_d}S=~S32yanQu4B_TSBjOTf1SS)=!0G@^ zSZPE^VVVgwu1ld??k%VFUCErehXW{H4}ih1Gr@VhO|9L~0#v$u61&j?f5drlogcDb z{PVk@$t6S6%1>N&iYsOXh} zWRvH-T9PF^3%u>d!*6LLuw1(hocEWF0Q%R!M9Lf z52yNX!HL0Ec(Rdspj%C0Q*JIq=tn}=yhG%th&A5d%t!lZ=CqIxp_#4qbgE$@6|5Re4r;?W?VBM2Yh&1ws zuBm6Z@k*@s`}qlmhnC~8c>x`nZ2@~16!M&QVGQsZPvmM!{1H5lh)#>*xm?qy0e zCv2wIl*I9^l_DN#enId3XFxTFl;Qa3N{F+6PK5pTQ^SG-xGkUrW1d9(ud_D~r|NzG zzLha!Bnp{TDkMql>s%%ENl6152oa@dqCrYh$q*r#B2!3_DMa>lu2dvbhz8OiNg+i_ z(}y@udA;ASHy${eNCjTHfXl>mxSM7UZR@k? zwWWn~MhY*eY(Lz?U~qjS0zY_2M-7sYHLXFrh&U{1@}(EHX>gGZK5%TS zBslHQryHc>F;J--D_5Stq_<7{YT1{>Y4-r}bDKz(xF^uI?JC$DxeO28G{LZp`}FU# z0s3=VGCijyL*KP#@tgN)({1_DG*zaGZk-%N(@XPFyp)f7omSz`Y+qt4&oNK9^DQB6 z2G!P6AmGGaxVL2nTz{NQ&zRl9ELTZ(&Q%Fk!!91Rbqi5^S2m8f$D-ZCF!Wj+jWt!7 zXc1A4wcYvn=+6u6-O`Pj3mY*%J{6lUt-_mqFX%VJ9{OR+2{b=E!i}k?vLDV4fk*oZ{SNnVn8+1p|Y?=%UF@spwcoHR_1@ZL_$I0Pi^+Y`BGgauQ$0@5N zSc~X)sC(1|O{Ch%_euL;(WNP{=Fn6eJahv^pVZ@=-Z<2i?4E%Td>lvMNQe7OjIMn08UwQc5n`6Xs^A&d8dtuSv$HZch z2|jdwhi0oJ*r{#xIQD3OE}UFPHXHpVC#z5JXWR-vo76!xSSG`Yy&6H`*yE`9YmDz7 zVonx(Izh|0_QFzX7=3q6Ws_q^(EsTPF01MTJ(K#1UjJtmdJU#>bN`K4S<#JK4s_wX z@A+tzRF1M?ndo^v3V$37Lp`rp)L58}{pN+Jb|4<@JSEt;b&_oM?=t-7SPDHSI|KCk z_QIndvJjyo2y7_Vhfwb)k-o!p#ppxy_&b%A7*t?8IL5+k$tM`dYsIFrDopDw#ZvbQ z+^yJv(wZ+(PrD79!iCsAB~f<4xu5vUsScCQhU4`7#rWHT;}JJx;l3Ja_HT_co9;W6 zoxG_Wi+MljySSPB^?AiKS)KC;8;P;aPAcrj=|XIQUp%gI+eBaTC}|i!O^#%%VpIDI zeELF;^*lI?$r1k`XCWYsdG1_)M;u-&YhnvGXDxZS5j_vbVdB|Fdih8u3Cx%SN!LD- zuH8uGRQ1qO+Y$LK4mgxL1Md!p(1F>1_*3TJrQ7CR!Jt$3>9#4~_@>pNbc3r3+SNMX zd?!aVRMp2KBT6S3_7koAxlkpNNlHdIE;#3#7`oVqFIN_#In}^{773U?Z3noyRg%WZ zE75kiZ*Z_?PI znf&ADe^R&XauiaT%Fe#5%t|#$v&)$*d^nxp0h7hJSR)*tc-7&+te^Pch$ySzFx=C91lK*=iXYxN z;JkPKxX(Ni)tXPE=c4;q+x!SS+W4p(6o!*OavFvPbDE~_M5n%7h0U*<(URMbyNM&H zI_!^y%~Sa8;Or4pQuJ*ch>Jw#68Pce7uA146Y#kaui`6_UJ zWe5$>n1qp0^HHZp9sjP%rxA5Vyr;(;Ny?k~{P8{?tbC-)e{JYQPL3J!UOC^OV-GZN z&jJHfJf}57SMHXX3+_SL@8?%Mk*}LB0sdu)hzA~A& zWMd1dSRu`Su6qIlZMZtfGii3Ca0`Cw&F1)V6;`0HkIjLw z=6$g1>J%91G=uE7iO^ja3VAM%N$aaRDsAe9&UUjf$!8LXPvr9~ZFlknq8E_+W2t1_ zU?CCBIz)7KjPi_350HqA0vccbhh}UoqM<|6c$2S*gKC>6xOOOktthS&ZQ}2I^pF^|Yl25N$1n5zO>JcAh>xrmWVh9k#HM!s6*W=x z*s}nIqq#OGwuv_X`bBbNmO$sXc2eD+Olu><5JM1sYEp28*L#}w%oe`JWpHDrRq(kY zg|@se#3=tRe5w(Ng_2IZ)1Oa+V^#`$Y&3%Rf_rFAcqCpMr1)lDC{BvK$q)YM1R<-^ z;JDEixO2RYKeqn_rn_X}ge|MMcI+e>sCNdvnO^Wkp3Ap2P~?^DD5kTUe$()fohQREC$SsA3#&iHTW~?3~>hAsDcwgzECf{ z-`7Qqd-LJgjW@6-tN@m})RTf)Y8W|q20OCmUZIKGRe<=tDp2gGDyA9-~nF-Vc^^uDl zt7f~YD*o!xL7}G~=t~E?%7ow{(tc?X9Jn(|j_fa^4me6?a{XtM{4eC~Pq)f@A|I*8 zcFxx|RSk30KGG8HLGq)JV*#c%k=H`;^p6%-t6p7!mw#Qv>!FqO#U>wU(JBMxj2|dB zwehqHEdYo=na_C^IW3y7DjJSAl8}7`VDA40 zURUMAsjwI13foI<3jp8DbfVJ)x52u@Igt3I2D+JxaK2CtPPl9%K3=iBv92TJ9 zDhoI`O2BA#B23s61M_UBK^$D?e_HgB<4`KmiihVw$xnj&pZra(F3^NoJ{uu!?p_$` za0k)Ab?~Ug6Q*RJBLm;A^X>#6C(^;Qq4(Tn=$KUow`ViC>J}kGiO_ z!$%U;d=r#p-otF+Qn=FHN$O8<{G=(tct^ts{ZE+@hePMVYPbRZ+{%R|>d9o(Y8Ot{ zsKxgRsmM5+P?z@Yup{F-=xB4b8}~`*=zSIKTI;Y_=Mo+{{hmsR`I37N#KB`cf(-te zjlml%F;c`D6$_^0Lf5%;ZATh;EyMBul#|KVxQV#?P!f^YCCO!#q!R1$*>v-&>FDNT zjd43IIUUg)j5v9Y^h8KNv!yS&GulhF6)vGd@C z#@~dW>juci*jLo)!D1BVt)NP(UJx}@118O-;47U1LuK6YedkBQ9W!~BJHyH0I8T_> zxfZ%b+(CHiKA88=4$f`V0N&i+ZXJxt6W zej?|}S(2dhk$j`hGPY~xc+{W^&hKzGwcOBFWiDjo8v*jPJ{4`f>7Z_ zJD#-9!Pr&abY7h!*wtKvJ|!F2l_-Ha;aO;rdmrc2MPpOJT{@7NNu=C4CZTL0>HoxK zcd({-&R`j~PE$bIv!13YaP4D8J7I55A__|LvA|A}a10xwSb2mL9o|5tzE4H{nal9f zVN(pUSH;=q3dvn3O)fVjgA{n(rM(Upu%2VQHk{AGH--|p)zAiBR9u7T9ZqoR@;+M7 zn1h|q+L3P=gu$&^#B5hQyx4RLE>?SijQxvBqPq{*cV{Ex;f61*w20ghuAQ8FkIT#r z2b<@Dq+f3pt`Q7JSr0kf=_C#@i@E%mfYi#&CtSw*euBaV)6gvaCKVZ2KtE5M zOGPH0rj|;J>2&k$sNNciMW43g#o8wNuk8+UjpzbdqXzpNrFcF^!!RVV4*k5-@Y8_= z>R7#ytM|l%*kWt=>ZMB01|?$DiALP@Ck{K3chV?73y`qS0Zs967_5utrCFXsviAv& zl%?Uz#=rcfgQvma%pG`~8U%Y(uF@T!&Y+YQm&?1u3f!i8Y!hrK0AQGx&G?HJa%b1aCOKo`?1s$YllT zx}h}GIsXJR{LkSsqd1-fr(?V7nge5+%V3fA4r+8g4!2Eg#2?qV{kBr!>b}+xXBiJ$ z!;L{LJf52VO2yK1byzwOhW*DBdFjVh!KrpO2!lIG*KVSiz8%wsLh<=ySJXdjOfUa? zl714_rBmcgsoH2Ey_Keb77jdAnXiLJdC`@*iIMpJZVpbBcfi8F_xzwP30ODF0usAK zAli6{K(Gp2{FFtqv6%MyN8vn+Mm#xm8*B8R(r$-HZ1zmU4=2v!$=i;|`0C-M^qJ`J znQ*@0IcVKff{zRzp@Q`_^u4kjOX(gu^M^l4`{hig^psHf1r)WG{KDnjYyO#|g2$se z$R2Mmc=JaJPUKh9jVDXFyo?^)>za$BbMmNLixK1-WH0iG_x^9fqf?Y7UzK(uv=p?)6$HM7rmGG=87M`6~B;|_o z6)U<*_;YGID(^Q0lGOK7aK3v6tjP5t%l8_f$@W7e*J=jH-IfADgFud%`l+&LZYBTW zA?=C_?ux{L#DbO+AEZYw!bb5sf)LjLW#5a(xmk!<0r+8M^ zqR^c4o8P{178h-~%xl^h3NPMVfsE}7AvY?I?wFE?(Mx-%MjB)kcf*L! zGxGB04BRQ-h=DmjxjA}n+{CeF)<`=O_n70P<;Gs>X15)4ALU?DSre*HDdjRm=i-Nr zfNcpnINYX(_kK8{$fpSOJdlPWVv#6c@`SDvsKkg@jTm)+)AbwNq+{>0$TB@uc*q|j zr{}#TMtRaO>*!gsfHcshb1q=n{s!EoR)JC8Pq=fRfL2)rc(W`6S8hLz?rnCst86uT zk2s=)&}vjJ=kiQK+VR((Cj9+xD7tQ{r$5JXc(+%?@+|%S&_AV*P{>@AJsa7LWBwL+ zIq))BXTAmA%^gYaYfBMg&*P%MD zjueX8M<$bf5jWsh(HB@6T@3pd@8ZQXM>>QAeF#-Se*T{n$Wi;P>jkw=3f%~85Kh^^5M>AWkI>ZSH+^^^kr(8Ajkj_h*ACH8>}_d67wdK$DQdti_N(y6 zWJf$?uo?pJ=;s3&OCL>AZL8R(Ng68Lk~Vf_^qUTya$dT^dK| zHZyUo`SF2zEl$FZn;OuywFITLw9%3rZ8A(C9G4{A%C_<|kp^y4Sey zO&Z>dkmpaD90#c<+hG6Y3vlxiH_y|&7w1k6!sKPYY5k!plGc45>NN^sI+u@k@nt<< z>dQ7dbHgs0;(ec=amj_`cgRAO#a!5WC7pcxqK_iYDa3}E3nN_4yPxoOA`|_PpJcy_ z2HkR|XO!>pyEKnM@Z>^BN<0tSqpFCW%WvxH8HA@o_HuR7Q?#(;0<14>16j_On6OuY z-{GB(PyT&{Hd~TVJg=EwpmZ2s>L!BWTUAJt?WV7|dU?XF?|2MrQ5r;X-sG<&_me%W zm#ZKzb}iy^Z(eZy3?bIz%6**mw2)r<;Y&=-&Jlf%G1`#09gE6J@G0jV(VduthGTv7 z8(#wXpT5ytMKOHy#|+csx!QL089epD3Y&e^c%j;7QM3ID?z_!tt}2hwt(=G~>#_=L z?Buj$evT^uTmqv;o;_kybWtsP%Wi#{>0t^o~`~3 zV)WrLN#j+M(fPwfV%C}smPX&;cmf+?BT#Wn4v!#xA0AQHp5XwDXE-a zWE+9=PsQUD=MbE7#R9jQhtQ+{9wB~(J*3OIjCWBm3hR1`c~gCR$ld*iNzC@MG+WaG z&#n(a=YV)@n-GD&WJ~#4Nl}Ep(&g!o&BpZechGM2M0U7o3}3OSXrXtB_;7v$=QL4x z?yZH#LIyEQM~;1y{uG;I{!oKgvmtT*G5F##NKXFZzAqv7@MP|HRMKWqLAimx_SJ*O zu2JB+c@7MvR?~$3d@T2C!~OlIG5XE{lFJ=O;IzjgCjDf6ACJ1b&Ouq@RoHRz1GT+3 zM5dYDCzZR~$Z5koB7dcm=lA9>KYpPF{b?P@QL?WO|s=X{m7y+5+PLIEUxm_ z7JDM$mrOo}?;|$sU7~kW9Mi=d(Q7%Ndgv*7H}nqi)>#LPC&0$wue`ZiBJqvJOLX5@ zj?5^>9iJ-;LK1g!itb;`da@uv`d?$?SH4>|rvm^!=n{y0V@$6<2(H=1Z3MwgW= zK-ovV=oZq4@jh*MU*rZJpSlGv+%BMsX;L&~Y@4;l2pQ`m_B*Lo^7mtFyKj6xcY+R@71vrFxsjiNw=E zUZhX}CPs_0vho`2Qh~47J8dx*kH06eQ!OE7R{}TwFvJh-ZFqgIAZy~=jA1)PQ23V= zh=!enE`M3b6ds@#-O6$3RTqXuN8;|b&%D+M0#%zfKZe1$Qb$Yt&h0s$iv<#d~$8o0RQhPReUUKi<|U@=%p4@m`|ru_mC#4xFeS?_#MGd zTd{=f3p`8&cpI;3IP-JHZrUn$_u9}@#e&8KlJ^CKPzLg@o# zQo-Mg_V*^RDyYORck07jD=qA}APOR0&QPoNf+Reb!rbq7QCQ>~%FL_88BW~wpF0kr zJyDPpU<^u%DYS{r#FT&r90Pw05%MKYDh}|d(;fD${Y2!S&Z50JI=FeN74~p*?5A5F z6H%*VGPvG{9J$FO<-!;FT~>Kis;!ak)lj13j49l99imp3Z1F*jDxN49Sg#6bKBaku+M;wt>e0cT&DSg`~>pB=>~x@!1RO%gWD%cEC!PQhyPaq1OsPZj#6 zRA|3gMr1O!5=XN<;?0U+*7c3}c)1>G$1J4xkAEQ*H}=6E@!hbjv6!sbwiRvND$x0D z6qc=>LSr6ohwq}-q2sh0xZRmTUoJ^ThntVkmQBJ}=11t2IT_f^l#QGP!*DJqHtjfC=03ymJwv#L6r-@c9QKR1@m{ScAij&M z_@gdym{FdHC+>!0{m%`!clSeziw~JDioDFTYC)Q}x&^J~Q)+LV%!_urXnMBx0ky2y zfI}YPD4CXsgv&ZVTvWt=7k+~Td~V_?{VRumZWp0W>=1VxpP{q8KaS}Qk{Hh8qrO@f z1|D`%i}VM0_|-4moK=iftzlIEpbK1nnG9{y8P3b%M}>xy@KW0&ylS3|e@@C%`#WwB z6m<kzg}6d@{ZSknTIj~r}5i!CoKP6NauV>C!Q4>LA`qmY>q1ftW7Wg@2o|j&{I$J z)s1;)l;+XXagKChlN?>Pc`wwS&4Zc8qQHOb3HiRNm5#6QK()0iv1CAw#@1xPeYXL) zzq56l^xxmhQ1aFWT)s~p%^I^wPLeIUjl>{Xu@`-G8)?w> z31m-CGx>FtPdeN3$Z*jWQsZ;g)T>t*6+=Stw|5q9i#p2Xe9a*ufBwP74`;E|+Z+2- zm!nWN$Fkm^OCx21X}Rzp{uB8l823B_b2z@x%}vu#FYXCX5H;Z1yLs@U+JKk@`(mtD z6>hAE#g+r=G2Rbg&@9f7j8U*xyS8UBJ0VLGtx8JX#x4t`I5!fAC$ z=CAK-(CXEMcS{vurSLQ`Z#9Q&yMrOc`zlE1C&F$}h3fe=bmz|*V6ZL`UTR&1V?M#K zWOym$Y@G(nYZc&bfEEbdeGPt#BpHL_KVimmPBX$hBeh4w=#}cT{LXv7h=|%(=&Vs; zYzzLv(v|BWwXl^+)$T>z+V6C2yCh5!Y=pBFpW#9sSEEjeBd2&vF<&heLvL(HQ;T;z ziS;3{^GgX_IJ%pwRcTP(jaY0yTZJ}mM=`l!; zKTTg0==sZE%k@t+qjG89)p0t@YdLyVd1FxhS)9o6>4ZeJ$#0vZ7?qcW!%3ldU6ac% zh;B4xo3D^G;|+Iu7z zMbvFDRzn_@#G5z`T^4O^w4jD?2Fni?An#BbI`at5{C5dIy7f0vdM*jYZ};+)4u@gr zomQ-|$i{D#7wPX1OHdNchHs}1!HRJe-us?|mcPZY z27VkIptBR)F(mRT&0FgUN=9PLSg#Hf?J2;lm^>Mt3!dUl>cMv5_X zoX)%({07VRTS0J+C49KVF+)oHU~fYPyziiJab5u^y|jjoeJuaE`dYZNqX4$wrLgix zI!sjLJZy7p;4xzfxq4QRI`R!n_fBVIe@HP(OYZ{fHXBYBoZ(GW4dh9zmIaqx0!;SA znM_557}Kxl3D@6Wr9$@ZSaPhN_J5OxR~Gf4xn>9yopRvLAAj=Rirdf4Ik>j?5RyIf zcq%&&f%A_nD9W~k&}Ffd(aT2F7_JW+6NYc4cJZRkq(EoIzob@YCBJl(iwWJ3hKivD z_#y8MPDxln?Y87_+{Q*)HztQ{#s++||14&nI)R?57AW~<59zeqk7Ca=@!6FSEEN?) z-*YcbC4`H}{ILpRIQI$3a(qS1;^z>J2PO2v3|o}pbXI*Ir(y)>6*^~8OSg(!V5#Ox zl**UKFC)2B_l+{Gp6ATpYUafo`)P%du3Yx>q)3b&)JB<^NoK1QB;giM8N%L`^2&un zvA?bzm3I_kk6{&k`dbh1d>s4{wSgMDF@F5@7_`kclo9rubp9IP@F723H|*hPR!nY)tl4A>8IyKGT*VCD-;5`8+naXU7jvm{L#qOKzc>e5cSONmvq^AW z$p{Z|T!4_oc&w;COAnXUwiHta*_Zd@IDb^B*z8@)n+$Fi3x17lm#2jzZTi zWk}SO#Dk~0(8WiV&EMLK*+ufWBUKh+S8juFuNL002I4>3;%vLW2z&Hw2@b9wqQ<3L zh}&=`>Hc<}*6l9EznmW@G0qu-wFhY9uqdx}*9Q{vRD$y%*^_I9YcX5Zp2Y5!1f9zF zq^w zd78u9Kx?-gjPH`i@QuA#nkvgaIM#*IK9XpRN)R6J2hU{0AoSh<-Qsr(pU?ToX+H|_ zq;(8k>$em<)+E4n8;(a{b(XSNp1d?zfdu?2$|Qy6xYgz}SD$-~N-MB$znF*nwM^0-=VJj5q^b1msa+l6%S zoE{OmEeTTICZK#(3YHtDkkxz6lWFh5_=6m$@c7vfTy2twF}D}uy>vd=k~|e_%J<{C zprcqbVvC2leq-+YFLbIZ(!(+Fw81(Ez1~+~zj`GmI!5B?!>O3g>|DThr>n4gcugp~lEJ~x<@9T! zDSAe{#yWpvRw{inYpxA!gv$sn)!vD(p6Q^}j573}u$cWO;m(d7a$%2NGi4tq3$x{C zs&U#OKHd|Y!0y3i>@iU@_C}iy`?6Sq%@uf#nJye3=+Su$W=b&QhY{=k>?V$KzPBG< z>G)dmIWm_d*cO?YY=p8IYyDsuTPDhlEww1N`BtOA31Qa1%9QDgP1s_U zGE7ODi8~{9;!({JY=2Eyp8saH&fAy`E_scKwTtm-T{-=Da|zPqMm%v@g+2CR0qgo= z8Y{l^7B-HL(cH*;{09*u)LEo}t6NL3hV7hQ&h9mahRw!vJrk+kWo25kf}4-?ibTtR zN*uaffjcIi#gqAQ)Z49+F26TK0|(SF^fIRdl0Ax>R_sT^r}7+Ui4qZIV^mE|#D%Lv zFf>;lZwUtRm)VDtD^00HX@L&erLM-y)Jdl=y|?0~ErobTxdd%Y6Y+0}6}6T*jxyt! zSQ(pzMS>UcARfhDUmHAdfScRh5e2qn?K-yZvo-r#1X#^1ZB~@a_ZIQ%M7p;g=U05e+X-&$vhFWvy1gD3 z26y5+3o$n3mo~e*maw@xYuRO+*Rdo@txdscD#87+g0Vk-nikxo*n~M zD7+Krz6r!`bB;^5UYPZLx0H=NzL`zB$Z<{zMvxIbjy>Zu@mBazOxV(ijx&{6zZ`uw zxkr(mSyGD?sf+PrEv23B47h$WA0_{hWFK`*WmlZ(!|Ka!oW8k(zDoT_*WBEMTE}uQ zz_Jmq`!}H2xhojjH5+Z+37WsO!TF{~aZ*DJT3TguVXIlFdDtKCHLRxQcN6gndlTPb z0p8Pe!fSIgsP+44ymPm;$e~{t4j(#fCJ>)|T@c zhoHD3=ashQcr0p5aiV}KHgejz^7-AgeCbZiZi>c(c2O8LX)8|e?xDXc3i!v;epa>( z*3sSCF~|?8L$R7N%voWH|5QEVyZSFAj|-pj!__xn|7t|-3Eh}HO0n8^Gp?GvhKjwr z!829Uqo=w4{frlHF;7a8{WSOq_Z6MNez8F6RJNMGXJi_VKB`CUg);1QLs|Bi+6R={ zm4Ih;&2h2y0?vmLj=}l8ICu3oTrbs)UTzed10qnyb~z5LTZDlU=W*M>0(PMGJSwO! z!Ud5QXnQCEEmQdDe&!8IH;m%q`d)P08IB>rhFlK2IX2jH%&x*dL_t|rYL*P!tJ;9G zT&LrSb}RmHb^sN~sp7QU%v_v(!;`+wh5R@}V7zu@qW{`qc? z{bf<;(i4s6>UQGIDcv+{wJ<*HQboPiC1^csFDmT_Me*JcEKs(?V_}B0pd}asz8B!u z%@OFc+#0FaWqPoG*vu?x3X$LFM1nWXA}hD<;5+Qm)7D5pbgXriAoeW~M}VScoBW@Yr>EZ+T3$EoR-2D(YIh;q<-e$0n3(lU!1BaIrs z%k^z!ce4^b>@`B8$C7D9@KLgOhCRHhjey$UJ7DGa9I{qm2Ce59Zcn$_^XHCFfXf=8 zU_AQKOHV|uXAb54LB-v1xD&!;Ny>cZk(3P_l?%% zS8tjOHI^ALWq%tyx>g1$XBLD0ZXte$)J?k9P?g{1JRQ`y9Bc{w8=zPa0wHP>K_=6N zKYJ{bZt7E^tG%z2fdLQjkBb0}*IVFt)nRgf&t*Dc(Px^!YdW3syN&3k8^Bk=Y0#<= zL3}me@-ktJ^FTD8(ec$zn-g_yGVx?l}=Tt)PCCn!; zBoq+D9r1g zktYR`1A0VT*}|-5sKL}@qX21}>_^O(RuIeIeZ=#NGzc3NkqO4ZBxPw1S+!dNgl}$! zEB-Oi>6QjBibKKc&>X0l{DC}wtHS9c!=S~#9&F$BL7U!lP`1AU;sS?Yl&j&*a<~BD zeNAwxN0>SBMuBNfmSbWQM!>l5Hgu?60jtF|5c5TXIk#>hv&?Ndm)CC0RJ$oL_xis; zJM#rp=1Vg@b-K*ailxj=t(DBXtH7K%JBQhEK!%AM5@-6QCNn=DC^7l(G?)f4CC2I7 zC*Y0hLF124(0D+J8ST+v`fe&Q=euMWZ#fA@K1hZcUp0q$ZN@O#%u41`_)^9wUYF@w zCe2(=`U1J(!{B;SiAn8R$n0LVoRRij$lO$vWDa=NKmq4#T`_zcZv7bnzXUnPqe_9f zkSNUPjx}-l^cO&B&;=S=4}py86^Q%$4CKT1L2~cdvCi@ozlY@4}MEjUD1T=mmlTvOI6HPzTku=WRv`WaVAY0MQ z{@W~a=d~pK+kKj^?U6w@?7d7gKkVSYZIC2wmGLCK_cZxwdxmK5O(I1znqrG;WlL7$%L#e zneb&3$8*mZ1eI|~=Ic&nCUKQAv!_#>$#NfsuYK=eWubEoH-7z;3Ck2JLHF7#I2qapR+AnFDSch18q?2Fv(h#@(36V&;P5cIP34b1vqlr(5c1s@_V84^5d6U3v+5ibj zcuTUHhKRD3C~QsT^bO}2s1H~MS&z-YT2`K8<_f~>jdH;MYYH+8JfYS$0EVX?fydEn zp-;vPmR~gk`wv@S>(n49D(Cn_?Khxq<_+L+p1U#e2;hwcL0dvR)Ck{(Uad|zq}LA) z-F+~{^92lyRl$2Z7F1gwL*$bo$UZiK*&QIlY`-GH+$s@bj{NP1Yh3%*g8PkM{-$W z+b1w0t3Si$sclf_&Vu#bDvn$70-j(W{I%-`UPmV=_Eo@x$MK-k5d=0Q0(P!Q0kzRQ zFb>a$j14Ie;1L9q|89ZU;l*(0lPPGeTMLt#4})rY0IaXr1?dK6u>GJM+@355Q{$$< z0v9s~;`DT?stic*)IqpU1X?3LlRds~$?1-M;&+7`H_jO&0+oG4V*ClY#ZXdx>IV7J zRzY~f?ZkGK6qMaeBL-bF$y>Q8WIWA*sPT|J>GF z2?_}ah=~dOe-;^mB?7KKZo76L*$O|IfR(rh06g$i3Tt zZtJZk{O9ifyuXT4;FIN{x4e^MkD|L diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/data/OLD_r47-1_s28800_adam_lr0.003_wd0.0001_xy1x1.pt b/research/part03_learnt_overlap/e01_learn_to_disentangle/data/OLD_r47-1_s28800_adam_lr0.003_wd0.0001_xy1x1.pt deleted file mode 100644 index a5c2fdacfe885da267e098e4458b8eab5fafa5be..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36920 zcmZ^~byQVf^eqmOBB^4df+98w2AnmI-HBkg0xBjdb|NMS3ew%(%{`Y!N&yA&0}>)A zU4kGe>N{`ze*b*m8}9;xanCq+pMCdUd&Qh{>1`Gh7ZH(`7WuzkiXtOLj$JZ5WpdS6 z)99GlvH7NF&aPV}FS7rCyzUP(G`qmxHUG#JGn2EIWsY1jK7PgItkDtEW0#JdGd456 zBy&;Z|8a|%@%hUaE*-h5C38`9=FGiP{1h(pgB?3*eDVJuRc77U3&+e1{=bVd7sd9< z^Y`d3(q5{&bjjj{OBXHCT69r-uNZ&P(DcehiLEQAnO&6Ju~}s1W>Nm<$mSzv7o}$M zSNUK5ov7JG>1T$97tWqFHZ(K2aQ?E)_De>_myC_pm>8PLTpT`2|Duf9#S!`!WuKn< z-*4oeT$I0nVuDzbZrs_dXn(mz3IY)uM>nzv&RaTnJWw5EFB>{d~1ww(^f^{ zz@BkJN%cv>F(#9RVxL9{V~;2c@4Of%RGTR#^fBn~<3I;k_6o3=WP3)0Esu6F$7UfmicG@C3geD-LP(Dt{oP$g%SFd%uf@Yh2H z;jQtC!lldPgqFuf3a9ld38fv!2^Tx63gZU+#hbeol6Gm*7{>EZt{i zTjlLOj)di^1Xf_^+}E5#mKSobeuB z+e>jf=mWOTD8r;H)lh5gz|;~Ews~+A+c{@Uq{zVDl?a~RtjS)IRSBq0ZC+nxe8_}FlM%GO@t5aXt z*t0+=<@%dbR7vGZf}e7wdHS5<1}ScJjEx|uXspiZ>`N({pPw(aiTjeWcI_3x%ncy| z?K%nWlEGBYL}nFtKxzZGw(&UU>3N8Y=GJq6Cu(r-R>^QnOI`}T9rjN4d^w*UQxx4c z5rrbR3fqoa;LD3w@Ve@armzq!9*RP>OBAL)ii2H1JZ=x9AYq{ZmQoy|-wNPu!NIgR z6J`$Cc;8fp3fTroUKeE#7k$Q`+F}$;&c&^7LMVlE_&isDi(>@Hm>G|Q(#bfkBtXOB zIOrQh;%{Xr`Z6MrRqKZ}0$aQnF~?qwb$AgfffnZonya#r6x}B=DtgldvB!M{x9;k4 zu~u_AS#~Q|JpTx{@yZy^_6EaMC8%=6tr3ER=^28~UoI&JwH66t#yruvJvt_3?z%Yw zeF;~=k;w0Yi`84WA%)vqTHkAKsbfD^eON;%GIU6IO2%1;E9SyRWo_YI=`K!?p2Yp~ zC2k5Q!_|qa3wB<3uUlzi#>}dcBQeK?bXrS?yc8GG`{LOvA@A#JJSoH>*- zHtc!I*YHAtf7?aDUGo(wN~cZ;PAIKQd7O8Xi7)z(F}2=Lk=IkHC~G*hW^BN}&KsyN zehIBmHyp6{#jDei=-U;H=bV6$2bVz*OiZ`5VV{;EcCAupJ%2NHA(6P5{z6f*=NSw?~r z#+3PbMI)Jy4=b6b2~y-AILKT)p-qQYt|H;yUrf`shs?~s`eq7ABpPW)2D?DodOsExbTBy86S@`^1GN-Am!!?g77o3T<(hao_ zWEOa@qb*IxsDJ%)nzr4FVt?4t_yi+TNIg!w6lYV%-|3G%KeG-ZJcm25Ppr5~5k*50EeoU1@jzHg(@9W&_LjYFhdYCXa{^%+WfWCYPqHpm2ATJoXOT$sJHx8O& zanP2GL)GXw*pExb+1DIA-j!fOdKFGM|HGs)br?!{gBd{?ILG@aZLC3u zMxm!85~J3IVX=4==H`SV@>UoQ%nOF)Y+o2xIw1J%GmJ_yhRSuGEVpv)w6R{~KCvR7$&(KhwWM z?twd-+F z6*C0|tG$^C?dz%9$ch}(r zGk#h7A|Najb^C(xWmX*CJ47I5P9z*HW1&|Ti^Qi%_#u~sj%_K(gaGqL=izPDdwe?o z4LehYv5HJRbVAE;d~q&3R8#OWCA`a4& ziSBf{A(WSu|;_|Y(wdq`ty)?4Zd5Yq1T)})YSN<$Xe zOyu|Cl+BV0xk+O`aZlw}2>0BXDRg%W<2IB?a} zVz9^}5p4w=u4-{GJ|#q3Ll#Qs6k^`h|1g2ib-W%*u?NjsF!k#vJZ>+*o)>9wG8f{} zgj6VONki)6c*NgJ#IbKND4Z0F!;0~Uz7PZB6`>HZi^0uNJ}9{Cga*w=c<|pIe9GB| z-4-)Y_-Qo$mWaSEwu0op7gOM~K$1x3_u#LPqI-pud;23L`4`gTtbe5Xq@V7MlSR&S z1t>ZyW8L1#koqKt7Z(3WEWVG_vdZY0p%>jbqD7&AOxTVIpk(63+$|AcsOJq>xMUgNn}JchL-<9li>j1=N=iuYf( z>2biV1nh{9M2buxoc+Bqcib~PG(L}J@qKVI&<3n$Bf(Y)tHwzp&H5{??fOjN)5_^E zE2O{!MKo)$i&{o^k@ls3RH?0kbE`%me~k)4jn#2x^k^J@t%A4m^0+jilb#-_q_$n* z6i{%7_VjVgUnWlQI_?E$d1k6`h0aLfqQVp0sVC)0X-;xf(rH6O%m-pVRZ*ddB&sef zK#{^6ER35EJL5$Vb()FP&?)$}bsUOwM?kc_gEXxAsq}3HwKwFGvie7wYF$WWO+|55D#kt#!k5qEJ5Q&eMMsFOS&8_yCmy!XW6;wQh;mgo`0+7+*6$m5*=&sS zwOcUYya;0X$`C)Kgss;^(bL>cdCQCG)=(MU(`}?>rEPT7SOk>?GFY2A0aue$Q8;ZL zR6OV681GkU`Wy^pPQ+%*k*Hce4CD7_QFF)>+F2q^=Laqe23DAGSNi0IFI%LAYR;#* zs=`OQQY&YY`U`)m%deqgXE{`Pj6#^qY}{VD7#Ak77}=_g4;k8+&^ZedT~l%AloGV& z{h^alT{K3%krH(ZX!Y7sD!p1ok%vT(sVs{Qx3Tz9I1k56x4=Z*1S{Mx<8qA+d=g#Y z{M;WcE5jjJ5r>;=k}ymy2@SnTP>o8)ngAgRrBiYIZ8~aNgs8ZYj)-Ad_;b1pZ)_`( zb*c^7N0eCoH60MI{SUo*ZxFF18#24n5VA#xd9{2_-zfz9Hwl%B2^h0D0Y@7+1gIt= z>}nDgKa0YVssI#>^v2fJHW=Z46-?M^q-3)w=$(s?e^imGsDNLW6tJ(hj+&M?(u?E@ z`jRw2i5^nu+9QU)gQM`zZU#O?&w|OqB{=KMLVp&6iQkvv<7N$*Z<~OXI^z6y=%%c9 ze#CVRqqVwTDgGI2Isae>(MiB50;6WB0TabaKss-R{J4vS}YEx#bKXRGPsvXaCw}9h6~9swNHYQ zQ3~333sL(t6&cUcvEX$!&bSp~^3F=gP3pz-kt12HbDh}xtp=)>i_qYh4*x}2(0rW^ zi)s#T$2nN_C7|?766V&V;9o!re*H^C^wtDy^9sY)z5q0<3C^hA3aw0p~K~}bo=F>S^Z7ZeFY^H`CBXqGXY8f1db|LoK9VB0}#u*c5 z+?Djhoa_kP+8&F7S>aft6oXasVsQOZG!i0X(AX7^$peW{UlWIRRRLz~C_wm^Vyt`I zij&V3*&jQbQ0QF-hh2Hda7%*q*F@NDi^FHPc+_8yhvKCKsK1Pc-@`a0{*A-5JBg6H zAA^()f!Mvw6*VcgDA2rs;Iso6yJ{`!T;}4|!|5n>RlrF5Ix;-aK@09y(4DnuwC{cx z{WT~gM*JI%+1*A@A|zq>pAs(SO+!o3bo{GZfKTbVXx%ysac(LYWg&;u!~du+GN0sH z?J4!Y`Lvju$)Y2c(&l$ikccGKc&H#? z!c-jTpNq3$v*FM_30w6iqkq;virzOsDo*9}V0Q+cToXpxp_Zio$(hVXdC>#gDEh+J zQ)@T0(ZW_q94H@;vUQs9NnMQr%kwDje+;KkXDHtbz{69Ks4t1c#Lv-iO^iU--*A*~ ziAKR-EGi7+u`V|bMW%6BG%5)hm$RWBT8`3)Mm#t!&zesBir}S1$kNP)Z(lO{>ryat zVKip)^?=B=7_@WIDCv(vYFRV_mL)*MH42ONL_l_3Ao9mJ!SwDE%)WRIX501P?lK>) zRw}raDUa=$9rXChTl%xUf{tm0Q{O*NvdMR+LcaEjsf!~6qgtBy?mO9r$U!h5fk3EZ zbG9}@=FG+L7n7meGZCh|-9>-sAX~e))TjM`-j4gpj9oBBpeF0eJ=io!`25Fcq2C$@ zZvPK^!GyAS#^v=@Qjtoh4d1`e;j=%f$94o%P2~}yr2)2Z3QAHH!N`up`;{HU3@@Nz zyJBgLw-d>BTqUmPKCLOWCd004^ytQ8GN0o~7q^FzV!~%SAts5BeExQ1;e0&Wv8}c+T%!z%xm8OuJDVvqM*?xpqhO#l4R?l*-_f!Y|4`_L`=?kEFo4*0j#ooPvL^AekvMXe~FFk~3zJ zRDnJj=~+_p=Et;cVi@hwZlJMe#9{m2NHqVa4w*&Upqp)htxsP;0zQa3;)maN4M%$aF32PKBwf5nN+O* zkv2z(VqTyuG}M*x;_X;mo6qO>*MHLtk1pbVR?^zL)>Iz}3fd8^)A%t}P-GI!NijU$ zt~FjLkW1#$Cr;)pEgmkplqO9xd~ef0H4hWt$RgqS|LEs95lE$XQS^m>% z58GTJOD>31c7n%tM;LoOgKqc*Xc+5ar`0Gt%d4W{L$9ghM-<5@JfcH#w@J}@8Kupc zLIYQ1=%~&|M#ZF#*>_r(dM9qCeACPHv?Z9V6tk)DTQQyc`;THKiDBYIMXcQ}fk)jv zwAa0q7Kp@CahMHVzU0HK$n@6T*}5KbRlzK~Av7m|=@qHv!f(q`&tSSm*vSG*~owIF7z7M_Ex>GK_PD%Z85Jpp;-95V`gj_-r#z#Tl+ zw1cdS9Y(Hj#>@qdc>UELcgDHkKY3?NoZx}QN*>VQToL-z8)tn&fU8j`4@t+H{bwkITaM;owb3+^u_|+PpOddhh z=P({l7=z>gs;KYX3u4?JkhazuvbiytKdV|az4bSFj1zF3EuuXvmZb6n2YNwzad zkxI0p?GWYUU8TL6ain!Kolc9@Q|kCZiVGejn;9R;?PV<;DiYF!1RrXsxJ;`qL@}J? zXo1z+9o(cRdED;qNkSb%1z~iVkSky(acag_1<&W0Gv+Bq^h*0K9k354N7Z+ zrdA3`8iro8kJNgZxB1bp=#>0^N)2mbYC2ytKl@HIZG+30&lNGcY3V__>31r0FLjqN zwZ0|HA|EMIJGX@1ygxyHqo0w*v`qS}C5M>3`|(J^0&Ws7v2M2=k~TW?HNOkQJ?&s< z>x7;hXUNWVgH69XPHDSf=K*)P@EF9PbqFp`%|u#aKa`UAT>eHRcISE{*wGb}Mmgh? zrW^k5aYFuI2W;i>vXu8O5F6tF>UGBEeKz1cUt)uv1p?X*V%LJv7|xZ_u;DHgKGKSu zvXrTP|704cqevr%2AGifDNN!=3#R1EVaB!hzV6|ed}d&K7ZXvbL#qQf(P%L*>PgHX znKf@o&b68z9{xouPk$rHs5bJA&Z2;gFUh8C4+)xrnKlutl#=cZ+`#Ne&SsUoaNn9S z!fh4VT$Sz#PWh^XAht!DY0Y0n=HDMuo2&-~W|UCUvKsocFQM13nJa4UI#=I1viqKVL zV781@O%us1R{~pJZbH@OJ185l#-Vf@nB8>5RU3QUedz=za)jh+I|PQYZ>T)ZdhzTA~ zw_>)mZ=)=in>5uVhJt41(vgpKbZ~b)MGiF3InDQ!`ST?OO6{W0=5y(uwKWs(p^~!y z?N{AYy|aSy?uUXp+t0~1Tc+vgt+HiGyEB>ceKHh3ZxLMx-$V1l9?=?c0SRpf$)s8j z4H@SAnpq>b@)gQ#9Wm~R6LOw9qT{V2%Fj7KnBj)z6|RV^cEZeVS4e93BYrR%hq5!U zclJLBB(rf$EDANt{qSwJKXw~>AxhmH1;6Q-4o z?VLH%P4=R6izGZ+{K@dkGtwDnK&6Z_efOPCwc^&yF#8hbgW&__<0NUFQMX5=y!fb+ z60`j{qm?+BQO%Mh_up%1FyJZG9&@CI+;rOiUo$QE@r@$dddMZWjINA!qfMh9(sA7# zR6XxG^AVjX{!h1XUd;k-`$2KxzGqe3{A)p+snZ-z?DrAD*3N83%y%F89X6-fiZEJa zm`kSrzEIOr5$N-nP0p1JC)$!9PGi<}(~74%;P~JU)Oh>ueqe*L!;ZKd z;fk-0&iJb31p8zMynWz|(#x*c&10Sy+&xgv^B$Evzmi~`futBIHhEtb| z@0{@^+YPe}T(Rn!D~kEpA^q15kN7d&`W$idzAG9&y@GzYEs`Et;mVV}$p0yVt>4qh z{>fvS7_^7ZJ{?b&CW_Eq3puJ8%49stRx>xJTxB%x8|tVvNiy@Eg)njfam*K;X|!$J zDte&hK#N`m)8p23`V-kp>r(njs{eo0Gz9KkCjrG6>e)8dRs-bxeb)|NVk_xGaYW#z*E^!*+6> zXha@+ACj;lhLYlIDZOqYMyQ>_?EBBLWV;ix=QzP;lPey4^S~?%Kf&*^~>Vs(74J4TqLW5DfKwFm|##7K?hqC&de` zjV?HG#|y`AdEneeZ$zAQLui64{IXwQTjW`!Pn&|*JVsf^`BBZUSG51B0quFdfede| z($DapOx%S7%$EQ)CS>wGCM04m6KPn_IK6IS)>$p3q((QoX6a8J%PVP>b|>AC8pY>T zQdo0P0v|*BC{DSUl(q#@{e}DFQ&Yu!veiggo}Dh(9P7e)ytn00B+2P5S;yrcD-=x3 z7tDhSW2Y^ zvCQvP0!B4yA#?6K6`?(3j)|!{$K5Z}Y|I15RkM@xhreU&vbcAxGK|35)o;Ydz0l1tr6= z_dOn78_%ADKBvUd@5kNEuO zptvU_FWDox=q3zSPsU1~i#RA&OnK^&)IZ#r=6t_R_nldaJ${HRbwy~Db_273Yh~=y zqnM4$MpNEmS#l25BZJKkX{u}#E$=C%wljZd=YBzO zAx|VTjf{#x+f7XNrTRXa$MD!##SnSfTFh$n-B+I9v>Ti zCKz@%Ll80|7$(6!*dE}64~9YT(GA2Z%|N&h55Q#}KaHE~f;P+C5v#ggu)-6NSr8!2?3GQEoW#Dum*ffVCJ zu+m`^4*TzbOomViSC37C~u#4%FCq>{N@uuzlf(ycdW4mw29>$EbZ{!tpRF z7LH|+INcbEtql=ae%===`fV^;auXajs3TwPAFT~5rzt@hR5rzx4#_{Es6bwSY-oI+^@rmQir;eLC|zgF4Q4QlpMMcFCyWn#NMR?hZ6#XNTs zm57R6NjQBs1}652n9AeJlTRdIyg-1Hrm4KoiU>yOIDOY zVOl=UI;CO4v3Lm2$D+t63FA@{VY4n40n?LktxSkI=VToH8jspbiBP;3goQe8s1-kn zzw#{B@f=TWk2uaWl+m@y3`*(_qgg+nkm3O&@=sn)o#PHuvhFsDQ8ggj(8pv{8b@m$ z)|0xn1Ws8@fcCJZu$Np7y(wFGY(oz>vz8&`^ep%)D?`3i3NJ?_k=v~8MBB8;q1lvf z*O_3B#(P{e6oIXO4COkc(u$rwl581|vxVA76IqMQ-G&HKJqhFUhWNA97{5!_W82LI zSmLFI4*??BE|Epn@(-!y;6BP=Cen`et7(Fg66NbykX%+Ay*v~{2lm#G%P|>r9vO>! zwu|t7mmXG+yat6tcl=C?!EqiFczP%eIWu{U+V?b^aL9s%SQaD`1n5vq$LU$wDEg6w z=0imgpHTr>#YWsz7GuXuFkt7`NU-XnbEuyqePM) zbRl*BOZ4^KJ~Ff3MSIoxxG7;v%h)%x&T1Gk3>5J0F$?a^I)v;z0qc#2q2PED?_M0l z$=_Sy%Ih+AyqN+K>)%vX&huBJgtYK^AJL2woSUu44j0)6OEnQhTpEq{h0~z?XBqZv z+Kj@gljw`QhRN4&qt4m{Yy2+ZM8F;#_ud4WHXk_}WAIC+gqTUbw64I0`c@n!n=PiK zpZ0*>9I_+bPL7VKmQb|yUpj59f_86BydT(tu_G_y#P%0>DH(}xw}m*ZQGhb>Y&cGN zi%I3hu)kIWt1G$qESC$Hqq+DUn~U0oMX;Iv5xuKFBgwxL^HgP7^&2Nxtr!`0Q%)-m zo7W+?xE#CBzkxXwLcYBaHEZ*sGFXUSr9woO=Aurj7=H~*&?r@ip8qmX!#}^=?iJJ~ zUV+G}UHFZqm~?R}CV!Q}m8EU;cS|MpCMHsJ!YfkJdqSa4FVpW1XIil_o=S{+XtIh7 zdgC>bG{*opeGVag`~{@b4frtkab(ST1lS)#PnH2{LUiy^eg1*aj; zW}D1phv_`WI{%Hh8?+ME@%nh{aU9zu&O<8u0kn&r;r9tE9(#I@N$t;IzU2Xi6iu*L zZ7rVm&BCy?GWa1{L1ue0DDZteS(rr8?9~wzv*{fz@oOc09vl3!Tn$>WOEIv=0IyoE zp}W=^V|MwY=xs87^IFW!=^qfp>pZS+_<;7o|L`Q>14@J5@p#TV+_o=AwDt#d$X8(I ze+?*3Y{g%PKj@k;nl*I4%+8-VmUYefi={J~;k>yX;(yCmEWnz3Y<&#V)&G$xX6yivKn~=9hJmJzitY+ zECF?q5m?wT5v%?#z%y146;_5AdUgZP`s`6y7Jz9U9CloP2eDz5P)w+W)Sa(reqD<- zi#~(<_8C$^HJBSwhxLgKNE`hHM%~|d4!sv=r~Jkg4GA_uZwmWk?-lmlxT)-YO=;HY z*AVUneaC(AufW$jEELw`yk;E^?yW`cwg&vLZNN#RMtEPWMgPaoxOc4z$-ad!c1p$s zu|OCse}&G&4^X%E9NaUuAl;J1cm0`IU8R89Drt;+B?X0jvKTT_!uoNev7E<06ASjD zIrlP{g-;Q%p2wu@+;DZh4{m(&=6OC3e30X7!yT{Cd(#ra9k(!AiMM0%yRey@h-Z)I zv%ah%`(8Q`>HBQalJx2_!o2qfVVbcQa`rp%bkr`WjX#U^AI)(3wHfxleg?~bwxAdVnJ6UJ}Eb1!``p>u=y)OH`YUcelw(Vzv9C37Ifa| zhIQruRDGq{{@k(bWc{t|Mm1%2=`(S5Rm&i1JG*hkq6LzoO)%xxdF{1t=+v&qvKb9L zcKnsEVZTB6su8JUzanEz9U_*OXvh&29k&bZM@f?F%_o2~liq1P{ zFy_HtoX*&VF|YT)$6zN0mK;Urj&mr~e}Wg^ZE#uF6&~N*uqVeK&a?esdo%>w=LF)& za!-t(;DTztmM;5bgQRITaCNamxG)i8Hz~3X?UJm8O(r&wJIq|1u$`~p4F`O2BgP;5 zeg&cAUh&qj`Okm=%t_z7L6|53t(#7AEX8N8lE7 ztl9Ahha#;|b=(DYgZ*REc#z>zm~sP}3@R<5Ot2G?sAvzV z4i~JD3gp)^2>X|XAu}%md(K9p_(UXXRs~^R5T7UC=W%TnKiqHjg|oOHta;tVluQXW zrDGWTVRk;IDTHCe>QERu`(w`25Qtrgz`yf6hgz9{CGS)4!z%_mo+V<^KsaJ}Zu?1d z5cVGRM{5(0y_|f3DrH`0vCSH))2$#_@*Ly7IYEOzcQ1~5VL64sD<=vo-t#){TX`_9 zEkQ_04el@hg1KLs@#R$$s@OI(rnI7MR|~F0weg%(3l26kA?9fdTK2bM=b|pSZ|TR{ zk$qyqkb0NE7z7o5$K%9Nm`Ws#Gy%-R*Lp-Guhwiq)X#H2*mu-ZaY#WYc zwL^VeDCgI|3Cv{~)_36n?`(u$ISZ2jmm+ zG(8>;SSPe!x5JG`FR&oQ7Wam` zfob+b_ib+&ObLN=WjrLOrDOfpT&VWG!?(U#NZ5SFwx`V)3Tp$~*bAeo4h+3%#e>{d zK8J6@!Q3lewZf~t5q%Sy@pV%>uJ7)|ZO=AL)&GVP%{E+k(SfFeO>kXOkI_-3II$uh zSwGUCpqz;OhY<+;5`cl9zUVsVj1dFwI8tJZ8l4x&?{&n*)|WV%?t<&8VHnuyi%Z(k zP`?y|Ql}J1&P>GF1wvla7KJjNOUvRl(Lgw=T>S9DC=eAL!6;eq8aqrx*}L|`SeZF* z5ET%KpzA@%(Fp=89f6Gpcy8`HuSMlGMVoHLxj zkJmamFi8?4@O0avHK?VqvCWYn4TV-uV}|4K?~C38sR?J0*};oRNJ@WgH|)lds>|Kj$5q!T2oC^N!OzpyT9-9#?xDTj+rH zdpyUW;e=y3o^TBg!I-dMWOs){^==%`QF6S^#zA#`B393fgNs8fB#onBZ559DDZyBA zITGg&L?Hk8TYQuhVWsYivQDLMFjpfQj>#d&xD)}W#Zh>;G#U?oB;%t}B7U}}!1upY z_v?P3`r&shH*dp{1MSe{ z{g|d}GXmyxVFs_;Ola-FP)-{(EW4ogs1c`ke#3>quMmV);MnM5JQ7KV%IjoYm=Obq z=@HP%@ejUGddXyKXO=H z$?LmjCt~PDGW`EVLT6qmQes2k@I3;DedA#HI}c})|Dd8`0NDm_F`tb>&Z9`oFb%_p zZQV&YdWo z-;N}y4r~!^g>xV;eRb_9OJ{09uZVvcshm*@1c$w!lf2HCrvl z-qQSqzZ1V>g5-CYA8beNloos!X@~Wpukb(73C!x?Ym!!UcXnWuWIKLlx8PsbSEMx7 zqv1ENc~Hv7sTmo#7n*=$<70UpXAsnbgW!D54P$v8`N$b(=&9e(`a^+!!naM=`rx3)ZXH6j{i2NL1U$DR&h3cTe~@jHOGdw%T`H%4K- zX9y-7=l5UJ9TJ=~F4Z|vqJ#wMK~5;w2d9 zFGEJ{M@VmKf$_8^^lGsz}ax%>wv zyc)y_PJ|t?bvXNKf(kq7unaqShZt);V-P1-_X9uL(DAw*4vX7iWY-FV9qsV*>q6Rn zK9(Hq!F>4+y!LI!d(jT4QX>omwU{=q45zB{VH2DRk4Xuz3yVPi*)Wt<^7@kN9%u-0 zfu4>l#$R&8gfJH<2wd>-KQ})A2td?82)v`C5wSl8idzI|^x?I0#VNR}n*i@6Nyy{< z$%yw+5XMB{pbNhr&4~!?kcp%9``~=?6=AA>oM_gk{A*KPn7z?XfU-OGbNt zGCu1Fu=jil96xZ_y*vTBJl54bBOLN4{7@U=g_+9EST&dDna;h0qOBcPraHqk#u>@c zemFW92*ojBSi|?7$@Ra+MXzGGeJO`)Z#^aiG@|QMJ3j?I*#U$2z+X%LZ?*lD09mM$E9rzs9 zfwL)X5D425-PDB8hmFvG--5=!?bv>^9qij?6!Lp2-q?nvclmvdBX;JvVQq#Zba?DJG0Yi$YrN1sE*R~@BQfi846HUK zVtBd`J$}5MUP;DmE584tp4U9jNW%0pG5B~k3aTb?7?>hJVO}x*8!f`lNET&Z{&SRJ=?pfr-^8eCzmvrsxhF zx9W!O?lz>(ZpZM@4(#Clj>V=9Gz7F`+}2L)@M?yB#SaX;>xNoJKm2Bkuqm4**zR4j z?D@m0tm+vV_FO9;w-)`uuG2qYo!5=Ij_o)W*pBwid`z`%120Ix)mPnk`lAa`UM=`N zrW-x{Gwb1i-qmxA=2u_)nqpLbS((T6#F;d>Ak4`o1;|6j8y z9rquk;GtRyUO4mpOEV)eIL{yETRpI#-Wg=!j3=#bn19s;se@iH*bsmrnP5J5jKvSW zHW=TKjxSsZ%-?*##I{B}&TPi7ab4&f{v9`}+AyHqjfwla5IwdBYx;Y!#j2gRk01CM z)qy9Edk`4a&-XEiut|2}?4?Dr?1>*!*~#tltnOI}HnIOFQauLw7~7AbDL?Q{p$ilD zwnF()2Z{|kpc&YO7{&K^|1#Xbyd7a^01hiY?@TDsbmkJXx(mxHW@1?@p zEgOawshDb(f~c2?P*#b-)5bVRUFCKEpHgwKxeSM^hq3v$hq0PY?=f$o5HfYicwZk6 zQ)8Zk<9lr!>@yH^Egcs=XJPC5OsHcLt+;WPAM}HV(`S_e$<%?%3K0I&ZfxmTb zNE`G;agj4Zm-|5OSSZ$)g`u~AKgV4{Bp77DKK~tNJ6B@mmsX6D>4ITkCpNF{Lf@nx znE0v-sUNyv_w)zKOMB40>O1o5dvSf|PsA(#!0oQz(6JF^duoQWcWx-LyMonOiDOD^ z=wS)g{k<5gHbR8`71)PSWqk-r>Vm?yF3djv9RohSIL!1Sb9py*@MlRjU;yX5dtk|Q z;1iF}oqkpW{q-f7Y?KSfQ3A;FI@-p}5Ukr1f~n=+h~Ddl?_YgjTjPxFuYDkM(;Hv8 zAXxEzbN3SBcpM@LgHZ`sdnOAR=9#>o$>4FlG(6$$-u8DqocTVQTKOc{CdH#8NB~E- za+tV^up%PjZ0Nv8nA@hqb_R!yQhW{dDg`lbQ{nWJ$8ei-VAK8@Hs$&Fdm#_ZmP`~# zWMH&Q3a+`tV1;rp#1nk5{2zY?S9svjDxQzK>xp}=0eIgKgMUl-9;ZPLn@?tA$f6iA z|0-ZL^Bb=8bwlu|5Bn+xp`ST~hs%GWaQ*-UH~R5xaUV>7^5TJ>=utk zM`aN2yFKvbjSo8bGdY6qU z5(0{WB7&lV-O8DJ+1-Vmuif4Hs9>O!BHi8Ha1I>;(h7(mh|)-kiNLqMUp8E>efFA} zyXI~RwAC`<<&lgy-y}?&pNKy77_1nG#MD(Wu=_6od58EsF;a@nEgHl|uBk`B&`b=< zN=3%h6zn~diAS?DQ1UMqw}*4gTkAc7(~2=(u9$OsneeN7i(k{z5cVtyYJH(tt`i8; zAaCv^@xj$%KLi$e;>g?}D5^%_>SG}^H)V03KmpE2e?WNdC(dcM<4eN;p6c~s*`MDS zHd2z^cl(8s4`M7Y<2Nj)iLr6ce_>M8hsp*0*r?Qx$^S_*-6CnG(In4K4;#WJwTxob z`}Nt+AEQ}e$Y3U1tjOkBr|K?p^GR!A#y8l?+SAlGRx_@-82<=jTH&JP(sc=5p^sHpE*qasP5Els_aQ zKQ{?(mN^i-uEMuQSr(%|j2$%WK+C#fOtN^3BDn&X-^qtCz7$)|Rbr;^M@%%S;~L~D z_}!|(|CWEiqrwuDU3-ga?)!7Nlz?8(D7ZX}#@$o=9o!ZU;k*b~Bnxr1CJ7c{X;|xC z1lNa^coNe9wN>4yIxfy8?UZHKS7n%Ay$o|6BgZpM+2ILdtiq68RxO^29j4&*--BgC~7 zg9EFuL4kX}4t&H^er|V6e~;`X`B1o> z1%>?5Xr9i!;zbzxsuU08Y7joY1qNY1(c&S_J`Yu7Nq>g3u|1=ij?HNH z_<ZqX%o&fc}IOF-!+k~jn!Zpn?|tHdsLZcz5>JK!Ax68hFS7{=$)$~+qP1X&6AdA zUd77ntd%ND=^M(l#|&emCuuUZmm}DgH%jb>mLv<9@e8weH{in5YV?>@VDidBgr=wC z^zl^89hQm#p4p(yIhRGz*{JM(kMD~sG57peR1R&1WlI|>_jbWrxf6GKzTxs=j%V!n zg6fS`m>pSxpQ}DYm!GqtlsMB}tIcA+FJo>J>MXUkpX(g^aq{0^)Y*x%^pBEkMBX4K zaX^V}8l=KbyQnbt(?i*^fgwzN<`C9qslY~UlVuq*B-n_+pBP>C6Yq}=z-+b@E74M5 zW*Wm-tfvOM@nbyG8#13Mk6z7UGj_59%j0b0kt^(v)-5(-_7>)=#8~mhxlBDN z#460S*ruql>`0X+`?6;ovy<0m!%mN9CN8=xanlSIDnqR8&IZ=B=^QJp3}O$jo?}&g ztJ%ey3t5ldbmpEjk&S(;&DNS~GUqLt%xCmiCU2?DRQ1NOu~lQ4ZHpee@o)~i-@A}) zlG?~N=H6uUZ7#ExlPB5T@EvT_@Kwxt&|LP|S%)#R(d_xPAQ` z)O7y9+t>Z*y(Pt(k14S0hm~2%Mio~3mEVQ4RoKX}%53jTdDeMGmTlu$Q}5tDEP4MM zhF|_-)IBj~QLe_WH7;R#o5!)Z9x;~0XS{RTm2k2A1Yf;R5PYfUxK0Zqdz){}#f0e&W(-he0WO;VYScbhDuFMK{sIb0j6}B&QB+IDRVB))mG3QYn zAKE6xcIf;=b#gP}kt;~ z+li*FrC7hV2&wl9Fqm^nK1H14Ec$?LrPc6=tV8F`uP_qUU~$O@%s>1dYj{mk)l5ft zd=d)N<4`0WjnI|h7^xQpAC8@V`IvzF;>lR>HVp#dTPVhUMtWvFj!o@B#c(lZBQc0c z>&dYTE-eGE-d6pX<~?EK5!euf0y(zulq9QL!|P3@EYm+O#y^MOvC^~-BHIe=omT*ZwORNRkc#(wCYpXQ z9JP-lpy3>Z*a^XSygmZHe0J*kn2DCnc{pcP4726$;J5w*X3BoViak|)<|)HtR)O(r zi=nf%5Fcva!pf)+6Vi(?_G}9@#D=mdpzV06Jzch zB$$J$I9n6n2P3zBcj<`Gd~6V$3|KANLOp;NsH(&JTCu z`SC9ps#=0lt333cN=7xGZ!dWBnmRTRa<6@GVmF`91KrWZxzqesf4sRC3>}V(&Eqpr z$o@>UzJ3Fp!Ntf6$;GPEMff!}AK_YWVR|wfIeQa1Z<_%r%@q9T&BTknpHar=ab_#c z!pD^%lb@r-_(Z(7kHrt!F!=flG1(v)XSu#b-7g)QQ{Q0o>l{cfO~m|65o{V`@U=e- zPPUw9-r@m+S)7Yq!TH2-T>mKN2IWCMFwYCbTk$a7`^V$s#x&6490bmYkocQOwDas1|!AJ;zG zr=e6m4LgVM8Q^6Snl~kYUEnyy(Lf9t&V6NCoL^YQ^~2##nA**=QDSW&y5@nq`~C2J zW;i;>L;_DmXy5b(8(1-XMpWalSv^{!I`Mt>cdVY+g9&rHP?6Y)^y57klk*$)w|`+z z=1;6z`v>vFc@dXis2S0Zo%~K7q%OyvEmCFkA53B|t}3&ZIkHUij0BU4>BINK|8Vpp z$2+~+kZ`mehxNO#R-E@954tcbyB#Orv}5Y@ZphbmavY=`slJV9TJRYg&b-B^9~tm0 zPe55_I0D}JBYb}#8q(ci)$N6$&Rh?@+lk`}_Bhz-jHlzhA)e!p&pHuE_m4sF77^#U zl5mFKQ@W%1c`Z-EIU^CuZp0%%G7LAnLm~bn0m&ysaORl8sA4f*pCp*c|6@CwW8v)` z3pI{E*goQ!8>3?p@+k=yxi9|ch!kijaxCo|pADoqC&PPp6=5Lmz6^!iPL3(GIOEJQ z2iS->kG9JdJ4QRg>6a@$+IZs4iV!@z5e_OA;pc-in49KftI<119InHZ;#SCfYlVST zC+rq<;pspJGL<^G?yMWNMxETx*#>KlIh1|s!9T7Kc{KGm?ky2#?q*W#xVZwW71w5N zuN2u#4@p)%mCwB2f1+Ic2f|Kwa_+MoE8jI^%&it|EN;i_y{+8m`3=t29q_Jf!V`|A zwdgb>`~7G9IaCV0z4V69Z~S$JnM-2QBd!U zN9mek+~Qo1ROEKn`e`?*{ATc07l}7SB&Qqv4Gs5_h}fZgUXiYI%;6X)HEu z%|=UaE|Q|k5t&kltv8xcbAn^Wa^KM`*^UzjT5)Mo69N}@a-66gfn0NN!@Udjue*8A z@CWT{e?eCNH+FIiYMGBb3zZwooH>8>md|1p+k`+aISwbpk}!txJcP3MTz^v!x$3WwJ@*ZFm$ss+yc4s}ccA)D z3$|6ZpqOV>?2u~Z*c9h;4cahSstdZV-|Y{2N<2Pa8aeb20yG#Vw^an5ML0?9WEf@dk80V`HEBHgzRYuA-x zS86sIeEb1thYboN;_d*EM@?^}}5{8O$?vtAv)Wa&G`SBLbiul7#L2olablfqnd)OTY0I z6^FTI;9?Vg%Je`rpcOf{+fX9WhIgFv^Qvn_2N6pF#NF^Eq|guiw?<|!t?SuU1qlcSKuF;Q8CV0iFn zuzyz+G@7Dta&j?FsP|!K#6P&t$-|+ok*K&Bg63?0xcG%&QE)hZyytvCiI8W(#G;vF zky|$kahLbI?%tueP#*}Zhu&yw_QZ!x_NYH;j{+rI$f(%zp56wt`dtw<-Umrqe%Rs3 zpJlKR`(?NW%RUztSBjv&{ws$6sKh6?0^R6 z66fvuiLq5}2pRPg2TlIK)=H9H=RKOoM=dtrRi0@iNw6dF|8SVsiS7KEO%iLtA-z_7 zc-DyQAfpOYr3zeDHPz~oAv~#ZLJ;-Ztl@B8Of^jP)2%0Bj@ou*e z3fjCs^$dk?P9zrkhoI*|FjQQE&>IyAi$kF>kITcT#r=@z>4(c_o-Y{{jSHH=7_SzB z?c;)Y#%CBdi1DmS&lp&Q#v}560`~H{ZI>Q~t208t4g^B9-5bB$T+uPt4*mR{{Gern z{w`aLuH!z|O^yJ6Hcx;1;bn0+&xa7=%8_)GPtSqnuXlJ_R)@`P^{71Ef~%ZU`bO~BYaRx5@SaSa0Juqve*!8oS0$*mFZw>5wQnh|dB4Y}?OTu=E0 z&;PxHT5d5cnv+qvGY%fDp@=XIMq#5j1gl+8W@C%qpEig(YlWK4-TJ11g4hq5U34z%EDh!ElBjKBshp5d1 z*v0Rh7v)*_ANL1*<-B$i^T&$*K(t;Chl64adbDED{y~VvoOhJePQ3SbPt%Rhy0)0H%O20tULiTt4u5{zU@7m15Aa&N$AQ=49ijNXH5M*B(@Cx%pIf}j zu;O72#yzToYFRT}X1613X*-7US^NaosSR4(0=bQ?a2U?@f`8jktMU!UH+LiGN*5GM zd*S0I!F)a{vA~-;?C$NstZcF*(_Hr(o0f6x!>0{4r7f@%@5DJ#6O^8|pvSlcB|}>A zBb964oEs4JyoGa^jo3Y}0Y2qbaJXEEqZ{AC;$AAI^h9B3O&F?0zFfoS3N>#hO#5h! zuRISx$bZ~nY>S}@c7TEtO27J{m*<`&Ye(P=?^A~Vih)CUEc7pPzV=}(zFS8@=Wie; z@N5UyAV1vWe9LvkP+ZGOhup}1Brca=vlqQV%*hah3xkmKg#W$U0SMsrv3plI4k^VT zU{)N*;p1^FJ`o3|MB=SZ2!d|}z-fjDqMV%IGlpx1_FJR2@)gGHx5R<~EA;j{qxiWi z^wa|2d^`|`-J{`Gl8paU-eB(9G8E^1Mx0LruEaM$u!L))ezt&kE&R8!1!H1%C%ET{y6=f>s0xi zb!K)vYS%`?Ts;=s_+4}3csLY(1>;VK2TC0M5VI^8B3@4$^Ky`JTbw;zAwA)(YhdR%qFmlJXXs(G{7Y(6(3f3&h#_h=^xFB1LV$%lf z<2`<0L??#w&t}cjR(w`!hWxE|DEQaoY;r3eD78RyY$x{Ev_USohv$~|!C6h3J)f(_ za+2mS^W&;)*0TJdlc)RN*jF-HEQ8d?S zTfKy`x(%w#U*nXe7ydpA#H~86Cus~p!@O7o^GsB+4Y7D~CKArdeD39QkoXL)OWW(g zwHsbAc@ql%kPKLVm1HemQf&SD49L&&K%RvY%=x=?X0|I%b3bH+PawSKhr?}eED}#e z;NJcSn0E)j&O88aoF{Z2Z3UCgXDFCqiX%}^pp*Rsem{(0nEwbrDvcqcmw0!dd%N#( z4H4IQPKo0B(8VGwc%BOlw`yeU_>8?;yxxX4<8c?i3v`>ghN2N8tiHk0uo0%(4ajWb z??Z130{{07sVBQJFryb@Gi90K(xFW53$Yc#q3lzg412?Q#T}*XaM{s@*L-IB5>bx> z{9ehDZ-y|h9<$#zpQQP8srN^4O2}`2H|H<^|Q`E(lWQ^Oz3Tx=9Mr+9b{Pei_Ujb4|jyQcI}$yu?*= zGnD4N!rUXy_{XzmM(cay(NJ&HalL`Z2Tz`x<_wc_&S>_13HRql_;lkE1{4lM)$at- zZX7`5mE9bF-p8}iE+JszeaxKq2!rOCqVks&Bu=@aG$Rt8%A7w_%7f$MDwJ}3voxq4 zdsa7Kd~7|6FE-$G8b7zKycaiX!8pTKEK6v`fw*SGDs>`hSs$dOWZ0Bj>P+ObiD~T^ z$zJW{c%-l&9<#pV?6_t;=g%=!wG9iMns96DXN;ZNh*$i5y0yIl&-c_}>WxMiKdwWE zMFled7V->}cv#j1aNf!dt3TLdX}1Yf2cDp+>lQpGoWiS1yRk}VCj`&;K$ZXB9{FyE zXWjSv|RxSsWeA3yIY?_8n4{dC5!Em4+lfuXHt(41w9 zC?&4lspNRFN7sw8LhWgfbuzyvA6FDN7uJb|vTpLXI`T$uw&*4AsEtqqA zCR`H7quOjZ9y^NhjE!zedMpLcKYx8wuv;@iYAi9<8v%z)d^Of4D%i znfn(exZu^3*T`r##|8e*BnBH|@1E-@dS?trt!Oljp3c@@pUh6*e~mj+Rzf9oDc*cA zzni!h3F!w0vNmf z)!2xe|4__zP|0!CPgQpfPZZ7q8DFTvDh z>$soZ5@)S+S=Un)X8dC_j`zvq@dau8STPnC`T^CdHSn-KifI9-;G|}V@XdE{l)Uslp z`Sunbsrkr0P>LaHwdn5d!uBHyOyb*VcA-d#RV?j-iE=G0*S^QBmLdoqXJh^4Tr4}2 zit4Ua;v=-xU974$9yD&<= zjH*vJaQX5j7`6Tn7q6^^lh0ykKb!^yzrW=6N*V#LmSNq8!R-2`Jgf-(M;eKy^kr!# zt&SB(xv4h#m+0ft^fkyCxeG5EE}(eb5o`@Qg1jxuF~&$2Nda2O)DXjpQ7JUH+kxK2 z8B)^Zz2tZ7JWZ)OPf-&+Nx#aMhS=qitZWfEToT8Nc|$S%l0LTgZ$^3010*sJj(czo z=s*&5lQ=HjnvCo6sTlq?8PjBwpt(+jkz13Y7MzNul!;;CZ?NZ3Ir7(hMK9MithCv| zUS~)#jfr1yn!hs{d`8TEoDQR+WRA%RaqmJhzC06Q=LgOQ?Mugt=Seu_83+5>LWJ^N z2NA!Vd1lxX1lSs&MxY0id0M<*7sK7yF8V_$6tl*X?p@>?6Kt20j_q;!(6x+mFFmA_ z6R%U4iVJNW`knr_OAd=wW?=rU^;~;@3WZlrV7S9sxYliijnoPpojnK5Cno}%_#L2~ zLr+zV=|0Z~S$i=LQm0NMNq;+4Ict-UEvDs}?zDU6UuwNM3W|BtVCJzB1+J^`L2dDI?7utBjjebr(LvqJwQik*xlA5@ZG*0kbqqfVmr^|v~ao_W_ zUs>cjt(*3n$RPKI4wRP7$L-X!XxHI$dbS^eAB3PSE)pAh;=!`w5VJoSGUd^j#Iet@ zr3o-?isd=EF&NLeQn{IVd{!(+omU@%g=?9_s(&!s@&O}fzQxs~B)ET0#Oh8yqmPL} z5ZA9;Ka0d|-za$bh2rC$XoQ%@V#@YNysHU9lZOpHt~-lC!{%`vpE{;|S47I88v5~& zXYlzrlVsp461JSDA)|FEJA69rdj!((+em?PZ&RsuFbz_8OD7YhaQ>AxBDAK$XxIv5 zAKik}W*ZPNcqwGnr(r}j&-Ru2N{TOIsLbj##Vt-0w9nd0*B;$NP=70lDg29bK7B%e zj+zvSeKesFspa(ei#X2(9f?h0I&i$s5I1xY4i5n90yUtfpbWEsdUAG1CSUO&5?iy4 z8kbF@hgO5BuULx~DAx-5Yu1zPg@cs1=N9D~S&^|(7=60%p2q2mA!o+~-Ycv}$Zrc| zt@B34xImt-90@185IAoQg-?AblJorWlxuuI^*4BTx( zvTz|YP;JGtB{}H4!m;<)Ja4xtl4srp;lLB_?^?*`>FhAX#|OcU_k5Rff_PRJ*EqfR z!W#!yJj;E7{Iy$9dPWn=mx!ZZ>JweIk0fUePfFK*K$p*MA<;*D>WWjMf=k~7mZk23 z;9=_YbK3}V(fuF&@{J~=E1$`%R}q2v8p!zHOuQV#xHfVIhPqDXSDz2&o$C;e?MGm^v9hD zUr5JzAmoG}-tjqnP(Rl@^WR@k5sWuSc=m#GGHy+L&ovX1*xjS0XyTgPBVnAA_~VbZ zC|~Rn3&7Y0A6Rnyx~;|&qB8-wx+wse`T=+v>V?qVo=}OmL-mXM*k(Q-f$yZS=IdA5 zr4>OYem2z4?*(`B`BZaoDs8=}NrR3`Q9sX--g6>T@V^cP`grl5K+$(U*>u>F`_n99 zA3xBtTzQla8IAEJS_nzjfuMd}+uuD#2y&5X|TsirWx}&-M?r zxIUE9bg$7lr}-p`lBF|?>je**%>`?uS_GofSiw3{r={N~QFqZw`m_8Hm94)>*VVnL zc4R7Tlb68U7G1bWKfr&@FOhrG3HltX9oprEJm!rXn|*LIknaYtaf253f5=C8z)`~s zLM1G&Yghd%bs7>bnJO1aBOT5t)>4PZ&o(Q_?3F+h0$JxCzQ2NfnZg}T3)j0o7An}O2q#|57WzJM5l&VT2&Im1NN{H5 zf~ezX>1nw=ot{}nCEXIZTOo-hdk0}_yd2KD)ROuRA*Ig@A?say--eea&Dvfo_;BRC zz~shWfk1RkaKUH4;9`e|VEtt!x}iUgR1-8Q%;Y+${bx+4lf7u^;BqSR8-kxrS2!Q} z0`K^}`z^!?Cm*<=?>omuQym~~?u2SFC!Sa52D@xGd@*;!FgJHh@(V=Y;uy@CpNGBf z!Iib+F?h$spEdgeSF7C zwFwSL9E4Z$NL(x^qQ&pM$aJj*{S@q?-lMCi(@=qC&if^Ju=Tm%?m0ujj&*YdHKzac zi|cZDeru9om-s|_eP$0COmU<;OEM{cRToK4lfi~Y85DjM$D3W9Bvp_=|9kL^!j`U~ znZIiUt3GW?m~m~Z(0$ZvVXtD8a9&Z3@UUuvaE!uPq4Jf&_>nr}1no~3Q|ob4Dw-Ne zFYZ@RV0$+mROz8vJ9=o};%Ks5=}*hwJ*GnmOK6n80{!>fRAAIDup_c~N? za+^Gvw#=for}yY&vOUeP{YbK3d+4sscZxYtL+#2rBqtw8Gfv$iv#rOe%1tEDSz?)R z@a{+XFDVih?;9<8b4e)dJ#b67cc+|i^>1gvncpku)Z}MmDdkNKmY-?JjBj+J z<1bB`#Ct0FWI~f08T`CPYR#L;T=tJZeUgJfrstLZ{PRlj+e+@}k3F?haHaB^;Nqu` zf|(X=0;NqdHn0k!eY$mIT}b;G!Z#UzRQI0jJlCmGA#gYcGf+fHdNP&na; zw{5Rc^`AYoc3X1|m=)&bIHKaK0~)osFE7m++INj%^!GlBSMB6JWJ$dLFNrFwpVH22 z2dJ@m6Xg!=5%{ky6ReOvEwFffMqpTAp`ZTjas2WZatX1j-TGf<$qQ_a#E1?_kX!0~ z`lisrQqhVQ(WS{QD~GMCERed zSy(PULbUepd*Ste(}idD7$+3287b(h9ZkPCKA=+v?J1@`g-SM-QEFH#jcLuNmz_zZ z+~PnFPn;(MnPK!j)=RL}ZMMj>d zSwE4UD}i2Yzd)f^lSn&53`T;3klV|5jg*+dif0WxxM$71a5lK_!1KX(*`ac{HQLD< zH`{pLD9^kcRbqz>DG$iZ4Te?#&%t^23(61Uu)o?DSDieuZG{cywc26tI7fUrYmMA> zrWilwC0Zs~V?(Yb+;&+*Ue*Hod`>^9Wr#-!J29zP2HrhFn%rzi%e&T)phA~se@YS9 z>Vyc+%H0>(3l<6{ylB<8`#C8=;)+c|<=+GP6#-iXVYh1qDJ$pDW&LNgG{cgXhb7RS z|9B?4!h5=RE}v#J3d#Gm8)?10NqwD*=*06vK}2*!!uOP2!W`LR;e#PVMJ<*?MBX>j zgyA<<3;nNIBt*Lz3vv?HQ0*Om`YYu^fsfK@sP$j6@NcFUtc#vKC?##aTc$SKnjE)Z zBnL+++H*Ts;9=`0$h|f}@GZ<-|DuGiV1jpq;8RJyAh$z}jEWWLgvSv&Uwo35q?ps? zweRV|puu>3=O7fC@4!#m5>~%nXZ?`)4 ze`I)IlzcG0pH4)Y-Y*=wCc^m25JYn=mhmbl=29(S zXcg`j1f6ozcUpB*kUpkDpeP?FXh>3{1dUZR{GlDiw}sRA>tE@(NgJuoYNAH(21*pZ zC%cv2)ED-W&M(_c`kx2UL#?2Mm+$8Yt*;4%O%aNs`dOny<|ng+!^Y|h*S(cZxN$`! z2z_ZnLH$l-ES*X{4rNq5@Q>aqOCjac51z+cM-$UBY48Xqk|@1Q8`BokmZ5=y{F+=r z=(6jA$GwL7OENPAGk0eTXm^1?oqZMLoYEjY(Ifg;`iurQye5rhX_VAD23;m6@V@Xd z9$evmc~5H`J>i1w=C)XT$sU&v*<;au8|3RaV%1$|sJ(VU?M648ec*#>oYRkwOULe) zzwllv1>=^6^4uD4Y&gqzW!1YNG}I039nXOEaloA|wy@`ZuyvOm@G61d^|p2>y!#S6 zZyDlb+;L<$j)ZqvH9al1p@YRn^nLCX8h%=v{?7j=a9mv}SaeV%5dC~9&~7vmg#9l{ z|Gn1-!SN`0+Hrg#N!k2IW*kddGe3nI^Xo`M)JXv!ztKiM8)>{Nr5QZ`R@u~%#^ybw z&R5j}am9l8bS+(B&OoYguE`M5>}%si=QXp1uiSK8MDOi1MBfDiawCR6IWH`f;7FO>gJzoLI^y`y@dYnM=T!-Mtg95>d zOaCL!tFtL7!hpu}oU83RA#`iHh_Q+mZ9oHLz;Lqo3f9VXAP^O7A-pLc~hKV#`B zuGq~pEtm1SdghHA>T`I`G56a9^BF+g%>!?rbB$4X93D@5i|&|zaQyuSj{n8+-9w?6 z!1EvsZhAuA))THTT=C(PGxsAo;rnJ6WRyGMz&{Th8}5v0S+-oqVUD7*iySwdi^`{i z(6^$Tj5O28y4{<6rk|#Sv75*sVmg)8YSV(xy@G;f8^LyQbHVEVCV}K=ZE;rN(hf=j&H7p-iPn z_~g8{NGWQBsPaINFz;SK!b!V10{6M&>Cy^!k~dGHm_w};6fF*eroqtg9D-D#8YHay z=y%_5x=jW2Y(gB}y7!7`#(6TbxkJkuM$@zdlgYbgId$~jpz#Uk$jjD>zSb6z`MwGo zcC&{%r^!L@pdO4=H{qW7W3(7qAz8{5H>SGcn1ly*Z1?0^7gt=X_r(QyUubD_9e}q# z&w=v86hpqhAvqTHBe@sT;2i=Yq}a{lh3I^j%4d!kjGGjQm7KS}{K^Z_j{_iK;0kA< zJ1k87(OAj3Wo<8PxZ}=sBrfPUWs42bhM1_Z2~o54uxOPE;sXBA!NE18Q^-8$wQi%ag3^Dy(#aKC%wDxLt>igRG`;HpMEIeN`xY6XHCMPWt%W*-%TueYld&N zt~heS3!j2HN1DO8MV_m8O5l&^TyHoYF<)xk?eL78GLSiAp^D zt;jY=enw1ZE~cj?gPBBP6Zd5Qy%35Q!T`u055mYjp}6yuXQSl!!niI3!_WGmV}lRs zcRJz0;uko#^)g!Ctw58_6x5v>k4E!<6f~@dCU(Ztr3eq&w9JK!t*+7EQQJsv)DDUc zu%?sE4rJXOKn`7e*OgHx)jgGgCJpP-u|109+rads4+M@ zS_`#u)9_kq9*V5SL!o0BHYCWxMqM0#dh03QDud=YCD5c;4=S4PLhidG$TqQny5p*8 z)>|pORT+*)Z?y1ScOHUFSK{Nc<2bR?7&^R`zH@NLlcHdrnG=qbv61L$ih$LIP^>WE zI{pznU*lCQ0=gqG?`8~sbH35aAPYOL7ogxkBlqgVHfgB2d+(YTDve-6t6rP1> zAb;f)q+im7pOzM+hbyC9R*vUKHqncjvGm`?YgBu>P*6KdBSCo5SU6|caM8$dnxf~< zrb0VK>-fCuqv^2K3wl+bO@;cM^nTV5TwSFF(*`Y+Nle6T-TBDfJ`MA9C&7Kt2-rs| zVMXRw>iS(r!c}E7zBr$neq~ej;XKNf{Ylw}r4hVv5bo*d;FR|a{5D#P!~1uma?d45 z=kt7{7fzV_-3Nx&f!OlDP@csQh9>E7=)8|WNlzr~rty4HzUN14X)M%L6L3#18Q0Ed zLuqw66wANh@lR#udAI?A>q}twE)z)ylDTJ;dxs~-qh2Z!avNe0nHq*kTvz;fAPl{W zBJeJk&jwtp6*bKV_FL?6qs9UcCq976e_Oeyd?oA+CZhek8kX^lc=cOd6kqa{%tu$z zoda>SV|X;}8d^#lDsyOyV;fD_B@4e5W1u>13^q0M9apUjU@Jcxk&`uHDn1H(d6uzp zQy(P=zfgw!OS(T%g&xd*njmI&S7ri^3BWVhAsn1M? z=jV;VeoTSp@fp~(fT7~tJoLpAMypT8!1eL?yLt##H;7|b`EQyxO$rx|@fm<;1MXJo zrE;MJe0L0o{giRAvYd4N|b;fALt(eXS*2*TQZJQ6C!1_Z3FN_G27!wnsoVH3Bovhd`&0Yvux-5q!oJ zF^v~6DtjM}wk`qtFdYrQhQW8d5|r~4Q99%+mE|?ktEH9XZr4HA)PB(AYDwHORD_LS z1TI%ihQG{QJioON^{eJ^t;Q6-FLx3)N)5$-@lud{R79(^-6(a(FM*Ey`}lcbCc=p7 zA)?A~CDG>A7s4SsAH_XkBWP*44_&xjOD;$M(V|I1uvu0Uj}8(R7%oAx>LTQioQ_35 zbun>_4x~(!U|q&DPsfO1iE%TnU*Al9H{0o!RXv^VRD|Zlk(lc>0Sf~K&`Dnb+gqm* zsQMVs^2`uE+z}MvgV{cOM~rPOjP&?tb!;3KU5VwH4AFR!pMX71DKJ}^2qoQQxF1cy zz5H|>*Ura}jw+;>wPAwG5Y{`Z4aPioPB^y+H%rsuz&*v&G!sylkqB@1IGmM;g`0IO zWWU95{xKHeo>7Pp3i0M|D5f9s#(l1tzF=a8xv5vNe%EoV?OG36rMXx}V{kN18Mk?F zJ@a)feF~@{*I|W}yhsvP^Sa1Hf!Bq*sz@&#hJohs+-o)+{YwS-S+xLao8~}N$~lfz zYIxx+1x5cil=k#FwS{e=8>$Nh5@jcZs_RCI68uMsx|1FX%f}s0=(8R|MfYxz){A;d z*A)Z1G88vWN8{5NT}Zmm!p!W)-x-KVF%g*xUs+p$eAE zxlfFHGgNw0pp>7Cd;cY)Wd0Rd0TUJ811T|zYE(S-30V>z! zIh4yr;!n;voap47MB#MYESZ3F6DHvF$PuuI zy;Jl=mLn&M&Ue`fYw{`+ZvO2NT)KXnyo}<>C%&2XPaB~6RnqV_oPb~a3~fs$!%u1~ zmc@?2wn$}oZR{tv&mE-GT}=mri>Z2L5lsv(r2naYCS79*csonrX1^L#_Vnao?t#@@Z}}?B1UEfyK@_zIJ01a}9VX)7-$5Ah zvV$BKx6xrc5&cvCKmp53{4RL6`%uEmyhp;}8KXtY-ufcTRSCia=T(K$3seQO zQ|D6AV?&bt;zb6#OK9aJdCcHEfMek>?xPxkabMJMcJc_UyYP?dLizXC$l-a+A4%q1 z1_>Vd(yS79ac4p&FC`^_XVNXmvKf`GJ;O8GUGZFIKZ#3*uKHt~)5i&;KFyO-d zi3TkgUGM?wT;Dw5Z3^F4n9Oq=gz#OHfQZ;=z&Qq!hDX9=dKA=VaW9iE_lU@ZV~tb* zE^;q}L@v)HHMxt6zYpT2&N}Qh(SyN#6<}-!)r`)es=HD2M8=C$r$&&blQDI=$I;Ef z;k4Q>ors_Foxp7Eq7v%>if<{DD{(#>&J zS!C^&z-NtK+H&bL=|xu1n)G1$`P!Bq3B$>KfK5XMz#+Nbvtp)aQyx{jYAE^KIgV*>l7?_3N56+|qIR!M?w32omkjC$M zzp1BL0{0EtsAC(ieUq~()!2#t9GXP)116F2-mdt))0YZ+6l#PmlbFasXS*o#o}wr! z(^EK{V?9^0^7Q@H-U&_{?WUysKyuiXOj7C@bZ}Qa*&NNJo3Tk0cRQCVoqQ?$z7?Gv zI77qV9-z>$6TC)Gr{lpZNOXxvpzx6520x*7%bh7`a|Mmzv-ZL9Iw;t@5SwnD$JWYc zP-wG(N4+OxE4^?n#vg?m+#kesaBV!_>Pl$@zMlxk)$#~T-xmSe!N z#GC8YxNb9m5x=V8D4)-B!Bg?lJC6Ij!ZF!73hW`@MR_0;#&Kb&yB>shT!Z~>2j3rQ z?u&DrOKf)M`&0HEohVwBJdO@=sl*t&c9!sD(bX)!dsF7Y5Nk*=!P(R&xyQE%pA5rF$A7w7l>s`98lw zJ-0s#hS}YSPu{I0R2XV1e7HkG^zg+FQCa&H(H(tl(U!V2VciA3Q_|icA#eM4{kch1 zf~WP`WOLb&?igB=-6S8Ysvjk%vr3(sQtkOU0 zVJ_JH(_0WbF-GugQ?Vd5;Uv9Ie?$vS!zod(fl5BBOTzPM@F&trGahRUNbCfO>U}j+z;%@ze-rXvU zwttJ#B}q86F&?=-Q8?EVj+xUVFza~;u5$hR9qvmoOYy}EKF`jb$+gzQJ(0QH8C&o3 z-3Qn1<7Dg>j8ss^>=Rv7JS&Rw4&Ef=-0_^Js298{3Kdix&KJ~&j})ZmnsU7AieRi$ zlpt4Kn*Oh|YYmFx%EAMKFwCF=3g`kN8kGo$3Ifx88jNm&Fo=i|cY_LuyiqI@!PRAy zuxJ7SAAl$*@=yeWC_@mY&jn-=sI0U<<|hv8AmZbj5Y9`2<=Ow9h<+jZ;S(|yi; zRrRf|)8F~-KmFZUgBTgRrz?i_c#^;lq@QH1T~4rOdD*Ow|3UVC=xR2j$(8)AJuCTeFLh?&*lEVE z|1tB;K||K-vK{LYUc=m7Ud_CcjWX%AD;?XkOdY?!T(2DQD^Rv6o%r7NEdIg6r~LI4 zCtgrhS}xw0D`id&Fr5x@>{z*y6}*zbpM|bql}CLktqEYiB@~1JJ3uK*!XuDFR(;9BTDzn0<^nP9pg2|o9a1MNjl zz*%U4T1YuNv^#>0KW@r05qV70_|MA0v!48xbGwwsl2iG`!P)$#nvbp@e`CS>-m`Tm zS(wcjEKg&GliQihCSCR~edA2n$M3O!jhN56)y`raUKC231xAEN` zUrVcUs+k_QY}RF(8U+8t6~;Ekf^PLmFb&KFJ&I8-)GC5!KPX_xw-kyg50=Lp0#U=) z!LCdRol6lc6*P8LQU!*dH^H_<1&ZD-khE%WTB=^SF24(rbL%1cXeC6_Z^`~-9^99e z!%V#jkmi)ax6uli^)}&wb+pDPAHNHoM5?yjjA& z-__!{^~@o@*iV257nx$!{2rgeW7tgF&ggz!B>@C;A^syp7zB-Xtb@WN$ znpi0nAFfETx>oV>fVuM6J}orlDN;J)2=eEvaD|b8L>eq2|5_13?hK}pHRlK^)=wd~ z<@?A5XJ^t%eVbzP0*}Uihd+%MpiXWgs>P&YKxZ%p1)V_4kOZ_g`wF+GUP7A>uV7eK z1NM(sqR6iu?W0>Te^(Rc=pb5GC1Ti@ey9i-;q$iFmhZo{f#L>5VB@d>(uF7C$Ko<@ zcTs{kxenYuy$v;+>foBR5f1k>z$;5Co8JYE-bS#kybluj17I>#^gM!AkY#qjmo87? zl$U^OI%L3EZ61f9L;r!GsBUnyZGo&o6$}PYALsXtuzsKkN|xP$zWEK{|Kv73f4Cmf zp4CC@n|hE(-hfW?ayYiG1R|v;Am_0={FwieWg54zM-%jv%Q`hNe!dO1!+Fe~lZ4*M z4Hz`zCQ81j!Q8{&V%JX!T(GwgH+i4NcoP{`%Xi|6NBi(&Yah%|#$jyh1#FjG!N9Bc zF>7K7os;#5Kw?Sc!tJDJB#A6;EGLgzhRD?lC3$3cmS{)Ck!ho!5sSaMk@dPCko1!G z$tr6D^3YzF1hh^k7RLG{Mt3%8=`|uz%6Vk-Y!h-ebS{zSnUdVgv&brsS!CZuZ9*eH zvG@ELjE$LwseUS^yJvvaJQ4@4#T?O23T_Lyd)+m?+OaP9*hP`HEV837juH2(sCL?N`TR44RHfnKO^`~=< zK1(?xi=~`zr4HAueiw9l_R|>EE*7G%GWH3x@$d4BD1Lk!3u-jT=6oaabDl2A@X;Z2 za;B40%0H10b{LZpv6#HNFq?e0Tts?Jr;{%>=n-daLvm-`65`=*OA<8Jkq5!MNq=)J znX&LRX)7Y6arG18Dbo@e)Xa(3NTSSXG<(W~8UlexB>0~+Q{XJ{-y0AT8Wm_06}b1) z(8%2%tziY9{Qt>TJlT75R`6zZ+S_FIl&MV?Z~O0k_e05wEq3PjNh@eQxm+M%cQl9)XvsH>R`WWg@c{_O1nQc^!p*se5aw*aS`<&2 xrl!@D(mbK{TXo@7?bq_=l&pX3mkPDhnyj@>$#jTxJ*H_+-V@SmQvVm-{{p8)S@*i{d(E>|RFacZQIY$o@qgnyzv7avtrz$<4-EU2etyI06)0ytc|pjdt|*+TgZL)zQ;+wf6?MHIDAi zp3a+Hy<9z2J>>o$MZ8?MY}@MT=wq(xAwO)`GG$p4w#h1UUgzrZ|JSN&AGg>1{W_?k8Z;kc&K`LbYI}1 zwtxNqzUUF^p}xRFWAr>*WtI8w=UsX(`YmNV-`&QUl zDqe6^4i*O*j1g&oyjXZEnY_>GQog>2(9`>~Pe1c-}iBm9KJ3=1E}mTe z^`}`ULj)z0y=>@mBh3HHp|Q1q?aR{?=Zx>5bhC8YubL(#*RMhs^*c!Rs77BSFq0P= zqPW*m%v}GT7S;r?{_}Ns!x(FRAoej@3v@(}AZKxat%bPqqoOpNlght8 zQs#>eyNJW*cBOIt2he@jYb30?jIZa+&}-ot!7h9moh*HZBg>>r&)fqqhDq`EW*NjH zH~cWV$nNYg#>A$%d}FOKRI4wt>K&eVzqSkun@dss*&XK>HL$X>r|_>^L|JzdgxkG` zBC6LV3=&`A$?iy8&{d%)Ue03o|5Uigq%{6@W*JWu*K%f~SUT$cIPvH&2hr_;ju;j7 z2+879o?oxdXDW|ob#a}PDp-mSnl;2Wm-iCmd*yhWP>(l>%W?ivieQ|aPA-W*XsN!o zc=k;J^XW9ffc3qhb!H8Fw>wlAsWyt%zmKD##lfOsthun5s=9V~`qH_*2G zUaZwqfoo|G;nyA%W4XS*=)7g6cuWhT$*duyR;tO@=LGWBN*8|eq6-H3?G@iYIYlFV zquDNx!}yec2w!G>U`Lv3ggwfE=?v@>TT`lAEdM>VmcHIc%;kzv9T z9}TY35g}xL-YZ-kQO$zFUEsZa2G0JT%aSb{gb2k$7(X|VtbdjZL80&2JL3p6yC1~( z_1P?BMLeZ@28#+8+%Z$$h5uO^#0My7@ff|K)U=n2|AJSFI>U6u*4jLb_%f8={PYXS zqi3+}MR(|p#ZYm$vZ~0#6%X5>`V2yL zh&ihAl9`u6mDIFrmf#<6LKp1J#iriBDgMk>!O$y2I%lIDGbvNW-X|+@v~m$H6y9dX zXFsAl^P;Hx9!>fbv6NXa*^l*x^>FBui}#mJ+28v=X!3-f;xOM#+PU=r(+hruLDjlk zC;JSd^VG$a8|*|kH4Aa^uN`!}wh*Jj#`4%=Encc+W%Xlbv}oY{f%ab(g$J{{VE6lJ zxbou~leVS^mH)g*MQsDpURY4TNs>w=eQ{#A3%R&Yx_g-QGd39R3rx#myE6RjG@6>P|v8QHxtQzrbAY0A^N@OFMdN zi_*Z~B>!j;J3KobqgK|!AvXk5LI<;JEk=}_6hleMFDUhI4*Qrl31;7xAgs149{uL* z-tAK9tnupvHdjv6{gFs%gSWEJ4@=o{-(L7=;11`!qX_N253M(hv062oPT5YOo`0tc zA18cdQ7UJ#_wz$6|5k?His#wIOSi}?_Z5X*@}ihzJ=|6O4AqM|{9b-KGUmUc6@?>2 za~lKkcaZ~iHV5I|CN;i(@MAcZx(XB2BE^?Vf9XJzF$K71Gu71=Q2x;s6Dqn1I|`=I zra=x!oj!@)z3nEbjx|6_vnz5{buiQSgLK!C24TwXe^|e}Q8>Tkk@T~QHde+#UYE#^LwABkcagzI0RdA7xq{5g&B*6YNYM9aN3%1gXJ}lw z21Q>Oge8t6aeJOD-i?moKl4OPn39NF#ZxeGeh3vMR|)rTJ4&oE(y$CG zgiSw$md6eV%NfX;?|BPe6+QUiI#;1rxhdOZunHw-55siOWJqQokfwx5sHxHl|0Z;y z)(y*8xQRVl-v>j%=^0CC?nqjz>1z79GJvbRO{H8ZR((t_lgKCQ`%5N0i5Sk=cId)(*3rtf|n zFgk;TmEjl{IReX@0!aUwR2bwbl|&gc^m<-~9siyqe`7WD&8{wwseLHxhDQ##UD}69>DjH5e7sXiwDI0H1XSP z;hV{Sh&-KxTh-?gr>hOULobEKol$J(&u!E@_J#D<2ut)meGJWKw_x+)C2Wz;8KIkc zSH8|KO^~x&!KOd;fb+$}(4z6M8DT7ZKedkTUkhPLM_&l)*1BK=(xCl43;!LKV6Uo| zU~)c;+-4e!vlT{QnBik=YI~0-5BK1A-DzR{r34bgMe@#EMJnB&Vda=XJhFEW-m5qS zj|OB3lDs53D#@kj^Lfm9VL3*3d5i^yWr+B0g+3#vOEhDz3QuzP((Wg^aOfR|-baq1 z^~heZNwx?|GlIwcF=%;^EKIrZLP%>NCimqKn;G7~5?XpgvVSTD?lVTr%&{y=A(=K# zHK*qdn}wv7i|pi#L%6trpg1TJU`#$9-X8of@`QZ^n(Mp69WP@&I*95!xSGFqeL_=~#=+-{F- z>}8k>S@~b3MGr$LG<~=bw0AnDc0GexI|K0N#Voe)a+Q#9`VWkpuL-)kZp?ag0Q^rR zBGb_ZcUt-icPau2%F2Sg+6+PS)Nv@jRp2%!<#}{!68h_%Wq%!mgbUA(&;r{cScZJX z1B=&ao4p+?BwZv0J*Sh&y-+$)tVkvwKjOdmp}Z!eC;wOz293fv;p?`uRFZO;+|=Z) zcD5db$BSa@@(jiPCSuo9q68;rOFEU3LEUCdWo6I%VygQ@1pVrRL9Ug|x6_j)|4m~X zhsn{EHU}~Ev6g75)B_Yh%PkDbU~UX4?Gd@%Sy6b)W~jrMGeB-PUs ztUO<5A!VB`&p6YEUwcxIdpU})<99U;Vn5dI z5QZJ!Lvnj>)46|L#l+}Q;tk_$8c+LZ1NPJB_1UCc`;^qzb`^(*^$|}tI*UGimWXb1 zzmr=^6uQ$NQC!d+m ztq;O2$M2-0^og49h0~!8Gc9KnL?A}~5`GGy_-P+R{y%S#_Lbw5>a8QqIF^H8JsrL( zSC@~Un~iy;Wi+49rGMYe1UD-~Y->G^45MUZKOPG^Cy7+?NIP45HC!0xeTQkeM`L+q z4HjO>z^(Lt*r<|JYPh&3pVmIA)P2+dgw|)_)502T?0FKqobR*4CVPda{_>cS(N}t9 zhXtYrC*XsBBD5Eb#)KqIq5s=yv}<@Fh21U0iPc*CK!*-@>y?X9o7PA#TAZety&9?b zdN8T1-V4`;YP_mEj6IJBNrr6SPkl1plAYgg`nKbP5Rmka1)FN)@MTrz*s~ioCPfjI zS5ro%lGyAynY=DN5MnYqg9)$VgA?QLZ)n@&sbeY zQ##zpE9sgf{`3K8`<7!`p&wRQMAGuRFUjg%5t+?DEkwxe!|2X0Xh{ExtN%73b>~O2 zKa@aYk_S+qyix4l8;*n{lOQS_U`zd_LYG&g*@GRU=yvH(X=bH5Zo6*4U>ggJ^Nue& zb-h0+M_$LwkA3OTV!yJvI%Y_Bbi+y=9lVMWq{Z7u()Wv2?An`B!65V$tFxL0T~!Gj z-Y;jK3Yru*{WK+gdq*2@`oS>vJI-AFg=;DMaLD?cQ2M=)awoi?Wq%LSUA`S(msDcl zi6gjnNr|lq*hIhc>Zne+k>nz$2(PACAmHu-RE)pNrh2&ujjyJW=k}xYFT0VxwuTF% zyBbPfJsc{zYa1bWc=e|CZ{F0r>O3{nnTz{h-=xrACk6RhQz;IET^|2~`Me&7Shg6; zhu>tUhqw{?{1$XJ1qeQ0qFIBnJ$^UEVX5*Fm=n(fev-_}#X_Sp&GZfzXM|0|PXsyavvO9cRVsv{X z+HEGoVn#A!H^YVduMOGi>E=}6q9rIQ_CSTY2huN2!Tss-Z1Jpnf^BC9swFMLu|sk! zYS1)9&EAUIZR%)Aua!C;8cW$bt(f~+e_`>_RQBMqHG&_RVCMWz$u8H?)cQ7y`nt)9 z6J3C5Eq5^g<2@XhvksLPt_xw4OG)ihE7?14qgMS0e0x=o8=+}vX4UNS?%wphFot-a zB>LmJja|Qf2(5-^;BM-K9|@t%uIo18xwi^!QQk(oT`HK?84G-SHx7}#+t_@wrR>oV zE9v*TRN+G1ZJOb`K?rqPfr???m?pzGBUuXSUR}VJN5v>gK7hY#A_O`2F>KcFN9^o7 zN5uUof_B+Mco|)XLzp!jye3g&$!PkytzDY^X%X5dG{frEKez`Mp!D?s@;;bBUik;8 zC83WHr4R(2qBkh%`U)W@mf`mBht&3RKTUXjO<4b^g{>0z!n`vEnsFS18x&cCMhZ*F zeIdTmQ#|clGdQre&kyFed@aLQ)~ww+iFO=(F@K zuZ2MUd=@yMKikk_ChUEWVwqVGMlQX_9u6i!`{+K}cm5$&S=(bw-g6|}dX0^)dttMv zyATr^Mzk)A)-Tnfso6z%k}k)4>DO4INJRG0^iO zJopAQnU{hQsnSZ;t@_M=>OG*J#+3$gS#!FQ?YlXO4nbbbGAND z7~h3ku8xDrmlJGzW3;fPtxPcXJ`CZ+XEc0L;AcNP!gYt!SW>kDek1$f+!C;|>=N`! ze2OQYFOYfv0fyYWf}dS#aZddiB zEQZUXVoX!q0hM=tthWBK(EU*$%geQfv+HGqHGYA^q!ujs8Hf#?bD7ak5+<5#CHc7- z=vSn~8y|Jyj{X_Q{+P*V&MjfhtbU}k`Hf`I!e7|s-H)%RQQ>zz&cfkhA&XmXD+C@- z5C*O~h+w<-_!sd7QQhj%=6nPgA;XhF@7WZpv$9Y+jKELjkULk6oVp9xp_YmC=LLAM ztpfIwr=mxl2^s4Bl&t!z!M-W#v#3{*tjYKu(+bI9(-Uj9P)q6-MRwO*+9T}vt27S!1= zMmnft1x#mD!o#luQ$DP~{?b#_i%+GQcbx>iSB5C~myfLa$5_6j7}qWj#fi)Z(y3pT z(bCx6!uRE$+1CVbgl*c5o!@$(L)n8B;!W2Q|RN~4}&ms=5hJO{zt#7yXtHlCEkWHRr+91%8G;9FjeC)XYD{mdq5 zS=xVeL~2hBk#mJ{o^>$!E5o>>ZbN_LOeC(+5lqa@>Cr7$GBj#qlIlZH3yAO&OhR?y3-eHP9A34elJ5vze{NJ{RzJp zorwNb1gDp4A$NQ^Z~w3!;T|QpRP_}~>E98fRfQFC%P~wlhOPU&8~&Qfh^?)IuEs4W z_e;QE_JXx!mkGf)UFq1~VhVdW3jzI$acbsSj8S~aOz&$@LR$_M$=#vG@MC1C{E5l! zn29-864|x8eW|775@m;V6Yp1QiN|g_l2?~FVfOcQ;qa;n6#D8cRhc!@6YHtq$Rqp_&^J~T&6c$V`xH$IVsHx7e0JgEUeC+O0$2fh;J+u#kUrhDL-Nu z?P}l54mOTKXxMocy80wt`u2d*%d2SAP&3L4KEMuX$ncM5A`HyBA*9z+V*As{*kThc z9~CUHnk%gL2nOM@5%h1=v9{;OSRcQif@$2A}X%yacZyuUIY7dJ@Qu`6|gT3QE= z533QjG}*E@8?3N)_3 zNUSBpFnKY&p_}OMETKUivxWWT!Gf0IXmZo|POqh{_byZC$wQRw7AM@y|HEu1Y{Pv$ z9gIKjPD?uL2!9Ib!JCoPu;wI-`F9Xs+Y%5mND~2x&uPZ-Z2FuTNKv^}!oPVo?9duj zv^{^p_J*!uCYyg)4YnCiKC4a(bYm5p8?Oxe{&MiQiL`wBN0B7D9hffr{P;paV&Nx` z#h&W;-t5Zm)EyF{^YsOO%JIJ{CJn8k6#bUWO zo@AxKC;bTi#HXAzu`5%S~s!N|(y^qU~Sjsi9 z_ocSbpAnC|#w3hxo`hv`11UXYG}*pBE$m3V#g-b!pmN7GJlh-(&lEuzadSKAdO1?& z1~=jI=CPQ4M+)2GdK@|sgftn)_U@iD{nx2RZnq_pWtpSU%f(xUZ|t!xTm_HH4oZe^ zc_t`l4l6S>*2V{YFB#Xn9W~x2*rgcFW~oYeKdU6>U_Bg-aejzs--Qo8`j|GSi=^A` z$3n+)KS}gTWoTAyM1A{iY`tTG_qNw1|Msd-P{%R~p;J`U9)!*Q&A4w;0qwF^>3^!>jW!ro6H)gFJWyPyOU1dH=Kbc z|I;DIgI~Tz`uz?(O;X|+c|ABAt;yG|^M|I>0dy^lgw4_nJV|NA+jK?VtEmhB&*3^u z^iDDJ$!@Y3RlsNY7^I*7foeHDUQ+)Xh3$8k@&-HM_*Q*kw_*9VxAUwVh4au>iBK7o$Xm#hNzUgr{>Xf{o8HH2r7!X{{!@rNv_HrAD;< zeuil8I)qAdaDs<(G-V?rp$<9p0(~Rzpo@MiYVtHN((SO7s!}RDcrF%~wQs@w&`Sg! zB=|~}V(5Ec)-8Uv)G_aga53#Z4*%BSarsJox>p)LZ+*$mDfSVPlO_qlc5UqFTzTH_ zpdR-!|A`*WC-7K91#t(Lv*+zG?A-IK2ya&5 zi+__-;~_hgct`VR6e+#MaxUZ0W%`)sl&?_mkrlZzna_PVgxh`9;gMbS`1*T&dCOh~Byhu7!~Y#!$$8?NA&=Ob+QE(G&B0k^Z6Xj*?6?xP-J)yq!ox7-8qLNLm#JaK#2 zejHs?f;F;r>4Jw}aChYeEd70pg%_8wD>i-cGcz7x$7EP!TpwPlugvERHb&NokCLo2 zwUR1m;NA2bd^YdKtt$HP>*ik(*Va=w_-#8gTlt+m3l4#|aWiZVcIEXGmG~c(X52K& zW+&D;NF1Bm*<*(wm~<3F-@FE8Viu;h#9--#{d|n=NuX~I9M;$1e&15m$Opk~a0i=m zGfMK~TpjyQXhQb6t~}dRmAgD@LDq`>u-NyTjZF(?S8Eu^UkQfclmL(n3pD9H!KKyLkoqqb%hP0QuTlAM33-5y=2{GyY9{_ysUz;{ z&=qgD_7G1$eM4{RQ)t(6N18nIpWqV^Md>&9)8*<2`Y~b`?N#qf>t3q~TYJiV|a^gh%6B{YHz2y%o7yV5l!b0ga|$28zd7>{*YdO)0-}A+(8Q$h0)(_ z;naWi5xQXUR|v>iLVLW=QD2K!^l*cQsCY(K+&@HDJg#jf>a5x=zIfs+*16e>tl3Wd zaCMYekSZ^>cUF+sl2G!`swb=Rn{>tb4b7E2r1a03WH-t|R=)v-<*cSwqh`8&_z5j* zzCtzO{?zu|O4#@|lT|!?B{?-IhE65d(xT&1y76BC-B@QMHJdaY+MACc?|eUu9CC@~ zy983GlMRh;@DL`rJ!gJHLlCtu1?s!=5t?a&+MbhG?M7nndrU!}M`73@L}LBFWmsBR z%&t$A@C9j^EadNUBt42oK%p$pE}f32YcDh1!&}&zyBq)X7>^k+L?KSidFWL^O`D} zJfn;F@8Br0YtC%(o`$_>Qs^WqeD@R2`AI~%atkqDb&|M!qlH+qrLXw8@C8MbB~!^Q z8(LKoL1vAy^lhg|3MUK6dtU^N-_(bSr@a)m6_^n2ea}azMUy@GQwutrUCWi$PUC?o239B?r6L#y!<}jTU!EA>U zo4vL>E{yrdx|gqDBlkv0<>q(^M{B-GMy4)?l`-&O;%{~%HnYS$NsDBh8ZG}FO{S6e z$x*e9`8Ef_?S};4(|$<*>+qus)l$+`tfE)1<0(nAO!5pFY*FX?oYy5?{QDi-NJ0%Zvqi=P}RI9mL__xTWth&!KtH`zXQs0s1 z1afZ?l;@5V_V!xB{@PVBnD{nOD7|#HY);TQb~AbaMvOLxn)h866Mfb4-Cn66Dn;vX=Xc8Ft4Vo-%Zg|hVNHa zZ-@Cke`d5&o^~$TLGH@lq(UHrbAB<>V?y)>bXsWKj<+JF{YnMs|z zN?5Ol3)ofZcws|vix8%kE(mOhbe~hW<5hJaft$?B^34|x zy>7y%S37nJNwA(jo6dOsm433QmIOP!WyKkuGA=F)Cq|TDL&iz0e!o>{anWZhqPk+` z2w&Vfdk(AHiqV>N2In6J!`&;8cUia>YMYbL5mtiOD`|L>y#u9{J<%vkWVULHg~rx+ zjJhjD)#Y5wN!S7J_b-^j{A7umCrR6VrjXajH0)09#HopQu%(ADie}0DG8djomk+xl zHJs;PwryS!e1lqW#N`|s9*;-W%hOD&a}cb65cTmof<6qxo;!HTj#Ts&=$rt}bW ze#k+gUOt$wKm0>?V)}$V=rcF~+mGa<|3FQ?KhKr(w~2gW+)wTuSjK1c4&=-7$MVvg zZhYqLulVWx4%-$!!DfMGX?%9vkLOQ37ES0Ffu+L z!Nu1CLUt?h16loL`e`?QVAp>*QZNN}+a>6&AB;_YO%UpPa~~~Zu92(2I~4jzdbZEP z*3unt8gL%Zz1|^sVRx=BJ14yVfaC!uEMHrgpz%bE%r|idk2flD{S6v?v8fvOI^Ts` zv}W-s6T0x4obEgzPn}QyrpWytHbUb<9Q&8#6@$g<=ihx_r-VP9B&XChYb%))Hp9{#IW<2&XJ z<@S0b_^|mJe4X+Uc3Y;Ek7~bwpW=5&uITaB=SKY2lY!i)vKODzR>V~X=<`=Ujktlz zVD4bwi(|=Oq}I!PndQN-GmKA721mDE_@J{-Qxl?s`onXM*Gk>6@pcVBwH(;RIf?d8^{O2YozG?P3 zKG^mpe}8)!?_Ht9J=zu{W?;D0yOqJhB~>Sh$Be2%)-(a11SK>#c8A(EZ}`Nv;8dqN_tX50X~Une@>>R&|LG0eC>Y_Y zY6?c*RpL*L`tdPE|KR*PTN7^q^ylMl=#nqYCPOaj;CmNv*(}sp(be| zJQd?GeQPx$9a@q2tp>Y4pTsfq5We{IDO4=J2){1PNd9>VK09J@p>7_=@ZspwHH2+k z{R`0>d+<%qmH3uZ^_V>PFp7uU;pW?E2zeDKE&KfsvE%#k0cP@ih*=sIS;?^N;-{?m zmmz9i*0Jlmz9ZL0jc-_f54wN$U~Rbuyk@E6*3cQ)RkaSw8d9*|DF$ymRv_%4Ir=4S zhQv4rZ>z53l8h6NxG)ftX3Ud3H$Ei9rade>Y3qzlsZ|tJUI&w;^KC zgQMcy)sf;=wXLGb)xo0b@#AC_e2DT6m(q31cQk*Ayy)26N``ylsdtkPeepj_d|VlA zuzNt)BOlP11&Orf;5*@0eyt>CLN}qqwSgjzm(rl#59m+KMJiLXqb%$05-q*A?0&^5 zshXFem_Fhb9nii*{p!!jIM|_->yqfl<^oDMA4iQQakMGv2Ak`b&Gd#! zEv>_bkj=|SrSz{QfIj-5h8&t-?-*rHTG%ydcZwOBC4mF8zHy zSomx4PjdEGi%?j8nm(F8peId_snNcGs&^)nsxXi3&k3qFs;aqL$JOR?@6alcp!q#Fn>|mGGBB zZEw+N_Xs*OCsD{hkt+=BHlNy;ohADZxs=@zK|u~Wq>|Gj7`ofgf15Vbo49l;IF(0} z3moa?Zb5qEm;$={hp>OA6Uq2a0L{xfN%e~BX=qce^sdzmbWuEmI}X$EL1>^$Ci--@ zt1^A}3KE9-UuE{~0hoNe5E0j8{vDSfmU%+~aXMd^m8A)`hwYJhx+1V`NQf+U`ynJ{ zBEMrW0H)D<5Ir{XvLA8+rn`WFj?K?U9E zXi_&?7gA5}-%LdO;Y27sU5rahuSu^>bfM%makM!-fTp#`^zeqgY}${1Y}oXf(o@ck zre$vx0RZ zGwH_#8wJ{#{9ec`RS9zNc~krX+Bhn9@QBi#)$EzAdQ zDN8WVMhA(8H7xCGEj#X~E~t6rVx?9b7Vle)uzM%jx0?)n_Z|uCy4YtHqt4rOTwn? z#lnG?wg)ZPDEcPR zp7)5YpslErc`vVw>W5)h?z60{W5WKaomK6O#W)<8iFK=k;Wz1};Ba-Vu%fXy z{achHjA(zy($dpWSJQ%?(V_V3mqwi*HVRt*LWQ&;urxQ2L-J=IG)T`PFew-6QYA9G zS;%f#ykc!5jPY^F8e}{^fZ_S!C@9{9r$guPC$^qgy7V9-a}QwCu5~gW?@&BXe92sw zRx`t{e}s`M^D%vR1}-cQka5dDS^2lyRx>T93jGb7gv_yLDXsY+4vl#TJF6^cuKUKu zOgblA)96jHP2R$xLmqJr@r{PDAe9cnrG#hUxuxiXE3%#gg3-&{KJX`Z{ZV*xsBU z8*mBZHy%Vo+yso=YK0tcUl<#>A+p+^18#b~*q3&hf`+jG?nxojy zE%)Is<0y|EErF8bFsP<@vrobIn45*n^F5~oQ=3{*6jF=Eb9 zxb1P7S7v`68v1U>+%pW{{3gOSBMcc^s?pPP4ENal0k;x{V%x0O%;Z`b+woKmN6MX$ zy5>9{ERg4|jhejw(bKpjZ-d@HtfNxE&FkxN?4mY$zW2h%z_oE#Z z=RU*8?hQf)-$wMqBD^{s4|XaX%T|0BJ~#h`ilZXeHGGQoTKPDa;g3Q0oYAQkiZCZL ziIA_%d#xV9gYtB=VGXO=iRhXb53!pz|8ZD>e``yJi_#hl^#9J>u3l!b zH|3BtYbcbrzr$IB2z0lN6*LaC3X=z)6OP*S7S@*qvwLa!XxSZwhHWpQHGLF%{EB9y z^kUf$JB|SzeEtD8T=h4!*2;hEbnN#6Qicz2cKdiO!N zGbRE3a&EF~UB|KFsXg#-Of1ey?x9ul9^5K{ktYhF6F3)t_e{cq4=1qq>pMi`^yFru zEUu3cqwz9NrBNh0`)V5;P+Wvptws3aZGp)LWIDp} zCz$m+2Jh~7hnC?wrtW)-Wp#~3kF}=Q<&}l4Rj1 zORJP^ZhKI{@>(?Mk;5>YTdm7n^tkXS$xl#Tl~U%pH;7%;TFoBsIZYAT(d^Kk8G^l4 zr_ke$I_WjZQ*E_Wxch9Qpjvd0j4YK%&Ci#t71C%%QZ-3)n#pm%Z;Gh55_KD*#hSws zafaJpTK1xuT&GkL6`UcHt{#-5{##gR7fL-hSqtGsg7A4jr=X)ffTG+#3HwHj5z;~< z+1u@LWRU8|5`M)ny+ie7fu1V_pLlbj*u{ud)sDhjbv4rJRwdK#`#|ILSvGt|i{xPX z0!exIMAqMwK(%YOI@IXhnqn@6o6%ManK?1Tboe;-2u;U9&%#9LO`iz`{>#w5w} z<+AS!Gg%iOf8k#4PQiM{I5x<7JUr(FB%I1f{ zO!$tdM5^~cVSaKHdiOYtZr1zpSv&=OSx(;5P=S>N+4!@+QI==(@Y=l^7d>(?XY(oC zn0^S~I}YKlehmDdwhM7xFJsHfBCN>_gSCnoJQv<#la^d!{E;#|i;vT?C&qZ#doK=G zSfRpfG#kIjL+EX`RM0uRl9ecgkWc*)Ns?I=ThUEdGTiuy&=G7(iErD5&G`{*PqXa# zFFp~BpZE%UH@p(GqD|@Z?=h6NAXq3W55b^s!KB-^i84=3qEL;=^jT*Nb(?ix*th-$ z8?|UVKd*e4yT5nG`i0|J;;mxo`lD7t?AJ)?+4p0GUh~a`^YgwT@5WVDp|2o3Xc;2B z4bo(X>zmoQ#bep=VIe|r?h@R!St5NE7%R+Pf1>1*w+4zHyCW*`4KtjyfV4JWVjCWH zN*8TzDa)VO9f5<7LoP&$Bj$E!eqKyD^&gn4xyX(iG%@L)DOhO|kJQhVxc>Y$dQRR- zm6c%#lg-!rzV1c7-$^7-EG| zk>?Lc*LT0dW~`k_`=g2nQLj>n(ISzlf<=HL-o! z2Fz9Av-DZdYUvB#W)_h14YU7@5wfj%2ptZ`rDv5D1f@40rE~tHY=585mb`Ih>j%z6&aYHqk6RF& z{=161<4)jRsusF=d9v*h3g}i)g)) z5=lJP9$fLiC_3+OEWbaFTN#yIlvODWl7@=syiZA`opuT-_0`hW9#%?1g)%Y|QK69W zobw3DD%pevrKwa}X#MWrA1;4!8Rxq1&v}p6hZZUpftPK3u^Wye9Y2NxbF*x!|o*P5IV$9 zc09#}tTz06Schd*3+U#z)d(nkiwhCYU}pLNx09}8&5{_TG)1B6z+Jvm^#{t$I-zj0 z5qF|v@JnY0YR|7hvaC0Dc^=^9JB#pN$3Pks)`u?jO~$CJBQS2i5;8^xV(Bq^(Pom1 z|JKUU%~xx&Pq#(;wGk(UuW{mubpP5&x*=#gUe|JR*d&YOy;X+VBzu&N+zL-W zcUWy*K^OWYW54}aymYd{`+lL=T=f8z)*0C5yAL%dzhYFO1@;;A5WNyll-jmJ)j@@R zjW5Ra5Rfv>^Du0Xg=zQ*cu&p1--T-QL!%L$Tib-cg+do_X$V|oj|d&~(`Z8$oc_JW z^US}3=Osf8s$03+&70VLHw`L(CD5yS1`Cf**fOgFGp&m8dgxNRYH}&o^nZ(4)?K*r z&Y=^=Pip|2%`zrV|*R9|XBG8~7vZC-`PE zg331wKUXlr<8*UOx!xC++XQV=XD#VhQ3k0!Dm2Ew9aA58LD^LvHG`~>kop~-YDcjl zEDKvJYhY{}0ndfA;s0tfd?wsP+nrUku)jO9Yu(WAsy*6g55%~PD$+f)n#fMr1%pRX zbk_HabmZO}gk3n#s>3o_!g(21wnc|1L{1`2i_J-c&KP>2c`yuq-OKen-6eXoa}3;{ zg}_Mh5L)t8p(>pJN9Pw1<-Bv23WLT%C8iY4A~`x+B@GjIs^?CwT#D_ra|utaBtw5E zV%+P#^lE|uZF9JfM<0WDz%vhgUgV1RzlC0LHAj_x7d*}T(B`)ablO{K$zCvc*)bf; zs(}5A=U8*&E%ptr#ri2%5H)xuRguJE&+Z1uy1aqq+*cTStpa@>NpSsiAx;kY$3r(L z(>W@N)MUj^th`i!qA))k8|jK|j^0=;Xga+~dk-4GVr~SRohrjq zs~ebx*Iri9lEnUo)UY@E`_TL6uI2n^X~U}`;^ zFkX=ltMn&!Z-u^CdN-{2CG6OKoQ?RKO;nAN*~cTExdlxgm~JP`?aG6x)$s_-oL)Q93?N-BLiReGB4sfAR!k1`cM{f6X4QZ(^_B7KoBOO4+B zMErvQI$)X#H5KwbnPpu#JjaCk-C08inAy|wUuM!5qwMHwzb+{Cyv3>yB3j^khECaj zh4wvphEB+vNZTG;2xnM^4lNl$4>owx*-1BOzIP_Is_~}YHJ4!HN2%G^>9oOlA&oox zAN{4bo6eAIqN^U8(camYB!j#j;#Ho|WBF=9BepD{H`gwq6UHo~|4AL8Vef`c>-9WJ z*Uqw|hh{IOOO7q1es4tduc-n3QeT4-or{t*zlrpHq6_sNzK^DMyU=-^O9ano8jTk; z&4Ni0c`qd`NpX-Rn${5IF}Ut9-~O$KC&oqVWJUxs*U%{Q%MIw5G4qYwy zS{o*wfyNg#(eX3kY~#Uu#GuQSJ)Il|`I=t*JCKRHf5M5$qd!c>>K?Nn?1}|Sr^&gQ z#z<I>A!)1NlP$zmSi3ytZE$s4S7)cgw*wq zM>g$COU#DS^vwH6kPqR{Ke%JA;dU53@WC^iJV=gx!jepR>bA2lo%Z-I@2?}qrilz1 zJ1fz7_Z4<_zC}ZBBT{3ppy=*$x~F_47(Z(xYj?VmFG6qd zrF^F4)3glX|J8@;$Cu&SL{q$Q)e%W=Sxww_~~x)^aS@JaJ45w2d<*qILc8s0RoroybyPsl>d)M$Zzc ztX@tPZ63j^s}mzj-yvjt8v-{M;C6HdX0>GCdh=uMOXaAzzZ|`^;60A6%*BewXOXq? z82^n<~zPgupp-4vAl1froi2bd~yBfYCK_RO-DpJy%eG`%<3VUC;{49<9LRz-`zuI~ElKsv&!^1O2{! zhidPAo?v_#v!{kZV{-~F^v}ocUr%s7_cdnHdTbcHgwn$eIKSjICiZW^uKrc%ElS0L z=a=ESHWp8gb3U-;JKm^$$HfOVh$xDO?!;XfX|e);p1b4g$^HD_++qauYSZUW-eTM6h5wUcsn@|zjPBtrSozyH(bbve6PiYA4k#Fp$^sTmH5=qgU_XIfKe{~tgMDw zdm_&DY{&N%TM_WQ1Xq?g(zxa*JZigyhS&>uo3k0^Wj1h774A{?S+LQ%M%!1%(ZzN- zNbjDH6FZeLbISl&-!(^G3qhhfa7^eI?7I62TRvxES~WPCr(5_ z?Pw&gYnqJ0&jWB)cm~Fkae`uIpGl#D8b4rC$P<BJl=N1^N^G_#BO?w_T3p6)GYK7 z^Nlg~=|KFmoq~Z*QGDaxi+J4DjnAu_Fy_%=1SN^dr0g@~dV~$MGQ`a3m(X8WScXm2 zmoYAIFn%N)BwpL4P`&0p&N+3F-}!F1+HnzQzPVuXEEW9ir;S(Vf^cyHp_vx5;Q5^3 zsS{vzQXlIE41{>h5`?YHfzyBSG_5C)`XAbZdY`Fevu~MLzC?;WbG#&0Y#PXNMvi6f zzTa^+vXV@jUn}0VUW$!3QzFh^TgYd(Wh6uBzWx|!2Qq1^c${(!`#kbN&XB@^crt$< z{(Je8uw@H*o?$&XoODY3DXuYhQCmMO%e#cv<#{-fzEsd=@9>v#-$}hm4nh4JGWz69 z{M3wq%=c257&TzcmtA~m!dbi=<%PuSei*}}@%(xkCW-R^;a#_F>by~e*O4H;{V*{Cf2Kw z)bB1{R!3m{I~C~2rspoIOB2`FloL^d4UaVaK_;c{g~f0^OmiI}ns_se<=nFq*SI?1 z`Tjj@#UTe`b*q8gj?^cUV%69TpNHaE#X;{BO?jo&oBb-*5;`M{U!R(fO5$RIYmYXT3x zKZz$E`ocB`c(E-Dp0XPTtNDdWUvjsxlZ{py%}vHNvF}YQM1vKI$=RJjBAGot{9ymR zWMt4HwoB;%YdT{>0^SHLx_lp8Z5e^d*-^Z~QxWUT?~$a7m85On7|cuvKym&Z=uavF z{r8%^zkLaZCZ5AwwP3tZk41@84jh{EP(m`0)KD#CnNzXM=^lix7;gDRqaghp0$MMi zH$u1r@J?3WQ-t5L>6miG6AMq7I+WJuuzneWplY(Ty$car>M!!pfS^kZLgd2lqB_rFN?Oo$}x{Jh-7_DfkpjwuW3 zm`G;dn}7>igZRbH0@58l2oIu@N&c@3qLeKQ1f^MowA^~M+2`Xh>V>4#x~cp2Y$ z(SyhK-^DAPW^&`@8vL@?cedBalZBi;iEY~si*4d^*opEBOx;46_-+`A7l&q}UELVz zGr}OXtddmMttDpPlwmEL)56N3IBnlP|Kil)XXQ~g{SY8$YYN!Q*-fbU~y2;Qar&~NabDNmPa zE7Sb{`clQw=5&yAUz!%9ObbI5sMP*$c;0vg)elYZ-du`ngXXip8S*rR3(O~FX__>! z9^EZD7`QA8W+}CJGq06=cq>gO1#8gAoX-gBh(&Y7LNqz8L6hJqR+&zP)j@%~5!i|k z6=66LsgAB8_QZE(4N*wG@a4ytjRcW-qVXAIcrf$!&NOWN)Obi7t-Z2qh-&JE= zsL+`@sY%Z_D$}x4$t-py!=c!642`eH^uFKmGp-Mf2vedP_V%S-I$Nn~kOKWUM~Oam zm8Q`$-*MBu8N2sA5?BT4=va7%CHGUIqra%r87pPzJ^KdS+LV9`W|tA%d<%;%=90|m z%G7AM3C)OBq4l@3@Y;7djAMa<57*JRz!OFb6=?lPRaz@Bjtp2`DC$(OQ>%$HDC%u>Wu!g3Rv-9KA@)m?Yexesr^Y zbEWChHF7k`={+9SrozTf(9nz=q4&WD2h7_n)0ZmH&fQ~#^Po!OpWnvSog* zPowm%GXi21>4BZPuxaxb~W~WdyH^hb^f2{SXhm>#SDKhY)KLH{rMa-f84{+*%GWeznT7c zFG0hc0?7JF5dSD1PyhKqWo}56fmmL(%}u%Yi4-^Z5b6J#G2_rbC_K%=l!Ienv2hbQ z^SP14$wuHwQVx!IG-HAFJru`zqsVqN1RNW7?+n8$Sw%W0Sc0CiErydU_9Q zp_=^k&Nz1KR~NarHwC>mt+;!`4=S?~`8|CLw&ai#Q+~Wolt~nzeb)o2!3kKAbqo6h zUA1h=EAsx_b7GinijD{u6bX7E5BNj;gu;4Wt7?F}zsmU^zR6ysm-7Nrzav6=;LY}$`UY?4wf&vp zbgqd^c~c{zrskxtp_ZTnq(VIZG7O(iMU2o>O~x7Gc=R{fo8^n`f){W(W+q*$ph4@D z&f|bfG)bu~VvbIu_|+MGc#6vb_H11mxzMMa*e#yQs@k>qkBkZYLcdZr<QH^=xNO-25}$cvQ!UWtQ=^4J~b#rK!maQiAxe$`qe#xCwOF1Mg}IhlXbT_3Z1(07wy})#s~Lo_ zh&7m7;(>sc9q4{B7TsH|=+}mcnCi0wr!IKl^(5g;3=leW8n=kiy$8f+hAH#8oPmOW zk+A#mA52Ci5=(Vw_9E^LQ+XlH-QG9wK3{FnC6$VCi;m&P`f$;uffM=F)>FLd!91=O z?ae!ydPtS`5^NpnMnWGe@fr_LzDQ{&KYXW>{g<$W_0^w8woF?j3N&qDlQvuM8^QX# zB7(4>m_ibg6i>EaCyY^Lu3s>gTdF-|(&i!LezGt}FH@z@e=Q*$1C)5)nMmH4^O9?^PMV!}IU6A2t?zwA>DEp?!_l z$lvB}F^+uZpgZFDGs>_GKS>6^h~P$lJb1j2#opJ)nSa)~D*l|k07DKXVt~>dc%(k$ zCPUSEbh;uh3o~QdE9*$Sx)+qp1z%#kKm`5jPloyS#Z-4Ge3B)|PCt#@MPYDoK7~_0 z`dH;KgEj{a74nOoI7=^~r_2M^9Rx?qd&!KyDuQ~xSko+SZk9K;(_*(^G$+m3W zHdP+3slp>|s(ANQiaTL3aPyvz-fefpNw1xSSv-u-z2nV;|Hg9VIv-MR^qcHaJs_TC zvxzH(-sGc>rg1)gD__2E7I&$uVrhPZ_#LMJeoZZ1$i>I;Z97-V#&vLF7{}*%Y4X;W=477UTuF7=X31-JABifxDCsLcCz(2OpF~n&B&qR^ z;1d>H;29Tkx$)l5+^Sqgq95{v>raj3-f@Sx{GxcS+Q#@b`8K}#MXS&yyv)B1eZq#! z98TOLq}U>zCV|~h#D5-o!o@~+xtr5se)nOjNJH~8IkhNNtQDp&$&G!$J(pJUecp-u zmzN%&Bj+l57bziE@3)Yb+ELv3R21)N67uxV!ujbLNBNFx>-j#r0G@HMf={uDgrQC; zxAh6&wWfAlIb}aD{CJuF7?r`Z`$zHoQxV)NznNTkd4p)4^0X|UsK*oMT;=`DE4jt3 zDlWD)kl1XIh?kqRli-l)qF*By^U_cGT;s=6zV3D*-?6h+;D4yIfZ2v5ub`7ToR8xp z*R}GRyytvreJo6hz}{VG`Vq+P5n@788~w|%on7? zyS@s~kKaJD(PnhbHAkB1f2dsCpH#&KC0&eE_CW5tW)vEz(Dpup z)@1XQeEV#O8-?G(%~2dy_j!moY(j{#uN96^7-nixI2-Sm292faS+1(9u40Ud?5k>M6&P15eN{ zDa9tUaHx-1i6uh*XXmXWWY>YeaQdW7^V0j!aPkP%Lca64)(j!jHw%rkn#DK$WvP{d zK8>>xdMP)P1pjjsQdfT_+Si6)&YM=E)9?kgG5u*;WF!7P_JIALzWDl02CtQ-Vw%8R z4|p4gJ$Hq?-T9?>6+Z@ty|+QxEfto{6^P8gh1F+u;XndJL#q4)E=FALGjoC0=PpZc zS*#JfrI)m(Cz0yvZ=mrC18BmICLt%$g#8^&(AfJEZ-?tqjr9X*dCeDusW;$GK`Wwi zq-a=$99aR>uiEPtAkH zzkg^QI+&U-H==%*hS5GdbSbOPqE2;#>Brn*R5oiQUEel{)=ZP4>(lbV8a>b)8A$p( z?N9r=j-t!>XgYhpDm^E83h$c&S3NHdpKrV;Hm42fc}nQH&!eb**%xfck3$1H3CrvY zxM+D4R?BqhjUqETMMIr7mDVHUUL2~^qwsKMCcc;#!FGl$b>AXQ57o7z(61Uv3g7Uc zRf8sc(4j|We8cBYO-S{uM&i?_m^DI6@V}g??#@J7>(ws&4SQ&W&@=R%;zj!yPNM3A zmFO(RZp=Q{hRoD9boOe}IqNj2UcX)xm^{Ojsc%qe)`wPG%F-PR?!k>k;Pkn(=hr(U~>!lwDscDl}OJ zG2MkV@2_LO+*?!@s?rgy%Cur%2WkdB%`v!R0W<4JjM8ex>jXI}F;SsnGv-ILe#k=%m0Vfy)^oB+->{X`Af4;-58#mCi z@u-jsor`CN?Un=MWayQ=5p;RX06OVO6*jy+fl**?V_iE|4rqk$(;qnEqE0KVHRx@Hr_izc1e?+x zG;ZrlUzts&IZHgL{<$1lHMX0Msg0vyf}go)w+8)@o{G5b-Y6^o{&^|U@Yx!jS}j*CL4e)!HypUBc6D?B>^94)HIO zlzDKoFL~VJ4IksHLhiyC&ANH~@;0I0R4|!44Zq5yKJ6pqwiyl$S_J3R1*ly+nN6wO z!@3S%ww#=wN_u-nz;wkpte@0QV*5_FEC?P=pWEHFpnp5b36}{__cKInzdPhkUWw%{ zi`8s+q9t26#Rdc4EreayP>eLPC68P}n5*AJ{&d?0e(6sEKlyhIj&{Uj#MTqo_S=oL z8L9FO=HC4G{3ASI*dnf5q>IsQYw+t}CefMV%jDXpa_uX#c*G)IE`Q2fT%4UsF2rsp z5_?;wzNw9!?zqV`m)VLh+uhAI9x_1e-{He{ZdK)7=eF}FG9LVO$Y>t*M_a6yp@|K7 z(}fix77~Z=(tI?dh#ogi-uQ~4kfo z2yXA+*K+Ty_K!KCMu1N$;P zNW6Q32N~5ahhM7&HgCcaQam_;by&&Lp?i{8(|T8;Bc}wJygnGW+Mf*hd|j-%bp{K5 zGLoF!y9}lVf(CG?hq#KLS{l#R6CM6~nH+L7W(nKY^A|mt{PMLzel~VxwlAHk{&r#5@AIp+Xjdv0mA16=0_z%yEjOJ zJ~~S59G6Ja#p5MED`X{~NFl#=$eW*fUCs4oRB*e@*WCZe3*K@%l}C&1xy~XzUMv8s zpPD!D5$3Ht$>%=TPdLbpt!J`%o1@97`tc;lGJ?}O3HKeI$EUqH$a}2zi(k)OfYrgl zNZ&L9cYhV}J#G8>(9QFBzj|Nxb>LSL=W-fv9*CEMW$ZKv5fWK%qoTDpL75Y|COvdw< zr_rtPiF6knVH$DPe00en-q}>hr$r7#jzbW74d=sF^g$f?;|O0XSHv$pDB|`Zm$}*V zPom7kaB|h$iG6>wkDt0-&Smbl^VRE{d75}5Pfr}mwbQopy!xBGzDt-tSH9;}A8NRl zVOId%=H~`J zd_V-Z{N%~atzR>}(HZ28k2Bsrafj_83phFV=WZ;8=-PYBr|m$QTa8` zLiO7oW^X!)Uhcn>4f&#HDN0*LZf#2wZP<2$HRjrKhb|xPVy?oCq%7br@JJi;o$*N} zk=$GOl(oO{=BtM%@WdI9xU|l5BGAyG_}v6HW6H$~wyfYJA)0Ry?l8a9Z1|Nq!=N#1 zD?TXIlBLF(tmexhZnFI{PjSD*n_PFWfE|9KC#q7+M^=X$_wnXCr<~_wJiYj0?GQft zw+!DHxP||4y3XTuKl4fX{Uz7$YfFr?2T7`LPLOmRErYjRGnSuAMOk_nR!9;sq_Y^O z=DfyNFJVW(!ow)c^h8?aPGmNmf{j8p#*Fxc!f^sCOTGa8Y}<(B{4L_zIT&WMV&Je# znszB0(Dx%1Xi0=IydOxBWSN(ucN^89bU7Vsm+xx-lc520n!IRkpZ?HMrpY7HAfKiIEj1Z({CBm8>@mcmgg$hAfeu}=xfj-+;ZQOW*lohD zgj|_NBrR2fwh290nDtTMYURTCqBkaEI}QolD3Oqb(3R~1U)}<@UV%HIlYkl;fjpNq ztQGwJLC*@|S1zKt_h(XCwNB4C&%CM)YU01Rb|Z@zS^uy;tf1 zPZ{dnPn-5W(xVSde_~9{G~69-gh8KnW7)1UlzD5@YcD5J{fC2Ty=V)XM_(iRgpX4e zYhlvUf+I>p>F^GrV<)33r~n4UV75F$)0X3~UM4Q6xc3PHR8?8oN3gzgChF;`eh^KqJdNCOC^v^0)`sTYV&GsyX-Pb*+Q+q@r zpJ-#ng*F&=s?aaZgJ}PaQq=BND$*KOW9BL~tT--5@*ejWy8PqmycPr6rPqRqo7Nyd zBbL1Umq2>qgu5ag+P!Q7eNn1KofOJoxkJ!eFB+p@IYEI=H{M(5((k(m($0CmVbxHF z>pvxs(z_2=r9#}kZ$$ggF{BQEhtRau18H=_KzfRfqrG07ZNrAsNHiVp2 z*iGI;4hfUP0kdxG^3bAH_ybM$hLrZ&uq(OhY{I&S#8U4XTviKti<_EMsrU(|YZ zdkbch&e93C85p7V05-#g+*Dfw2Jei7i9{D0tWJn7h^~^jf=KiTnzYi4HjEYWH+L)T zk@5Z%an(5?4$ZA(N~KMLX01WLOzcCW9IhhkKqK)n8pUG5Y+1{wX7b1JH~w27oVD+N zkR%&||9qv<7{5ti4kQrv`XPo-QlL9pWa#kYjTrXe8iGHah3~^a+|WIV`V>`aH&2;9 zekDuCo%{g9zaNn|w;M|}1umoF53~s#)HRNl0_fNAk6mCe%Kj3 zlpT%G5ak>H7Abz7dW$gL*5CRMy zE>~}|zEckHb9Ni~kX{$|&U7-K$lMijSAqvs?ujM$qPT1FdVXcI953?q5Oukm!B}2k z8$EEuw!WIM&6+IAzw(Zaj;EsW&e|AaeF)~&-a?nh6ztelA~hBCPeB;*4I78#30`Pl zb^@=a(aKv_v$E^Aj?0&92Pe19#e`Z}1G67L2n_G_9-cX!2%p(WVQ`o4)@w_K)G&iW7 zg;|-g0v|dQ50#e+@B1P$`22m5^sKd_TrVv)H!J{mr@EtP)e@*_ej!Q9=gF0OyUD4+ z7G#h?f4)>~CAmAbn{V0P$Rl5-@p)g5^FY5RQFS)PuS1~O~C8n$nIOa3!jNzV8v zh-&Yf@ruDgtgv|x`IM@Ov^qIR3baIJ{r@l*gHELD{$Wk>^F)CnY3RKkfELv)q&fWz zQ)r#SH~RZo>do9k=JYg?+&oPzH#bc0-qc5j)=W{u4yozJezT|w(CsyFJpDP}}Nx!#761`+K$?BV*_^}(g{N(Hyp0eR8UwKAD@;v<&S6o}pC!Gl4$;yU2VO9`p z3|z((9!=%)@qHynPRK~CP6)e{)|>K&Z(K=k(rEM=-68>>F7uk=RvvaJpU(~uyyEDa z2V18& zMr?dv2D(zKaY%b694|PLg|7G68MB{giY;I>Q#O!0rn9knzaw6n%V5}PdvT@fXZF=F zll;2YMDEp@BJ%!7``@{kufyyfV7E@h`7iO@EZsC0@XFAjQ1#EACnD`e>vM|7qa!*0bd zn7RIdS6MdHx_1j1&o%U5$96Q=<-jED2V7tMg@sQM zUdZ^xz`3lJw1?bbo7YS&N|MTA4ao_Oz5JXzW5WXTncU@h{NRr`zF^{G9;YWInKFE+ zMAN8(n_s!bv-f22&(Y0%)t#UG*;qx%>8a|HHR`sK$h`5~Azh6;Zz>j<4bmeYgJX%! z@)nZaE=JHX8izp}>8d2GQwXR>ncOcW>hV~E!|Y<|@bM?S6NdZp93_Q;_u zFm4(iMpYti&2Lnm&BO1Vo0#KXCtez6$^-9Av+U`=7Ky9xLuTAVJYvpxrIpABPngPI zbz3nHm*FU1#qc$?74<8{XsI^?$$ljsQf$u?V43mcSz{T9m2=hfv}x(id?#H zO@pd^$T#~zsJL<*2bUi~;2mkm^mwq16V`I`r&(gp%cCH~&*Ag04b$vy@Ig= z^9%$5Xl^2R^LE6AW%tneBM<5Nt1-9BCpU5A6n-*g3!gFn5c`}_fGr1mv9bIClFcm8 zxz?ODo7wPxNeB50{f}g_jz8k#17Q7I1o`S!#IMelHQ+lFRk`zPKC!vcHkrg^#0w%_ z89*E@^vF!_JkebtJR_1!<@Y*f;PDj+cBLdEZgC38`yBk~d5Ezpcj2@r7}hl_vt(_% zN!ZLu7|sHrY+H$Me}3SGFw>q274o8Al9}P)xvXpYU1DXO2;0T-ln9(!uV=z8gzg`r z*RM7Cg<&PkcWEM-t9u;|rPB1_3OQ;Toq*w%2YJb?6lR;fomg%>i9eg4K>mIYZhb1o zluvUZ{pRFJ=VQX&wQSsYmyJd5Jn_bVENa#{P@5Q2)W;sjr<++Q7{M@M z!*L9pypQZ{`@}wk93nrBXTfh-F^mTM!nGOAs8aPr%1T=js$9W7PCCHPYbQc^l{DS& zElua0zYeKzG5M2L!k*R&dxSL4iOBOGxa+4w+sYK^qO~#Dd-E3gExn7qoma{vZT>Kq ze~t0B-w{3L5kB|xLxrO`vYi`9e8%HkzZL#y$Y)R*cOO-8S$MlK2_?HF=PWI8 zZP85V20p@_Yo)mTuN>8P8elDaKa;iXaF5N#%fPWjvPK{IB`aXJDh|_Ip5u3y41LLd zpz``TyqTNJeDY2*AMcAq_EHp%&z7Z&><80=D{Xk-D#vW<*YWWshWvWT0wlYMX5qgBmDflgA0F^s6SB_PQ!O?Guds=?gGM$AMP-n1_za3;0?h>^m7*k7(yeIJ~SR{nq<3FU_|kr+P33 z^cTGH@y!^jTm*}~4u}oO5cM_Gin>I<)*lh_|-;fiN#2%NW-;a->%QoZSKqsEI$m1nbGAU|K_s#S2HvT~ z>PUF_QtW`|F zFww{*fm6r+f7d4A0GPisDG46&pVEb<6mP;-3v^LT#eC_TKM8AzP#dk zIdl73OR`3v0ZPK5d%y;9HwF>~$1IYxw1br@8xx6(9m1C-qMThucGg5>>g^*f(@m+B z%Wk6EIuVcOMC1L>YbZ0>0NfZ&=Ge8fxm!|*iQy{EwbSOW2S*sR>1jv6Y6_!!$i##kX@~vw+kJuv+z$>hpd(^VP%b70!y=s`Tw3woCOx-Kb4(ms?tEn zm}aqSkSQM+Jf6(d3})-5-ymxb%!Q1Q!*%$TD4LQsfoJT!!-LF)9pRfNKvTK_epby` zRJsPEs_NO@#o0Vq{{@fPeu6KU>mx7-s^C|h03WGhVzYh-SNS67v(w|b&*c?lRn-}E z9Sy^iEMecPj30UH@4_~n)!^Sn-hBP|8dBFY5jCgGF(&N`ah*AV+%2jVhg7ApbhidB z6FfoE5tGCR746{n;|+veXdY~n$_|mCrm0A)K7hTsQq6-KFLI@;wp``OGPXPG70G+; zh>xcxVTOqbtFf);=k?2YMZ!-0{Ou&d_FO=zbslz)>7m@cqz_g*WFwV)B^lQpt2Qw&9R~pAHOM=DA zUMeHA(iP8*ry%l%jHR=g0pBpL8m`-P`R)G_EZbL4zzVAP;8kLy~m>GN)Y>zHj3{VdxS3xzs?5@pCpkovXj_U_Hf;( zBJR?5ksr7x;)m%wHYcTsi7#33u(fiM5$EpnQ8O;^zYVhd+Ge4z?5B;l7G31gmv$CD zM@sV4{|$fJ;m?gaY(=%gp3iDljF&2h@ocL<|9$K+ADI@-4Z0d~RZ`M$L3IF?&m2Hg zr{Bh*tX#g?WF^;b6*2kgBk|y9G@@MuFZAtXxF36%dv>ohf)ae#6ZI#=sxl0Nib^3r zHWj@!hNyBJVmW-SD&74|-SXk5p>UVIiJYw^5M2{^1L}`SR+%3wI;n&;CjG@5Kbv7o zZwzL>2}8l3aVYzz$+Sx5@GLFHAJE&l7CnIG3Jg~3^h8udEEXHqM)60Hb$r}0KR&(w zFqYIv@ZhX4%PjA;v{nz{;-(I6xJ^nD3@M)G*+yEjRH2|ZhQzK^<(gra_@Myd4p5{Z zu?(Hcn_oU=j-DOtcl<2g7#+s1R^8|M**!ez`znc5%z8=nR4vJ~U-A-NBY~wDnaXd6 zpW$tO9^7F1b^ddMv81iGk0fVyHCN6&%;#Epu{pgHN#903b#AC9m+CV_GR#C>61q2& zzZ#{>{M@~u_l}_|>nQ$uU*N2D6sRF zR*=cRud%{4XE~zlp;E0&mu((E$B$0K$)6W;;)5@8yVMF^p&i7hy*LPs7rAH>?h290 zYHVU9`zAn0&n;h*UIdaojBGr10^A>MD zz4Z)l?!CoN41dA@9O@(axMF~0k>VzaoAY0Ga#=D_I2S^~e?24yLN7{ugW|B_7`#8= zMyBfucPyCmqSHYvQmT>c$vTSwr(C=>35B+&nrPGXBHphzmhbjH%@Sn&F*04y8F!AM z&doms)}8aeA)j8ZQJ!J%{S}pD41vg;)2(ct*+@c5nGT(o+zF59~Lz zq-E(TjZoN*TETp~`mxXqU;gCxAhz|#2BVckwS~o455-NDWw!? zQ6!R_j&Gp-+MmiobbTY zf<2FTaCf~<3@l<<^PWk#>De(pFemy_rP8D#CE!{ng#F#Hmn z#X@8);8unacs4Bnci~&WB{>7$mu-iTk=hWY8;$A5?=v}veO7}OSwYH-a>#0LhBcEJ z#Ko480qG}E<8c;lzIB|Lyov#*KzSP8G?osKY=Y6+i%}`&BHDk-#_X}ltnp*Cpuh7P zCakH4dWX?)F-Mncm+izlTR&E;L12dc3E0tB2r-sJ1pgj_jltuoD9q}WnKCX2h(mQ*!{^p&B9h4z~>%EG0DA@kl)UMtVRPY3_A@Ij+nrhW673b zfp?j$y5MVEW*oS*Zi94<{qWv977E8Wfkow5m_m&p#w(faxj7PBJx79H=0tchd?DPq zl_cOri=kaI103q?p;~1t+BrRDE~#=>t-`;>gev%CXpoUcL4vPQCZd?`A8JcHv( z6;S8G0@$i4tO4t_XrT8E7~YbH;?pGgXx4%byXA5EqRlWttQTA=YX$6kf>pIz2sdl2 zMYknV*scAFtoDk6qien3vU4CAm?4GsGBZH-o-LlW93wKjKLhG*;vvV)6H+e_GJcgC z8y7K{4%;}Nb*$4RDq_BH?doo5Xr2aUTC+r_@@z0R#0D1HO5^rpA;e|qQaD=?2^mVY zI{0!Hy6*4_=`Ja6<~j z)gK1Wi7C(-v;cO=Xu$vpb-~uBKHhK1LBHVXq|T{`xRnow+083pY4IMx4qiMAU84qd zp9}DIyfsFxU&m%2S9zmn0R9#m>fFd}@)*OXqH5 zGMUdox?m8^uaAK+Rc|!R)a2V+jQ9<^ci8&00QeFmI_Fp?oKBR1Hn|PR)H?8u`xCqv zx1Y2fD21Q$p2C%t`Or}Df%LpdWVQ`97`*Q!p1P|JcjC4|l;uv4n(qgZ0m?AN=NK_J zTTJYpWn=ZYKHU8!noa1=DEIw&6zO`a1~#>xK(pt8am9I}R$GUw>yokTj1C(8K9YDu z?SNdh8?f#~5#0PdiEIq`9gWPD_^iGn?DU8r`AY3j_Ev=|$_hHJmlU|7l@qTTZ^0wC zM&LQGQaGYBnd(HUQeD4$mVU9R{O$dA9C%8^KED_Qe%Bmf$G+>N_E7=*bxbgZ9}9v( zPfamPbWN0YUl}X}{Gf|p1sT60muYRjz>avH?1c?Pk2cOY1F|!R@!u2|^N;|_?Usx~jVFq*)-pZkC}HF)sH zyN2=Ww@!lfRy|tv{xe|hcciuH9_o)U;Efvx@wZJqR=t|XLDS+IH0lPy{G>RMaeD~v zQ@)Cu+ogECM*?v-FoUj?;lf#29-)T*qF&d0HmgYw-)frj5vOB$rC|a3z0pPQkE_{} z5=)ZTGXgpX8^V@X$4SK-Rc<)?1D41iMF>k}XGeKJ!;xDsd&dO`>M|r@=M{NrwFQ@G z>%=3yp=6_z3boLkK_k7ML+8?=eA#UezVWF!FC&eX(*pgd$E7lQ(Y=7yYDv>yzxwd_ z6Y?l$p9pUnUxL3u7078#fLm4jS)BM>?3z*snqJk!`NnEUaTesDD1vzP2toh7lC0Z2 z0iwcPXkn!>Xw9o8g@RtPZ7m14`5ZcPL!rj^200|)^aie z?gD;MLk<_``10F=Z5(Ug0=hh-oJPklrLnCC%RjqY@C&njxSZ}#URQS)MA5c%M3)k+ zGZ{;IUVK8&A%weEEAub8S|oaS1*}^33uvXef$i%wHt$R!>Nj-b7kxEukXAyXeI`R# z+As)qoI?ut#WH1c1H5+nFb=TY!nLv`DiY3b=H^>Xxu<_0?q8XSzH`(u`AY%|*&>e9 zFMIRIO$&LZf*Dr{xQ{1Xg4vy&ws7H*H7vVogfV4vxRGoOS2ncfDLyi|oqmATk%X%0 z--N2CM*MJMBu_~YT#y@SgPLI;{UYE$#a&xyZr&oA5ipyZ!(vi&P>~jD+f#3;NmT1u zCj`t-fTDA|VY*f}6Pd_`DS&e+rJY$aOl=0M(A+4IKT zi|F;6O;l3m6ZOx@r3)-|VW{7%Zy{B=viWr_` zCs%C6=H4UA)58QAYNM!-55q1TnGx=;|mTDJCW>EA~G@QOsJ*J9Mf4inV^mYyCpk ztO#0dZ%f2}`2WMCLH(`QWG{CY7h3C&u8srxlfT+`d+l2JQYiOFrTz#12!Hm**E`JH zC^&HCzq$WDF3o>%nhy~BYb^cT#+K%$W>Y3lv9vTXn_@Ec-xdA6i;DlKC^hlF6@8l& zS4U~-iQkAuN*Mk{F7Yk*XL{vZS$WqV8G%oxKlk8US?HoLA5zl&_aua?pZ*Wp{{X0V BnA89O diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/data/OLD_r47-1_s28800_adam_lr0.003_wd0.0_xy1x1.pt b/research/part03_learnt_overlap/e01_learn_to_disentangle/data/OLD_r47-1_s28800_adam_lr0.003_wd0.0_xy1x1.pt deleted file mode 100644 index 15aee5c6deac99411f5626fde6df3642ee3bad35..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36920 zcmZ^~XIK<%&^3q(f~c4kQACWW2$EsC4oFUtbIvG1vY;XoB{Kt(lVBD>Q4Ao{h?p^A z&KU#vC}vO$+wboFd3UdGfD5MS>FKGv?o+3#PT5+H5)~1VkP!L*Jtm4w7V(P-SQ8Q- zq#WoM=cg1E7H(o7CF1_S9w{RO;v%`Tlzi63g@ngS`oskJuL}te^ojC|@mm`d7Zf8I zE%JXjB`zo;HZsO1UR5%Bgo1*rICl%N+{OG>2SxwCSCupgkMxUk_z8ZGKNiaQt(wJuuBN`HP_^k_Rv5e3T;+?S7~Ph7OP0(X@A=AMm+ zi9m^JIKhw3L1H z1SMNbaS88#f7o8`eg5ai_Wx+JSt3*Zm(7&__r01JO!)tp&)qORqoXs}sbe}%t>gJ| znU4H;v5tH7AJ~YXyX?=-Gpt|30amV}p6%(|#;Vzuva=;B*qc(@Syi7M?6%b1ti0QP z_V(lkHt5VwHZin@U7MZD3g3IN&W~rZrc2fcr?y{X*0*nA?L=7q1oMv3|JcMp#v)Erwk?h#f z`Rvb#8n*vn9qW>|o838PAM3YrC)=yBm-Sg$&yJj5%ZfP^v+KiaS&8IbY-H>KHY}in zeXn+lwcGU{``cNx<8`KV$NZ-%9q!{DJ94eJbX3f8>R7BcuVa0iXotkDTkM2c``Pcc z#q9RJC{}5kG5hAX3_I3zGOIjoP&oZYzwq~QDb`zAlKs{u$7UNYVNdQbU=w96*;$KB z*|=;Aw(q(gTUe~j*8G~px~;z`TsAaSSjaNWs{QhmJ7+Hq?HGZej1~Ac=mZZ1cQlO; zz_R$Y&@>7IKOqLH`O#?hkHyi_IGEPQV^>oG>K-NH`Au9tTy;7l*orLayjnMm?faVJcu-_Al?Y%MZSRR9W2H|Mj7!I{vp_pbC z1f3pt^j~yB$sHA(+#&*1{tmL0l&6Hrlb9!0<_O1%whLRu#U zd#c@neX6U^#))#rH_5Uuc~a~(ts&uv^B;s8Xa5$C*f@!O?mdTPoyh(^7Pfcui{0_d3|V-rZ5ye6d(qckqDFRriuG>cI_RQ{!pj z-JNa15xSMaWgp#zJJwHY-?MKV^K@$ulVj#hvm*DCYtBE~&^aG--)f<^&IF+gtWcw9 zixm0Jp3t^)hrFE|oQ9krW^IQKFDq=)GsdE4>Ubfu z04=YEXmC_B3AVe?XSu6PXo3i{u3El5HF1(KCo@;L;n#6tP1Y4*=d?S*is@}afA{mk z3)j+x2U@g*5(_T2%ZROS&)6!;Luw80U11T=QM~Fz*XyEox3(N%_KiovuOSOriHkmL zIlqE^y1Sb_`{^Hh`@BkraFKLJ=7f9f%aAfwzr}`K@nr&gc;P-_@2;lywFM=-f5ISU zY*PgzINil;`g@9LI(Ut#w=8GouPS2JmN4JX8(_rt1=VMla*2bm2e@%7<(_7A-R1O zcp^4PIB$=4&2~7H>x_F9F6jQ^j@|n`;osv6o#E9uy(0=z=hD&OSqf#VGz9I4gJ<~~ zJo@f~8CyNEu+RgUx1F&^+XW8e9U-S?k6o{=u|mxX-ZL!`9b$%RMZ&ZxN(j`Q3GJQ_ zBp$w>#LkD%iD4Nsx!=UJUpHYEx*g$dam+ih=*0H+s9y)$^XYB7$djb@n0@Lx2j`{m zfN4x~k}D(qw2Zmi*U2n)eagheUSVPzk1?C7z~qFUVgoLj3S9$)ZGkNT%-_`8%(r!tBzZ=G5^6Q5 zV7C@2x@ggo@VR8YVgfOX`Wa#0PR8e)3-hlbfO))8h1n?*#q2GOWj1fgX6B}LF(y?% znfUVy$Rs$7@`434CFc~)>ic3!5e|$qg84lYB)}47n^&Q($_8_n+2d)B zBVK-SK-3H;cynvespEyKzg8o(EE*E2X*fEm1bw-w_^B6zq8UM`SNBG#ge!s%aX6A- z2fmX%yd7*&p>2i4N0!*L&jL3SOz`Bl0mhaSPG&8~+9T3fko$|I49?O*K{>fz@}@>j zMS8Gih#6lbWHzi8FqiK8GE3g;Gg@zgne@CkW^h3olVz}zd9dX(6Y@ifRz%8@xx+k~ z#M7t10d*=qv6No_9ZQ?<9Al1&c{25JCtKxjCkaJAim}Ee{_LnZO>CN(ct@wtM>Zy_ zh&6n^h;97YBwV3Bru|{z2FAbcKZZ4zqvA!zv?4ixlJ0DzzD?^%<82tV^aK%BI8pag z73y&wPxYI{=AxYK1g$dyHLXhvlyw@krndu?f!jo$rqFa&J5i z3xw|ISd?_6p-#FS8jsR2Zg(6GbG9!((K@0CKyBhZj)Oy<$P zv+o#}Yf()2@W&J3$IFEo-b-2ieWh&0g%51oggfkjVGR2`_pb1V;=6WhX*oH1p}xpUvbvW(}pJ8I#H%6?zaik9s2IX%Q`@ zz_Y59G|`BJZtk>SG>b;9EuqWF&D5@Xofb^`N?GO;@t@Qp$b8X+zMUb`O+9+oO7vJ!)@TU?TT^@zRD!a?-}{5ej(t zLkf12hsZ{=hvtSIq|v6uw8AKZj6)*H_qZjc9#*H}P3m;=hCH?UDpT}ohID@!(*u(L zQj$$09rJw3_bs8wCuJ0Qsglku&nHcTG`eEqOv}s_$o^(OGf)!2WK6rzzGK`y;b(&| zHa7G&J5a;2lV7i74KJh%E$r)gp+i@gvgSo(>*Y@0Ju~V1-5MGZu#4vB?W1D%L-cdg z9=aj5gN~+F(1!R7I@TIVmh)|>ro)cJgUo4Vq!lH-@Fvy9FxvSmn{0hbX-|J6)y+On ze72XIibtUKL+-JgBI zd*zQ`VXHC0HxkJYHpAm;4rWc>j?VsEM98KkSXlg^Bg_ za9uGiI+aQ<&wEgS?0mZS`W&;Z^8)W@Q>t)`F2fq6b+MM;Ti8$f^Vm}o!|f?O2~71= zaat8-MV5;*C|$CIy#DN_vl4BzyYCbgr(C4V4MOTVf0!P>X`s>iWi%L^O;^MdDL5d8 z#LujuQ{TfWA|;hfPv+BCohn*U+C(Gnc9E>XW7;|6Gqv`OLAvoA>`2grzmp+$9$W?0 zX^uF3!vzz{T(M-K8|3zRKt;(5R((ELal;#}mwlnH5`3nbWMtR|wxfiCFxMD_@Gd^x}!U|bO-08GN zaijtIvQ?2VVGi7Wjl*)iFSL8iefsfV2Zg#flC4o4WuuTnR;7}gay%7pT}w0e#8P=j zG^yxklXY_)8R+gKJ*`%9P3WRGo6b?u+fEvNwvG0V+)Zm6N~r%<5*@QMr>j>c(B+H1 z%;cz3vxPMFtd;*!&z?5S2rsOVa7x^DXp5p@3AKDk-IZFBQog zBjpX(=zPL!5-Np_3;DKf9wy*9kMrZD8kRiVKV;f*;O9oB0HUjU9eZHAWO{x+NR)7zaGt@rQeS*8$Q&wy=#zQweohc=ff)4GxSY3 z;rKh9#ZR6vul`dZU9AMNtSF`knFDl>-${E-u2SE+N93IKi25sUP_FS6DwFP{I^_d& zzo3jP4D;w#K_1E4rqL6*3>u(9$~sp`52W|g{<=2WIHiYb-@c~y{i851WCFBQ=Hr)_ z2BiO)Ai~}Ob2VJBeV!{$wzxx8!xJetz0f$)7jxHpbGeW|Mur5y@$wohlUa*$kM;07 zlLqVH0>t$;qU1UsjYeru_FD(v*AbY%F&IT({b1J=fNDv9RHS+1y^S|Es&KLTqB|mQ zxZ~+$XV^ToLEc&uY+a#^&i(VjZ<~OYMh*{k`^Y%9hZJ^pQRJdVs!FXPX7*OvV^~1- zRaNQQhM%3JB|_wSl+j*PlBi@A;6n`dp(^9%pH@aT87VE~VJ` zIAXhXC|T+vb5eS`&g1u=gd5gWv8zSO+4b`8g&N@VHov*f_+B%lpMTa-fMN|~**Kmk4S;*+ae zJ@s*G*Xk*xI?HbQ>Gy(MKm8^)NfP7a<={V60}jf@SbD)4LsMK3a?2H`rQGq{%M)(o zfe8ja&<^&*d_^B9P4LHBn_!#{i@;rrII!`lIGa`g2e~Fp)a0Z2L@JD%;@}<>0)(!C zr==f?PxxT;Vs9*Q_Qn$}Uo4y8g;GOTD7d=8VV)B-qHR!QZ;IA2S{QL$4l!THftSwZ zw##1Algg{~cSeB&hb^Y#*N zbty$!#L`ctrDV4IDx+2$r}J{mZ(&fzc2-tU&-yn?u?uG`YR?bY&Rk*Tsc*XvDHr6^ zf;Bs7py)U)UT}&;IGnfjx1ryrsiWX z>*fdA`RP6#PVAybeN9xMQbFh4(y8t6T1waTpeH+BXw_PK8WrzD%N*TFYRv}v6TSi8QON`d@q5mIh|)meHiS(VBS0ne?))lmg=$DPj5%(pb<*SH(|Kld+H* zUhbp(K{pwn{|XiuRUt@f}YvyA~1GI|q<{>i5n$-Sgx z-bxl%FO$TMA7qy@2@?jFBI_)nFUthqcbZ{ulm%`#SYa^R3OehoAvxCu+n(EEwx~U> zZFGXc9S`)555}rB(eON&j^LwvaV$IofjTj0IqHwYC*0xM>;SbiM_kOb!KV~U6w7k2 zzq$%HH(OxAEE_m4F-5JvDe?vx^juWJk#!R=&-5!zKG#FTL!ROrzbJ3lIv+d_JNr8IAS6X`BELif+Ic{e+D@k8PfDOQ>g7x1hYHPy50A^49jfV&lb+!!%Ai-v0ImV3)ef0Vcw>_ zXO1jVrH95bbRsvC0(&b+ea22|3_U2!Nf2p#fJrgjT4dVF1y zoCMF9Lf7xi>=A#N@#*8}TBJI;JL=JwTzmTbA&o-jm(t1cd&sEi0@;`Uq?fsKG5f&^ z$kq_7F6*K8^-3IYG(@_L33%^~Q1j0KcxD3C*``<^VTFTtZ86f-9SeVj;CxpS`n0#f zNhAr^cLd?t0av`8Y=@}ns~~&W0z5HO{CZ@>#Q~XniwG6u z{hfhq=>BPCD!~|fShtVSH>qa)r^hkt{!3?`7Su5AU0n=+%S*;6 zUxH@8okGtR>Coq7cUr|^W>0nvwMty0yE7*tzjYxjwKXvNxi-9Id63WHVPqgMJCAU4 z4Pn)Ou3}|th*8akD9Sg(_wP1%@ZA*~^dm7MyA0(^VqqfmK>K(OPfnSjVV)6WU+7`d zJqG1-b>Tc!7o|&qM-B`MAM)@nT>}Rr)GG|jIy2z9ixHW2nvr~Xk1;-`NN(kUbl}KF8XhjB z^G7&)CsdNhgi_jKl11l!hf&ceOVSsSp*_zJFppo$@XoXegrDPd*rw7ZcJ%c|_SzvM z_P=Kh!dqj)+xJwaGOYuDm^N)qI+ox~CoaTOw|5Q|ITTRf-(1piPo?r@0pu*EMaz80 z(&V$P%)llUX4+b5WE;s(y~4h6S>;X0`_H z?2n{f$M(?dpwU=yPZq)3m%(<7GMe1fVP>QbmqXe(`cDTMPt@UDp^cgl25w`3tSCLa zZ8pWbzm71T7>tH%0-U)Lgq1Jtpf%G3YRM}hFd_uD@Q_@tg`MeIut?U%nMGQNYEna_ zmMY>SRq$V^B0{Fg;o|~H>~h^t!)Ea`yiSD*cTJ}k{-RVQ$}(3^?qi%(@|Y*X#5hlO zXXY=A=AE6!^7ds;VEQ&}VeV}`&U_k{p!MD=B)`UkcB-e-Mih|FPyzk;R6viFlBmts zokUudXz9~047)gz@qE9ieVT5U(0+*#YpvVJW)9c0Wl0w7?Cqz7Tkr2^FIXGMRQ-C* zEOlH#m))!=GA^3rH)hk->3QUGHkbaTZJ^n=>?yNTiNZQQGeP^RneU;anB#sgc;=%6 zc$VG*UWNP>-rzAU=IOT0OkT$h=3xI$=3EBnL#L3v|4%@F9uQVS-7%M z5ykDw$XKX`lYdmvqpyMPbsES`RmTQ3ElfY51raG8()s|o`CQ%oxE(TG0^s3N1h=+8 zgrBv6Oq~(zzUX3sA`hi!dH7YQfpg!rVUeSODMz%B)~$j6zNz6;&2kLRQG{T479280 zU<1d~e?of3TvI( zShPDbtgtf1njLMilWn!GV518yS*zif!j~hZgiCL#Ftyc>nDcArP?e|!C0nnhtA#04 zH;_S_lhY~lUM!6oG$M)X3+eFhyNu(25p%=kAn%2!4DY`h8J#}AARR&49$xjx>paUk zE9T;)DCVN4fU)Sh!BkKG$c)XOLZ8bGXvvIHQrjbjlp~s96JnNFu|_^UG(12 zfrYXb?lq{Q?wSVNV%4FNt$|i!P0UkP!PdD-T>oSlW-pzCfpMa^a<`I>Yq=0#d@}i& zjv&34cbKC=<;^1?Q}gXDQ=~nY zJ|DKE@iI~5|1pC+PiB(L=_I;r7ekGg%*m@rp8hR+#dPs_%wCOK?R^V6g&Nl_+2gsj z?0m0m_I{Tp8#J>+xN%)wyGd;T(|Gqc^I^j>DqH17wOcmQ-mmG@U%}PtmgmyP>J*CB zwWIiPDkQSx19Pu8m5EXv!Du~O&dWMHnzt!?ozC@VH+5{rPhiYv1TmhUQkhdjyOr`ROSA*MG4J0j4h4?CUct8u= zy|wUirZ(=x>EeNr5%NCTpzvTYUbR-?^6VfeiaQ`K$_VxHz=L->aNewi;>&87a#J0# zS2SR`nA=BH>WDa^h9h;$5xi;%L>J5A$UHIV9IvI#%BB=KVLk;I3^KMpFPUhL3o1UW zVYWBhF)Q6AnEJNmyio}`ZH^nKX}crrJaUNo$Qa)@XZlNaweK|-3jgKlv%holS-nS|?7}cPwyddH zIAf1ryH!R6!cl|70yj~#Bpm!Scg%`)j->6~IGQ2>6ldt8PRA6V z2keotDio>gcC5Y7aF-7Cyh!g1@3R)^}^;_7zRY zj8MhOUrKl;D~BngB_MUVhOWQ2C69e`$hBlFtsV7{+1}H{bj~Vau8oRh`u{Fw1AkmejxqT&Lw4yV(JB-L@pN5b>}Vg;)5f7 z4b`NSou8R|YeJb#0ms^>&uSDtKcU1P$P8lD7foe94vk^uk}8DNlGSY$>#CSkrRgO0 z!IUI**O8k_0V%Jorc{$1bfT}8mNpfUnC=F8pkqVlUreGLlReDRMGP~zbUssLA z;q#T7I#}KgM7Q&B?-+w+(F{KR1QxCYzHHM)5XV2a3|Im8Tp@Ko5KNA4jCb>5O|*km@5ON(S%JOdGO@;l6$ER9tHBCyjllAA5<}t7Gl5icszNymrBk? z(3m5jL*a`k^qUwlJMS}xf1P4F=ImiA27Q>`zk19nS81j-?i=rE$9SeGMwO|aeS-1- zHklr_7}DukNtCprn3jjuQ;)`WT0F9vYG)SHCWUq6xxkJ(?nu#$j0DC)cdU*qQ!5;d zoW}M%oW+h1+6%+Rb_->srwgr*@8zj)>12E)RVX;bjXsHHlPT4Z(&q;19y~zm9R~=@ zx0B|xJaXL;MvX@ok+$eYJ~SoP4P3+0o8mTv}JC>$>&EQ7Pt{v?LHX4#11)U%@MZF0MEDTdPho-%8u8J zL*FUJ_t$o2Pjx2anYV@sD$!vq+Ts{t(aG!~FRG{fvzI4fg%N?zE(YdjWbYjVV zG7#)1&%0IhNhFh28VAzhDpmUZ<2rNx&rx1TiJs8*@(JNROBG?;z8lP|wlTuxP8KJW zJnfmm*a1d4&XCrgO{Jk3)%1SGK{^}M%Eh5}+VzyH?aV(!7H#D;{#Y6v_cJ5CcO%K+ z-vLH2P|xIyThBj|iWO7V#rNI>9BQ5YO!w?Gt zR>C1%9fA|HI9-3hm~uaoTjoRYwR5R&=`}_mbBOVCC}w{Ctz{IsJj+S! z7PIT-J4ViaJOzdFC`HSGjEgoB8C8&_(m{H@uAh{(mq5s+u1Qj+_b{bPfeiB^c|v%-L=HPX|q5UOB@Cpy-U8(4+s2^Ls54_yAW5Zk}Kqh+=&wBN0e z1}AQ$s~I*FBD`xmK z1#~C+0A(w5k!0Ex5~;gFV)9q%;q}w>_(TJ_o-3oF&Jaq|ktK}{XBhD+c}8h8kMYc@ zBTd~|i0zlASzR}o=20?qd!0XJ4U~~VbPFlTpP_XNAJV-oy(A~wNAf%0((A*Is7kks znn&#>iGdWZ$EHQoq$kj+T{oBt-hC#w^$8QJc#qj=EkV1TW>K#MsAabUMV{kwvg;+J z{JoK$#^0cC3&jvzpoYvn<|y-Uzpig5-ve!5-EnxFC!9w3 zAha|9cOOKesUZcg^oudZ>OTac7-Od;W8&lp=p_bXoV_O^Ydr9Uiviaq+_7551BRnK z@t?6fdZpdCSmKDiYwhskvKgLusUd!pILA|O(Aa+mDJ8#%3P;6LdX*E2De95y^5ryh ziUetXy~D`%USr^`-pouM1+4wB+50sVgNLid)>CbPm%%=|ZVNwR+y8kQ&F+~98d@xhB+72;_@MF~0o zX{DnP*J#Jdmn0wXlcKhb#17|?_|9pXw#5ENg3LQK<)4r;*KDP~2_B^OX9hj+7Nf=s zy-d&07v^;HAj5w%lkO&fG+JEf_3s$+Dk~<#r!#mbs@P0HfCjlcXvLNPPg(&gA7}-+| zmE~Jd=dcOO#G-KUVh~PV3&4&>|Npypc)24Od3^zB-w=d1x@(~C>WgWYJkdMO0nP?1 zF*!mSu_wOL#Hm;5jmSZ|v$&KjH4-RmiYM((AfnC%v|zyq3VJq@id|nbJ*!1&^?)>4 zb6S#!(lXkV!&1iid$e-VC$7geM2!zbFy441!uDkLv?0c~0B@`$RS}qPVbG5{Jbn!Y5%8>=ef1 z%Yo6DH|`6$vOQ#GvzHuHQ>k0Pqaa;rvKgL47wlt(wT+ciPNz`IqNN1%E zQet5@c^iMGrIC}+rKpNUIriv!76cdnwOF((2LC3;LXy*e>==rLT~-ttS45-RDh^>W zvA8IgfQY`$_#m|fN$~=F)!2(CbH?gslge{?$KD(!1-ptUda=-#XK zG~tau@uET3?BzH;zzp&&)B>lsrv|9wHLl59r&?Q(f8dzWVi>9l6qMnIE zWUelba8)ThJUSB#tLNf(@O*qaBM+NHb1|i2CNvL8qR8kA-O{*3YgcWj`H_({SnEnL zmn>;Tr~_3@wxKQY3A7_5pRAwO(#-D%srKS6>K+=Pk zVjG0LXDsk>*gE84|v871m zO;rD48%2%1K>eRS(+`0-o-N|~L0xk(&2v5!OXs8Dh8)gaoQ}$;6Y$YN9Lj;esi^b| z*-8B){nslnO8pXA7F>n@L1zS7&WCgBOgLVZ;cB#Vk@HRgBlDH<$zKC`y<9CyRSUmX zt6`Lu5(3`IVTR8{EOz}(i~6q7vJFjSw|55>ay6Q@2e>|sb1m7YG*ZnjmZGHYP~o$W zRC#hVhDJ+6qIL!DuQfxZP9RneZRGNvbjniI9;E!TM?Dh=@```F?4by%-Jg@Bh(! z>07jFE=y*del${{npRd+(@V)p3TdpO#aH$dZ`p0S*F8vgW=lb7hb%UkC?QKj9b@`5 zFlJa28)H<_z|{x;PFFx0$%6@+g+p1>k>NHE9b8YyV9_-UCp4l{-2-xeHL(53a(Lt@ z;d_J{Y7?}fZKaE)I}IUs&m8xUo8b3VV;nrB3svsgca|z8KC5Ais zVyNYQXG@R9@lR5anIw(btK^`hxEOn$tK&xaN-Q;Wf@-oq&oQu`b zIavHQ4`KW~{ISV}Wq1z!@8@AbS{}sjF1avC$%TkY9voG3aKf8g7uh@{#Bmzh*%_Gk zX(Jx(4TeO#Bc6&IVkE1MW3dWYxN!~+-JXVqfC)&>5l3;@Xq-tFg=6SwRF{fkV#8G2 z8k9$GsWPV5X`wVw58k>4aNT7J4KhYlj1lJk=0UGt6KaQ5aCFWJ^i(WI09Rv*5%Bas`a&Q^MNSnz-?gU~FcFFcBx*<}|{` zC$EE{O**VjWnzYA9&T3TBX~>!8Vd?wY|lsU{d~BOC_q_6KJJzkK;{)6&qftO)VLT@ zoG$-;NIi0Y9Kz+}uMq3pg6_n6=%`l0E}$6w)qG6muCsexJ_e5EL%B5{%$`tsheP@n+d~bIk1uBW5Sz4tWDvgl_$VgLE%p#Oj3vgo> zA1&=pn#`0Y2gt1W-TAhs9Dp*6uIBz6m)||7s!SQ;l0(J?dqREvz?~;pjv& zEKIimaa!PI5%v(}G)q@K+`0ab2jcVG@aCuk%x~D>>;+3$=<31og$7RcDZ=KvBCu#V zI*%)1CfARvHqi#J9uNc>B2#J=%ynIm8{-c}?`ZI)QXqCJ3!1ID*rk_`Lti+X|0zJ? zQ9feB_{gi{<43rF)2Z^&ku1P6qe4tvTm;EwTM^|^fu;ZI@cQ>bsOR>e^1}ffLme7S zDscAqR-}j*;p{U3x<(g*5%M9os{o!O_}E>-N6&!*EVL28@SOk_=L(RUl#8#`S=>Es zg5JYO^rZV?^ej%hscDIbJqBpF$l#5;7Vh>cW8y7kgo`P`NL3jLBB~h6)#v2R^pRL- ziu1WPxK+yOAgtZ-=!XjiSvM?u?Sz3YuKv+)fg5uzV4Goz%DJmhx!(qdN3O*^nQEvR zRzb*du1*6N%<8j*X1xWP&srjLoh`O9Tpluun?so9fpl#ztZa41)N*GmwY9~f77OSX z8{*Y+9k2(~Ai7Eg7PHi0SF4K2E$S#8=HWnv9!%4W;Sy#8e?50(i1}mmhiKfqoq{c| zGO^zx4_loIus?#2f1GW+@2{`SOLTr6v5=nR_xC#!yl_{ zsNm*xOnzR%z@!5ZDr`ebcsbhN6l2Jv5DlC7Sb2nxjB}jLiI1`#l*r`I-0Mc(-wj=Sbz%+msdxfddIK!AiVeC%m1gdE4;*K>cruoED&UH}8m zPrt7%!pMmw&?+fMqGlaFi#4Jm=ptx*Bm7IZVK-BWgQCUw(O!tfw*)wz$4B``K3AjQ z{JcVdZ*#U{Ca(~SZt~%5T8P~<_z0VokLEc!xZ|7()z2|-8VJM}XHGZ!!v;oi=8%4B z2s0nx_j)ay>Qd)&Zgsp)Qpdq9YWNe!px?;|3x+JQ)XEkG-L8<4@kGsYPkgI#$G{j@ zNLag~d#62aR$5~gH`5_sYlCwfZa)=`=4K5lac*`c!dv1{aM>GD3+*sq&S|hYoFX?T z{1JCWh0qf(cKD#b!VAkd-_I=ObVh@YP!X|)U5X)UcQJVTLK7=$)zHoPeH>o{n{H{s z=m(%$tcT|&{3%&fq!P3w>+ z?SqMX9MQ7K7Q=;h*t5U|388KnUf_i|KW`k)_QRr1FD@5wN7zkA%=pXgqdWRo{|Ttg z)Py6)&EBukz!!H-%u?Wb6iS4PvidN5!Ob-2Sfiiwr;gEq2uX{Et1uZhMOiqdo&(8Y z4(~bK`O9*+bD;nawH|4+FW^@$=!!UHcA)`=uP-{5)TPH9~HmalC19cODYi z9LVcuVqS7GUh-lfu<=LLE;qPydcn{^Q+!h}g!gnHutXb$jT#s|UJHj(G_Z!_ zlqpqW?wShxwM@XKKi>Ej=Yq>hHgF2IMWl~Ak_J36KHVE*5`3|CtPiUC{cs@B4cBiu zVaFA&HW+7&H7^NMxSn3gPE8#7t$}5FTG)Swi$mWTSROY(hQJJJ-)!*azB|sm_s2Ee zSah#V#iNDUcq*0$*S{Q(wBtiMi4Xt$0<3EkU@RABhej4)S6~5}rx)P!1Oa|bDnhpR zR^&}CL+sZY{Cv9)YC30Nl(!cHAxU(Cnz5d!$? z6+)52p|}NnT;=e8U|s<_nLJeN$ij^$DUe+i2M3J+_-S}RgxmYwFvEV!mFTPGLGFhZ zww_Z*BG*^jI6@8AFKeNQ``mhWINi!J8{9tT2)jTJ+`i<8i8($fI_=Bxd(L+4p7=Au z6-ARB;5gC=jSuaid&m?0cIz={SP7?dmFWB#k2+@$%(&_dTT4ewtaFB6vPY8)zOuOZyg%?uV2;Ri$9X9Hqi?*f8yYh>U89~o_I+?cbA}JT9bS)@KPph2Q-v|Q8&NUQ7xA}U z5!CF;Y3rPEn8S!wAH5-e+8=HOL8u(R8u}c_8L0YTjSr{ib+bp{N;4RQ8RGMhHf|{D zVElg0AGn@eem4(sQo88j8R1E)8J7duL2}3&5v4(p8e9ka18H#0;9{IbKAtP`ae7`M zihpuka&;j>J_^uNF2F1ft1G^7_B&C4v5yKdhs&+Z3QBM)lDp@H+aS`q9|23wW82yN zNE4}pbX*109u?#KgF+|=39yjkIg=jo@mWHEDNX_ywH4xmTp>2r32l5yb%x{fHfohVDy=rGriys+4o-9b=?bY+}g}jb%N|)7sN01!_x7a zkZx3iJE%dpTry-P2VmwYcdWSU3Q-qNbaU4@;_C;&@6|Y}x(5HELQtp{iuVQqIC0Gj z59OROr_=&>gN=~d!u26K7+g531D*k=E$b$n64ytWFPBq=SmKj6H}lu^KQkmDxOZ(m zIwoY`5#^z|i;wo@g~*O5LjJ2l{N=D_RY?)fA1*|Vc_ADwa(U3+LhyTv@Y;DREM!U% zbG{PIS=&)N_W*`YT|$b=0c4%5N8N!cWK1kYgP;fv?+dZ~v;bzIT%73U_KhQ#ugxmL zP$`!i2NuDaJ13UoLQ~WVVC9>OsO~hZdlQf6gCU6X^ugp-2fR9DiTQ3O5M8noW}L>& zNJR&M;#!D*r;Q662}?P=*}mKgjpv*(F5d$VyZrH@HVC%St6|)-8dJvx;zh1M5O zRCb5K<6t-%!oo4tCj0?k7J6Vx33uTt|ge(UIFPNwTRfh8)lg;7!`Q~HH|H}Zm=6^M{AKRSqW#C61AWh)MBDS??`38y_T#>b>0v~XO?J7)`C>TJfn{0Jzo z2-$Kzd2zEAzAzQ8#{b39RYq0SHf;;LunWNs6cx#{ z=R!alq`N^7QIST(qPr0Rkx~&!gAn1MsMrBwAa-|Q=i@i;_ro8uSZA^Bz0ce;SIrX5 z2z(t>j3@c~n8%4leVQMp_g;-p2R&dJz6w7+d1Lx)KWyawvuJGy9vBAWojUvMRRZu! zbshd5_P|aRduZyJqOYnRqUUJA;E5JW#M*Fvp^H}vM)>!`4C33i*j(X;w&?ZHS+yA} zvtr=-IR#nK*-%}X3!`g0FuXe#HnVnOYdGU%b8}$*ItNScXXDw19cX@JGH2eAv!r}FmrmG6$qzTSwK5P(QNqc!(}(K|F4YbP;AlIa5SC?QSkW}firgev#nvMeCCh36RYv8#08&2ESTRg#g;>+Xm2w|{|*a$p5urT ze>X(ttw!*Y0PNWuft12n*uG7}k~?{LvZn;=o)kl8KryN83UjXf>LMZerf%?W$xVcurGN2BRd(Ogs?iai^I)^=44H&hp02=auOD+z)dfhoa-r zW)%MyiGg;JxE~({>E;NWFW&}*s!&wU3uYf!AO=K*AbR2!Xfn<*%K9KeYmVbhPcByN zi^q`AD6A=uz!AoFH>bsMjZZ+#qa>7VWGt(O@2xYbNcB#_(y{Rfq8Mb}+Kg94ept;I z;05MV_dewB*qGHwD)EK~_cL#^88fqv!Zpi8*bm5JZfqy)MEh{RqZZ2+iecH=fVGW> zk-q&PB1YCgCZ-Z+ORF%1@q!~O%b?Y}96`4$;54!V`n_wg(54QybH&)*dIlOb51?x? zLQN(10bIVF#e$wD<{WEbeYP4|?<+7tzZ?a2<*?M?8BCJzEpH_}TAAc=|3FDP&1wQ8&U}4Q7y>0@uuVB0)3+yK4`bKOJXiEAOvSvb$#^?65t|#>|Em>+kyj!y{Awib zjEl#gVY!fVYQ+MlrwC9wfRh1vxUHRyxItWN6L+BXOFr7hmLR5285EzEG`2PAwau=5X%|DuN$CJ@hKiaObh!{S5kC zJcX&JPomEE6n5#gLH@`&s4lq3bHx>GX}yUf10G@LpdZlQH%2Xc=4iEb+_F~u4Vi;%SL-?{CA8%$M>GgJen3D(J-!(X}?g{=SAIC}|7pRJb ze@8T&7e?ZP4ZrtYu~>F10g=;_(SKVi-uz6#VW|4&3?6YaLiowM9n{Xu5A9vSr z#=$%fIJ$eH_K+9+C4KPRDHwyKw?Gsa3DqggVNA@#h=6?PEZmQz=vw$Gw_uiH1Dx6q zvA*&EOhT(+H?0!=EGuzqLM1#Ol|kjda{M=<5}~^)@n}>Hwkg&^OQ{(Ttj{6j^?k%` z8=>a%;~}OEIEUOnO{i=;j1h^|7+z3;bd_=xG4Akq&wh;5EJwFW6_Q#j`Oc|^`0fGz z+{0+NT@R_o7IX$2#{4z=G51?OUX^AsmzIcuM=Dv1>@M>MN0QKYXd)i$iNWuPXnd%R#3<$p z?ij~nKzI)Bm9|2&_8=kyQgGR8GqPd=aZR7^)&aq|D7}gO_FHlMa3qx2zZ#gp9M9rN zbWRKBXBr0G%K>oz&fdQuXRNuvzP)Rv7$$9oG#N7#m0P1G-4V4<+^~L?7xwOE9^5V* z2k*yV+SGJxxSNaIal0`yw*=a=i!pul9@PCT!pO%(SZ`W{B*w;iJ>P|$&V?{2EyCfy z#puZ|W{u50%sEkoSEltS={yUMx=-kT?F{U@>M+@%3J;V@ap+1xGTSZKCJIBtOUAIK_(86hKUB_!p-v?kN0kpCV#_}K_2N5zpdWLHt6;$WOZE+K z$PM*@!c#v?>&<;l?nY>7h2XMO2*ykJBH3^adRjbiTHY3U$)>P1&_kDj4pIi{!j!qi z#09!IRBMdzGgfH-VaIdZO3eA@iy136;*C@c633;%AwL_gojLHFnFp^yc{p)B2MYId zpkb4Tw-a-a@0f!Zk8_~BA_rso=fN&-Cw49_#AdTSxc{gEi_@C1G3Y)PZ4%?EPZf@? z*@I-oLOk)xgS6WY>^r{$=W=(zXh04mLUzDYG8ZL(vXSbVgZSt?EK|+J8Lp8XkvovY z=f3em3`}-};f%f?_t$QSnd$(aznnwz*a%ma>EVy94t}I*BT~@DnH&0uZ#9KefIUVf zu7uyrwGhb$;NhVVwEJ(sh#Nt;))5GUTRzy>%-qNX)=bY^g`6j=FnF3TvO1$7yPy<< zb&3&I9D#xIYcR5l@wZZY?4IF8u%^9Mi>?!#7{LfCG|LCJ8=)KSlZG|ytA%`*{Tmw_AGvLLm6J9O@5 z;-LoP&?mQJ>irD9OEd7FOgbcO;_;X@4C5KwO`7F_9bav*$;lMvyY=zZRSOA!)zBu- z65LzMaQ}cBJmS=``nVo0$eZFomILmbaD$PLC+5!Z!Q--ZaI^G>=hSs5v001oZSEM^ z?u?iHT@W$M9oYfi*tvi)^p--j*c71T`eq#U^h6fpp32ORj8AaHN3jFKl~;1DUWMm& zYalzu3lmB_(UtFplq1eCv$w|D|BMiAro%HSVEI>IADIZxUZ}#{Q$RzP8kCqT)cebM zIAs=ib;bpv=c}>2aswhBN5OnzGG6Nd07c=h<0M^v>kX^l8KTP8MxDs zjt_fM5n#=jbbAKmyE0JSlaAf_{JD(Z%NR4x-VddnThMf37fyZ1hg+Xb=sUy>H~QLQcME%g zS%>%2!w$jgT@lIo9gY8aV97@}42t)_*=lFhF>bVkzmE}X3}ALs9mD1m>rs@UC&xau zzsm5`P(dSW9DcQH@yx?q4A&pUJSXUkS`F8tAUs_Z3B`x;cxaY{qhnH0_%Rhqmg(q- zO=T`I9gVeVa864@!G$zTXZ-jP<7mG8OxAA6#3;Ti#_uUWkWVH4?l}$dzj6fpEWk_F zhz#z?Lak~#XYZtOkDP{nQ`7LcBNc1*rsMGSbhP%Q;XCsK*Xz&pv|I!U6E*0X8bW!t z1+vs^VO!yh2bVpt#=sLgTRj+ecf*Tc&Tu^8h#D7LyuZ!ZeTF^sKDc3zL>OFt?u3DC z9uEHr#psbPc%);C$kA44=w}0EF=r2GIwOSZPl%)^UTj#2wB(f-|IQHyep|zQrYT&9 z>Eo}gI;;a#aKVHB9`<)^TBm}s%w-N>Uti52Eo_N5fR2(0k{@y=3%?hgrU57`3&)f# z@hB-wCsyLx7dwdOn!)$4F8sAMAlxSG9;yllJtYife73_y z1q%$6asP`tq9kn_HKPU5vM9{EsJBPD28}w@#kS zqg>?kWSq@bb_cTmE5Iq9nZE2g#r(m3Nca|D;F=tqoS%vLi!yPUvFO4vDQHhkK^5yD zK53`m#mF?s2#g!|@mCn?~f&r$@kpXEABLznQoXGX?waj=BcU3=u2xM2m?)?U{+Kc>nF zp;H|3^^65V{H*Y-n!PcMKc=a=V!+>E1YFMrjmyJXy$#UOX3zFHOAHuqhMg7`*zIEl zzhjIQpK^k0oEw((;Y^8s{M}HqL7HHNiXTRBWkXcWXv3!isS23P^ZhhUeu>Wb`pb-X0x{uhfA5DMDeqGJpRRkbq!*^soe$jW884phPbd-IW=rADhme{xlrs?@N$x3byjyx*#&>j!spvZ>8G~mfat==% z6yn067!`omjJ100vd0E{*8G1p{Qo?$35J9dlI(ZfsEmOgig?pU3FU*8v79};YgwD~ zLE8jUHWqj{jWZw8oltG$0!0Hp`+p9Qe`bf2ZgV`nV~QKUO`tf#92ztEZs@ZCXXEnl z$}A6ak^}JgjV(U#9YZcg*dkhiT+T1s{U2lW8|+b+<_PPxPFNghj~B*v=;z1TeTqhy z|4|d_&YbhtSD7^xO00!qzRFJl_x0FcRw;tLwL1K3^wI05DFPxL&@$EwLoNrx_h~o` zhR0#4P9hdGq~KOq8m1gcL*DsROk19Yhey--er3*S40C&*6J$1$LPfV)ifgn6JphhXomUEu_P3C4bJMG)QrspL{G8U3zJ-;QF9a-~Li>afT>dkM+eS-FerAv1hK_iV z#dnE>1BP|kqV*8ZuBAq(WUjcimx#56>`4~O!~dlM4)#?5W-1`>j|wIvtHJ6TYlog0 z!?(&7TDhLwgZty5Ry%8Hyjg?BwV~FJwIZx*eVxlSh|g%= zwsfpw{nWE1SqNv`H(*8~XPi~wLiGjwaH@p2O92$Mau95miLO~0Sk#q@wgw{(Ka0X8;E`zIFs!f_ZU-+@nWVv zf=8%h*L49S&MUxpggnx}ErG^idF+0$1d;1S$T+2mA4d!zk#EkKZq_hr=PcB5j%d)c zhh>Ki{H9x@UEKsRR}2u7$Q+WjA!M_;Ctd7|*36waCXtUB@;6mp)s*I%Q(+^=M-3{r{Y+2Dr|Gp zP^p@RK*mjr-=ttAYt)+Na=+-5iUrrwq3^aG^DpLLZF?au|ER|Km`=Pdt$}TK5lr{y z;mn5ZFw#tiXMP%Vn3qk8Nx^tC#-I{Ya6~^1ds&0^;Cd>~{z(M|bM917B92PMp~`v- z1}yYLeU=-J9JOU_urc=C(ni%;5rRJ|!ozL}e!rK6Y>+He`pF^kz#^ELvDe)e*!|oP zvkiF0O|pT1m>r@O99ZLFj~f2Jl%W;w%rrr-2z{iS(?!rvL%3cthn44g6nx9W(w+ju zuULZvUpY6WQw!6!X~Au#9zJ#$!86JN!@VqVsL~c$C+wIjx5BwQ=8zs?%(#m#rey)6 zhb)ESTsat@nvWlP3-D7{7WV3_o13r%qjsob;4pQBhU&xj4P!np9I@rudQ4`lXii}a zb~`6PW@|Ejv5rH5-|w6f=E@JIvSuq4c862opv1jRPb%V=_q$S%j_%kjxOOpj#kgWH z9mGhX8%3WEV(5=zY^}+|U1Oe&ZPIaS7wd0Uq%wb*ih?Dn*vq`E#i2C3xy4-hC+=DM z^1LuB1uv&2U>$P`H|n-xril;YGkIq8V{CpN=Z`3>V`i{2K5dl8_fvB48!C%;0rT+w z(|lM2%3@2H0=@_UvDVV$ju8aTUz+1)hh053&h`-lhnQj5j9x6d#BjX3m@fnBne0qeCm#BgMQ?;>sfB|;DF-CTOGsqX4fX13) zQL{eE{%WIw`Fm5-rC97e50mAm;*XLPrgci;!Ez~hWlG~f{4A{TmSueb*GWe;h&=US zYR>hK{X3h+Y=CU9X!x1OLG6AbJbY5HXif@iy;G1{mI9YM$v9HV`VQGt7;*0t^fm=g zb<(h-IupSWd8jQZW*+4*&MbL?;ckcUE43IZi*hl9y;qHVZ?9xs)h_8|OyA2|WYc6= z4M@g}&&deOOhIvlVp+#W>`CO@tsMLeh!3hMFXt zVhs1j{zR-YPr}^!$(S~Sb=<#`ar}RnLud{fJ&Te4rXCM0zr*8fJz|9ttn|*o&|~RH z-j;%VaT4|}PsG}`1h{NW#Qneo{yrrlgMYrdI}vt25-@*t98z{f;$7S(1aI# z{M1o-WhrJQ%|@$^6w0i};nLg@7#KJZ?XLYHf4(omK1e|QJo`uvjbi^6=L^nUh=LBz z%($h2vpffnOwxwLN*$Co@?GdELfmu_EK}r}(~{$SY&mp{Q-qDb3zQaB;+JzfW`^3J z_q6FaXgU#!Zzf>2%Tx?|J_iGKEW+#>Mcl7e!Ri`i*gso>EqM!ZvW$Hvdxm3mLobLG zF45Nd8lstnRCjPE4fM;W{RSLHWX;~9_*QBOy+LuBALxl$f7EG?!_RwisF-g6YYom^ zbl8YF%thAyjX~bCSX`2bXI*6+h98RM8Wszgp>eQHj7R&JM6^6k#A4Rk47JRGOh^fo zRK<9cHAZcLml%Do6l3Xy}01xKVdNv_QX)TOR^)P<#5}ds=0ap|HW0CValGeXP>wVkkPRjurCR;!r`*P^- z^IUT9E}&3@8ai)um9!uIpiAbXU~4RmlU#2n&1cW9feP;A@+{_{fZbgSptN8%%pXZ( zweJ)RTR0Kj8q;uPvNiPGuW(-BdFb?A2&ad?>D|BA6dm%H>Muzkr+y@UYfMC5m^9|k znGVy?8QgEkVD*&o*s^~Z*0=P~%GKvdeo-++&-AA!PL5PkXF#^AZAj^*866q#L&b7& zH2cp^iY%`qlQUPyQ2Qe(izHFqJ`W2I>0!YlZ#0b#L6-GqY^@B3C~_MjKCnLT#5SBM zIbI>Z)t8o2jyL^ zB)>ah^tVTY78pv?@AngFpWZn7H-0ira?qf6t}E%5dN}!R%%D5tt4L|WMOrNTp4!5P zqc(OS9{jV!2FB?WRx`HT=8w~Lfe5P!#HRWFaON8PE#4o7>jUv(b|7s2Y=C%dI5HXg z{yLO#u`kT$+6+-!D^Z5qncRO?#9+$ztq6V@4EGm-DE0A&wp9>hwEUpp&Aod;Fyx&A zVfn@neRbC316JYkQ)}o7OR?QllJ(O+=_OlP@yAGevzu}PnJq*rSgD~iGUzGTL zphbP}(eKz+y3xCi4zzOanN=Um`(1^IOIwh4rIh?vIC2)`D!Oqyi^{zZljF(``a0wV zH5C6PoBCdGEbE7!wq7VW`I-8E>ZHVxjno#IOL4BYWIuZ*O(@O z3U|c6bH`y>4>Vp~i6rtsNwNnDkFUYMBIYRUH^J?0EJ~a6QNHs(EdH5~U&mt6#=P~K zIDfRZc%zT@Ds=4hMEgC~iS2X8%ULTqgWVJBoZT=$%N3gpT=7ZX9KA27W8m`%Q1|*q z+qPb$0oDg-l3^Bg90;bMrS{}7QjI>hPb7n=exy3{w-7h>mvCock6=7|GEE!pL?GkVi&*PlY(rwhVy-v>hb=|{rT$wR3iZx$tgQ=sFmoKt;o zBZY+Wb3Iu>4o})CPoqDgDirX6J%=riEn!(|gE#rM$Z5308f!Z!J+Z^K(RT1%Z3~~* z_UOg4V#@<}d};B;c8zVwe#Y9VEpH%yg7sCiw;{xqwOjK%VRhe$wJHvHvDF@q(soGf zWrs%>9Pr?X6YluhL;H+1ww$%Z1nA@L*rj;&p)YjDU7=^U57O;>+o`rPjCP8Q$fI0= zvtOmj>uMib+xLo4-*sD%I@%)Cr#}~N&z0Z|L89`aXu994m<&c7r^$JDNRH2XYTsA% zd-p3+Ki^GR`%Y6?Xf5r!nL`H_y3w)IGSqkEB|))d3Qg=i1#fb9la#-ca3pD{@GBISFOLia%Ec>yyLjy5?jm-86^Zuhl2>xvMF#ZwgJETuaug-qMcR zIgpzLB$#pSPi7y>DHF_7FvU9Vix>1a#fF*{`1aZa{u9hF!^(^^z-`bl+Z`=6{`lT6 z5r125;!j)x#=Qx^R_2b%Va>TA=BV&7#qBT?%n+HN*xZnNQB#OajG-H~f^}pDFqYM0 zUk%|y)@)q6`;vYHAE0ZQS){TejE)^JqPg!ElhNf#q_w^`P5p9H_}I5tNd22GOz$Za zuJ^4J=H6%*T7rg>*>+c2P?SJ3w8}`9PLksKi?rj+ZtCl{g81r2i96XKGO0k1b^TBa4RI zucCbyj?$gPV-(Ca%XnZtHH^-sAWgY>2FFqHL6MBO3@Nj?qlE9`sGmhq6-o4 zAc$CN2yc4>JU*?3M~%ApbXX6QVh!NcVFHIp8=RiwfkEdY@$15AYyoSL;yf|F)&{aO zjdA0t9yTTGLS~a5>%;W0ub&>2J9M#hhyl_pbrJVZ3;De?(Ee5)uZ*TZb8;s+Ox;cA zr-##xKqqQ=CQqxsPvY$1;dE!fcR}26Lb&=NSqNOSSMVPhFZ>KE5zgjZ5#ApdON)by z$T?#(^}U@z?xUo4wu%OOTbQ)khl z=4F>J30Ge&Zmyly(mb}nrrD<8k?7R^4k6>k9MYKLPQ5ux-Kl>b?chwzNpmacpw|Iv z-?*E0aem8nV;3q6ktJi*w}PeIcHv}rxZs}PDM;Suy%ZcGsVa41lf@9`;0T%tK zm&q)8K2wWads~y^j|5ur_&A;4Jr-s6WU((%6-!qN*r=|F{Ck>ciq_zaKMh2j(tzHX z<@g=0fpy-jH{{%l%OXoe&2>TQ@l7zYX~e$PO>kv@fXYiVlsOqd=9ebSQZ;#QTaLx@ zTJV0tcS@HQOs{f2X3jErPvN;qRs|6wEQjpw|kqTh3A(1PAn z>h@3g)Y-uq#e0SMJN<;F`XHfGY$w>ijuzyCvV{C~?}gI=OUU?_6PYberK`HTY3+?N zlKfdkcTD!vblE~0UlLDmGuP0}cMB=cS}b^fsuazQo7DWn_C)il6&B4l%V&spXpa^z zi%e`DS0*|-`rdv)Cv6~Id8$V*>_RAFb}H2r1rXbf=!)k|>K!qNJk#rhKKsrKiXMA~Ze2&AXT(Th zpzCo_#rFrI;b%oc#Wri4|^t(RdEuvXshs&$Q^V(&^(+^ZOm%r>hx_eW&P(Apr zaAkxFN#6FR=z?g9mPn=0zbTZzHJwgAilIT34%C*klukGfAe+hpA!@)NVS-L?q1~j9 zu(>8nq%pKN?*$np%rWs5e2n%8KFVdnv7h&Zp9y{GpVBOPkgiWtl(Xpx4S@DnY0N2D zh{p$)U=z=AAs;y>?ScxL^;M9~J_BFo=oGdIIB->f#BLyLoi4gm+4mw|gNd}8{ei5X z7MtPWAAO`fQpY9G@ui<9sm_=Cu(>x?K;G*lF3a8R^wqonZg z^(~QMexcCk{(C|3of0MPbE8Anu@pUjJE z8*}04>-oEzC!CxucD`&W9zP{f{Bf6t_|Uhe=6g$vnq*h13Jv0~LSCgjZ7;Q@Zz&NJ zvoL`+`lgUXWdg|;1yJV_OWKgRh%_X=2#&upgsYy9MP7!RL@~EUi>7ZlC<@X#Eoumt z5u9Tzg!;&2VcxI(g1h5mL5}-S#d{+t;eifS73Gi=`~T}ArEpJMmhsVr@bX)N>CAoE zFHpp*JO%uyQiRkX&Z}Olg3%&^ETwP~5bHeZum1GKcdZ4$Q}h*eTHPlYlsA2OX|5A~&T; z)aBfl?*6?ec&L^OKkpU^pMAZA%6e%*?p}w;b-JOdjeLWu((-E2*SBv)Jr%yfJM%}v zJMm0QjZ}Tmn2c60rt2zigs_c{f_rgN)3?Cp<}1SHuk2reA)>l===ecSH_f3b-xC$XLaly!bd7n(O96sZ$OSljWgB@(5ylWS^V@&TuVy zd{_yKnv}5dCc$pD8eWdm!O&CYkdmvDi(jU zS}s1Hb*Z`1ue?d#Q%`7V`5;JoFQPwvt!UNgP+H471;)%vp-CH3se5TSX}r>-6FrOQ z{<=qu&NmO6@yA((aIOA|zJ`Z1Wbr z%#s%RtqB(WF5V$h*gr?)|M8D#>x$VzcR`j=mH1K!Zjz;5Ox~F;<=sl(<7me5B>K0G z^D%XO=zZi03O%%d+WP$#+75UL`x6tIn+k3ZyH!8{+EA1TwZ1_mX>O3PJh0= z>E6f;!D{Xc;n6b{`f$RL#=Z}y<9`$Bk!Kq1=$A_Ssy378yBmF+qeNCmhSSJOu^_cp zUr&x3WuIu7M{2K5H7k-ruVX{)F)~kJy=me z!=?;G;^wL74w{c~59OGbQb51jrBMHRnGM$j_dVrLi%8JY@EycsW>A) zQPK-@`((nhXB9MVGnZqei}EOS#4YD6P1Y5pUR44)DWiRr5}xyO3?9!pr+t@V?$jlC z{(*7$@~OxV8vutL2WeryKzdcbka$rA)pEx3!JyMZ`@~G)TTGIWs;ny@dX(_6VuWy1 zzFG7qS3^*(Ru<0576?vGBk51Ki03Fby4Da$buvk`es2oB<$CX1yM+!|ThW=Dvh?1# zKdF@m3)<-in?CifXpR}GA@&UPe4$Rk%=hDpdIERVZxudt8tm)tlbmlBdV^ zHl!gJL!N^(>HD=DQg7QyT@~51_+T{kec?kj!*!{|V=^uAZ59@m9#U;GOmBWZNJCs5 zTqS;7eNcSeYoU0}T;1m3zO#kQoR`8}FP<-MxYOh_v6Ol|kEB$0Q$;;ziOwh|KkqWK zui8llD|vSl9I0oGEYA@VlsE35pssyGa5?uxI5M^`4T%{;Plm|Q$R(;&B3(7aH3lgWOX~(NG6$&|xoYpXFHdPZQ4~v=PSI+KHSilg&EY*DeOw zx7Y|3vrTbjj}7l)^+b~7ChYVmK-9M|v`$?G!HIE*L=*VL8A76lvmPy$BVf2T4sl+l z&2LS}v@b{fLJg#H|M^mm^Y*VV0wv2}?}uOXcwRf1jpOV+^X+tXYBX(7FsCnf^k^Rc z?OXaxBIiRtgg;@Qg=sxqf@ETkU^IOw-Hn=0>l$rnMp!Hre#@lRK4mn#cNJYwt|H09 zd+CSMPEtJ0xtnTURL~?()fUeL$)ntV97D6DuB>=-=w5N2{2p;q#w4+Ke6Qvamz{(= zog?V&X%lLh7fYT2+sS-m1vM!)QZI264X8d&gKBEY*t(3i{?4XO)2;L(&z(M1X)tak zM+Z}7X#eM#^gC09!2=t&nlnlyiejFnTy^GajzM>?C7E4m89yRYNFr8RGd>Z7hmp zt>2>sc+qDBf&<=B;)07LIkJv?iuaO}OCptdhErqXI{LiPfR;@WQO)f+B%isECcDbh zx>gZw%(bT*3;axr_=;oVRI%!r#_ak>OEgV7iYL;|)?r;j8 zZzV`HzH3SxH&tvjYmeAGb+U!{0}IedD@=_x z!|Fbk%(GZySqSTM%N=nlko~!iKIkPC3WL!Jh-IH=>CJe2_!WkA`@A8g;SSHgc9_x6 z9>IOA@$8Z%S{7R1?N1B#f|}!&v3 zOoMpOjh%Tm&FI)nL-RLLeq9Jn{}n;Kv$(z-$R(rhU8E`0(L>(#bZ}ocjkwrDvKRZ| z?vhD(Ah7@{w;40|&j2f8tntU$3S-CFU}1y}d=2e5)5wl@soFz_Ie&xG&N$%bj&bW& z;qCi%jJ$`lmxuTJ2-Ucyn22?kx8Q;k>*hbK#^OJ2sPkscV>NqOJMAGUuwQVD4e#*e zoo-3i2*1Eu%R#0Ht1y7uQ8lD_EX04|j3e9jg>S=0>W=NA3D&&VWk(HtKa$6p*(vnZ zGKT)Gk0GDRb!5Ial#)85>5E%BO+UAfj`b<^z*0ohpS=v$ZOSnQ)_dKOz%mpNT7>0&NQn+|u9#zg7*y6eFAv6fuE_}Yb@=8=zrR41v_3E7*MQh8Po4P2Z&ft7C4K0XPfrh}(eon-^d~c(Ci8B+hJ-9?KemhB@{Xv+_02Tc{|4o} zenILfy>LI5ch{Lu$EcMmung72e?Lr7Ho+PLKiXmcYI|Hh>3|L`7wEBGcd5BMs%l)J z{>T#pCwsA;*9W4XLD1hAiF-*Y_z+fwkmD&R+Y^Z%g|>oc?w9;1jT#aYly9f1XVd*PJMGx{vsNy{yp zX!+P0>WD2Mr8Sw9T%Sy}^J3_3P6A0pq|@t)9MZQhqm*Vb4emTg=cZmGm&W_Fcim0; zE!{yL(XF&@cOALS$)VyMp>%w%3axFc7e)rAiem5lX)fBFFRqg+6F1x)DmJ@i*))A; zo^bQsOuApXk}hc`ljnxLbn|8d6-%F`Wg!>nKI6XU_Fkge&Q@}s+&~%m`{;9LCfW3h zCiR#oTA0W@+p+{&-jzt_e&kbI|8m+mznMOLx=0#+Pf0=cAJxy4#CWwCkd#s3Y;HX~ zi#6jcPJ4V~?dL$w&n@s}ZP!3&%w})>EXkG7RN{PzR8NTR@Y$=aM;mLVCE3UNRV58u zTq{_2n2K~`_D|MrfV!VAN_<%3-|UG@U(S@6;)*C)7YttM0t3cA)8;zCI?)=Ic9!@) zj`!s1>tdg|DqH|0nEfSmTlHjFSihxgvoW&b~$K^qI{&+3xgFR7a z?8+J8t_V_gVXrdpWa4~^Rfp}cgfpI5X=9 za`dvru zD_iJppVM?|Rx1rOXeQH3)f8|jn-q40Q@mhF(_03U^}i5dWa_7;pyrw4!-*B*m<+La zuUKE~y(h7`Cn{28qI5;r#5;y2oU@^}fEYTWwSz{DE~5v22Pyb-6%F07pVTLC#=db5 zJ<^Y*$-_2~$95-5cxOtxYnD^fFGH%lVND|KP|{zYOeCL6el?YJF60!o4SGfyr~Bak zg>e|BwFrh6*}HLHA3d$CzdCP@C}|tcX0U?5nqG%)*6`kQ#Pprcc=p{DqdiyR#9qeF zBK*-+8jj`KoWCPgf?h`x(C}nL-! zGih$HqoF@^sCSDM^G!B1R^&$w*JDX3C!ZQWGxx_@fW4ZBN%XaxbdHzOAnBc?w=I(9 zOXN58BeO0Vg zW$AE$A|?DZrS0TNdq-}ffm|DhpROTQzO&;(K9jN8NZy$ti%Wq@_>jk$WV?+JIhOaw z_?qKDrx`piTVgMJwDQl`;&+V$wstunrpy^JULN@Khws~tP-NbYMRG6pfu_cx;=yL5 zS*^jGIQBffcEXN8JIwyc&rxKDa@N9)S20Da$Q-*q7^6nl7@6#)a(2_gm+PE8lR6ik zsUvYUy@#61uaM)%YC8B|9;w?!Qf#9;Rrl!8{w#rl$Hx>`IwOzzCFWA@%q)s&ilm3n9Z6@1A`O1?Qkb7&A&ir_+GKO9yZL5p zu(-^kLp;swoj52ZLu|8OR{VW#VY7MXWKrh1m4bHXDUB3$ z&+uRxAUJZ*r9o1A=g{WlVf6mb2SMTYb-|SPJkilUwLa@>;i?){Ndt zd-GCg=D?#gk@Y9r^d#^~kMn|)7*~`80(7;p&5!4+>4xZ(GKHayDNe?)msQCU`(&(e zY=Jf6Gi=c(-WlH~vgeNPRjD;muyxOYRedD$U4h7+yb98W&d7@3{{FQUM#u6FLrV*s z9b<+rH4|j@F~W&whS(7MFyx!E==dYkr|3@gfKbXENs`1=nL8_rHWWQ)5t-2RZ<#~QI zdOvFzc50FPg$eZi(g)!MKDvh_?0Nogka#=ybmYl%g|)cNN|gs$TCF z4BO5aqdro4V?Gdz^%;^N2gigWvVIeX_;aAUqe*#vrYc2 z2b-H(aDA@IJ5N}n=QKM2$9i=LeSn|!6V8` z2pOdzn74Wgljp4!o_r}4a)#U!_79Py#d3`O%ZX_IS`p2BtxVdX(j+lyBz2dx2?@j3 z3%OSY9=UYbrFq$+Kh5`6dyAuj4~m!7^=}ziJgLR@ShsjXbFMhl-9;=FGD1AxCAV3+ z`d-rmsZvpvvbm7;pkBa#Z9=qqmvHiWt+2Usub|RxExc`#5dM5FQ5{_KUt@i*Q%xQ@ zKby)o{5kq|&-SJXlLMQky=fB#?$Q^6N;(Dq-XqDo#fvn(Dk$XtIy=*#D2^zOFDxjh z2xd_PFGK{3fCz%SGtUD&ibGvFgON))geXf1Vvs`uh(XRKWCB(5U&Ap6crEnZ9N|%uLU`|L?i?u_2F*A=umwH& zBX>9-L7Bh9sP2I3U!G8n#&loGPR{U$^e}(WH>C_Xs-=0jI}*Gp=TTcX0~!Y@2R1PS zT+Jz;V^suvuSee}CDgybEy~Gi@q;VjK2X?4nTM|4;B{ju4BqmD3l0mQZ#iYmc(U+> zuB*158Uqs@UbAynm9q&c8`uSN1#Fe9j=4z1dIg3LL~9?#39mlru06Q6MA@I6t#m!P zS{eEKSY?D~j7YiOfXRwm&$QbgV?x|dF}<31nWvX8GkJFpF*kJAGArZPi!xRwDSL|c z5Zy*2tX#YVFU>lLif1=*iDi5Ve;du1AHMUjVBf&4yiTLIS za(h#ja-gA3ICX}bDE*RwDBGg5Hci)rG+ zuP(cpe z_aXNd8xRjZ)@}=HoTXKCPN*nJ7HKMv*WNolf@D42NovMDAr)a`u;oA~x(pt`!QOMo z%T(ivmG<1&Y5v^QHm=_MHtX2#?UwOu_eF_yN}Gp-YXMvZ)Ar)2c)iR`82 zb`0roCbyq8k%O`ND5;!^VOmZYRxd_Dl_$>9UxJ$#&BN4sE1a-<45kF@;6Cj)8Q#ua!$ zT!yt4MOgh*ffD0vWCK>BHfxQ3*1^QCtC5*}pn#35G6gqj8u-SSf?#wrOb}dyWdBn5 zYEJ=Vwkash6 z5BV`2kny}8Mz>Z#>Ys&RB|i*SkK|xHTMl*YGT84d10AXn8e<}Z7Rp34Ny~+*$&_(n zco_0Be}H1WQt&)^4K{f2xKr?PIZFZ!k`b-h#HJ zIhdB1j|Z$u(2x3pobOwLlLF~Cv0w!T868HwdO7Ni&czgYK4wUZF;9jVU1d_p=EL zjSsbr{btSQtv^05(@$DDs_{OYi>kGq(9v<0^Gu;|#&jp=nKM4`=;J9GeA3bWUH^78 zbWy|}!-m-mRT|I#@^-> zd|dbLJM6Z2my55~upo_2F|JNwt zExbjV-)H%fWlNUn8Z0+lrn6kn&|74SFu&2&%gpf|Mx8&)) z|NBNN!drTSx6C3Ni-}@i_h)bK=YP-tzHI)VHd7Xu{6B2A_A{P5KUVqh-ZlIMor}`k`ln&s-J(ElUb`3<5Ut4TiLcKB#%mih1oPA}wrQphGt_C3Nt`)lZPauzqlmD8HkcF3)*2Q3~$ z{=7HG!3nz%_;yQ_F`}90Cqw(Zdh+rKLsjL^VAi85vW>Tj6sj&IC=9z++X{g zvUZ6O7Aed7uSt-1WuY)koBa>-dX2fZrGi{W$piW#>I*0anDO2^%Jb&l$|B{fCURZ- z{kcz>)3}-MzA}Ga$?|kNEqTX8m3W4CqR9B{+1!A=J9p7pkvqmyA(i{&c)Lezc+-`p z^0v)8PrM3daPl96I7^tpS(K)cFWFOh_v&nTVI6Wj>x)Vx?Dkadv498HXg8a?!;2=C z&!+Q4n=E-@#j-q2@nJ^g&ooYSe*kx&Z6fEr1U^#WFmwij-h`WISEa<^ubyuR`$h3$ScrKbf(4CDy68;i79X_;kRTuKXAT zc6Ik*!{IV$fAf+=AMr$SM~M`VyM190&9<51)UH3$Zr!rRF9v+?bfYC z*8B&v&xVDqc7GwgvH>dQDUfv=+wrX12b_4b4$rPV%ZR?Hf^MJB(4AQY+mtpEYkCn~ zx+Zby`?4|P;3g7WRt?9#eS$vmYKXDFz{ISq$DQdPP`~~z9y>gl+}?2u;=BKXQY?i8 z^Dm6sr5kuWb}F}Jw;$S7`;a+1%V4$j2%N4cf}E3=-=fwj=x4RSq zi|)f{SrGJm+C`0BqVQpS8*VRIft~rUN#p1_xb1ZXlB8YW_^AZyLW@z2+mHW7JLst1 zd9pIy3NqG)fOhTykj+XaKmB?zVaI7~Q*;2greK)2!yZP?8UXLoUQ%;@DYgFemabLK zM9sZpMkjl=!P5D$5N{F+D>Ocnr5BRuzt!FtYQ7oMnhMxGoz_qr8w;76PlKM55WL); zO_j`xG5y6F49Y4oYArYbdn*bder*`Mp5jY%zO~ZQPaKx7-GJ$lCCsVckbJi$jXe~~7UxJscvg!Sp3GiJq0$diyLC;AWxU>ES+udo8 zc_+N^!(a*xnfZYntqX>8Y0(h2d>h=E-f2`Rnt_sjZ|R+=#e84mMg)uu;qRvnU~wb_ zKCQEdwMM6L5g5{KiWYEvsx?gSUJBOf98uN^BIywwY-4o-ZU1P3LhX*^wWkGq`@R>Z zcrSwS!u#a3R|7l#R0%6js^c}IDn@e2a>)Gd1ug!juxVN+SvR6V-9j8t@0tL{Z@9}$ z543|g=`&zfWeT0of=Jz)Od6r@h5gc6n6AEtG)38gfp{9E?sJ1V_7NnxoQKZuw&Rie z%W%7c1iAFw7Mi_cz_Qr}#176T-#S0hX-U$Ukn@>_ou5N`U)jM8<7oIV+y({}#K^tA zrTB1zBmRqDiF%5mC9+^a53y9Ypr;HUr^RXW-J2 z4RC926Ekmt^FPus;ht!^3WFH#iZ|@{3bD6Sdk^_le+imi~#Sp$kT0^VH8aPy~N0lrZ$j!n` zLfUhPHM4`%)&!6!lTIed)`qr?mC-5rm)LsM%cOB!0(4#eBVD<5WJqL$$!kuas$XPq z=x`DhIblyO=$lf3 z<5TX@?|Tm5;&rd-^lD?${}Eu(hzkg->VoF!`NTnajOLvSL))4WiqbR4_`yYR(8mT0 zy%xf`L}l{fqcA3kC}XLRC_YnEA@1`Qf&aJ_817gMl@F(rH_cyZ<)tvZu~rCQjm;ey2njMrQ3-#@$@TZ*pL$Jenrl{6@w|xm&mAI z39Iqw8Z}?OgRU#T!I(Y_B3()z#Q8uDu^mYx`)Y0y%_;>*(pN#T%@3&2?x%fBJFQ<;t z|FY2FuLdT)6C~TVOM{2M8tlF^0mgbJlWk9u=>(nIc-T3CdX8>pLLPh|(k;{BaJmfq z=lz#Cc>EjHN{z;>EsJrmE|V3E8zIrh4Pea!8Mr+Ar_r@%t&bsZ-*~b2rz_wRJ2~9XkNK{NUu2nqGfd3IW@i1QO=P9YPZDzZ4c!@e ziB4YTL`9t2+1WSs==QPu)aIKq?l`9a4-MAQgnyZ+vB(3@&HF$-#pCG8Q&(x;{rB|1 zS0P*x(?yPH`_W9fUYh>Al4dGYQ8$$-xbLDhiuSBVU8fRul43W@dy{B%^w1I}^muFS zQbP$mU=oF8i^9=n?_@ks8cnB6QY03seZ*CAFA1VY=x9bLDump`Jz$J~JO!wlSSk^} zB?R#i{C$3&FYColK&!dkczdxRuI|5BtE1UKLf21&?%3O;?P&}vWE+l&8KRu3Wdstp zT=vD=7LvnFhl`cB$Z}zy+KGQhX+Uik?mQ%d3w|zR!@SRtPj7|boX0uRVJAdisjtFJ zD-JEy!g0r`BQ)sM9&&YF7g=4UNMxo)(acgAy!$W=TPtHQ+Fu-#awTfNE}Ov|;DU_& z7Cc}#3*@lt^Gxx?DJxvjErVtMu@uU#Q_ZP-I|&`8!Z<{n= zbxS_J=#(N~a%#cA?-nGg&H=G6)@aI)69qPYz_%YB(BHcLN-WtO6#7jYXy z7wT|YDs^~AQi@8*XG4qJKL|W?AO6V8k-GeT6izVVKF#Y#@nKn_{;LBfr}E?Q#W^5I z#A!+#hmKx4oGUcqX%vQS_K%?a%TqW)4ujRq9%`cW4mbAA;QG{Uv0|MTxX-Qwb!Qf0 zq$J57*Fwzgy@fujVlehYBb}mU4PW+!fn8THnb@a+er>0)-)Des8kfRTK?x*ni3l7% zD#I3P=wkTr5$u?!jejPbBBGv;N%X*1(ji{T7SA=s@k<#PI8uxgv-jfjJXY3#x|*m~U%YeS-FH>*#2cWA-H6VpJwON?69 zhTydfGi`|6p8ICPL5pC<60O6Ao)GA|^MZ=;@P2s5z#M zcaC|{?**1**fWR3KG{ut_gEX5X~k2450aP>=!OplEO35v@sleeTdvp-mO^cKwz~FNem`^wl(zKcs@_pftWnK9GLJ)SJ6(GSD z7vav>Lzw*J8hHHNOtK6M(B?usE~rSse8>IFTk~|dc(e&dHfBL$RsczzuoAydyMnJD z@$l;|U!pcA3zolV0-Ze>U>E4kB-*6nqpmpYio1*}*KHyf?w7;Elt*y=)I~VD{sr^L zCkXcsJV7nNWGM`m!~nUp7ryR#MYf6*GRbK*R54QrZ?9iQ zf&-)BduS=-dX<3JG+XddRAN3pCHUFDhaUP~P~#ln3m>jkf&8fw*mG|OB(NPdyFA0t z@Wf9lW&3x;BhNwOLlIa%9w1j18`7iQmr=rRCWfaZ5pC0}uuyam)_8M}bEkqd zsf3{t8;x;e$FSBUfLz^q5kA63iC(eR5XVbu4>k3E>MMJilI%Fpq zvd7ysvAe;X_P*#QqRSIuA5#vi02QXbzn--LI6Q!3qFz)^cvL!FT zt4u)%_!3Dgex=}W>3>w)QW%Dw6oGxu8*n$OhBMbxp!CinRNH$O1NF4AI=_o-xRnd0 zKRe)$az1DTz9yUI#nZp$|ESxWSXz4UE$QVhgJ|kQXjzd9YYHEeqcgSeyK_5+*Db`? zSEj>n^D21u<_+u^D25g5M4)R~Kb;evf+s}7=pT1MSk;gPRf3)Ht|l9bC-xH6;3ZVM zZ6Erkeqv{>HGta6i=c70987W(;7LLcnR*lGiA+sg^;DD9xv2p;fw3UBrvw(8q=M&D zEht}W!`eK%K`+;*u+Kv{;$FTG0?I?c*!BVhL{@?Fl3Spl)(&pFMd4@Af8@vLFmaiE zmn6H5FtzJT>845YxK>*jULKzUy2KPRrfdLKRs&voYZ3DXLA1@X#-f=M*qY&I%7=fs+E~p-5K%ku44dW^EqZ!-`*t{byRA>mB9@lTQmvM z55}=R{%Y8|a3VL=axxk=hB0%dsKBWZ2N3Pl2lovgjN+YGRQe#rs zt*{w_zbZjTLkv?as(@!UPvjo#(?Fl;No?~9ajaZtk7kLJRVg~r0uO$Z20$Ny5_%? zsOD0PW5Xp>%4HV!I9HCF9NdcD>R!0AQ3X?0PQi!{IjkvLjkfxdINcx^e{>$fNxnS1 zn9KJwjn?=#_bdvYs6w5ayRkRG64yDsr2n#hvf_4OtOBQl;?Au&UFkbkY^uYnl~Jg@ zRgubNJtEO5k;F0b2fba*qWonwuEXOYPMHt5ZWbXGLqEwm{TpPxWIOFjNyn*1fAFqd z6cSxQoIRRB@^;OHw!2TsaoJH?Wcda29CvWapGQ!&=P!+k>L&Vy>TvbtIg+|i5MwN& z@n+?3RE$f*O)FZdN$YjeH|Za_D90o3#t824@5F-D>fC?{i*YVL>2eZ6B%VFv$IUZn zjx3v`eW>Yvd+qMfo zP1eV0Ph1fs?$M8zelUmE|0PWlqC~@f4z`dh7`E>+UhqAQ@594td|$c!>} zC+tyYw+Pp-aT|?#C!$o(7ovVR6y8`nK~e9)zzSmWnQg*R0RGi&PTcBS8&<; zIVd(DM3(3NBnz|NF`@t3>ES24aA1xO{4Zie#@ z^wT=stL)4NJK2{GiFEMy4E)fr1&6YtaZzm{j!h}YjRud7gWMeh0f_&ME%d5N!D`i%LqYsr|IKmpp#lN9K=3Gy7UCF@2c}&Wu`%@gH^Q zg7Md+#(Xu{XAKgvJ%>(n&PEM`vp8713jZ_CC8ptCaMH~k+EbLsjfHYZ@_NzvZyKrz zM$iaW6d-p}^=kb=fp(&WGdrp!J z=h5G1)?tUY8@`sZK-;4+%)ytvq-EqO>HT-QrX?nmZf;<(aBUz4TGg>F(NSdnnVn>) zyO`OfU&^Y3Gqp;1K@(R!M#)d-$yjUxtZWGavj{$S-(d(=3A$ju3@wL4#u=Y`Go?kGtO!Hm^u95IR z+tH2rMNQ%xS)olw_*`5a{DCC!`6~=1_rb`?DGifs`SHk2Z2}UcsE+XoB4V0Yuan`Fe*rI)kXqIc>9EDUIm)K?0?;r|r zhabO0-h@$sqmVC+?AF{bRQu-+j4drChE7LeW!yzj5Qv9=217*n$x`ZQ-%8!JU(xL8 zpU4WUAh6t#04MeQz+PP%M8}@jD(GwBoEmu}SM|pv2XrC)p&_W4*}|1y!63ae0lpaL zknD^_yvahcyk9f#!%O8jP}V&Jo5mLbHVA_GDm&(#%Ml{qWdSSpTmUsv1>eQWq1`bR z+)rr2rsiMlB%J{!GUhocI~EIbEpLMM!JDwrAq_qyPJr3N9aJe^k{S1(1TWVV!h@1w z2)un8z77JE2+k+QxA`3UTuHj#emOkpZinkze?iZ;x3I~39YotIpx)*(yi%u(Go%&Z z>`jE7VQ*pQA_RK{eaPCflL_dvAriHlnO+?|$S^{=S?UkXzRrOo(pd_LKCWI5!iHp7B~VQ5*M4-tzc!9QgZ zWBBPFHUA_G&J)w%vp)x4j&QKXHUSQwct!?8hZ(i=AKAHHS}=cC3Y=V54ifKfz~wYP z_Pc5hCTIMJptKC*bX*XW)fYfqqYu=1$3vWY2b@ll)gmAGCqIE2mz4l_1FhXnK zgX;yb+I|xrPpO3mV?F?ng$eWp(g|Wn#v^rMCqVx(T zWaSc)>Hti2nJ#p%ZibNb4{+SS4x}`7Az?E|Q=b3AMW&fl*XJ;J?tTsZA>&Xz(grTK zZQ+@UBAe%9f}-awSZyT}5U^^1&epe(q`|?_1$y9HUQHStg6Q;XLS&89A@KdL76kWm z;QQtRY|2^#q>}seMiMeGr93UQfxb4P{Kj*GQVAB8A5f z*@EJiDezg+00#KHLw3P3SY)J2WJRCQxaC^ctg5P6v-2u zfqv|gr{U?&%P2n9Nq@!NC6nzlpwotfkBuCAUmy+DB1nB9%0*7A4K*7i}azAp6GH&jeuu1@pqkBo`cr=`N zQ3Cp932?M7k35fIX->%`&ddKQee~ih`E@Q8JWM!vbtny1s^2BguXa-Lo0qWT&~%hk zts^(L@NsufElhlu0ON%X~nu-h5-Pi;Ze50}Qay$LerHT9B9HkettZHvb z53$?_HC(jvGWx2v;kdUpCfeuFYs%$BW}zq~O%);HWGc!WZNx~4uekGmFdiT4qbWjf zNy~a$sB!2ff*UT;|4!xN#Apd_%qh@q4!9Ar84;a3L$lOYbcv}1lLY}j(eom zW9T+T>h$gw>07D=WaSs49jL(PHA^vR`T}lESQ&2Ku0T&-{7SxC%m#0MKQk>gx}#_< z7Nqs!0qp>^Ql!-TaTt-@r35nqI!UAKZu-qT5z~H3a2Dmc_|P|tI>bCA^Q-tdtC3e^ z?}-<5aY``iO!|s(!k~P&oFq@pp@Ofpk^9_=ftT_yFGU^8EkCj? zkpi_}IuFpq8FNt0rk(Z_MPurF6RciyiK5H`%FN87A@SbWJ8cwaB};Q|@4~o-oyJ^b zh!DqXEyklln=mA29Xd`n!X|?VRLJJX$=h2Ij#QwKUKrY}+>dZ86thZhVA<;LC}LKM zXVzw;LGX6e+TTW1x#RTuqjjj8JA$Wll{v}J6S;G)otPdbif69G5;I65bm@dsl%O~QWhDdhhq!J?3zJ&<>n~Ra5JvjaQFy4*6hEjEQ*j^aKlwA`Amo33W zWw9yVUoFG=jl^=+b7i>YHm0~&Du|qO5{2LAk1+8Q?C`|Yt0|Ks66?mCANm+yb(~qZE0zR(n1H_h5Ane{F^;!InakY# z1=HlrQMWOe>RfN5d-gaYLoQ-{LkTuDeZ#ObRk&C)4CS8eLvRbj!IA4YSFat{%SBKu!7jgTn2`LHCcm3aCmt!Ft2B-9T3$VRC0@o^f6n5vrQd002#?YD zT1C{D2>RPP1ef`XaX-5sqN7$mZrwc0j_sHWL9H_IQ96O|oE~xtdBJrO-(Lvo8DD?C^4k%TjA>V%)R`-$OaA!z_O)n7?zR^`Xe~@}k4@^5l$=;poky-Eo zz4`b?&?^+tJDCQ3kb=wKwc!Nev#btz=r?{J8+s;k?~_9CjZOqT-Cs>yjf5GOjGy%8 z;$+mCbr}om>hVX(9ot4njG(dxA*1X(r>qLJW~!0O6urUpDMb1SOl-$ zF-1!}j}}h_IE9Kp&e%(wYiRGFUXk7GvA}GjD!mrQf2uB-v|$M&JQhVCuP?$SF$=It z$Avy0o=#d0c9AK(Yh(yyXy3mxxWFg^^KY&|m+ggY=%-2WeVPu)PpTxQ+I_Tr!$VxR zAs=I}@$iX$4LLP@1dix$1j#Eg%-Dzd*lH)hHGRK^dlp@yh7paR6pxF z+K7@nJTcGhGFp|FQaQ!75X#>Ra$1}qBz!v!-IRja86up_y%ZchXiJB_JHfqeso>^o z2FBGz)MftiK&iC?+^Nm;a86+iFfes za2?z^z5*VlsxbAJ1Mz%c1?^=t;Fd!Z#4HrxJ+yiS~WYV7zI5w1;kRQk_N1- zBh^|dux`ObUcn|MUa{H~9;aIbp>cQE@^ne=uWdN3t2_iR@fSp$ljc>_{)CTwPEt8C zi})Nm0ykFX5av}h?3@1!YClQycJZHksUCuemqD~sU5c}~U%}=Z@^c1uQ+TQw(|Nn~ zCi1LBlA-W#4Ou$3lol?zLdNDsgW_TVo|?sE-Vb;Wwx0rEbB8u0e2F5#wKkC5#==I= z0a*WM7`z8>!oEodVMnnu=va1<5XU3%>U#y4&hLbO%FkiK_9l3$N#V`)T0q`2QmMZL zqMz-ASSAWiKgtK!-eHhr*6`%7RqzCtW$~6MOy)VIUWL!EA|M4;gV-t!kkb1|=6~}d zn}65CczF(#u00QoRuL#|;pe96L!kNPEGT#wNzCllL&nqna9bq_)Z&{#>Td&de2WJQ z`RByQGn@HvUm7H=?P2Fg9ejE?4ts(-K$ATJ-RhOhz?NBfb(Rh>UmXGir@DXIw8{uN96#Og6W!60JCU>ObL2{%4N;WjZW1}R{QQilt z-`7FOo6|&Xts2y{gn<3^EQnDq2J?<{;Be&lTd{q(y4r(xt<-&79Bo8C4Xwej30r-a90)i ze5{{jdJDoSi9j-FE>2&1ex;&H!o+2UEO{p&3P&7yux5%0JWsD9zIU@}eyI)~C-Ky? z>C$&&c_eHj@j_Ja4{V&Mhd13zaR zBt6;>scg`HC{<8RyH_`mfz$}ttdp0%2@#jav#G}#&RY!Yx)y&Cp%9Q7FVWDL`S z36|*-i$yW?UVktaHtKMnQ$um|a5OzWZ7w-)eTqn@IWa4Z8tK8Cb8uIE0vhkwj+6Vu z@sEHR>m+DIsvqoSKKTk`2!x|~#t;rzdg6d~Exq)uj`=WFPFglCV+>Bt#X7GL{JXy! z$8Y7Ll;Jd-G<1L!SUi`A31yhn#*591;iF|Md~BWF0{J$Ea>pEEo#g?qTM4pXYV z@TIW_ws=Nk+9nPi=FH%3f0W{W#y&yUB?Tz_DH#)vUPCAvM#DMkoZ@p4PABpi68Tho ze8v`uiwR!k|DUgYm*5Pe*KpE3v$#3V0^ITHK(xAEKw}#Y&@8(4M67VrAZ#G1-i^!%Lf)T+oB z>+0X((VN%#IrShEa8$(r%m3)Zu`xPBemh3od4Q80o3MXhI@T=I!VR}XXxGkVEc2$5 zp2$f=&3)3`>!A0z`E&(_^k1c}cN)k7vXJZ-`$&bA!f}JR2Det~3l7_R%ufIo&<&Ix4?@hgXO+*Cm;RQrA1^t?fpv=S(5GZtg5JiA%?W zqPOwy?+55$xdR^x383Q6K5CSwhBHqDVdL~lY`^>-n_E^Q`*fJLJ@2Elj%nhzZ#GC5 zS7ArQG%h_cn;SZ4Om(`2iKw&yxxO)SDJ$zlF?>&;KQff3~$x1xmDU1O>f~ci$AZeVt9PT@AfTbPnWV@dyRXH4wQ@t#4 z%fCDt{7Vg@4O8HGMlhJYI!>f6jnn3rb$Gwq3wLG=7#WT@LXt!_7%3-$_`XQe8gK@+ z{A$o|_hFonU0|eHdJ;BS6obT#a5&|)i2S;=17}#Z;F7`}=qjyBlD0-bqIDsB7C8zN zqeL0`R(~8jRL$24^YvM`B8ZVt0?1sr2ylNp)PF29>fdq*XSdYhyd)uHC; zdt`EgFg%^d7kxPOF#pXt%WggSgx*g1&5rKTfa<($3jxR*bg}H2p;&xE})em`B<#>|g;~*ar1G3-GQQe&)+>BeoIJDjX zTB3Sj@XtivuGl_k@!143HXdRP^g>AFS#!3hXEUr@*az=RCh~H>bwf)!zzh^YfoYj!3S7d^#5385e7vg#4OnY@+7I2X<&gc;2AsA25g(H<*sl@> zb7yx#=k7o7XQlve)#5tH%vA<2v1`Qfr9Lbviv+WZaVR%a;%(hEiRZm29a@&~h@lMM z{{}RW^BR1-_DYTyt2&#vL}MoJoWKnbj!k10x@mAN+fGyc=n&|05#=4+#IGIsEXpf% zO@JT&yhuq`0Ho&mkwtQe;PqRS=kG9`H}9SZZ~xz5xGfV(&CjZHUj{EQ+y7JmUYf}( zbkpSRYL(-OF35&2mz#*DXg6JbXBko0p9ZUCC3*cG(|M&1<8bTN8F;r{m#>2Zoz!;;l4T;1FzF`5oNsyCJ(FJOXAu{|rh}Q+QK*`rx|oLD&@WgCxz}M&w?HlLD7i_&EC&oT3BZ zXK)olpZI_o;{@)Z6{KXFC@8o@fs1Js{GNOh(tOI{Q*<6o6?p`Uk7kjB>>wHG(}O}6 zSFo=<3$ycXLHhS;yt8g4yl1-CA>wWn>2s7K=LIJcF6AUiYc(JOvmKf6`8Mq74iPZg zY((DNaway$Rph#{GQgfTvZ5f4y%f?!x5{Z(e}?spti=;j`@|BqemV?SKMs+m@oJiK zRtuXdhN-XK5L-WMHz=hR!JC{^@LAMIcBPA>-P%G_ezy|mc}9|XZn0px?>fvGyaKC%3;eD?fS z8pxbr4I~=Lrx_|BKlcXN`e7>(_0c2mODDnu7YkN4#e=U)TS#Pr!^xtb@+9HhB*Gm@ zB;gJ>;G}UT*Bh$E{rY|vYwnyzhwFgNoq~AtWgL~7DT=F`8u3AQGup4u$I8#4_-FA# z{KW~O>QxQAG?a>w`gu64R*5$xLb3e#WDL?vrLA|a(8pCrF!^T>mUq9z<$<>_T-*?s zEJ5~Ly>G3YRW;Rr=Wx1;NukhLJNL)F1p0*C0WgK**=+^Od*y(r|s|LF0cHex| zHgQ6l+rnCupC{%=MM+f8bkt0GfO4nJxf6pA&@M<3E#w8rlZbS3zwHS-f9-Z$Uz?AP z{5YYq@GdI2uE&MPmFZB$C1$DUIjT1$3g1_~#F$4i+{v09*fyn>9+q;dWkb5z2cL|v zZ$bkKp6bKL6+L(~@CaJTmC%6pOzPt-i_23dkXRpmC|ti1r3}uQv?@IUW8F~ zp}6zZWxh711v@r1;AJD__%ab~sX675bHy)4gu0`RM{q5Mia0KmsPT+oT{f#}( zKJsfC&*0L27xe#e1@FK6h0d2Ias4UJaCg@T$`)s#o`?sYo_QbL_DOOpgk`xIynbx{ z5r-sV0&W%FfU8_<(Eg?fm%K%Td+aikvs^ZWeKozbQ12(iNo-`ao+YBYnlR^Eq{5jl zlj9mC?jX-o5%Z}Gw{=VbHMClB)k9e>*Ho2zB`(Zeop=`8G*&SZGk$_dUMH1H8A8{g znH)~m;JPhDIPv;wjLzMFd1_*KQz!v1Z0*PVmol8Bx+HhEu>%w052Cy51vFVTg5I;9 zV@2&GPJi|vkSX+-6or1)=_r3C7Ei1=h_-P` zC?m_#E$l5C?yw#W_}ZT-Q*Yo{+GU)&cop_`1ksV1o9K#}kE!A>O?+V8gns{3pi`DF zE_)!1{>gl7IM_zz{MKVzVLom~2S#k@5Gw9jhaQv5Xz{P3>^k%5)YN4f8qB>r6#)R1^l8@VWnq#wm2TjchqdZ4Z9Fsqc z<~u8JCh~C`AD4)jJ*L$fwp5@dgjy63EaAU1^R27U>fBZQGGjejJKm&b{wzJhPRHq~ z2XSFGsUjfRB)uGaLAyim-mfBT+q0Lf; zxaeUh)=jU+$NLqzwCYsuUkBI`E!Lalo!ye(ruzDtD3Oe4sfo{8rt*3sZ&Ze+;7 z0fw)6gT96|w9PN1k#-7rKh6%-8dPa&dktA~Ne|{|D1e9VDq=k23|%%OmKxs|!lm+w z%;2|T^6X+KIo;q-_Vzn5JER&}e=8ZPumx+KJ!Og9at(5Js|ev;&Sf+V@|jV|DdcCd z3`rhK;@xbP@b$ETf^HpH`bz{R8d2hFlml_+VxXfS9R@-#!V8aF@bWti ztL4-{$@U_@2tXf#k63`Qcmm|Qb1==g9wzd6Z1;UP$?GJGwoV!ieXa}l$y&?n7S zQ2FXEC|&;!G^P{cDtE%1)*UwCeTTdU_h9I>4d}R+Qj0~x zoU+w_Y{R+~s89UH*GP!+Qro*>LC-Px6;jUBELBAFUNgE3$udk_BG_5rFAKf^u20x&+K2SYcx?2hwPMdH%Tw7G(a$ zlZf*gY*%Omvo(?HFLwTdS05$$J|@HPDkL6Wt5=hZm{^EysU-=D@$jy97`7@*{2@ZO{g4YRg_ zo>e;N#XbQK!`JXguL0KmXoiFzqmXEz#`Tw{!cp^$uxLjJEZva{N9zXQf`=`Cwr30f zros)r(?u2j<_+bLKAAi3ueXKC_w}LNRUQOCH4@9&O(3u?8JZ3z!~1h7u+%CO?A`~! z6uu#BiFrw?*EvIp^C9>e5D#k}Aza23xNc_@Rr2IIblbc1H@3T{Uv-vjcL}G28(z z-fDq4Bf}7X!XlFVN)0fQrqNM^q)L@j;_d8H$;!r~6(@^^Bq zkS6T==u6lGLxjeJa^r+XaNtJ2e^$pqY2E=y*yRk1d|r_6Ervh@1L5E_3e@4{qkxducA@+}jAHD|}%V!|`Q{RrqegH~5R1&7uBg0-;xT5J4h9 z0#E7^(atI4@|XZwX_Lmx$@$B(zCX-+AMZs3f8~%ReQ!ziuQH;e<-{N3=ai^nyuhkQA|R;F4BkD zT>tUC9KKEA<`|`0!EtW{9KYiWU*w9&tnw*1Depcy%u~UbAU)!IXfLegc(>qt=q?Bz9~O5$>14>u3B>>8%=@+TMmK-j3LK_!J{O z?=We+Ucyv7-GP0M%_w<86^kAxV`3T)U0zM(%^R#Hbv_G;LYEpov98CQCF@wbqIx{@ zQx)%7%^{^LtH_BH0!&>k^M`IS+5g9tm`%n)HB~ip08g;8tnhW&Ea!dxxO&=$zeOz zzd(fboH&Q(Ui1Vt^G{^V?ifClHD;H^$Fjdu9a!b=yBI6wM{FjYg5!7JlemwGnDo|) zeJ>%$u29fr$1NhUIrca4aE^f&7qnps_jl$lTQ?92Vq zta$2943r+_I5u6h9LUAG(8;Xd`Q@x>t|-n<{a~)JxdY_JF23Zsi3p+QN%y8>bx~3vgYHEW74XCmPwupy$J*RL?Gh zm{{l&)f`C-yyJ(p7u48Tj*(jQ*8#h}x-$7%N69Lqd?wo35u0v2!J7#x=rt=DW6hS~ zZOMBkR?F+i+Lrkwt3VAS1nco)<2p7iTjGwSk&;xxnk0IUvNIL;PUor#z*P4lo*_~ zvJ(|uWLdMN1-L5ZAIE zWG$Rd{ASJ|c_qTcz9p8Kxloqq?fS>ldwqm#7P&#TmUfbZZYAWAU?49t=nuW~sGoVc zU?Wp_vYf1SFovEOV-WiKlB}^+q>gLMQD{O7y{zm&1DY4W3N{o{?`{Pt;u3ut znT?kEviK)y4mlEP2Pa}8;gFsm)QS|6nZ}wpu<;(+yQ!n~`6=Ytw>{9w)i-0uk3#=c zA9A)@1-k^o(PgJ9_Uj)bW+N_8-5d{HTlYfYLp74h@t!VH_fSWD3hHpV#JgvF;Nr3< zFihA2?X6nm@jW@zugb#L)1v5U%q1>y+u*cxC=?%92vsSoXnE#+?yO~wDJK=_v$A%w zZ{s{jUBfXthGnE@>>Ts?+ZeMvcbpEq+RrZCpng9sYEyO9H7{=6=J{s0^tj7kdWj8v6J-4Gh=O76SkOK z>~VodYun)J<{_BX-vYw9_8?cIMCa_1XKkDWh-^kOj8y)DKSw9<&op&`<@X>^oEAf3 zRmJekGDC7s^BBx?eG1nO48vmoI&iDm0Z*1`z>Uwon{tg6@?l^cp zj9}HY0MLJ#0g}&Bz@4i-)7PfJcd;i>V%13YtdoHjZv)VBSPurj ze4x#v9G?AF=Icq{;NOyK;a6<8<*%JM0G?-ap=oO{+_G2?8|h>yjhzH%E{(#=g-^hC zqygHeyaHpFC$RE%8uWkN3`dx0kasExzADDRpiU!ve%}qoYM#pDD!hu2qcaDgsG{L{A-JcA=fM(`qv=&%zgj+byKoy@l|MiH4Kw?Nb+SR ze!*ex`4yTN%)FVS!Sescki{#wGtnyrKKF3&?dFN|zi|E3BOdagn(0FY%}$Vzmtl}1 zCcu9XD#I7+8vuuv7?`?o7erbJgSoUW7%=(pQSTGH?dXBNw~a8+9Su@Dk|4--8h2*f z1{1{7fFUhl7WNwU6xPEpXa?2TQKIt)g2U?i+ zt{C{_KLodxMfvM{UVzZJ7x<**^5XSXvHRyf-kkpZ@P6q7n8)!|&o~Z>y=e~XAHL`6 z?I>QXOg#~r775#L*1#-r1U4%IJ|D0GhYllH`z)DAFV%pkyB9#PBn?t>QlQ=>9(eOY zL7$rkeMqk*>cZl1C2Ii;rf-1f=G@%#aRtn+m*?+#c$x30qD1CuX%a`NG~ULw*Ld*{ zCy>B12!zJ+{u>a)DQ7Ry#b+bw zagP-occKhpkBp&JshEtCnRKP;MhvM`#5Ubr`XxaIE-pF&6WMJr{g)GQNw1*?vlCD{ zhhSs(kcsPF6Zq`x2aiLKf$xL!L9SF}4iq&}# zlRIGYv2H28TbY0{ZVl9Q$9_`(#2)x=?jX#QgChyq)JQ`a6E<%`!NVH#mt;Qq*3Hea zf|S7G`7+XeDxB(S1=G(T1#tSNccAil2P)Icc&+U_jQUKS>mfqR#wHb?aNfzm1;GpDG`91wwPeM!Y}$IHkO*t z57Knuk-kDJwA>rMHQD5X!TeSM-tA{ zx}qjpFJ{H{Ag#zPgx*6KAR5 zSdB6*#305|u9OCzvcz>axjv@$PF@yd<08H_%7s6s$D@7dNi`9C;dc@@^_QStdIK68 z>Y|G6bK34;MF*mNX|AOiuAh{NnqP0>)Q3fwIn5fKu9wqme1<0JspAjof=#hu*d`p$ zvEspa;f(?=s24!LS2Hl@%NZnbG04x(!G7~<{C8dm=Vx4{@2rRE64}N0a*jVPs;)}fya0H<96=svShp+pYAYXS<4>WJ2w*dHizJ4 z;fH8z^c@c>iLv7afADzNXWXvCY|BX~&v91Vv<&!oW-k`hLTvfmDeSGca%`{1UHpAh36I#zvc*IhkA~mHF&{a$_~sP0u}+wk+H?hV zCR_6yE)T)!q!+Zkb^u?m&|){~PGc98i?HwSRG_o(CfuGn0WW+FN6DMr=vpAl1~XEu z8+WF=@!ACiT$1s+&o`8H?ZDR9iLA+v5uDRifXBw2@JiDKe02Q_)@>Ta{Hh78uJc!X zF#jP^{}2p&9f7Hby75V{5qkyNam%zj7+&U&ss;8~5bcBcKl;#T!GCOs{WyDNY65#< zuO2(~+e1uRbrHWLF2kJq1+E1E}bX zb;52us3ynmPk4g`tAkN+=QX;{B$k)IU!De+TVc-bhZw)0gC5?IjjR9d_+O5LNx&!B29bRA-GKuH!g;XYU|Dr_SLGt$a>I zKUl-bWDjs7KS;xaTh#I?_c@&YlOAY)#aN5(hrLU0fQCpMv=y*KLQ(?PEzQP9e@t=k z?aK+eICtaome;@nG8MYB_xLDfb)!! zaH*Rk*32#=jZMk0zJP_HBQcPi;zw@#T4P_(0P0HEpncD4va&57rc2&}OZ{Q+OX3oV z3pc@yA=&>wJ4>J5KukGSp_f^YBc3y`z%K&-+21?LH18yH|s; zRV!)in8fSVmN4m^I;rj-6zlYX~4{q5)%Hxfwj%Q7o!W4BZ-hYv2emn|Bj|=c;G-&d7D~t0- zR~NuovNZMGq{Wt6-=lus$6+|%iI*zj#_yE6e7r~c!H?Z8l z8&-uDlWoP4uwvvAq&%tMGRG`P*A@c9_1Hb#o@2T<=i^$4 z8=1sU$jRfUe!Ro?;jQJb&i(_7LrdXmUkvCCIl_%wlOV4|1m-Hd1owW9>)ziAHl3g0 zmwXSbv@Qk@)s4`4@gHfG3IMgV04Vz23SRnupvp#oFOX9WrcUax`3}T#1-;6p;Q2{F7%U@xO3&>fGgi zut3G1RxF&xE({9gJ+9{ffE!c!pLXf;o5SS!n%1e%_^ghc^m;+(k8L6oisNCPffRqz zGIjot|37ezxWIiE7(q)$0ZD392k-VO_!lR@w|(&k)_HUMO;!N(+55ri4S&h^Ra`!G z{3f_T55(>H0`1FN;Zo=w$jEDl=6WFzve$$J&Q~OJoa3G}i(ygdDC7#Put-iJ$2sJjJr_V7W%O%B|o63B4VNf2y>%gc z)(Ubc{uepvpaDy10@OWh0-@1|aLOwl);2d2v9e9fd&djJMp7LV0vaJAP>jE$vK^FV zoxtO}CGXGHIk-jT3X|4k57)fjL-(_BVG*q@9gZ(%O$Z--r-oltv7j6dVh3n(ol@L#bA-s=};GZd4l zXjLe93kmSuo8?XcQIjBhf~2-~~Vp@4J0 zwU&7>%XU5{<3n-qbooPgn*JEdKgWYhjwZNU43VRgR+7>28F1=nBm|k{!lJMOcqEnz zH$zT9fZ1^ns)#27X)>T&#{<2TO~C(l3FM|BoVq2)Pml5CZ{50z@+3aXtdY z9YQw5p^6Z<_$$5K30W2t&C6$6G3XF$w!bP)t!2cdGhQ zJM|01_T3UVp}GNd=~Ht2??Jj^ek9J{y%xXixlW@WOoN5{V<1ex57x{(MchJ!@YAzu z48HA#x#F43#l8p>@dewIbWSk zGSlqQNaHTbpV@(@Qf89fITyH0M=o6L_JTFu(ma*_JTP>)0xt*#qnBj}5%7tED>GB! zo3bNlJeK1bHy%TUv(;F6NCYp=I!n4nd?2+e8tjC0;Pb6ij(N2~+mcAsR(GLXW1d)t zZ-mSP9FN)PODUI>@uj%3a^$TM(9zUmnxk9wL z;}dUs??!Us*;+ECI-1wHK!|)6{mm0OXHCyvPG$dQtFe0~-a>D+^=KwG9W}Fh=ys0* zD*b*LX1n!a^wm=Q_}m+p_i3VC#s|82_%{`^wZ~;!o}k2*JLqJ26@wm3$MmQPw6b9- zVg(winYR#XE!_H-~66eWqX&h)2F@vng{2G27K0?;i|f zz8$_pdSkRlhc=dRz}|1u^DJfmW-WtpGfh3U#19lYHf z&$-s_V8@FmxH`$6bHx0m%D20zl!PXxuJggI!DT2q)`x>vmgDp8Zxr|S&=;Q5aiQ@> z^i(Oss}IyzGp|%O{ar6>FzpoUZ>`Mk&8osDDXy4y>^P)3wj3^O;#lrD`zax;DG_=Qj4untt59Cnt|m&vSAZso-MLVQ5G;|Jlo|(o;j%UFoRY{Q?hJ_~QfZSy<<#&s$-ZN32)n z@`@IeQT5^@xXr2`^JN23?~Wj9B=ULR4b90f)fG(8J#mx~cEklu5vbkCImzDFQpUrc zr#aP^JT{#}4t@!xdQ1Sq5hGT(*$ekz6?Cz04eztwO|sW@2JevJEX)z} z$Kfk4aeqfT?mw-9IBIrO3#b$Q&6~bYuYozy-E1A8^ zg_vIPAwSk6@{UhVV^-feO=XwPfQDVZq+m`JS&kn#Pqi?NrMi(*zvU@k`Wvl(CPMW5 zl*uJ+VQ|Pbg<}g>z>lYOL|rkHrkyuNjpoZV{Ix%?&0qzn28Du>^(mNXUPW$smD8Oi zTrOzdXBu+YmmCXpg6_z8@VpZQjgoa_i&q6*ZrFpG)1J{~bN7>^s>5($Z5X`lbpw%m zWyCpL0yEeChu(%VILEVyR6jii3nIe6t!E$Cr*$SH`E6A9L^t}lF*NyaHF4Q}0X}J7 zh5gSCz;K?>Ds$^3dk1AoRRh3B6r9Ab;x< zX{bC(WIoiusi|o|l#}2_S2n=UWVopw0M6_*a4QTUty*j0!GA}<+x;>OFK>j!d+$Jd zL^zB|b`ZglBwmG=9CRPw0{eZcId9HCc;xy7q_3QUv8?+%@i<*<)LcTwPjI=vT`wSa zbRz%J`mgZscMKf77FoG#iX3ZKzKQ-@ybUV)?}PjIaX6;h3R8dE0rSIzyb|pt8~7%i zLv<_1t3QQp75`w0@O@aq&E4C%d}3h<#~+&3RkpRp0DI;ugcMEW$J(}oe1adx5iB9A zEp^bdtC(lZa{0T^78t1g2l9ujz*J)s2%5@(#L@&_pyz9%>K_hYV(MU;SR+gzmq9Y^ zKR7C61Io>Tq}pp5+|lrd=6}ghcHsu}WhTO}mh0etts1si|3^lic9XmQ)4<`CIc$6% z2$G3hf89obFF=y{{dOgM0|yKK6#h4u-s~9zmX|`#K2MnSrm%&8-b!(vivJIhaoL73{|Ezymc6|HDvgv5+k6` zx`DS$AYE7?#ZGa(!{m)*1OJU8U*1lgKkxele$3|>`0%NgoGZ+yyME>oweAoY5B~?z zZxr}mD|&&q>J;QB%!D?XFrw3A3(>17l#6_XvP0iNx}^qmbX?&~@lGiH%eg7vpMir$ zrEtiaV_(+1hU0E`fQ&PM*_9wU>lInpGY=%RICg@2&oM1oV3+?L6cfz(FN;g~;}2E& zW|@N=Tb&DKzT7Pn@uM>Yvrcm|eE%7G2_Rh(Zc3a-0!lh%hKSXnDZKe|;AgOMAM_Oh3AW>rI1 z)^hlrx1QI~^gs5tl!mOG1|DXO@MhUR=+J%#zg!MN?kp3$cD5N?#+Tu(yE5FItP%>1 zzd~479TXf{1me?OY4L6q`mX5^E#JKeihtF?hy7ncLz@9}MLBTvHANYzCn&prDf;a= z1lzRULZqJn|0mM`>8q!KU11shJ;?|E<}Kjexy}a=<_z~2&{S>t$x5?&~FHGA3 z5&Ra&c||8hLgDQya9@xOQcu0$#LIMYsW7ebslO&2@X~=mqcFJPejRvL5nyE#1hZGF zz^`r7$kY)T63)uNl7RV8ZoUc3R=dD-t6cE19E7s}RQX58#o1p;P4v#GR%XlHJd>2( zW?sSax#X?YWS-^vi&SQP4hoIV#6^M*^n{5L8I*iMeonqgOkT^=DJMd3L^KF9&j7kk zC}$RmNWpz!W0>%?gw&6AP_4*T)CkGK=<{ZHX}E-}dwv@1{I)=ObQCWmmaA(j{-NYR zF>d&Fopzm^3|>ALV7ZP9NX!~#I&A8&)Q#hTA7!GGcrHC)XU?53IiCWv8%)!zY30{g z)O;br9vFzh0n63&snBka8;u2Tqt&4MESKJ%kcs>@XH@g4#jnD@ndrU$L9@~YsIt%i z9m5nlFR=t2Zwj#Whv(wyTYkI?r#C~&8&8lpS4^DdE=TpHIT)GTf`6}d(JfMCLyNN& zaiU-y=B>Vt_k-QFlOm%vUhwZWcW@6xvN1>vRO}2*G`-zk$#xn=R<>e4K zBfy$m^Jb^M4qz23m$OSPgxOA!Je;-69^J*Q&~!Er-|Y)PW=$y0Onrd*FG_HOTp-TP zb;cr1e>A3LSlKa%mg@?TKj#_-`Z;hO`v+9(lQ)gESdHZxKhV2GolRB}XTP0#iYrY; z(I+~Dq@BCOtMM_y4TUeThpYd&rUo0W9f`9Z+A(9t3Q6?(-DHcwO#I-}fd?zUZYq~~&c|sphVZ{T<)|O$kN1@i z{W(<6?>aTQzYGs#hT?CrChGH158Dnq(;yF-O1sH}j4x>8;7}$;O}@`D z=QB|3R{{-9tRTxPMPOg792pi<#@7eyFzD5HT>Xpl_iyi{tr^`!ef9>>U)({4xV-=T zEgAS_(L~mkcNyF357QS%o{@{vli`xj1@i9UO{y{GfgMh-@bE<&tQ?z4CkNG#UoE;I zRrr(q)ls4fiZ`+S&|LP=%6vRkFH0Lt`^nS=dcfDLBl~vf(AzQAc+~6_>QYakxgWa2e}TB-z#6xk=h7mHPI-ByIcGL+Qp&^8NBVT3+CbZyUa2 z@Qpf@aZ<)BOT>wVzX&)R6q7UD3_ws*7d<8Jql#!I=d{;EZ?TWelT|8}dqPf7cTZjJ zS!<@Af?*grwgQ*!PNwFoW>RKR61`h{1f#pgIA^CUi!cVFHX^V_yqnkzio&O>Z5&Hp z!gIwy+Vn&MUAY}2*7KC%ME4S?`r!y7It!t1mM-TbeoaFb8e?siJQ=(^K;D;o!oBHS zf706>zI9wALYk_0?u<1akN2l)ITa)%A`-0Z88~q+0`y2NiS$y%%;a?JRd`AxT3X11 zka*CQtbz{)v5=NJNE!l*X_@szHYA~vT2z&ivs|A?$15M2)FVLBv6Eyk9-;5o9l_Sp zf7Cs>i(D@Xg_VyBfKG`7xxqr>G_{I03rn%zI9G`Kya6)BJRZ*3Re^F=930)zOy*yG zN~dOL;Haf4PBW_|H(SDiZepO?HVT~m%E+|^KGc2Oipwu^{><&6L~-shI6gHF5_fsQ zV)aj?w_1#FyA3e@{tG&3x&g8ATMW-V9l`GF64W6HN|fRfUOFn;DAI2gW#_a8jq z$KGiAr)d~}HdESHV+3Y1s^O8rXXvrH3tEbcpyZ@Auk3+25$~SL^G-8@J-T-xO`hX8 z3ajCm<9rC=GHS!0hEa;kS6vzQfJV!Auxp9{fA*wDF#Q(ivWYZgrWCBe8HZe$+jh%f z)`U7R>;C{N8#!J)#Rw9Amy@~40hFAUCI?O)gAd0UaC*!FDy2aBdP|rq`zXDwzOieQ>F=b>0`AAr*wR7p-7+YY;@++=QF0g8Xlz z$M}+m4Ea=Aj^FV2G04sghrqOB5X2Zj&srhyU8c&rU2~em+*kvJvMHbxUjh3HilG=| zzH4?!buc*9eQArAq&Yzl%VL53U(aHNA(^> z6qdDtY2zQkiu{4(q$aqRvJ@m9O`tMrTk-Y3ndG+w4@#37;QspWa55!}bL~iiD;n^u zUfrcO@gktOD;^S;R)NK}D)8Z0nD`@ah+g|o-gAv_OmO9N5ZxRD4X=x#t*9K+1TuGV}(j+yx(Q@I=8NFT$yF_z?wa3p*=co#fh#6gJ)#&7C%^qDV9*1x|B4fh&hWkEQI*jSN6Mn^f((=HS%^}#Tk=fsqoTR8N$LgTGC z*mK06nuK39Zuo<_k!nQD-De;PJamotRRoMjYq*MGK%OwRU#miA=$ zav~gNaQlGV`E%(d$GcQd`yI`T93bab`@*Lck+4$w1PF@B!oOu2OwrTnD0VQ<u<;W?T#bkGCZTdBj98_aLR0^Xk`&csIa0+Ft3 zWNz1Z<3mlrISGmM*lYbtd()R>-GUcnG%Jf2J+v0bz4l_ZzBz8{SxmPX3M+kl3vl_Mq#6ULh060mTGu^hAFjY;6?G?X-Yq~cw ziDm~uvM?I@-miw??~$}AHw;U8VgJ={!TLudJXuv6 zNISI~emtCC>FjX<4_?hgU5|Wp8P-Ed2{E$G`!5NP`OOopxlfh1IpLZdV_YZUghRK2 z=#t$6jArN>8k*8Y4-XNXUTT3kF<+?Sk`iV|oF(J)F_KQ;b}`s1ZpG3Sp}3=yV=qLC zG2_g1_ED}ptLpFo({+xa+O8?svsxY}pdy+CTj3chiNni$@Yb7CxQ?4i2Xj9KW4GQ0meJn7^NrON6R%o@%FoFoOSLJ&ReWW_w}@s zWnv+uH)D_*H&^1#6m3>c=?Ol4!N=saW&~FKBmKKe$lcUNT3r=~@1_6XpT-b$zBf)S zloLo+ycXnoJ}34+{!#tb@A#u&2fH4A;Nl~H=$Bt@2cihQGE& z)3&RPjC@oCbw4GCU)Qfgmy=PXy=aIz@4b>auv&m8Bsi6cN)y87&px<7_zD``38Ymi zS9v*S`^npR(}1(g6Vs-4YTuWEtFvz5WR7?Fr7cIbDyKo!vC}ZaX*#^JR-$p^QOMu& z4F>{u;Jyj5~hHxryZy_8pEC~ zM|jSX5qPpdnq4#+hSm2s^O{!}!7%{`m^oPkT%V`%K6a?!ecuVJ;2dqd?-RhNFOh)n zi#CA5hFRR6o@f&nV@G^E^bL0oT)}%L^7OQ)7F2xj1or4OXnQ9`)htY~W^V-^U6p}q z6JOFfCf=krV>;*+4v>?(6L=@z1kmSj1?8lD@HEGz&Hbvvn4Mh83-i@y1Z>5q$>~V4 zG+-T?x^uIfoJe{xynuOV9!1%Vl_)bW7iYQ`(TP(Zaei(g__Ff@c`xvR{0_TIPIH~o zTlK}%VzL5W_YwuuEo$I-cNMIz?bl~Qo7HQKEMy1EL_)|@d*~)Z~+%#{9o)QDq za$7*^#0+A;RU0>Y1)%DZ8+3(h4)J}?W%-VmK?OJC4>@>*DE`qyYu_SVNr$P+=mSzD z9tHbvmcR+k5QzJBm#iHT#+m+KQTysB^}nA_e2-lKp(E+gy*3aMNfn8&lfgZ^_hNXK zG`2)Dq@*(df^Vk5z#gtouTx68+XV54<2O7ZCX5zt_sLs{5U`Fef|2A%$ldgm{1z3! zE22f*PL&yW`_vKQ+~^NI-lcHxO&Elh=8^;JZ_p-gcI16yDz0!aByruFpg%JfI#>9B zi*_gRIx5ce39iG|907b|r%LYbmAE|g%ux#VTYXpN?*T8Yi%cxn*K-R z-{?>B!|en4`c#PXkvxEEuUZK5E{FSl9Lrtl12-+_!h_!xWXiHghCHIQp!6`aX0{IJ z1&M`R-8}gEBNRR#(gv%o`pk)!I+$p0L1(FVkY2?|*ri+ya>}V7b7&z1-TA}pJ#YlO zh4ty_@t5S2W*Xd^$@v{+Qb49l2edbO7Tf|q8#Umy z8sN>(> zJ1$%OoZN8GLpSG#xJ%y2Vf6B30SI}X z3@1#Va#=XeRzKcOJjaab_6BF{iW8vco-BeEj>o2zg^tQ0&#HVvJQB7i!(1`DyTM_%DA3oiP(ji5VhG4=K3UoQ%MD^o?Z(h z0)|BW8s{Ie&j;Zl?wRx43k{pzkj>{)c#$7&(V3gMT_5ax(sVuq1RL@pUpNmUKG?wd znl~n^SUx)3d`*o;l`6BNj={?AayXU9G3J{bLDGg3z*z^OW!YD%udPJ3Yem2>-PfR* zmjh+$AIV<T)wRyLf}_Z%Bbw>rVJ`I0YuYJWK8!KZk+;f-$E! z0CnGRy~zpHU@`m&WcC-p>WoNIS1=W|4rO7$$P&6u_C1j_y8-tSJ0WOl1hhPuOpH^1 z(m7E9c<9CY+dk?$p-(ywwxpGV)Y231GotKvozY9GzQ+_l_So>Ya{FC#-19(Vc|PcS zgn;lR1EQa3fVBMu)$#`DX_b% z2T9A-kfJFGPo9Eb^!V0$1L^h|SLW%Xj1?RS7OF(p_v%?>L94$!uO08lV$0M42W_1x#m(dI3cXlcig zvp$&7{(*OpyJs~Kt)LiG1a?*HNa?Cg8ha9JJ)Iw@OtvHN52+`9Mu)ykA)3169-6Nxf9%`$bgrT6-I?S;j>rLIAC~%vh@>8A}$^xBTF*KC*6(Ym$)lk zesB&tH!nb?$(QM&dP6cnO&4U6R3JT0g8E4N<2Gg$o)zZf^j=lwyp$Cf>T%A40Y#`% zpMf(}A7RMAb@XmCK$)v*Fv*KTLG5)gZ}26nWcDFjEyFI8DdWDc6PYdcC!s~`4#?a} zhn^|RiPwLPsK@2(l8th3&x2a#t7!;i-gp9we|12d(f22d$pF7a_pC=6Mif5EL_GJ1K8TPBj zK3pv8Pgb124(~=OlyF>Bh|g?1|MMEsxCi)Tn;y<+Q~}@0OQ2F}4Msg5D{aS@BHpt= z{b*lo1xY&gPz46_^dRS)85s_}L}kt{M*U7*oH*!8*POE=i;dDr#lS%#*T1ze|rNW^4;tuZ2Hewfz zP)y!5nf5{|_~ws-LGTkW&Xgjmx8I}E#pSG6KsPRr6eAizO^|SY6hebiA@Z37o$6PG zng@(HH2Dt3=ZiwY{AXa--42RMCt&4)cA7QNi%DNJS$CIp7|HPfDqJ7v;)zOdm@

vH|IguP1B%I-&dAn={N>5Ed$B#EK_mvag+Vq}uRxhI~x{^@y z$`O?Q*-Lkbgi~9YQkuivQ|}uhDB{sXeuui!r?qeCpXdtOe!rZ4kdjA#?wVf|n`5<8 z9#h+Km+>xNVIsOsmv_Hob7jB21Qu}pXFvS_9C#sx9~&>yQ<`eTF0+&DTzrshKHyC+ zy$nDxy()AaH^q{A5qf<>EcvlP7`_f(BNtC>VkGZz=bw@`jOY6RuUHpA}OB!S-X`;clv=sTX~IBGwZoll$?}6m=Ewl14a+g)u0~9Ln=R9l3n^ zy%D1pkfLu@+aL*#DU`T08=!{aI(?LRKA?L%F5d!bdf|rTvbsI3c%DQjylA1( zS>5E+2@7(u6`2XuCm1}QE}5BJNSikCwpF_^sA6U<(N2ycl_TqkPR1Y*SGP0nBO>U8 z93$+T$6#dPRVp=wx5+ukfn%f=#F@V%+pfh+Mn){aSvzB~cVh>A`%52FUn~b}b0>n? z@eWemx``3)NTA(uRT#u3&?jR@lOiiAcsExC3YBF+q9`JhR!qf2w@iF$JO$r7_cLh* z@(}7g8P+Y7f;+oMk^Z@2YOhj-^`qiwm`ODAjkFUV?@2IY_9zhZ@s5(=?NnqLi!U4J z;T*eoiO%_UVjZdjla1xzNKco(ZD$3YmAM!n++KT z+jx_@-%!TXrDH*De+7A*#>XlbW|42Oh5VYnheU?mX6`(&p_+k3^u(4NiAhcl5!ep{ z<$pgXEPt-dGy9n_H8C_+SpiS{c8FGtT}NKXFMz|>=YfHS5Ymh?7`wh~nw)QfSNB|@ zPj+aM`GyO@YwSi?9WohS_SZ44rnjhBwl{wEc}HJ=(IdO!0IJK@!Ts^PJ=jo_%od1n z6OXs6o%4n6O`S>vtTt?JUInddwP0`b6jJp`fQttyVeC60o>`?r0vSzscEkp}PA-7V zwn;=R>ZYAdJU_)7A-aUjBsZM2AZDuzr2Wi?Fxy!Z3+WabFk}-lmb{N+S_2bSGX*@q zt%q0AT2R%XP5hgY##=4J2TM}uBE2F;wnYFvkLG~Zu=z0lm@CQZNTegUF(`L5fv(c* zVCsF!2$TAPloX7BY?%{8>vOK8E$%E8tF5M!mt15b0z=8=JzGgqM>dfYBom8;m87kd z(DMPyVTipYT=riC9=FO#T}&9+5_elt#~q@&DLlW0yel!aH-pwP4=A}d8>U~VCeMqD zC0bSd{g0njP;OfxvqDD~p3U?Hf8OR~vy$hvZ<|8X*&i^dM2H*18yKZ#8|X8N1{?k^ zb#_J|8Lg8+b+dg?``R?@s<9=x-aIa+cM|V!bqCR|-Gn?@iq+0eC}woA|5qW|`pgF6 zOJg8VYy&@pYLV_Wt+Z&19G0iPrV4863E}x{w*wZy*SI+?me6W(Wf!W`EXP;$nD zgvJ7T#?Q! zpKXg;n&nuDHPm;U9}y~rgK0z_RLJnyEIZfeSM->oxyEVCKafceTJb!m4m^&d(*aP4 zS_R2RFG(7|+T-FI+i})~!!)({15qjsg@WO6ptgA}JRDx4FMTZ;LkeC{gZG;B>%GlH zfj^rLN)GUrkIVX+y#`z+?!%3;4VZF4Op*_5;jz|3ATZ4m{C`U$PRvazk@3V;$4{Yw zRyU#h7+ckt;pwkNV-B zWQ=nGwEJ~}o5E@MdG=jN;!o)qr!L~g-rt7HdYeiAkaN&~W9rgKl({nJ{4j)js;R<_YDp)ij$>KZTx<3V%Cj0X#*mpu zlsL^|Z_ZsygZupKFmWpy&pwG+$u0*u)@;Z}CQNEP_mKwnLaFYUiWzcdK z&UV9zsyga&M8K^~HRPnA7wuz(P*yHvXHS%4{cpxYjq(%f+i((nh&*T5n}>#`QlRL@ z+X*~Mpvq$;7{rgl=;dB0UZ09n`^Te-Z8Gt-H-`mB*1)HRM8d8SNJYr4T1qX$TF7q`-7)h=3!M3CAu%j0gE90rOm#j_lT)T~ zZ5rp%uAmlkKV*F;(g>dDcr8%iYCHCC0LRLpFojss~ z@IYoY*T=^}cWP#FAhDoI<$Kxg&Pui=^fcRRUJPFsrEu;m`?<9EBrZKyfjpH-Wc?iP zvEH-tS-%-3aJx5`+gbC8E26nvR){GCE9SFXJnpe;dlOh&ds!lmOyYhq>*w4@q;PY! zi+Mby)9ltoRqUx(d)Z-U&1rUY7H4?u4QCTz$LY!O{zkWSc0Da*<-RnDWXH?+yvFIa&WzIk7>X)*C9oUd3y~@fy$bTpERLFB%7YMW~>t%vxIvmA=$E`{dwQfnXq8 zTZje=1k%!i|4Fg}1A(jWdQUIE4fFgq`0nu9w$;Rt5v=+DllJg|(eI{jv>U{aHju3? zgagH&<9BV!TK9r)4`jK44}VL4%*Neyr|aD9o4x<3{nvF({F|nZQ1Itk2DInt=`7M& zv|zEG?xOj6TDt%2=&w_h{aZ(gasTb;`=(f1NJ-84E~FyT{8L=?z4k|0`Ms@h;P(oz bH*}y@|Gmvmk>8IYQUmWq{4*f`yX=1e|FoLK diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/run_01_sort_loss.py b/research/part03_learnt_overlap/e01_learn_to_disentangle/run_01_sort_loss.py deleted file mode 100644 index 81f3d051..00000000 --- a/research/part03_learnt_overlap/e01_learn_to_disentangle/run_01_sort_loss.py +++ /dev/null @@ -1,80 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import torch -import torch.nn.functional as F -from torch.utils.data import DataLoader - -import research.code.util as H -from disent.nn.loss.softsort import multi_spearman_rank_loss -from disent.nn.loss.softsort import torch_soft_rank - - -# ========================================================================= # -# tests # -# ========================================================================= # - - -def run_differentiable_sorting_loss(dataset='dsprites', loss_mode='spearman', optimizer='adam', lr=1e-2): - """ - test that the differentiable sorting works over a batch of images. - """ - - dataset = H.make_dataset(dataset) - dataloader = DataLoader(dataset=dataset, batch_size=256, pin_memory=True, shuffle=True) - - y = H.get_single_batch(dataloader) - # y += torch.randn_like(y) * 0.001 # prevent nan errors - x = torch.randn_like(y, requires_grad=True) - - optimizer = H.make_optimizer(x, name=optimizer, lr=lr) - - for i in range(1001): - if loss_mode == 'spearman': - loss = multi_spearman_rank_loss(x, y, dims=(2, 3), nan_to_num=True) - elif loss_mode == 'mse_rank': - loss = 0. - loss += F.mse_loss(torch_soft_rank(x, dims=(-3, -1)), torch_soft_rank(y, dims=(-3, -1)), reduction='mean') - loss += F.mse_loss(torch_soft_rank(x, dims=(-3, -2)), torch_soft_rank(y, dims=(-3, -2)), reduction='mean') - elif loss_mode == 'mse': - loss += F.mse_loss(x, y, reduction='mean') - else: - raise KeyError(f'invalid loss mode: {repr(loss_mode)}') - - # update variables - H.step_optimizer(optimizer, loss) - if i % 250 == 0: - H.plt_imshow(H.to_img(x[0]), show=True) - - # compute loss - print(i, float(loss)) - - -# ========================================================================= # -# MAIN # -# ========================================================================= # - - -if __name__ == '__main__': - run_differentiable_sorting_loss() diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/run_02_check_aug_gt_dists.py b/research/part03_learnt_overlap/e01_learn_to_disentangle/run_02_check_aug_gt_dists.py deleted file mode 100644 index 995cdb2d..00000000 --- a/research/part03_learnt_overlap/e01_learn_to_disentangle/run_02_check_aug_gt_dists.py +++ /dev/null @@ -1,168 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - - -import numpy as np -import torch -import torch.nn.functional as F -from matplotlib import pyplot as plt -from tqdm import tqdm - -import research.code.util as H -from disent.nn.functional import torch_box_kernel_2d -from disent.nn.functional import torch_conv2d_channel_wise_fft -from disent.nn.functional import torch_gaussian_kernel_2d - - -# ========================================================================= # -# distance function # -# ========================================================================= # - - -def spearman_rank_dist( - pred: torch.Tensor, - targ: torch.Tensor, - reduction='mean', - nan_to_num=False, -): - # add missing dim - if pred.ndim == 1: - pred, targ = pred.reshape(1, -1), targ.reshape(1, -1) - assert pred.shape == targ.shape - assert pred.ndim == 2 - # sort the last dimension of the 2D tensors - pred = torch.argsort(pred).to(torch.float32) - targ = torch.argsort(targ).to(torch.float32) - # compute individual losses - # TODO: this can result in nan values, what to do then? - pred = pred - pred.mean(dim=-1, keepdim=True) - pred = pred / pred.norm(dim=-1, keepdim=True) - targ = targ - targ.mean(dim=-1, keepdim=True) - targ = targ / targ.norm(dim=-1, keepdim=True) - # replace nan values - if nan_to_num: - pred = torch.nan_to_num(pred, nan=0.0) - targ = torch.nan_to_num(targ, nan=0.0) - # compute the final loss - loss = (pred * targ).sum(dim=-1) - # reduce the loss - if reduction == 'mean': - return loss.mean() - elif reduction == 'none': - return loss - else: - raise KeyError(f'Invalid reduction mode: {repr(reduction)}') - - -def check_xy_squares_dists(kernel='box', repeats=100, samples=256, pairwise_samples=256, kernel_radius=32, show_prog=True): - if kernel == 'box': - kernel = torch_box_kernel_2d(radius=kernel_radius)[None, ...] - elif kernel == 'max_box': - crange = torch.abs(torch.arange(kernel_radius * 2 + 1) - kernel_radius) - y, x = torch.meshgrid(crange, crange) - d = torch.maximum(x, y) + 1 - d = d.max() - d - kernel = (d.to(torch.float32) / d.sum())[None, None, ...] - elif kernel == 'min_box': - crange = torch.abs(torch.arange(kernel_radius * 2 + 1) - kernel_radius) - y, x = torch.meshgrid(crange, crange) - d = torch.minimum(x, y) + 1 - d = d.max() - d - kernel = (d.to(torch.float32) / d.sum())[None, None, ...] - elif kernel == 'manhat_box': - crange = torch.abs(torch.arange(kernel_radius * 2 + 1) - kernel_radius) - y, x = torch.meshgrid(crange, crange) - d = (y + x) + 1 - d = d.max() - d - kernel = (d.to(torch.float32) / d.sum())[None, None, ...] - elif kernel == 'gaussian': - kernel = torch_gaussian_kernel_2d(sigma=kernel_radius / 4.0, truncate=4.0)[None, None, ...] - else: - raise KeyError(f'invalid kernel mode: {repr(kernel)}') - - # make dataset - dataset = H.make_dataset('xysquares') - - losses = [] - prog = tqdm(range(repeats), postfix={'loss': 0.0}) if show_prog else range(repeats) - - for i in prog: - # get random samples - factors = dataset.sample_factors(samples) - batch = dataset.dataset_batch_from_factors(factors, mode='target') - if torch.cuda.is_available(): - batch = batch.cuda() - kernel = kernel.cuda() - factors = torch.from_numpy(factors).to(dtype=torch.float32, device=batch.device) - - # random pairs - ia, ib = torch.randint(0, len(batch), size=(2, pairwise_samples), device=batch.device) - - # compute factor distances - f_dists = torch.abs(factors[ia] - factors[ib]).sum(dim=-1) - - # compute loss distances - aug_batch = torch_conv2d_channel_wise_fft(batch, kernel) - # TODO: aug - batch or aug - aug - # b_dists = torch.abs(aug_batch[ia] - aug_batch[ib]).sum(dim=(-3, -2, -1)) - b_dists = F.mse_loss(aug_batch[ia], aug_batch[ib], reduction='none').sum(dim=(-3, -2, -1)) - - # compute ranks - # losses.append(float(torch.clamp(torch_mse_rank_loss(b_dists, f_dists), 0, 100))) - # losses.append(float(torch.abs(torch.argsort(f_dists, descending=True) - torch.argsort(b_dists, descending=False)).to(torch.float32).mean())) - losses.append(float(spearman_rank_dist(b_dists, f_dists))) - - if show_prog: - prog.set_postfix({'loss': np.mean(losses)}) - - return np.mean(losses), aug_batch[0] - - -def run_check_all_xy_squares_dists(show=False): - for kernel in [ - 'box', - 'max_box', - 'min_box', - 'manhat_box', - 'gaussian', - ]: - rs = list(range(1, 33, 4)) - ys = [] - for r in rs: - ave_spearman, last_img = check_xy_squares_dists(kernel=kernel, repeats=32, samples=128, pairwise_samples=1024, kernel_radius=r, show_prog=False) - H.plt_imshow(H.to_img(last_img, scale=True), show=show) - ys.append(abs(ave_spearman)) - print(kernel, r, ':', r*2+1, abs(ave_spearman)) - plt.plot(rs, ys, label=kernel) - plt.legend() - plt.show() - - -# ========================================================================= # -# MAIN # -# ========================================================================= # - - -if __name__ == '__main__': - run_check_all_xy_squares_dists() diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/run_03_train_disentangle_kernel.py b/research/part03_learnt_overlap/e01_learn_to_disentangle/run_03_train_disentangle_kernel.py deleted file mode 100644 index 77fc3d77..00000000 --- a/research/part03_learnt_overlap/e01_learn_to_disentangle/run_03_train_disentangle_kernel.py +++ /dev/null @@ -1,386 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -import os -from typing import List -from typing import Optional -from typing import Sequence - -import numpy as np -import pytorch_lightning as pl -import torch -import wandb -from omegaconf import OmegaConf -from pytorch_lightning.loggers import WandbLogger -from torch.nn import Parameter -from torch.utils.data import DataLoader - -import research.code.util as H -from disent.nn.functional import torch_conv2d_channel_wise_fft -from disent.nn.loss.softsort import spearman_rank_loss -from disent.nn.modules import DisentLightningModule -from disent.nn.modules import DisentModule -from disent.util.lightning.callbacks import BaseCallbackPeriodic -from disent.util.lightning.logger_util import wb_log_metrics -from disent.util.seeds import seed -from disent.util.strings.fmt import make_box_str -from experiment.run import hydra_get_callbacks -from experiment.run import hydra_get_gpus -from experiment.run import hydra_make_logger -from experiment.util.hydra_main import hydra_main -from experiment.util.hydra_utils import make_non_strict - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# EXP # -# ========================================================================= # - - -def disentangle_loss( - batch: torch.Tensor, - aug_batch: Optional[torch.Tensor], - factors: torch.Tensor, - num_pairs: int, - f_idxs: Optional[List[int]] = None, - loss_fn: str = 'mse', - mean_dtype=None, - corr_mode: str = 'improve', - regularization_strength: float = 1.0, - factor_sizes: Optional[torch.Tensor] = None, # scale the distances | Must be the same approach as `GroundTruthDistSampler` -) -> torch.Tensor: - assert len(batch) == len(factors) - assert batch.ndim == 4 - assert factors.ndim == 2 - # random pairs - ia, ib = torch.randint(0, len(batch), size=(2, num_pairs), device=batch.device) - # get pairwise distances - b_dists = H.pairwise_loss(batch[ia], batch[ib], mode=loss_fn, mean_dtype=mean_dtype) # avoid precision errors - if aug_batch is not None: - assert aug_batch.shape == batch.shape - b_dists += H.pairwise_loss(aug_batch[ia], aug_batch[ib], mode=loss_fn, mean_dtype=mean_dtype) - # compute factor differences - if f_idxs is not None: - f_diffs = factors[ia][:, f_idxs] - factors[ib][:, f_idxs] - else: - f_diffs = factors[ia] - factors[ib] - # scale the factor distances - if factor_sizes is not None: - assert factor_sizes.ndim == 1 - assert factor_sizes.shape == factors.shape[1:] - scale = torch.maximum(torch.ones_like(factor_sizes), factor_sizes - 1) - f_diffs = f_diffs / scale.detach() - # compute factor distances - f_dists = torch.abs(f_diffs).sum(dim=-1) - # optimise metric - if corr_mode == 'improve': loss = spearman_rank_loss(b_dists, -f_dists, regularization_strength=regularization_strength) # default one to use! - elif corr_mode == 'invert': loss = spearman_rank_loss(b_dists, +f_dists, regularization_strength=regularization_strength) - elif corr_mode == 'none': loss = +torch.abs(spearman_rank_loss(b_dists, -f_dists, regularization_strength=regularization_strength)) - elif corr_mode == 'any': loss = -torch.abs(spearman_rank_loss(b_dists, -f_dists, regularization_strength=regularization_strength)) - else: raise KeyError(f'invalid correlation mode: {repr(corr_mode)}') - # done! - return loss - - -class DisentangleModule(DisentLightningModule): - - def __init__( - self, - model, - hparams, - disentangle_factor_idxs: Sequence[int] = None - ): - super().__init__() - self.model = model - self.hparams.update(hparams) - self._disentangle_factors = None if (disentangle_factor_idxs is None) else np.array(disentangle_factor_idxs) - - def configure_optimizers(self): - return H.make_optimizer(self, name=self.hparams.exp.optimizer.name, lr=self.hparams.exp.optimizer.lr, weight_decay=self.hparams.exp.optimizer.weight_decay) - - def training_step(self, batch, batch_idx): - (x,), (f,) = batch['x_targ'], batch['factors'] - # feed forward batch - y = self.model(x) - # compute pairwise distances of factors and batch, and optimize to correspond - loss_rank = disentangle_loss( - batch = x if self.hparams.exp.train.combined_loss else y, - aug_batch = y if self.hparams.exp.train.combined_loss else None, - factors=f, - num_pairs=int(len(x) * self.hparams.exp.train.pairs_ratio), - f_idxs=self._disentangle_factors, - loss_fn=self.hparams.exp.train.loss, - mean_dtype=torch.float64, - regularization_strength=self.hparams.exp.train.reg_strength, - factor_sizes=None, - ) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - if hasattr(self.model, 'augment_loss'): - loss_aug = self.model.augment_loss(self) - else: - loss_aug = 0 - loss = loss_rank + loss_aug - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - self.log('loss_rank', float(loss_rank), prog_bar=True) - self.log('loss', float(loss), prog_bar=True) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - return loss - - def forward(self, x): - return self.model(x) - - -# ========================================================================= # -# MAIN # -# ========================================================================= # - - -_REPR_FN_INIT = { - 'none': lambda x: x, - 'square': lambda x: torch.sqrt(torch.abs(x)), - 'abs': lambda x: torch.abs(x), - 'exp': lambda x: torch.log(torch.abs(x)), -} - -_REPR_FN = { - 'none': lambda x: x, - 'square': lambda x: torch.square(x), - 'abs': lambda x: torch.abs(x), - 'exp': lambda x: torch.exp(x), -} - - - -class Kernel(DisentModule): - - def __init__( - self, - radius: int = 33, - channels: int = 1, - # loss settings - train_symmetric_regularise: bool = True, - train_norm_regularise: bool = True, - train_nonneg_regularise: bool = True, - train_regularize_l2_weight: Optional[float] = None, - # kernel settings - represent_mode: str = 'abs', - init_offset: float = 0.0, - init_scale: float = 0.001, - init_sums_to_one: bool = True, - ): - super().__init__() - assert channels in (1, 3) - assert set(_REPR_FN_INIT.keys()) == set(_REPR_FN.keys()) - assert represent_mode in _REPR_FN, f'invalid represent_mode: {repr(represent_mode)}' - # initialize - with torch.no_grad(): - # randomly sample value - if represent_mode == 'none': - kernel = torch.rand(1, channels, 2*radius+1, 2*radius+1, dtype=torch.float32) - else: - kernel = torch.randn(1, channels, 2*radius+1, 2*radius+1, dtype=torch.float32) - # scale values - kernel = init_offset + kernel * init_scale - if init_sums_to_one: - kernel = kernel / kernel.sum(dim=[-1, -2], keepdim=True) - # log params - kernel = _REPR_FN_INIT[represent_mode](kernel) - assert not torch.any(torch.isnan(kernel)) - # store - self.__kernel = Parameter(kernel) - self._represent_mode = represent_mode - # regularise options - self._train_symmetric_regularise = train_symmetric_regularise - self._train_norm_regularise = train_norm_regularise - self._train_nonneg_regularise = train_nonneg_regularise - self._train_regularize_l2_weight = train_regularize_l2_weight - - @property - def kernel(self) -> torch.Tensor: - return _REPR_FN[self._represent_mode](self.__kernel) - - def forward(self, xs): - return torch_conv2d_channel_wise_fft(xs, self.kernel) - - def make_train_periodic_callback(self, cfg, dataset) -> BaseCallbackPeriodic: - class ImShowCallback(BaseCallbackPeriodic): - def do_step(self, trainer: pl.Trainer, pl_module: pl.LightningModule): - # get kernel image - img_kernel = H.to_img(pl_module.model.kernel[0], scale=True).numpy() - img_kernel_log = H.to_img(torch.log(pl_module.model.kernel[0]), scale=True).numpy() - # augment function - def augment_fn(batch): - return H.to_imgs(pl_module.forward(batch.to(pl_module.device)), scale=True) - # get augmented traversals - with torch.no_grad(): - orig_wandb_image, orig_wandb_animation = H.visualize_dataset_traversal(dataset, augment_fn=None, data_mode='raw', output_wandb=True) # dataset returns (numpy?) HWC batches - augm_wandb_image, augm_wandb_animation = H.visualize_dataset_traversal(dataset, augment_fn=augment_fn, data_mode='input', output_wandb=True) # dataset returns (tensor) CHW batches - # log images to WANDB - wb_log_metrics(trainer.logger, { - 'kernel': wandb.Image(img_kernel), - 'kernel_ln': wandb.Image(img_kernel_log), - 'traversal_img_orig': orig_wandb_image, 'traversal_animation_orig': orig_wandb_animation, - 'traversal_img_augm': augm_wandb_image, 'traversal_animation_augm': augm_wandb_animation, - }) - return ImShowCallback(every_n_steps=cfg.exp.out.show_every_n_steps, begin_first_step=True) - - def augment_loss(self, framework: DisentLightningModule): - augment_loss = 0 - # symmetric loss - if self._train_symmetric_regularise: - k, kt = self.kernel[0], torch.transpose(self.kernel[0], -1, -2) - loss_symmetric = 0 - loss_symmetric += H.unreduced_loss(torch.flip(k, dims=[-1]), k, mode='mae').mean() - loss_symmetric += H.unreduced_loss(torch.flip(k, dims=[-2]), k, mode='mae').mean() - loss_symmetric += H.unreduced_loss(torch.flip(k, dims=[-1]), kt, mode='mae').mean() - loss_symmetric += H.unreduced_loss(torch.flip(k, dims=[-2]), kt, mode='mae').mean() - # log loss - framework.log('loss_sym', float(loss_symmetric), prog_bar=True) - # final loss - augment_loss += loss_symmetric - # regularize, try make kernel as small as possible - if (self._train_regularize_l2_weight is not None) and (self._train_regularize_l2_weight > 0): - k = self.kernel[0] - loss_l2 = self._train_regularize_l2_weight * (k ** 2).mean() - framework.log('loss_l2', float(loss_l2), prog_bar=True) - augment_loss += loss_l2 - # sum of 1 loss, per channel - if self._train_norm_regularise: - k = self.kernel[0] - # sum over W & H resulting in: (C, W, H) -> (C,) - channel_sums = k.sum(dim=[-1, -2]) - channel_loss = H.unreduced_loss(channel_sums, torch.ones_like(channel_sums), mode='mse') - norm_loss = channel_loss.mean() - # log loss - framework.log('loss_norm', float(norm_loss), prog_bar=True) - # final loss - augment_loss += norm_loss - # no negatives regulariser - if self._train_nonneg_regularise: - k = self.kernel[0] - nonneg_loss = torch.abs(k[k < 0].sum()) - # log loss - framework.log('loss_nonneg', float(nonneg_loss), prog_bar=True) - # regularise negatives - augment_loss += nonneg_loss - # stats - framework.log('kernel_mean', float(self.kernel.mean()), prog_bar=False) - framework.log('kernel_std', float(self.kernel.std()), prog_bar=False) - # return! - return augment_loss - - -# ========================================================================= # -# Run Hydra # -# ========================================================================= # - - -def run_disentangle_dataset_kernel(cfg): - cfg = make_non_strict(cfg) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # TODO: some of this code is duplicated between this and the main experiment run.py - # check CUDA setting - cfg.trainer.setdefault('cuda', 'try_cuda') - gpus = hydra_get_gpus(cfg) - # CREATE LOGGER - logger = hydra_make_logger(cfg) - if isinstance(logger.experiment, WandbLogger): - _ = logger.experiment # initialize - # TRAINER CALLBACKS - callbacks = hydra_get_callbacks(cfg) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - seed(cfg.settings.job.seed) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # initialise dataset and get factor names to disentangle - dataset = H.make_dataset(cfg.exp.data.name, factors=True, data_root=cfg.dsettings.storage.data_root) - disentangle_factor_idxs = dataset.gt_data.normalise_factor_idxs(cfg.exp.kernel.disentangle_factors) - cfg.exp.kernel.disentangle_factors = tuple(dataset.gt_data.factor_names[i] for i in disentangle_factor_idxs) - log.info(f'Dataset has ground-truth factors: {dataset.gt_data.factor_names}') - log.info(f'Chosen ground-truth factors are: {tuple(cfg.exp.kernel.disentangle_factors)}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # print everything - log.info('Final Config' + make_box_str(OmegaConf.to_yaml(cfg))) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - dataloader = DataLoader( - dataset, - batch_sampler=H.StochasticBatchSampler(dataset, batch_size=cfg.datamodule.dataloader.batch_size), - num_workers=cfg.datamodule.dataloader.num_workers, - pin_memory=cfg.datamodule.dataloader.pin_memory, - ) - model = Kernel(radius=cfg.exp.kernel.radius, channels=cfg.exp.kernel.channels, init_offset=cfg.exp.kernel.init_offset, init_scale=cfg.exp.kernel.init_scale, train_symmetric_regularise=cfg.exp.kernel.regularize_symmetric, train_norm_regularise=cfg.exp.kernel.regularize_norm, train_nonneg_regularise=cfg.exp.kernel.regularize_nonneg, represent_mode=cfg.exp.kernel.represent_mode, init_sums_to_one=cfg.exp.kernel.init_sums_to_one, train_regularize_l2_weight=cfg.exp.kernel.regularize_l2_weight) - callbacks.append(model.make_train_periodic_callback(cfg, dataset=dataset)) - framework = DisentangleModule(model, cfg, disentangle_factor_idxs=disentangle_factor_idxs) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - if logger: - logger.log_hyperparams(cfg) - # train - trainer = pl.Trainer( - log_every_n_steps=cfg.trainer.log_every_n_steps, - logger=logger, - callbacks=callbacks, - gpus=1 if gpus else 0, - max_epochs=cfg.trainer.max_epochs, - max_steps=cfg.trainer.max_steps, - enable_progress_bar=False, - # we do this here so we don't run the final metrics - detect_anomaly=False, # this should only be enabled for debugging torch and finding NaN values, slows down execution, not by much though? - enable_checkpointing=False, - ) - trainer.fit(framework, dataloader) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # save kernel - if cfg.exp.out.rel_save_dir is not None: - assert not os.path.isabs(cfg.exp.out.rel_save_dir), f'rel_save_dir must be relative: {repr(cfg.exp.out.rel_save_dir)}' - save_dir = os.path.join(ROOT_DIR, cfg.exp.out.rel_save_dir) - assert os.path.isabs(save_dir), f'save_dir must be absolute: {repr(save_dir)}' - # save kernel - H.torch_write(os.path.join(save_dir, cfg.exp.out.save_name), framework.model.kernel.cpu().detach()) - - -# ========================================================================= # -# Entry Point # -# ========================================================================= # - - -if __name__ == '__main__': - # HYDRA: - # run experiment (12min * 4*8*2) / 60 ~= 12 hours - # but speeds up as kernel size decreases, so might be shorter - # EXP ARGS: - # $ ... -m optimizer.weight_decay=1e-4,0.0 kernel.radius=63,55,47,39,31,23,15,7 dataset.spacing=8,4,2,1 - - ROOT_DIR = os.path.abspath(__file__ + '/../../../..') - CONFIGS_THIS_EXP = os.path.abspath(os.path.join(__file__, '..', 'config')) - CONFIGS_RESEARCH = os.path.abspath(os.path.join(__file__, '../../..', 'config')) - - # launch the action - hydra_main( - callback=run_disentangle_dataset_kernel, - config_name='config_adversarial_kernel', - search_dirs_prepend=[CONFIGS_THIS_EXP, CONFIGS_RESEARCH], - log_level=logging.INFO, - ) diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/run_04_train_disentangle_model.py b/research/part03_learnt_overlap/e01_learn_to_disentangle/run_04_train_disentangle_model.py deleted file mode 100644 index fddaa07b..00000000 --- a/research/part03_learnt_overlap/e01_learn_to_disentangle/run_04_train_disentangle_model.py +++ /dev/null @@ -1,394 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2021 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -""" -Generate an adversarial dataset by approximating the difference between -the dataset and the target adversarial images using a model. - adv = obs + diff(obs) -""" - -import logging -import os -from datetime import datetime -from typing import List -from typing import Optional -from typing import Union - -import numpy as np -import pytorch_lightning as pl -import torch -import torch.nn.functional as F -import wandb -from torch.utils.data import DataLoader - -from disent.util.seeds import TempNumpySeed -from omegaconf import OmegaConf - -import research.code.util as H -from disent.dataset import DisentDataset -from disent.dataset.sampling import BaseDisentSampler -from disent.dataset.util.hdf5 import H5Builder -from disent.nn.modules import DisentModule -from disent.nn.weights import init_model_weights -from disent.util.function import wrapped_partial -from disent.util.inout.paths import ensure_parent_dir_exists -from disent.util.lightning.callbacks import LoggerProgressCallback -from disent.util.seeds import seed -from disent.util.strings.fmt import bytes_to_human -from disent.util.strings.fmt import make_box_str -from experiment.run import hydra_get_gpus -from experiment.run import hydra_get_callbacks -from experiment.run import hydra_make_logger -from experiment.util.hydra_main import hydra_main -from experiment.util.hydra_utils import make_non_strict -from research.part03_learnt_overlap.e01_learn_to_disentangle.run_03_train_disentangle_kernel import disentangle_loss -from research.part03_learnt_overlap.__OLD__e02_learn_adversarial_data.run_02_gen_adversarial_dataset_approx import AdversarialAugmentModel -from research.part03_learnt_overlap.__OLD__e02_learn_adversarial_data.run_02_gen_adversarial_dataset_approx import AdversarialModel -from research.part03_learnt_overlap.__OLD__e02_learn_adversarial_data.run_02_gen_adversarial_dataset_approx import gen_approx_dataset_mask - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# adversarial dataset generator # -# ========================================================================= # - - -class DisentangleModel(AdversarialModel): - - def __init__( - self, - # optimizer options - optimizer_name: str = 'sgd', - optimizer_lr: float = 5e-2, - optimizer_kwargs: Optional[dict] = None, - # dataset config options - dataset_name: str = 'cars3d', - dataset_num_workers: int = min(os.cpu_count(), 16), - dataset_batch_size: int = 256, - data_root: str = 'data/dataset', - data_load_into_memory: bool = False, - # disentangle loss options - disentangle_mode: str = 'improve', - disentangle_pairs_ratio: float = 8.0, - disentangle_factors: Optional[List[Union[str, int]]] = None, - disentangle_loss: str = 'mse', - disentangle_reg_strength: float = 1.0, - disentangle_scale_dists: bool = True, - disentangle_combined_loss: bool = True, - disentangle_randomize_factors: bool = False, - disentangle_randomize_factors_seed: int = 777, - # progressively increase the size of the dataset, releasing more information to the model. - # If `disentangle_randomize_factors==True`, then this can help learning! - # 0.01 seems like a good place to start tuning? maybe 0.001? I think 0.1 is too high? - disentangle_progressive_release_data: Optional[float] = None, - # loss extras - loss_disentangle_weight: Optional[float] = 1.0, - loss_stats_mean_weight: Optional[float] = 0.0, - loss_stats_var_weight: Optional[float] = 0.0, - loss_similarity_weight: Optional[float] = 0.0, - loss_out_of_bounds_weight: Optional[float] = 0.0, - # model settings - model_type: str = 'ae_linear', - model_mask_mode: Optional[str] = 'none', - model_weight_init: str = 'xavier_normal', - # logging settings - logging_scale_imgs: bool = False, - ): - # some hparams from here are not actually used! - # we should delete them ... - super().__init__() - # modify hparams - if optimizer_kwargs is None: - optimizer_kwargs = {} # value used by save_hyperparameters - # save hparams - self.save_hyperparameters() - # variables - self.dataset: DisentDataset = None - self.model: DisentModule = None - self._disentangle_idxs: np.ndarray = None - - # disentangle_randomize_factors == True - _random_indices: torch.Tensor - _random_factors: torch.Tensor - - # ================================== # - # setup # - # ================================== # - - def prepare_data(self) -> None: - # create dataset - self.dataset = H.make_dataset( - self.hparams.dataset_name, - load_into_memory=self.hparams.data_load_into_memory, - load_memory_dtype=torch.float32, - data_root=self.hparams.data_root, - factors=True, - ) - # normalize factors - self._disentangle_idxs = self.dataset.gt_data.normalise_factor_idxs(self.hparams.disentangle_factors) - self.hparams.disentangle_factors = tuple(self.dataset.gt_data.factor_names[i] for i in self._disentangle_idxs) - log.info('Disentangling Factors:') - for factor_name in self.hparams.disentangle_factors: - log.info(f'* {factor_name}') - # make the model - self.model = AdversarialAugmentModel( - model_type=self.hparams.model_type, - x_shape=(self.dataset.gt_data.img_channels, 64, 64), - mask=gen_approx_dataset_mask(dataset=self.dataset, model_mask_mode=self.hparams.model_mask_mode), - # if we save the model we can restore things! - meta=dict( - dataset_name=self.hparams.dataset_name, - dataset_factor_sizes=self.dataset.gt_data.factor_sizes, - dataset_factor_names=self.dataset.gt_data.factor_names, - sampler_name=self.hparams.sampler_name, - hparams=dict(self.hparams) - ), - ) - # initialize model - self.model = init_model_weights(self.model, mode=self.hparams.model_weight_init) - # get random factors - if self.hparams.disentangle_randomize_factors: - with TempNumpySeed(self.hparams.disentangle_randomize_factors_seed): - random_indices = np.arange(len(self.dataset)) - np.random.shuffle(random_indices) - random_factors = self.dataset.gt_data.idx_to_pos(random_indices) - # register the items, so that they are transferred between devices with the model - self.register_buffer('_random_indices', torch.from_numpy(random_indices), persistent=False) - self.register_buffer('_random_factors', torch.from_numpy(random_factors), persistent=False) - - def train_dataloader(self): - if self.hparams.disentangle_progressive_release_data is None: - return DataLoader( - self.dataset, - batch_size=self.hparams.dataset_batch_size, - num_workers=self.hparams.dataset_num_workers, - shuffle=True, - ) - else: - return DataLoader( - self.dataset, - batch_sampler=H.ProgressiveStochasticBatchSampler( - data_source=self.dataset, - batch_size=self.hparams.dataset_batch_size, - progression_rate=self.hparams.disentangle_progressive_release_data, - ), - num_workers=self.hparams.dataset_num_workers, - ) - - # ================================== # - # train step # - # ================================== # - - def training_step(self, batch, batch_idx): - (x,) = batch['x_targ'] - (f,) = batch['factors'] - # replace with random factors if needed - if self.hparams.disentangle_randomize_factors: - (i,) = batch['idx'] - f = self._random_factors[i] - # feed forward - y = self.model(x) - # compute loss - loss_dis = 0 - if (self.hparams.loss_adversarial_weight is not None) and (self.hparams.loss_adversarial_weight > 0): - loss_dis = self.hparams.loss_disentangle_weight * disentangle_loss( - batch = x if self.hparams.disentangle_combined_loss else y, - aug_batch = y if self.hparams.disentangle_combined_loss else None, - factors=f, - num_pairs=int(len(y) * self.hparams.disentangle_pairs_ratio), - f_idxs=self._disentangle_idxs, - loss_fn=self.hparams.disentangle_loss, - corr_mode=self.hparams.disentangle_mode, - regularization_strength=self.hparams.disentangle_reg_strength, - factor_sizes=torch.as_tensor(self.dataset.gt_data.factor_sizes, device=f.device) if self.hparams.disentangle_scale_dists else None, - ) - # additional loss components - # - keep stats the same - loss_stats = 0 - if (self.hparams.loss_stats_mean_weight is not None) and (self.hparams.loss_stats_mean_weight > 0): - img_mean_loss = F.mse_loss(y.mean(dim=0), x.mean(dim=0), reduction='mean') - loss_stats += self.hparams.loss_stats_mean_weight * img_mean_loss - if (self.hparams.loss_stats_var_weight is not None) and (self.hparams.loss_stats_var_weight > 0): - img_std_loss = F.mse_loss(y.std(dim=0), x.std(dim=0), reduction='mean') - loss_stats += self.hparams.loss_stats_var_weight * img_std_loss - # - try keep similar to inputs - loss_sim = 0 - if (self.hparams.loss_similarity_weight is not None) and (self.hparams.loss_similarity_weight > 0): - loss_sim = self.hparams.loss_similarity_weight * F.mse_loss(y, x, reduction='mean') - # - regularize if out of bounds - loss_out = 0 - if (self.hparams.loss_out_of_bounds_weight is not None) and (self.hparams.loss_out_of_bounds_weight > 0): - zeros = torch.zeros_like(y) - gt_loss = torch.where(y < 0, -y, zeros).mean() - lt_loss = torch.where(y > 1, y-1, zeros).mean() - loss_out = self.hparams.loss_out_of_bounds_weight * (gt_loss + lt_loss) - # final loss - loss = loss_dis + loss_stats + loss_sim + loss_out - # log everything - self.log_dict({ - 'loss': loss, - 'dis': loss_dis, - 'sta': loss_stats, - 'out': loss_out, - 'sim': loss_sim, - }, prog_bar=True) - # done! - return loss - - -# ========================================================================= # -# Run Hydra # -# ========================================================================= # - - -ROOT_DIR = os.path.abspath(__file__ + '/../../..') - - -def run_gen_dataset(cfg): - time_string = datetime.today().strftime('%Y-%m-%d--%H-%M-%S') - log.info(f'Starting run at time: {time_string}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # cleanup from old runs: - try: - wandb.finish() - except: - pass - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - cfg = make_non_strict(cfg) - # - - - - - - - - - - - - - - - # - # check CUDA setting - gpus = hydra_get_gpus(cfg) - # create logger - logger = hydra_make_logger(cfg) - # create callbacks - callbacks: List[pl.Callback] = [c for c in hydra_get_callbacks(cfg) if isinstance(c, LoggerProgressCallback)] - # - - - - - - - - - - - - - - - # - # check save dirs - assert not os.path.isabs(cfg.settings.exp.rel_save_dir), f'rel_save_dir must be relative: {repr(cfg.settings.exp.rel_save_dir)}' - save_dir = os.path.join(ROOT_DIR, cfg.settings.exp.rel_save_dir) - assert os.path.isabs(save_dir), f'save_dir must be absolute: {repr(save_dir)}' - # - - - - - - - - - - - - - - - # - # get the logger and initialize - if logger is not None: - logger.log_hyperparams(cfg) - # print the final config! - log.info('Final Config' + make_box_str(OmegaConf.to_yaml(cfg))) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # | | | | | | | | | | | | | | | # - seed(cfg.settings.job.seed) - # | | | | | | | | | | | | | | | # - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # make framework - framework = DisentangleModel(**cfg.dis_system) - callbacks.extend(framework.make_train_periodic_callbacks(cfg)) - # train - trainer = pl.Trainer( - logger=logger, - callbacks=callbacks, - # cfg.dsettings.trainer - gpus=gpus, - # cfg.trainer - max_epochs=cfg.trainer.max_epochs, - max_steps=cfg.trainer.max_steps, - log_every_n_steps=cfg.trainer.log_every_n_steps, - enable_progress_bar=cfg.trainer.enable_progress_bar, - # prepare_data_per_node=cfg.trainer.prepare_data_per_node, # TODO: moved into data module / framework ! - # we do this here so we don't run the final metrics - detect_anomaly=False, # this should only be enabled for debugging torch and finding NaN values, slows down execution, not by much though? - enable_checkpointing=False, - ) - trainer.fit(framework) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # get save paths - save_prefix = f'{cfg.settings.exp.save_prefix}_' if cfg.settings.exp.save_prefix else '' - save_path_model = os.path.join(save_dir, f'{save_prefix}{time_string}_{cfg.settings.job.name}', f'model.pt') - save_path_data = os.path.join(save_dir, f'{save_prefix}{time_string}_{cfg.settings.job.name}', f'data.h5') - # create directories - if cfg.settings.exp.save_model: ensure_parent_dir_exists(save_path_model) - if cfg.settings.exp.save_data: ensure_parent_dir_exists(save_path_data) - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # save adversarial model - if cfg.settings.exp.save_model: - log.info(f'saving model to path: {repr(save_path_model)}') - torch.save(framework.model, save_path_model) - log.info(f'saved model size: {bytes_to_human(os.path.getsize(save_path_model))}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - # save adversarial dataset - if cfg.settings.exp.save_data: - log.info(f'saving data to path: {repr(save_path_data)}') - # transfer to GPU - if torch.cuda.is_available(): - framework = framework.cuda() - # create new h5py file -- TODO: use this in other places! - with H5Builder(path=save_path_data, mode='atomic_w') as builder: - # set the transform -- TODO: we should not need to do this! - assert framework.dataset.gt_data._transform is None - framework.dataset.gt_data._transform = framework.dataset.transform - # this dataset is self-contained and can be loaded by SelfContainedHdf5GroundTruthData - builder.add_dataset_from_gt_data( - data=framework.dataset.gt_data, # produces raw - mutator=wrapped_partial(framework.batch_to_adversarial_imgs, mode=cfg.settings.exp.save_dtype), # consumes tensors -> np.ndarrays - img_shape=(64, 64, None), - compression_lvl=4, - dtype=cfg.settings.exp.save_dtype, - batch_size=32, - ) - log.info(f'saved data size: {bytes_to_human(os.path.getsize(save_path_data))}') - # ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ # - - -# ========================================================================= # -# Entry Point # -# ========================================================================= # - - -if __name__ == '__main__': - - # BENCHMARK (batch_size=256, optimizer=sgd, lr=1e-2, dataset_num_workers=0): - # - batch_optimizer=False, gpu=True, fp16=True : [3168MiB/5932MiB, 3.32/11.7G, 5.52it/s] - # - batch_optimizer=False, gpu=True, fp16=False : [5248MiB/5932MiB, 3.72/11.7G, 4.84it/s] - # - batch_optimizer=False, gpu=False, fp16=True : [same as fp16=False] - # - batch_optimizer=False, gpu=False, fp16=False : [0003MiB/5932MiB, 4.60/11.7G, 1.05it/s] - # --------- - # - batch_optimizer=True, gpu=True, fp16=True : [1284MiB/5932MiB, 3.45/11.7G, 4.31it/s] - # - batch_optimizer=True, gpu=True, fp16=False : [1284MiB/5932MiB, 3.72/11.7G, 4.31it/s] - # - batch_optimizer=True, gpu=False, fp16=True : [same as fp16=False] - # - batch_optimizer=True, gpu=False, fp16=False : [0003MiB/5932MiB, 1.80/11.7G, 4.18it/s] - - # BENCHMARK (batch_size=1024, optimizer=sgd, lr=1e-2, dataset_num_workers=12): - # - batch_optimizer=True, gpu=True, fp16=True : [2510MiB/5932MiB, 4.10/11.7G, 4.75it/s, 20% gpu util] (to(device).to(dtype)) - # - batch_optimizer=True, gpu=True, fp16=True : [2492MiB/5932MiB, 4.10/11.7G, 4.12it/s, 19% gpu util] (to(device, dtype)) - - CONFIGS_THIS_EXP = os.path.abspath(os.path.join(__file__, '..', 'config')) - CONFIGS_RESEARCH = os.path.abspath(os.path.join(__file__, '../../..', 'config')) - - # launch the action - hydra_main( - callback=run_gen_dataset, - config_name='config_disentangle_dataset_approx', - search_dirs_prepend=[CONFIGS_THIS_EXP, CONFIGS_RESEARCH], - log_level=logging.INFO, - ) diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/submit_03_overlap_loss_experiments.sh b/research/part03_learnt_overlap/e01_learn_to_disentangle/submit_03_overlap_loss_experiments.sh deleted file mode 100644 index 6a70dd5d..00000000 --- a/research/part03_learnt_overlap/e01_learn_to_disentangle/submit_03_overlap_loss_experiments.sh +++ /dev/null @@ -1,200 +0,0 @@ -#!/bin/bash - -# OVERVIEW: -# - this experiment is designed to test how changing the reconstruction loss to match the -# ground-truth distances allows datasets to be disentangled. - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="MSC-p03e02_learnt-loss-with-vaes" -export PARTITION="stampede" -export PARALLELISM=24 - -# the path to the generated arguments file -# - this needs to before we source the helper file -ARGS_FILE="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")/array_overlap_learnt_${PROJECT}_ALT.txt" -ARGS_FILE_RETRY="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")/array_overlap_learnt_${PROJECT}_RETRY_R31.txt" - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - - -# ========================================================================= # -# Experiment # -# ========================================================================= # - - -# DIFFERENT OVERLAP LOSSES -# -- changing the reconstruction loss enables disentanglement -# -- we found the optimal losses first using the metrics from -# `part01_data_overlap/plot02_data_distances/run_data_correlation.py` -# SETUP: -# -- we mimic the run file from `part01_data_overlap/e03_modified_loss_xysquares/submit_overlap_loss.sh` -# this experiment pretty much is a continuation -# 5 * (2*2*4 = 8) = 80 -ARGS_FILE="$ARGS_FILE" gen_sbatch_args_file \ - +DUMMY.repeat=1,2,3,4,5 \ - \ - +EXTRA.tags='MSC_sweep_losses' \ - hydra.job.name="ovlp_loss" \ - \ - run_length=medium \ - metrics=all \ - \ - dataset=X--xysquares \ - \ - framework=betavae,adavae_os \ - settings.framework.beta=0.0316,0.0001 \ - settings.model.z_size=25 \ - settings.framework.recon_loss='mse','mse_gau_r31_l1.0_k3969.0_norm_sum','mse_box_r31_l1.0_k3969.0_norm_sum','mse_xy8_abs63_l1.0_k1.0_norm_none' \ - \ - sampling=default__bb - - -# RUN THE EXPERIMENT: -#clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours -ARGS_FILE="$ARGS_FILE" submit_sbatch_args_file - - -# -- continuation of above! -# -- frameworks crash when settings.framework.beta==0.0001, this value is too low! Try everything inbetween -# standard MSE for some reason is fine with this value... but kernels fail... -ARGS_FILE="$ARGS_FILE_RETRY" gen_sbatch_args_file \ - +DUMMY.repeat=1,2,3,4,5 \ - +DUMMY.retry=2 \ - +EXTRA.tags='MSC_sweep_losses_ALT' \ - hydra.job.name="ovlp_loss" \ - \ - run_length=medium \ - metrics=all \ - \ - dataset=X--xysquares \ - \ - framework=betavae,adavae_os \ - settings.framework.beta=0.01,0.00316,0.001,0.000316 \ - settings.model.z_size=25 \ - settings.framework.recon_loss='mse_xy8_none63_l1.0_k1.0_norm_none','mse_xy8_abs63_l1.0_k0.1_norm_none','mse_xy8_abs63_l1.0_k10.0_norm_none' \ - \ - sampling=default__bb - -# settings.framework.recon_loss='mse','mse_gau_r31_l1.0_k3969.0_norm_sum','mse_box_r31_l1.0_k3969.0_norm_sum','mse_xy8_abs63_l1.0_k1.0_norm_none' \ - -# RETRY WITH A SMALLER LEARNT KERNEL SIZE -ARGS_FILE="$ARGS_FILE_RETRY" gen_sbatch_args_file \ - +DUMMY.repeat=1,2,3,4,5 \ - +DUMMY.retry=2 \ - +EXTRA.tags='MSC_sweep_losses_XY8R31' \ - hydra.job.name="ovlp_loss" \ - \ - run_length=medium \ - metrics=all \ - \ - dataset=X--xysquares \ - \ - framework=betavae,adavae_os \ - settings.framework.beta=0.0316,0.01,0.00316,0.001,0.000316,0.0001 \ - settings.model.z_size=25 \ - settings.framework.recon_loss='mse_xy8_abs31_l1.0_k1.0_norm_none' \ - \ - sampling=default__bb - - -# RUN THE EXPERIMENT: -#clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours -ARGS_FILE="$ARGS_FILE_RETRY" submit_sbatch_args_file - - -# ========================================================================= # -# OLD # -# ========================================================================= # - - -# TEST MSE Gaus vs MSE Learnt -# - EXTENDS: "TEST MSE vs BoxBlur MSE" from: "submit_overlap_loss.sh" -# -- in plotting, combine the results with `EXTRA.tags=="sweep_overlap_boxblur_specific"` -#ARGS_FILE="$ARGS_FILE" gen_sbatch_args_file \ -# +DUMMY.repeat=1,2,3,4,5 \ -# +EXTRA.tags='sweep_overlap_boxblur_learnt' \ -# hydra.job.name="l_ovlp_loss" \ -# \ -# run_length=medium \ -# metrics=all \ -# \ -# dataset=X--xysquares \ -# \ -# framework=betavae,adavae_os \ -# settings.framework.beta=0.0316,0.0001 \ -# settings.model.z_size=25 \ -# settings.framework.recon_loss='mse_gau_r31_l1.0_k3969.0_norm_sum','mse_xy8_r47_l1.0_k3969.0_norm_sum' \ -# \ -# sampling=default__bb - - -# RUN THE EXPERIMENT: -#clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours -#ARGS_FILE="$ARGS_FILE" submit_sbatch_args_file - - -# HPARAM TUNING FOR KERNEL -# -- 1 * (2*4*2*2) = 32 -#ARGS_FILE="$ARGS_FILE_PARAMS" gen_sbatch_args_file \ -# +DUMMY.repeat=1 \ -# +EXTRA.tags='sweep_overlap_learnt_hparams' \ -# hydra.job.name="hparam_tune" \ -# \ -# run_length=medium \ -# metrics=all \ -# \ -# dataset=X--xysquares \ -# settings.framework.beta=0.0316,0.0001 \ -# \ -# +VAR.kernel_loss_weight=1.0,10.0,100.0,1000.0 \ -# +VAR.kernel_norm_mode=sum,abssum \ -# \ -# settings.framework.recon_loss='mse_xy8_r47_l1.0_k${VAR.kernel_loss_weight}_norm_${VAR.kernel_norm_mode}' \ -# \ -# framework=adavae_os,betavae \ -# settings.model.z_size=25 \ -# \ -# sampling=default__bb - -# RUN THE EXPERIMENT: -#clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours -#ARGS_FILE="$ARGS_FILE_PARAMS" submit_sbatch_args_file - - -# TEST: MSE Learnt (USING ABOVE HPARAMS) -# - EXTENDS: "TEST MSE vs BoxBlur MSE" from: "submit_overlap_loss.sh" -# -- in plotting, combine the results with `EXTRA.tags=="sweep_overlap_boxblur_specific"` -# 5 * (3*2*2) = 60 -#ARGS_FILE="$ARGS_FILE_TUNED" gen_sbatch_args_file \ -# +DUMMY.repeat=1,2,3,4,5 \ -# +EXTRA.tags='sweep_overlap_learnt_TUNED' \ -# hydra.job.name="l_ovlp_loss" \ -# \ -# run_length=medium \ -# metrics=all \ -# \ -# dataset=X--xysquares \ -# \ -# +VAR.kernel_loss_weight=100.0,300.0,1000.0 \ -# +VAR.kernel_norm_mode=sum \ -# \ -# framework=betavae,adavae_os \ -# settings.framework.beta=0.0316,0.0001 \ -# settings.model.z_size=25 \ -# settings.framework.recon_loss='mse_xy8_r47_l1.0_k${VAR.kernel_loss_weight}_norm_${VAR.kernel_norm_mode}' \ -# \ -# sampling=default__bb -# -## RUN THE EXPERIMENT: -##clog_cudaless_nodes "$PARTITION" 86400 "C-disent" # 24 hours -#ARGS_FILE="$ARGS_FILE_TUNED" submit_sbatch_args_file - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/submit_03_train_kernel.sh b/research/part03_learnt_overlap/e01_learn_to_disentangle/submit_03_train_kernel.sh deleted file mode 100644 index d8ff87af..00000000 --- a/research/part03_learnt_overlap/e01_learn_to_disentangle/submit_03_train_kernel.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="MSC-p03e01_kernel-disentangle-xy" -export PARTITION="stampede" -export PARALLELISM=24 - -# override the default run file! -export PY_RUN_FILE='research/part03_learnt_overlap/e01_learn_to_disentangle/run_03_train_disentangle_kernel.py' - -# the path to the generated arguments file -# - this needs to before we source the helper file -ARGS_FILE="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")/array_03_$PROJECT.txt" -ARGS_FILE_TUNED="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")/array_03_${PROJECT}_TUNED.txt" - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -# 1 * (4*2*4*4) == 128 -ARGS_FILE="$ARGS_FILE" gen_sbatch_args_file \ - +EXTRA.tags='sweep' \ - hydra.job.name="kernel" \ - \ - run_length=short \ - settings.job.name_prefix="MSC" \ - \ - settings.dataset.batch_size=512 \ - exp.kernel.represent_mode=abs,square,exp,none \ - exp.optimizer.lr=1e-3,5e-4 \ - exp.optimizer.weight_decay=0.0 \ - \ - exp.data.name=xysquares_8x8,xysquares_4x4,xysquares_2x2,xysquares_1x1 \ - exp.kernel.radius=63,47,31,15 \ - -# 63,55,47,39,31,23,15,7 -#clog_cudaless_nodes "$PARTITION" 14400 "C-disent" # 4 hours -ARGS_FILE="$ARGS_FILE" submit_sbatch_args_file - - -# 1 * (2*2*4) = 16 -ARGS_FILE="$ARGS_FILE_TUNED" gen_sbatch_args_file \ - +EXTRA.tags='sweep_MED' \ - hydra.job.name="kernel" \ - \ - run_length=medium \ - settings.job.name_prefix="MSC_TUNED" \ - \ - settings.dataset.batch_size=512 \ - exp.kernel.represent_mode=abs,none \ - exp.optimizer.lr=1e-3,5e-4 \ - exp.optimizer.weight_decay=0.0 \ - \ - exp.data.name=xysquares_8x8 \ - exp.kernel.radius=63,47,31,15 \ - - -#clog_cudaless_nodes "$PARTITION" 14400 "C-disent" # 4 hours -ARGS_FILE="$ARGS_FILE_TUNED" submit_sbatch_args_file - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/submit_04_train_model.sh b/research/part03_learnt_overlap/e01_learn_to_disentangle/submit_04_train_model.sh deleted file mode 100644 index 2db6b7b5..00000000 --- a/research/part03_learnt_overlap/e01_learn_to_disentangle/submit_04_train_model.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="MSC-p03e02_disentangle-data" -export PARTITION="stampede" -export PARALLELISM=32 - -# override the default run file! -export PY_RUN_FILE='research/part03_learnt_overlap/e01_learn_to_disentangle/run_04_train_disentangle_model.py' - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -# 1 * (2*8*4) == 64 -local_sweep \ - +EXTRA.tags='test' \ - \ - dis_system.loss_disentangle_weight=1.0 \ - dis_system.loss_out_of_bounds_weight=1.0 \ - dis_system.loss_stats_mean_weight=NULL \ - dis_system.loss_stats_var_weight=1.0 \ - dis_system.loss_similarity_weight=1.0 \ - \ - dis_system.optimizer_lr=1e-3 \ - \ - dis_system.disentangle_randomize_factors=TRUE \ - dis_system.disentangle_combined_loss=FALSE \ - dis_system.disentangle_mode=invert,improve \ - dis_system.dataset_name=cars3d,smallnorb,dsprites,shapes3d,xysquares_8x8 diff --git a/research/part03_learnt_overlap/e03_different_gt_representations/array_01_MSC-p03e03_different-gt-representations.txt b/research/part03_learnt_overlap/e03_different_gt_representations/array_01_MSC-p03e03_different-gt-representations.txt deleted file mode 100644 index 9fddd4a5..00000000 --- a/research/part03_learnt_overlap/e03_different_gt_representations/array_01_MSC-p03e03_different-gt-representations.txt +++ /dev/null @@ -1,48 +0,0 @@ -+EXTRA.sweep_num=1 +DUMMY.repeat=1 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.001 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=2 +DUMMY.repeat=1 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.001 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=3 +DUMMY.repeat=1 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.001 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=4 +DUMMY.repeat=1 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.001 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=5 +DUMMY.repeat=1 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.00316 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=6 +DUMMY.repeat=1 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.00316 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=7 +DUMMY.repeat=1 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=8 +DUMMY.repeat=1 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=9 +DUMMY.repeat=1 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.01 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=10 +DUMMY.repeat=1 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.01 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=11 +DUMMY.repeat=1 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=12 +DUMMY.repeat=1 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=13 +DUMMY.repeat=1 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.0316 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=14 +DUMMY.repeat=1 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.0316 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=15 +DUMMY.repeat=1 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=16 +DUMMY.repeat=1 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=17 +DUMMY.repeat=2 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.001 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=18 +DUMMY.repeat=2 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.001 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=19 +DUMMY.repeat=2 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.001 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=20 +DUMMY.repeat=2 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.001 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=21 +DUMMY.repeat=2 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.00316 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=22 +DUMMY.repeat=2 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.00316 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=23 +DUMMY.repeat=2 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=24 +DUMMY.repeat=2 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=25 +DUMMY.repeat=2 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.01 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=26 +DUMMY.repeat=2 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.01 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=27 +DUMMY.repeat=2 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=28 +DUMMY.repeat=2 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=29 +DUMMY.repeat=2 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.0316 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=30 +DUMMY.repeat=2 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.0316 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=31 +DUMMY.repeat=2 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=32 +DUMMY.repeat=2 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=33 +DUMMY.repeat=3 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.001 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=34 +DUMMY.repeat=3 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.001 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=35 +DUMMY.repeat=3 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.001 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=36 +DUMMY.repeat=3 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.001 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=37 +DUMMY.repeat=3 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.00316 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=38 +DUMMY.repeat=3 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.00316 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=39 +DUMMY.repeat=3 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=40 +DUMMY.repeat=3 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=41 +DUMMY.repeat=3 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.01 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=42 +DUMMY.repeat=3 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.01 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=43 +DUMMY.repeat=3 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=44 +DUMMY.repeat=3 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=45 +DUMMY.repeat=3 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.0316 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=46 +DUMMY.repeat=3 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.0316 framework=betavae schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb -+EXTRA.sweep_num=47 +DUMMY.repeat=3 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject sampling=default__bb -+EXTRA.sweep_num=48 +DUMMY.repeat=3 +EXTRA.tags=sweep_different-gt-repr_basic-vaes hydra.job.name=gt-repr run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=xyobject_shaded sampling=default__bb diff --git a/research/part03_learnt_overlap/e03_different_gt_representations/submit_01.sh b/research/part03_learnt_overlap/e03_different_gt_representations/submit_01.sh deleted file mode 100644 index 86b10244..00000000 --- a/research/part03_learnt_overlap/e03_different_gt_representations/submit_01.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - - -# OVERVIEW: -# - this experiment is designed to find the optimal ada-triplet function and hyper-parameters - -# OUTCOMES: -# - ??? - - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="MSC-p03e03_different-gt-representations" -export PARTITION="stampede" -export PARALLELISM=24 - -# the path to the generated arguments file -# - this needs to before we source the helper file -ARGS_FILE="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")/array_01_$PROJECT.txt" - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Generate Experiment # -# ========================================================================= # - -# SWEEP FOR GOOD UNSUPERVISED DO-ADA-TVAE PARAMS -# 3 * (4*2*2) = 48 -ARGS_FILE="$ARGS_FILE" gen_sbatch_args_file \ - +DUMMY.repeat=1,2,3 \ - +EXTRA.tags='sweep_different-gt-repr_basic-vaes' \ - hydra.job.name="gt-repr" \ - \ - run_length=medium \ - metrics=all \ - \ - settings.framework.beta=0.001,0.00316,0.01,0.0316 \ - framework=betavae,adavae_os \ - schedule=none \ - settings.model.z_size=9 \ - \ - dataset=xyobject,xyobject_shaded \ - sampling=default__bb - - -# ========================================================================= # -# Run Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 43200 "C-disent" # 12 hours - -ARGS_FILE="$ARGS_FILE" submit_sbatch_args_file diff --git a/research/part03_learnt_overlap/e04_random_external_factors/array_01_MSC-p03e04_random-external-factors.txt b/research/part03_learnt_overlap/e04_random_external_factors/array_01_MSC-p03e04_random-external-factors.txt deleted file mode 100644 index 2e1116bc..00000000 --- a/research/part03_learnt_overlap/e04_random_external_factors/array_01_MSC-p03e04_random-external-factors.txt +++ /dev/null @@ -1,54 +0,0 @@ -+EXTRA.sweep_num=1 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=betavae schedule=none settings.model.z_size=9 dataset=dsprites sampling=default__bb -+EXTRA.sweep_num=2 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-25 sampling=default__bb -+EXTRA.sweep_num=3 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-50 sampling=default__bb -+EXTRA.sweep_num=4 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-75 sampling=default__bb -+EXTRA.sweep_num=5 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-100 sampling=default__bb -+EXTRA.sweep_num=6 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-25 sampling=default__bb -+EXTRA.sweep_num=7 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-50 sampling=default__bb -+EXTRA.sweep_num=8 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-75 sampling=default__bb -+EXTRA.sweep_num=9 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-100 sampling=default__bb -+EXTRA.sweep_num=10 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=betavae schedule=none settings.model.z_size=9 dataset=dsprites sampling=default__bb -+EXTRA.sweep_num=11 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-25 sampling=default__bb -+EXTRA.sweep_num=12 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-50 sampling=default__bb -+EXTRA.sweep_num=13 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-75 sampling=default__bb -+EXTRA.sweep_num=14 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-100 sampling=default__bb -+EXTRA.sweep_num=15 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-25 sampling=default__bb -+EXTRA.sweep_num=16 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-50 sampling=default__bb -+EXTRA.sweep_num=17 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-75 sampling=default__bb -+EXTRA.sweep_num=18 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-100 sampling=default__bb -+EXTRA.sweep_num=19 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=betavae schedule=none settings.model.z_size=9 dataset=dsprites sampling=default__bb -+EXTRA.sweep_num=20 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-25 sampling=default__bb -+EXTRA.sweep_num=21 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-50 sampling=default__bb -+EXTRA.sweep_num=22 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-75 sampling=default__bb -+EXTRA.sweep_num=23 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-100 sampling=default__bb -+EXTRA.sweep_num=24 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-25 sampling=default__bb -+EXTRA.sweep_num=25 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-50 sampling=default__bb -+EXTRA.sweep_num=26 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-75 sampling=default__bb -+EXTRA.sweep_num=27 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_BETAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=betavae schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-100 sampling=default__bb -+EXTRA.sweep_num=28 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=dsprites sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=29 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-25 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=30 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-50 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=31 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-75 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=32 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-100 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=33 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-25 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=34 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-50 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=35 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-75 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=36 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-100 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=37 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=dsprites sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=38 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-25 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=39 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-50 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=40 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-75 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=41 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-100 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=42 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-25 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=43 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-50 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=44 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-75 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=45 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-100 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=46 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=dsprites sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=47 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-25 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=48 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-50 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=49 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-75 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=50 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-100 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=51 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-25 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=52 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-50 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=53 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-75 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=54 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_ADAVAE hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-100 sampling=default__bb run_location=stampede_rsync_tmp diff --git a/research/part03_learnt_overlap/e04_random_external_factors/array_01_MSC-p03e04_random-external-factors_FIX_ADA.txt b/research/part03_learnt_overlap/e04_random_external_factors/array_01_MSC-p03e04_random-external-factors_FIX_ADA.txt deleted file mode 100644 index ec174530..00000000 --- a/research/part03_learnt_overlap/e04_random_external_factors/array_01_MSC-p03e04_random-external-factors_FIX_ADA.txt +++ /dev/null @@ -1,27 +0,0 @@ -+EXTRA.sweep_num=1 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=dsprites sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=2 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-25 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=3 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-50 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=4 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-75 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=5 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-100 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=6 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-25 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=7 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-50 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=8 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-75 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=9 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.01 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-100 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=10 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=dsprites sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=11 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-25 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=12 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-50 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=13 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-75 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=14 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-100 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=15 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-25 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=16 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-50 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=17 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-75 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=18 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.00316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-100 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=19 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=dsprites sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=20 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-25 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=21 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-50 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=22 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-75 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=23 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-bg-100 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=24 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-25 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=25 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-50 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=26 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-75 sampling=default__bb run_location=stampede_rsync_tmp -+EXTRA.sweep_num=27 +DUMMY.repeat=1 +EXTRA.tags=sweep_imagenet_dsprites_FIX_ADA_RSYNC hydra.job.name=imnet-dsprites run_length=medium metrics=all settings.framework.beta=0.0316 framework=adavae_os schedule=none settings.model.z_size=9 dataset=X--dsprites-imagenet-fg-100 sampling=default__bb run_location=stampede_rsync_tmp diff --git a/research/part03_learnt_overlap/e04_random_external_factors/submit_01.sh b/research/part03_learnt_overlap/e04_random_external_factors/submit_01.sh deleted file mode 100644 index d8fc6d7b..00000000 --- a/research/part03_learnt_overlap/e04_random_external_factors/submit_01.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash - - -# OVERVIEW: -# - this experiment is designed to find the optimal ada-triplet function and hyper-parameters - -# OUTCOMES: -# - ??? - - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export USERNAME="n_michlo" -export PROJECT="MSC-p03e04_random-external-factors" -export PARTITION="stampede" -export PARALLELISM=28 - -# the path to the generated arguments file -# - this needs to before we source the helper file -ARGS_FILE="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")/array_01_${PROJECT}.txt" - -# source the helper file -source "$(dirname "$(dirname "$(dirname "$(realpath -s "$0")")")")/scripts/helper.sh" - -# ========================================================================= # -# Generate Experiment # -# ========================================================================= # - -# SWEEP FOR GOOD PARAMS -# 1 * (3*9) = 27 --- TOTAL: 27+27=54 -ARGS_FILE="$ARGS_FILE" gen_sbatch_args_file \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_imagenet_dsprites_BETAVAE' \ - hydra.job.name="imnet-dsprites" \ - \ - run_length=medium \ - metrics=all \ - \ - settings.framework.beta=0.01,0.00316,0.0316 \ - framework=betavae \ - schedule=none \ - settings.model.z_size=9 \ - \ - dataset=dsprites,X--dsprites-imagenet-bg-25,X--dsprites-imagenet-bg-50,X--dsprites-imagenet-bg-75,X--dsprites-imagenet-bg-100,X--dsprites-imagenet-fg-25,X--dsprites-imagenet-fg-50,X--dsprites-imagenet-fg-75,X--dsprites-imagenet-fg-100 \ - sampling=default__bb - -# PART OF THE ABOVE -# 1 * (3*9) = 27 --- TOTAL: 27+27=54 -ARGS_FILE="$ARGS_FILE" APPEND_ARGS=1 ARGS_START_NUM=28 gen_sbatch_args_file \ - +DUMMY.repeat=1 \ - +EXTRA.tags='sweep_imagenet_dsprites_ADAVAE' \ - hydra.job.name="imnet-dsprites" \ - \ - run_length=medium \ - metrics=all \ - \ - settings.framework.beta=0.01,0.00316,0.0316 \ - framework=adavae_os \ - schedule=none \ - settings.model.z_size=9 \ - \ - dataset=dsprites,X--dsprites-imagenet-bg-25,X--dsprites-imagenet-bg-50,X--dsprites-imagenet-bg-75,X--dsprites-imagenet-bg-100,X--dsprites-imagenet-fg-25,X--dsprites-imagenet-fg-50,X--dsprites-imagenet-fg-75,X--dsprites-imagenet-fg-100 \ - sampling=default__bb \ - \ - run_location=stampede_rsync_tmp - -# ========================================================================= # -# Run Experiment # -# ========================================================================= # - -#clog_cudaless_nodes "$PARTITION" 43200 "C-disent" # 12 hours - -ARGS_FILE="$ARGS_FILE" submit_sbatch_args_file diff --git a/research/part03_learnt_overlap/plot00_all/plot_p03_wandb_experiments.py b/research/part03_learnt_overlap/plot00_all/plot_p03_wandb_experiments.py deleted file mode 100644 index 02d6b0e7..00000000 --- a/research/part03_learnt_overlap/plot00_all/plot_p03_wandb_experiments.py +++ /dev/null @@ -1,556 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2022 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -import os -from typing import Optional -from typing import Sequence - -import pandas as pd -import seaborn as sns -from cachier import cachier as _cachier -from matplotlib import pyplot as plt -import matplotlib.lines as mlines - -import research.code.util as H -from disent.util.function import wrapped_partial -from disent.util.profiling import Timer -from research.code.util._wandb_plots import drop_non_unique_cols -from research.code.util._wandb_plots import drop_unhashable_cols -from research.code.util._wandb_plots import load_runs as _load_runs - - -# ========================================================================= # -# Helper # -# ========================================================================= # - - -DF = pd.DataFrame - -# cachier instance -CACHIER: _cachier = wrapped_partial(_cachier, cache_dir=os.path.join(os.path.dirname(__file__), 'plots/.cache')) - - -@CACHIER() -def load_runs(project: str, include_history: bool = False): - return _load_runs(project=project, include_history=include_history) - - -def clear_cache(clear_data=True, clear_wandb=False): - from research.code.util._wandb_plots import clear_runs_cache - if clear_wandb: - clear_runs_cache() - if clear_data: - load_runs.clear_cache() - - -# ========================================================================= # -# Prepare Data # -# ========================================================================= # - - -# common keys -K_GROUP = 'Run Group' -K_DATASET = 'Dataset' -K_FRAMEWORK = 'Framework' -K_BETA = 'Beta' -K_LOSS = 'Recon. Loss' -K_Z_SIZE = 'Latent Dims.' -K_REPEAT = 'Repeat' -K_STATE = 'State' -K_LR = 'Learning Rate' -K_SCHEDULE = 'Schedule' -K_SAMPLER = 'Sampler' -K_ADA_MODE = 'Threshold Mode' - -K_MIG_END = 'MIG Score\n(End)' -K_DCI_END = 'DCI Score\n(End)' -K_MIG_MAX = 'MIG Score' -K_DCI_MAX = 'DCI Score' -K_LCORR_GT_F = 'Linear Corr.\n(factors)' -K_RCORR_GT_F = 'Rank Corr.\n(factors)' -K_LCORR_GT_G = 'Global Linear Corr.\n(factors)' -K_RCORR_GT_G = 'Global Rank Corr.\n(factors)' -K_LCORR_DATA_F = 'Linear Corr.\n(data)' -K_RCORR_DATA_F = 'Rank Corr.\n(data)' -K_LCORR_DATA_G = 'Global Linear Corr.\n(data)' -K_RCORR_DATA_G = 'Global Rank Corr.\n(data)' -K_AXIS = 'Axis Ratio' -K_LINE = 'Linear Ratio' - -K_TRIPLET_SCALE = 'Triplet Scale' # framework.cfg.triplet_margin_max -K_TRIPLET_MARGIN = 'Triplet Margin' # framework.cfg.triplet_scale -K_TRIPLET_P = 'Triplet P' # framework.cfg.triplet_p -K_DETACH = 'Detached' # framework.cfg.detach_decoder -K_TRIPLET_MODE = 'Triplet Mode' # framework.cfg.triplet_loss - - -def load_general_data( - project: str, - include_history: bool = False, - keep_cols: Sequence[str] = None, - drop_unhashable: bool = False, - drop_non_unique: bool = False, -): - # keep columns - if keep_cols is None: - keep_cols = [] - keep_cols = list(keep_cols) - if include_history: - keep_cols = ['history'] + keep_cols - # load data - df = load_runs(project, include_history=include_history) - # process data - with Timer('processing data'): - # rename columns - df = df.rename(columns={ - 'settings/optimizer/lr': K_LR, - 'EXTRA/tags': K_GROUP, - 'dataset/name': K_DATASET, - 'framework/name': K_FRAMEWORK, - 'settings/framework/beta': K_BETA, - 'settings/framework/recon_loss': K_LOSS, - 'settings/model/z_size': K_Z_SIZE, - 'DUMMY/repeat': K_REPEAT, - 'state': K_STATE, - 'final_metric/mig.discrete_score.max': K_MIG_END, - 'final_metric/dci.disentanglement.max': K_DCI_END, - 'epoch_metric/mig.discrete_score.max': K_MIG_MAX, - 'epoch_metric/dci.disentanglement.max': K_DCI_MAX, - # scores - 'epoch_metric/distances.lcorr_ground_latent.l1.factor.max': K_LCORR_GT_F, - 'epoch_metric/distances.rcorr_ground_latent.l1.factor.max': K_RCORR_GT_F, - 'epoch_metric/distances.lcorr_ground_latent.l1.global.max': K_LCORR_GT_G, - 'epoch_metric/distances.rcorr_ground_latent.l1.global.max': K_RCORR_GT_G, - 'epoch_metric/distances.lcorr_latent_data.l2.factor.max': K_LCORR_DATA_F, - 'epoch_metric/distances.rcorr_latent_data.l2.factor.max': K_RCORR_DATA_F, - 'epoch_metric/distances.lcorr_latent_data.l2.global.max': K_LCORR_DATA_G, - 'epoch_metric/distances.rcorr_latent_data.l2.global.max': K_RCORR_DATA_G, - 'epoch_metric/linearity.axis_ratio.var.max': K_AXIS, - 'epoch_metric/linearity.linear_ratio.var.max': K_LINE, - # adaptive methods - 'schedule/name': K_SCHEDULE, - 'sampling/name': K_SAMPLER, - 'framework/cfg/ada_thresh_mode': K_ADA_MODE, - # triplet experiments - 'framework/cfg/triplet_margin_max': K_TRIPLET_MARGIN, - 'framework/cfg/triplet_scale': K_TRIPLET_SCALE, - 'framework/cfg/triplet_p': K_TRIPLET_P, - 'framework/cfg/detach_decoder': K_DETACH, - 'framework/cfg/triplet_loss': K_TRIPLET_MODE, - }) - # filter out unneeded columns - if drop_unhashable: - df, dropped_hash = drop_unhashable_cols(df, skip=keep_cols) - if drop_non_unique: - df, dropped_diverse = drop_non_unique_cols(df, skip=keep_cols) - return df - - -def rename_entries(df: pd.DataFrame): - df = df.copy() - # replace values in the df - for key, value, new_value in [ - (K_DATASET, 'xysquares_minimal', 'xysquares'), - (K_SCHEDULE, 'adanegtvae_up_all_full', 'Schedule: Both (strong)'), - (K_SCHEDULE, 'adanegtvae_up_all', 'Schedule: Both (weak)'), - (K_SCHEDULE, 'adanegtvae_up_ratio_full', 'Schedule: Weight (strong)'), - (K_SCHEDULE, 'adanegtvae_up_ratio', 'Schedule: Weight (weak)'), - (K_SCHEDULE, 'adanegtvae_up_thresh', 'Schedule: Threshold'), - (K_TRIPLET_MODE, 'triplet', 'Triplet Loss (Hard Margin)'), - (K_TRIPLET_MODE, 'triplet_soft', 'Triplet Loss (Soft Margin)'), - (K_TRIPLET_P, 1, 'L1 Distance'), - (K_TRIPLET_P, 2, 'L2 Distance'), - # (K_SAMPLER, 'gt_dist__manhat', 'Ground-Truth Dist Sampling'), - # (K_SAMPLER, 'gt_dist__manhat_scaled', 'Ground-Truth Dist Sampling (Scaled)'), - ]: - if key in df.columns: - df[key].replace(value, new_value, inplace=True) - # df.loc[df[key] == value, key] = new_value - return df - - -# ========================================================================= # -# Plot Experiments # -# ========================================================================= # - - -PINK = '#FE375F' # usually: Beta-VAE -PURPLE = '#5E5BE5' # maybe: Ada-TVAE -LPURPLE = '#b0b6ff' # maybe: Ada-TVAE (alt) -BLUE = '#1A93FE' # maybe: TVAE -LBLUE = '#63D2FE' -ORANGE = '#FE9F0A' # usually: Ada-VAE -GREEN = '#2FD157' - -LGREEN = '#9FD911' # usually: MSE -LBLUE2 = '#36CFC8' # usually: MSE-Overlap - - -# ========================================================================= # -# Experiment 3 # -# ========================================================================= # - - -def plot_e03_different_gt_representations( - rel_path: Optional[str] = None, - save: bool = True, - show: bool = True, - color_entangled_data: str = LGREEN, - color_disentangled_data: str = LBLUE2, - metrics: Sequence[str] = (K_MIG_MAX, K_DCI_MAX, K_RCORR_GT_F, K_RCORR_DATA_F, K_AXIS, K_LINE), -): - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - df = load_general_data(f'{os.environ["WANDB_USER"]}/MSC-p03e03_different-gt-representations', keep_cols=(K_GROUP, K_Z_SIZE)) - # select run groups - # +DUMMY.repeat=1,2,3 \ - # settings.framework.beta=0.001,0.00316,0.01,0.0316 \ - # framework=betavae,adavae_os \ - # settings.model.z_size=9 \ - # dataset=xyobject,xyobject_shaded \ - df = df[df[K_GROUP].isin(['sweep_different-gt-repr_basic-vaes'])] - df = df.sort_values([K_DATASET, K_FRAMEWORK, K_BETA, K_REPEAT]) - # print common key values - print('K_GROUP: ', list(df[K_GROUP].unique())) - print('K_DATASET: ', list(df[K_DATASET].unique())) - print('K_FRAMEWORK:', list(df[K_FRAMEWORK].unique())) - print('K_BETA: ', list(df[K_BETA].unique())) - print('K_Z_SIZE: ', list(df[K_Z_SIZE].unique())) - print('K_REPEAT: ', list(df[K_REPEAT].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - # rename more stuff - df = rename_entries(df) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - orig = df - # select runs - # df = df[df[K_STATE] == 'finished'] - # [1.0, 0.316, 0.1, 0.0316, 0.01, 0.00316, 0.001, 0.000316] - # df = df[(0.000316 < df[K_BETA]) & (df[K_BETA] < 1.0)] - print('NUM', len(orig), '->', len(df)) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - df[K_DATASET].replace('xysquares_minimal', 'XYSquares', inplace=True) - df[K_DATASET].replace('smallnorb', 'NORB', inplace=True) - df[K_DATASET].replace('cars3d', 'Cars3D', inplace=True) - df[K_DATASET].replace('3dshapes', 'Shapes3D', inplace=True) - df[K_DATASET].replace('dsprites', 'dSprites', inplace=True) - df[K_DATASET].replace('xyobject', 'XYObject', inplace=True) - df[K_DATASET].replace('xyobject_shaded', 'XYObject (Shades)', inplace=True) - df[K_FRAMEWORK].replace('adavae_os', 'Ada-GVAE', inplace=True) - df[K_FRAMEWORK].replace('betavae', 'Beta-VAE', inplace=True) - - PALLETTE = { - 'XYObject': color_entangled_data, - 'XYObject (Shades)': color_disentangled_data, - } - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - fig, axs = plt.subplots(2, len(metrics) // 2, figsize=(len(metrics)//2*3.75, 2*2.7)) - axs = axs.flatten() - # PLOT - for i, (key, ax) in enumerate(zip(metrics, axs)): - assert key in df.columns, f'{repr(key)} not in {sorted(df.columns)}' - sns.violinplot(data=df, ax=ax, x=K_FRAMEWORK, y=key, hue=K_DATASET, palette=PALLETTE, split=True, cut=0, width=0.75, scale='width', inner='quartile') - ax.set_ylim([-0.1, 1.1]) - ax.set_ylim([0, None]) - if i == len(axs)-1: - ax.legend(bbox_to_anchor=(0.05, 0.175), fontsize=12, loc='lower left', labelspacing=0.1) - ax.set_xlabel(None) - # ax.set_ylabel('Minimum Recon. Loss') - else: - if ax.get_legend(): - ax.get_legend().remove() - ax.set_xlabel(None) - # ax.set_ylabel(None) - # PLOT: - fig.tight_layout() - H.plt_rel_path_savefig(rel_path, save=save, show=show, dpi=300) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - return fig, axs - -# ========================================================================= # -# Experiment 3 # -# ========================================================================= # - - -def plot_e04_random_external_factors( - rel_path: Optional[str] = None, - save: bool = True, - show: bool = True, - mode: str = 'fg', - # color_entangled_data: str = LGREEN, - # color_disentangled_data: str = LBLUE2, - reg_order: int = 1, - color_betavae: str = PINK, - color_adavae: str = ORANGE, - metrics: Sequence[str] = (K_MIG_MAX, K_DCI_MAX, K_RCORR_GT_F, K_RCORR_DATA_F, K_AXIS, K_LINE), -): - K_IM_MODE = 'dataset/data/mode' - K_IM_VIS = 'Visibility %' - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - df = load_general_data(f'{os.environ["WANDB_USER"]}/MSC-p03e04_random-external-factors', keep_cols=(K_GROUP, K_Z_SIZE)) - df = df.rename(columns={'dataset/data/visibility': K_IM_VIS}) - # filter the groups - # -- FIX_ADA_RSYNC: adavae_os - # -- FIX: betavae - df = df[df[K_GROUP].isin(['sweep_imagenet_dsprites_FIX_ADA_RSYNC', 'sweep_imagenet_dsprites_FIX'])] - # select run groups - df = df.sort_values([K_DATASET, K_FRAMEWORK, K_IM_MODE, K_IM_VIS]) - df = df[df[K_FRAMEWORK].isin(['adavae_os', 'betavae'])] - df = df[df[K_DATASET].isin(['dsprites', f'dsprites_imagenet_{mode}_25', f'dsprites_imagenet_{mode}_50', f'dsprites_imagenet_{mode}_75', f'dsprites_imagenet_{mode}_100'])] - # rename more stuff - df = rename_entries(df) - # print common key values - print('K_GROUP: ', list(df[K_GROUP].unique())) - print('K_DATASET: ', list(df[K_DATASET].unique())) - print('K_FRAMEWORK:', list(df[K_FRAMEWORK].unique())) - print('K_BETA: ', list(df[K_BETA].unique())) - print('K_Z_SIZE: ', list(df[K_Z_SIZE].unique())) - print('K_REPEAT: ', list(df[K_REPEAT].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - print('K_IM_VIS: ', list(df[K_IM_VIS].unique())) - print('K_IM_MODE: ', list(df[K_IM_MODE].unique())) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # df = df[df[K_STATE].isin(['finished'])] - # df = df[df[K_STATE].isin(['finished', 'running'])] - - # replace unset values - df.loc[df[K_DATASET] == 'dsprites', K_IM_VIS] = 0 - df.loc[df[K_DATASET] == 'dsprites', K_IM_MODE] = mode - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - orig = df - # select runs - # select adavae - data_adavae = df[(df[K_FRAMEWORK] == 'adavae_os')] - data_betavae = df[(df[K_FRAMEWORK] == 'betavae')] - print('ADAGVAE', len(orig), '->', len(data_adavae)) - print('BETAVAE', len(orig), '->', len(data_betavae)) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - print(data_adavae[K_MIG_MAX]) - print(data_adavae[K_IM_VIS]) - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - # fig, axs = plt.subplots(1, len(metrics), figsize=(len(metrics) * 2.7*1.1, 3.33*1.1), squeeze=False) - fig, axs = plt.subplots(2, len(metrics) // 2, figsize=(len(metrics)//2 * 2.9, 2*3.3)) - axs = axs.flatten() - # Legend entries - marker_ada = mlines.Line2D([], [], color=color_adavae, marker='o', markersize=11.5, label='Ada-GVAE') - marker_beta = mlines.Line2D([], [], color=color_betavae, marker='X', markersize=11.5, label='Beta-VAE') # why does 'x' not work? only 'X'? - # PLOT: MIG - for x, metric_key in enumerate(metrics): - ax = axs[x] - sns.regplot(ax=ax, x=K_IM_VIS, y=metric_key, data=data_adavae, seed=777, order=reg_order, robust=False, color=color_adavae, marker='o') - sns.regplot(ax=ax, x=K_IM_VIS, y=metric_key, data=data_betavae, seed=777, order=reg_order, robust=False, color=color_betavae, marker='x', line_kws=dict(linestyle='dashed')) - if x == 0: - ax.legend(handles=[marker_beta, marker_ada], fontsize=14) - ax.set_ylim([-0.1, 1.1]) - ax.set_xlim([-5, 105]) - # PLOT: - fig.tight_layout() - H.plt_rel_path_savefig(rel_path, save=save, show=show) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - return fig, axs - -# ========================================================================= # -# Entrypoint # -# ========================================================================= # - - -def plot_e01_learnt_loss_with_vaes( - rel_path: Optional[str] = None, - save: bool = True, - show: bool = True, - color_betavae: str = PINK, - color_adavae: str = ORANGE, - # color_mse_xy8: str = GREEN, - # color_mse_box: str = BLUE, - # color_mse_gau: str = PURPLE, - # color_mse: str = PINK, - # color_entangled_data: str = LGREEN, - # color_disentangled_data: str = LBLUE2, - metrics: Sequence[str] = (K_MIG_MAX, K_DCI_MAX, K_RCORR_GT_F, K_RCORR_DATA_F, K_AXIS, K_LINE), -): - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - df = load_general_data(f'{os.environ["WANDB_USER"]}/MSC-p03e02_learnt-loss-with-vaes', keep_cols=(K_GROUP, K_Z_SIZE)) - # select run groups - # +DUMMY.repeat=1,2,3,4,5 \ - # framework=betavae,adavae_os \ - # settings.framework.beta=0.0316,0.0001 \ - # settings.framework.recon_loss='mse','mse_gau_r31_l1.0_k3969.0_norm_sum','mse_box_r31_l1.0_k3969.0_norm_sum','mse_xy8_abs63_l1.0_k1.0_norm_none' \ - df = df[df[K_GROUP].isin(['MSC_sweep_losses', 'MSC_sweep_losses_ALT', 'MSC_sweep_losses_XY8R31'])] # 'MSC_sweep_losses', 'MSC_sweep_losses_XY8R31', 'MSC_sweep_losses_ALT' - df = df.sort_values([K_LOSS, K_FRAMEWORK, K_BETA, K_REPEAT]) - # print common key values - print('K_GROUP: ', list(df[K_GROUP].unique())) - print('K_DATASET: ', list(df[K_DATASET].unique())) - print('K_FRAMEWORK:', list(df[K_FRAMEWORK].unique())) - print('K_LOSS: ', list(df[K_LOSS].unique())) - print('K_BETA: ', list(df[K_BETA].unique())) - print('K_Z_SIZE: ', list(df[K_Z_SIZE].unique())) - print('K_REPEAT: ', list(df[K_REPEAT].unique())) - print('K_STATE: ', list(df[K_STATE].unique())) - print('K_LOSS: ', list(df[K_LOSS].unique())) - # rename more stuff - df = rename_entries(df) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - orig = df - # select runs - df = df[df[K_STATE].isin(['finished'])].copy() - print('NUM', len(orig), '->', len(df)) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - df[K_DATASET].replace('xysquares_minimal', 'XYSquares', inplace=True) - df[K_DATASET].replace('smallnorb', 'NORB', inplace=True) - df[K_DATASET].replace('cars3d', 'Cars3D', inplace=True) - df[K_DATASET].replace('3dshapes', 'Shapes3D', inplace=True) - df[K_DATASET].replace('dsprites', 'dSprites', inplace=True) - df[K_DATASET].replace('xyobject', 'XYObject', inplace=True) - df[K_DATASET].replace('xyobject_shaded', 'XYObject (Shades)', inplace=True) - df[K_FRAMEWORK].replace('adavae_os', 'Ada-GVAE', inplace=True) - df[K_FRAMEWORK].replace('betavae', 'Beta-VAE', inplace=True) - - N_mse = 'none' - N_gau = 'gau' - N_box = 'box' - N_xy8r31 = 'xy8' - N_xy8r63 = 'xy8_r63' - - N_mse = 'MSE' - N_gau = 'MSE\n(gau)' - N_box = 'MSE\n(box)' - N_xy8r31 = 'MSE\n(xy8)' - N_xy8r63 = 'MSE\n(xy8,r63)' - - df[K_LOSS].replace('mse_xy8_abs31_l1.0_k1.0_norm_none', N_xy8r31, inplace=True) - df[K_LOSS].replace('mse_xy8_abs63_l1.0_k1.0_norm_none', N_xy8r63, inplace=True) - df[K_LOSS].replace('mse_box_r31_l1.0_k3969.0_norm_sum', N_box, inplace=True) - df[K_LOSS].replace('mse_gau_r31_l1.0_k3969.0_norm_sum', N_gau, inplace=True) - df[K_LOSS].replace('mse', N_mse, inplace=True) - - df['_sort_'] = list(df[K_LOSS]) - df['_sort_'].replace(N_mse, 1, inplace=True) - df['_sort_'].replace(N_gau, 2, inplace=True) - df['_sort_'].replace(N_box, 3, inplace=True) - df['_sort_'].replace(N_xy8r31, 4, inplace=True) - df['_sort_'].replace(N_xy8r63, 5, inplace=True) - - df = df[df[K_LOSS].isin([N_mse, N_gau, N_box, N_xy8r31])] - - # df = df[df[K_BETA].isin([ - # 0.0001, - # 0.000316, - # 0.001, - # 0.00316, - # 0.01, - # 0.0316 - # ])] - - df = df.sort_values(['_sort_', K_LOSS, K_FRAMEWORK, K_BETA]) - - print('NUM?', len(df)) - df = df[ - ((df[K_LOSS] == N_mse) & ( df[K_BETA].isin([0.0001, 0.000316]))) - # ((df[K_LOSS] == N_gau) & (~df[K_BETA].isin([0.0001, 0.000316, 0.001, 0.00316, 0.01, 0.0316]))) - | ((df[K_LOSS] == N_gau) & (~df[K_BETA].isin([0.0001, 0.000316, 0.01, 0.0316]))) - | ((df[K_LOSS] == N_box) & (~df[K_BETA].isin([0.0001, 0.000316, 0.001, 0.00316]))) - | ((df[K_LOSS] == N_xy8r31) & (~df[K_BETA].isin([0.0001, 0.000316, 0.001, 0.00316]))) - ] - print('NUM?', len(df)) - - PALLETTE = { - # 'MSE (learnt)': color_mse_xy8, - # 'MSE (box)': color_mse_box, - # 'MSE (gaussian)': color_mse_gau, - # 'MSE': color_mse, - 'Beta-VAE': color_betavae, - 'Ada-GVAE': color_adavae, - } - - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - fig, axs = plt.subplots(2, len(metrics) // 2, figsize=(len(metrics) // 2 * 3.75, 2 * 3.0)) - axs = axs.flatten() - # PLOT - for i, (key, ax) in enumerate(zip(metrics, axs)): - assert key in df.columns, f'{repr(key)} not in {sorted(df.columns)}' - sns.violinplot(data=df, ax=ax, x=K_LOSS, y=key, hue=K_FRAMEWORK, palette=PALLETTE, split=True, cut=0, width=0.75, scale='width', inner='quartile') - ax.set_ylim([-0.1, 1.1]) - ax.set_ylim([0, None]) - if i == len(axs) - 1: - # ax.legend(bbox_to_anchor=(0.05, 0.175), fontsize=12, loc='lower left', labelspacing=0.1) - ax.legend(fontsize=12, bbox_to_anchor=(1.0, 0.05), loc='lower right', labelspacing=0.1) - ax.set_xlabel(None) - # ax.set_ylabel('Minimum Recon. Loss') - else: - if ax.get_legend(): - ax.get_legend().remove() - ax.set_xlabel(None) - # ax.set_ylabel(None) - # PLOT: - fig.tight_layout() - H.plt_rel_path_savefig(rel_path, save=save, show=show, dpi=300) - # ~=~=~=~=~=~=~=~=~=~=~=~=~ # - - return fig, axs - - -# ========================================================================= # -# Entrypoint # -# ========================================================================= # - - -if __name__ == '__main__': - - assert 'WANDB_USER' in os.environ, 'specify "WANDB_USER" environment variable' - - logging.basicConfig(level=logging.INFO) - - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../../code/util/gadfly.mplstyle')) - - # clear_cache(clear_data=True, clear_wandb=True) - # clear_cache(clear_data=True, clear_wandb=False) - - def main(): - plot_e01_learnt_loss_with_vaes(rel_path='plots/p03e01_learnt-loss-with-vaes', show=True) - - plot_e03_different_gt_representations(rel_path='plots/p03e03_different-gt-representations', show=True) - - plot_e04_random_external_factors(rel_path='plots/p03e04_random-external-factors__fg', mode='fg', show=True) - plot_e04_random_external_factors(rel_path='plots/p03e04_random-external-factors__bg', mode='bg', show=True) - - main() - - -# ========================================================================= # -# DONE # -# ========================================================================= # diff --git a/research/part03_learnt_overlap/plot00_all/plot_p03e03_kernels.py b/research/part03_learnt_overlap/plot00_all/plot_p03e03_kernels.py deleted file mode 100644 index 3628455d..00000000 --- a/research/part03_learnt_overlap/plot00_all/plot_p03e03_kernels.py +++ /dev/null @@ -1,75 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2022 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -import os - -import imageio -import numpy as np -import torch -from matplotlib import pyplot as plt - -import disent.registry as R -import research.code.util as H -from disent.util.visualize.vis_img import torch_to_images -from research.code import register_to_disent - - -# ========================================================================= # -# Helper # -# ========================================================================= # - - -if __name__ == '__main__': - - logging.basicConfig(level=logging.INFO) - - # matplotlib style - plt.style.use(os.path.join(os.path.dirname(__file__), '../../code/util/gadfly.mplstyle')) - - def main(): - register_to_disent() - - # ['box_r31', 'gau_r31', 'xy8_r47', 'xy1_r47'] - for key in R.KERNELS.examples: - kernel: torch.Tensor = R.KERNELS[key] - # normalize kernel - b, c, h, w = kernel.shape - assert b == 1 - # convert kernel to image - image = torch_to_images(kernel[0], in_min=kernel.min(), in_max=kernel.max(), always_rgb=True, to_numpy=True) - if 'box_' in key: - image = np.full_like(image, fill_value=128) - # save and show - path = H.make_rel_path_add_ext('plots', f'kernel_{key}', ext='.png') - print(f'saving: {path}') - imageio.imsave(path, image) - H.plt_imshow(image, show=True) - - main() - - -# ========================================================================= # -# DONE # -# ========================================================================= # diff --git a/research/part03_learnt_overlap/plot00_all/plots/.gitignore b/research/part03_learnt_overlap/plot00_all/plots/.gitignore deleted file mode 100644 index 225c3815..00000000 --- a/research/part03_learnt_overlap/plot00_all/plots/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.png -*.jpg diff --git a/research/part04_application_to_rl/e01_learn_xy_representations/_util.py b/research/part04_application_to_rl/e01_learn_xy_representations/_util.py deleted file mode 100644 index cbda66d6..00000000 --- a/research/part04_application_to_rl/e01_learn_xy_representations/_util.py +++ /dev/null @@ -1,63 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2022 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import logging -import sys -from typing import Optional - -import psutil - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Helper # -# ========================================================================= # - - -def get_num_workers( - num_workers: Optional[int] = None, - default_max: int = 16, -) -> int: - if sys.platform == 'darwin': - auto_workers = 0 - if num_workers is None: - log.warning(f'MacOS detected, setting num_workers to {auto_workers} to avoid Dataloader bug.') - num_workers = auto_workers - elif num_workers > auto_workers: - log.warning(f'MacOS detected, but manually set num_workers is greater than zero at {num_workers}, might result in a Dataloader bug!') - else: - auto_workers = min(psutil.cpu_count(logical=False), default_max) - if num_workers is None: - num_workers = auto_workers - log.warning(f'Automatically set num_workers to {num_workers}, cpu_count is {psutil.cpu_count(logical=False)}, max auto workers is {default_max}') - elif num_workers > auto_workers: - log.warning(f'manually set num_workers at {num_workers} might be too high, cpu_count is {psutil.cpu_count(logical=False)}, max auto workers is {default_max}') - return num_workers - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/part04_application_to_rl/e01_learn_xy_representations/loading_example.py b/research/part04_application_to_rl/e01_learn_xy_representations/loading_example.py deleted file mode 100644 index bef143e0..00000000 --- a/research/part04_application_to_rl/e01_learn_xy_representations/loading_example.py +++ /dev/null @@ -1,174 +0,0 @@ -import os.path -from pathlib import Path - -import seaborn as sns -import numpy as np -import pandas as pd -from matplotlib import pyplot as plt -from rich import palette -from rich.console import Console -from rich.table import Table -from natsort import natsorted - - -# same as part01_data_overlap/plot03_wandb/plot_all_experiments.py -PINK = '#FE375F' # usually: Beta-VAE -PURPLE = '#5E5BE5' # maybe: TVAE -BLUE = '#1A93FE' # maybe: Ada-TVAE -ORANGE = '#FE9F0A' # usually: Ada-VAE - - -if __name__ == '__main__': - - # matplotlib style - plt.style.use(Path(__file__).parents[2].joinpath('code/util/gadfly.mplstyle')) - - # load data storage - tables = [] - df = pd.DataFrame(columns=['exp', 'name', 'i', 'rcorr', 'axis', 'linear']) - - # iterate through all the experiment roots - for root in [ - # Path(__file__).parent.joinpath('exp/00001_xy8_1.5_10000'), - # Path(__file__).parent.joinpath('exp/00002_xy8_1.25_10000'), - # Path(__file__).parent.joinpath('exp/00003_xy8_1.5_5000'), - # Path(__file__).parent.joinpath('exp/00004_xy8_1.25_5000'), - # Path(__file__).parent.joinpath('exp/00011_xy8_1.25_10000'), - Path(__file__).parent.joinpath('exp/00002_xy8_1.25_10000_MERGE'), - ]: - # make table - table = Table('run', title=root.name) - table.add_column('rcorr_ground_latent', max_width=7, justify='center') - table.add_column('axis_ratio', max_width=7, justify='center') - table.add_column('linear_ratio', max_width=7, justify='center') - table.add_column('mig', max_width=7, justify='center') - # get paths - paths = [str(s) for s in Path(root).glob('**/rl_data.npz')] - paths = natsorted(paths, key=lambda s: ('_'.join(Path(s).parent.name.split('_')[2:]), s)) # strip: eg. 0x0_xy8 from names when sorting - # iterate through all the runs in an experiment - for i, path in enumerate(paths): - print(f'LOADING: {path}') - data = np.load(path, allow_pickle=True)['data'].tolist() - # pprint({k: (v.tolist() if isinstance(v, np.ndarray) else v) for k, v in data.items()}, width=200, sort_dicts=False, compact=True, depth=2) - # get stats - name = Path(path).parent.name - rcorr = data["metrics"]["distances.rcorr_ground_latent.global"] - axisr = data["metrics"]["linearity.axis_ratio.var"] - linea = data["metrics"]["linearity.linear_ratio.var"] - mig = data["metrics"]["mig.discrete_score"] - # append stats - table.add_row(name, f'{rcorr:5.3f}', f'{axisr:5.3f}', f'{linea:5.3f}', f'{mig:5.3f}') - df = df.append({'exp': root.name, 'name': name, 'i': i, 'rcorr': rcorr, 'axis': axisr, 'linear': linea, 'mig': mig}, ignore_index=True) - # store - tables.append(table) - - # make table - console = Console(width=200) - console.print(*tables, new_line_start=True) - - # filter - df = df[~df['name'].isin([ - '0x1_xy8_adavae', # ok, but adavae_os is the original! - '0x7_xy8_triplet_soft_B', # bad, sampling is not good - '0x5_xy8_triplet_soft_A1', # ok, but A2 is a stronger case! - '0x8_xy8_triplet_soft_C', # ok, but manhat (A) sampling is better - '0x9_xy8_adatvae_soft_A1', # ok, but A2 is a stronger case! - '0x11_xy8_adatvae_soft_B', # bad, sampling is not good - '0x12_xy8_adatvae_soft_C', # ok, but manhat (A) sampling is better - # main - # '0x2_xy8_adavae_os', - # '0x0_xy8_betavae', - # '0x3_xy8_triplet_soft_A', - # '0x16_xy8_triplet_soft_D8', # good! not as good as A2 though. - '0x6_xy8_triplet_soft_A2', - '0x10_xy8_adatvae_soft_A2', - # '0x4_xy8_adatvae_soft_A', - # '0x20_xy8_adatvae_soft_D8', # good! not as good as A2 though. - # others - '0x19_xy8_adatvae_soft_D16', # too random - '0x15_xy8_triplet_soft_D16', # too random - '0x18_xy8_adatvae_soft_D32', # too random - '0x14_xy8_triplet_soft_D32', # too random - '0x17_xy8_adatvae_soft_D64', # too random - '0x13_xy8_triplet_soft_D64', # too random - ])] - # df = df[~df['exp'].isin([ - # '00001_xy8_1.5_10000', # ada not strong enough - # '00003_xy8_1.5_5000', # ada not strong enough - # '00004_xy8_1.25_5000', # might as well use longer runs - # ])] - - # make plots - # for col in ['rcorr', 'axis', 'linear']: - # ax = sns.lineplot('name', col, hue='exp', data=df) - # plt.xticks(rotation=90) - # plt.tight_layout() - # plt.ylim([-0.05, 1.05]) - # plt.show() - - # set mig value - df.loc[df['name'] == '0x3_xy8_triplet_soft_A', 'mig'] = 0.025 - - # rename the runs - df.loc[df['name'] == '0x4_xy8_adatvae_soft_A', 'name'] = 'Ada-TVAE (supervised)' - df.loc[df['name'] == '0x10_xy8_adatvae_soft_A2', 'name'] = 'Ada-TVAE (80%)' - df.loc[df['name'] == '0x2_xy8_adavae_os', 'name'] = 'Ada-GVAE' - df.loc[df['name'] == '0x0_xy8_betavae', 'name'] = 'Beta-VAE' - df.loc[df['name'] == '0x3_xy8_triplet_soft_A', 'name'] = 'TVAE (supervised)' - df.loc[df['name'] == '0x6_xy8_triplet_soft_A2', 'name'] = 'TVAE (80%)' - df.loc[df['name'] == '0x20_xy8_adatvae_soft_D8', 'name'] = 'Ada-TVAE (episodes)' - df.loc[df['name'] == '0x16_xy8_triplet_soft_D8', 'name'] = 'TVAE (episodes)' - - import matplotlib as mpl - mpl.rcParams['hatch.linewidth'] = 2.5 # previous svg hatch linewidth - - # make new df - df1 = df.rename(columns={'rcorr': 'score'}); df1['Metric'] = 'Ground-Latent Correlation'; df1['g'] = 1 - # df2 = df.rename(columns={'axis': 'score'}); df2['Metric'] = 'Axis Ratio' - # df3 = df.rename(columns={'linear': 'score'}); df3['Metric'] = 'Linear Ratio' - df2 = df.rename(columns={'mig': 'score'}); df2['Metric'] = 'MIG Score'; df2['g'] = 2 - df_all = pd.concat([df1, df2], keys=['name', 'metric', 'score', 'g']) - - # make save dir - save_dir = os.path.join(os.path.dirname(__file__), 'plots') - os.makedirs(save_dir, exist_ok=True) - - # print everything! - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 2.75)) - names = df_all[df_all['g']==1]['name'] - ind = np.arange(len(names)) - w = 2/3 - # sns.barplot('score', 'name', hue='Metric', data=df_all, ax=ax, palette='Paired') - # sns.barplot('score', 'name', data=df_all, ax=ax, palette=[BLUE, BLUE, ORANGE, PINK, PURPLE, PURPLE], hatch=['/', None, None, None, '/', None], edgecolor='white') - ax1.barh(ind, df_all[df_all['g']==2]['score'], height=w, color=[BLUE, BLUE, ORANGE, PINK, PURPLE, PURPLE], hatch=['/', None, None, None, '/', None], edgecolor='white') - ax1.set_yticks(ind, tuple(names)) - ax1.set_ylabel(None) - ax1.set_xlim(0, 1.05) - ax1.set_xlabel('MIG Scores') - - ax2.barh(ind, df_all[df_all['g']==1]['score'], height=w, color=[BLUE, BLUE, ORANGE, PINK, PURPLE, PURPLE], hatch=['/', None, None, None, '/', None], edgecolor='white') - ax2.set_yticks(ind, [None for _ in tuple(names)]) - ax2.set_ylabel(None) - ax2.set_xlim(0, 1.05) - ax2.set_xlabel('Ground-Latent Correlation') - - # plt.xticks(rotation=90) - plt.tight_layout() - plt.savefig(os.path.join(save_dir, 'scores.png')) - plt.show() - - # make new df - df_all = df.rename(columns={'rcorr': 'score'}) - df_all['Metric'] = 'Ground-Latent Correlation' - df_all = pd.concat([df_all], keys=['name', 'metric', 'score']) - - # print everything! - fig, ax = plt.subplots(1, 1, figsize=(8, 3)) - # sns.barplot('score', 'name', data=df_all, palette=[ORANGE, PINK, PURPLE, PURPLE], hatch=[None, None, '/', None], edgecolor='white', ax=ax) - sns.barplot('score', 'name', data=df_all, palette=[BLUE, BLUE, ORANGE, PINK, PURPLE, PURPLE], hatch=['/', None, None, None, '/', None], edgecolor='white', ax=ax) - # plt.xticks(rotation=90) - ax.set_ylabel(None) - ax.set_xlabel('Ground-Truth & Latent Corr.') - plt.tight_layout() - plt.savefig(os.path.join(save_dir, 'scores_corr.png')) - plt.show() diff --git a/research/part04_application_to_rl/e01_learn_xy_representations/train_rl.py b/research/part04_application_to_rl/e01_learn_xy_representations/train_rl.py deleted file mode 100644 index 431d3bee..00000000 --- a/research/part04_application_to_rl/e01_learn_xy_representations/train_rl.py +++ /dev/null @@ -1,138 +0,0 @@ -import numpy as np -import torch.nn as nn -import torch.nn.functional as F -import torch.optim as optim - - -# ========================================================================= # -# BEGIN # -# ========================================================================= # - - -class Net(nn.Module): - def __init__(self, in_dim): - super().__init__() - self.fc1 = nn.Linear(in_dim, 120, bias=False) - nn.init.normal_(self.fc1.weight, mean=0, std=1) - self.fc2 = nn.Linear(120, 1, bias=False) - self.sigmoid = nn.Sigmoid() - - def forward(self, x): - x = F.relu(self.fc1(x)) - x = self.fc2(x) - return self.sigmoid(x) - - -class GridWorld(): - - def __init__(self, grid_size, start_pos, goal_pos, reward_value): - self.player_state = start_pos - self.goal = goal_pos - self.reward = reward_value - self.player_start = start_pos - self.env_dims = grid_size - self.game_state = np.zeros(grid_size) - - def simulate_action(self, action): - sim_state = np.zeros(self.env_dims) - if not self.player_state[0] == 0 and action == 0: - sim_state[self.player_state + (-1, 0)] = 1 - if not self.player_state[1] == (self.env_dims[1] - 1) and action == 1: - sim_state[self.player_state + (0, 1)] = 1 - if not self.player_state[0] == (self.env_dims[0] - 1) and action == 2: - sim_state[self.player_state + (1, 0)] = 1 - if not self.player_state[1] == 0 and action == 3: - sim_state[self.player_state + (0, -1)] = 1 - return sim_state - - def take_action(self, action): - self.game_state[self.player_state] = 0 - if not self.player_state[0] == 0 and action == 0: - self.player_state = self.player_state + (-1, 0) - if not self.player_state[1] == (self.env_dims[1] - 1) and action == 1: - self.player_state = self.player_state + (0, 1) - if not self.player_state[0] == (self.env_dims[0] - 1) and action == 2: - self.player_state = self.player_state + (1, 0) - if not self.player_state[1] == 0 and action == 3: - self.player_state = self.player_state + (0, -1) - self.game_state[self.player_state] = 1 - - def return_rewards(self): - if self.player_state == self.goal: - return self.reward - else: - return -1 - - def reset_player(self): - self.player_state = self.player_start - self.game_state[self.player_state] = 1 - - -# ========================================================================= # -# RL # -# ========================================================================= # - - -def run_rl( - num_steps: int = 100, - num_trajectories: int = 20, - step_size: float = 0.1, - discount: float = 0.1, - input_dim: int = 20, -): - - losses = np.zeros(num_trajectories) - - domain = GridWorld((10, 10), (0, 0), (9, 9), 100) - net = Net(input_dim) - - criterion = nn.MSELoss() - optimizer = optim.SGD(net.parameters(), lr=step_size, momentum=0.0) - - for traj in range(num_trajectories): - domain.reset_player() - - for step in range(num_steps): - optimizer.zero_grad() - - outputs = net(domain.game_state) - domain.take_action(np.argmax(outputs)) - next_Qs = [net(domain.simulate_action(i)) for i in range(4)] - loss = criterion(outputs, domain.return_rewards + discount * next_Qs) - loss.backward() - optimizer.step() - - # print statistics - losses[traj] = losses[traj] + loss.item() - - print(losses[traj]) - - print('Finished Training') - - -# ========================================================================= # -# MAIN # -# ========================================================================= # - - -if __name__ == "__main__": - run_rl() - - -# ========================================================================= # -# END # -# ========================================================================= # - - -# EXMAPLE: -# encoder -# data -# loading -# representations - -# torch.load() - -# map back xy to dataset image -# -- example - -# send json files with representations linked to xy positions diff --git a/research/part04_application_to_rl/e01_learn_xy_representations/train_vae.py b/research/part04_application_to_rl/e01_learn_xy_representations/train_vae.py deleted file mode 100644 index eb079949..00000000 --- a/research/part04_application_to_rl/e01_learn_xy_representations/train_vae.py +++ /dev/null @@ -1,391 +0,0 @@ -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2022 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ - -import json -import logging -from datetime import datetime -from pathlib import Path -from pprint import pprint -from typing import Optional -from typing import Union - -import hydra.utils -import imageio -import numpy as np -import psutil -import torch - -from disent.dataset.sampling import GroundTruthRandomWalkSampler -from disent.util.visualize.vis_util import make_image_grid -from omegaconf import DictConfig -from omegaconf import OmegaConf -from pytorch_lightning import Trainer -from pytorch_lightning.callbacks import ModelCheckpoint -from torch.utils.data import DataLoader - -from disent.dataset import DisentIterDataset -from disent.dataset.data import GroundTruthData -from disent.dataset.sampling import BaseDisentSampler -from disent.dataset.sampling import GroundTruthDistSampler -from disent.dataset.sampling import GroundTruthPairOrigSampler -from disent.dataset.sampling import GroundTruthPairSampler -from disent.dataset.sampling import GroundTruthTripleSampler -from disent.dataset.sampling import RandomSampler -from disent.dataset.transform import ToImgTensorF32 -from disent.dataset.util.stats import compute_data_mean_std -from disent.frameworks.ae import Ae -from disent.frameworks.vae import AdaVae -from disent.frameworks.vae import BetaVae -from disent.frameworks.vae import TripletVae -from disent.frameworks.vae import Vae -from disent.metrics import metric_mig -from disent.model import AutoEncoder -from disent.model.ae import DecoderConv64 -from disent.model.ae import EncoderConv64 -from disent.nn.weights import init_model_weights -from disent.util.lightning.callbacks import LoggerProgressCallback -from disent.util.lightning.callbacks import VaeLatentCycleLoggingCallback -from disent.util.lightning.callbacks._callback_vis_latents import get_vis_min_max -from disent.util.visualize.plot import plt_imshow -from disent.util.visualize.vis_img import torch_to_images -from experiment.util.path_utils import make_current_experiment_dir -from research.code.dataset.data import XYSingleSquareData -from research.code.frameworks.vae import AdaTripletVae -from research.code.metrics import metric_factored_components - - -log = logging.getLogger(__name__) - - -# ========================================================================= # -# Train A Single VAE # -# ========================================================================= # - - -def train( - save_dir: Union[str, Path], - data: GroundTruthData, - sampler: BaseDisentSampler, - framework: Union[Ae, Vae], - train_steps: int = 5000, - batch_size: int = 64, - num_workers: int = psutil.cpu_count(logical=False), - save_top_k: int = 5, - save_every_n_steps: int = 2500, - profile: bool = False, - data_mean=None, - data_std=None, -): - vis_min, vis_max = get_vis_min_max(recon_mean=data_mean, recon_std=data_std) - - # normalise the paths - save_dir = Path(save_dir) - save_dir.mkdir(exist_ok=False, parents=True) - - # make the dataset - dataset = DisentIterDataset(data, sampler=sampler, transform=ToImgTensorF32(size=64, mean=data_mean, std=data_std)) - dataloader = DataLoader(dataset=dataset, batch_size=batch_size, num_workers=num_workers, pin_memory=torch.cuda.is_available()) - - # make the trainer - trainer = Trainer( - logger=False, - callbacks=[ - ModelCheckpoint(dirpath=save_dir, monitor='loss', verbose=True, save_top_k=save_top_k, every_n_train_steps=save_every_n_steps), - LoggerProgressCallback(interval=5, log_level=logging.INFO), - # RichModelSummary(max_depth=2), - # RichProgressBar(refresh_rate_per_second=3), - # LoggerProgressCallback(), - # VaeMetricLoggingCallback(), - # VaeLatentCycleLoggingCallback(), - # VaeGtDistsLoggingCallback(), - ], - enable_progress_bar=False, - enable_model_summary=False, - max_steps=train_steps, - max_epochs=train_steps, - gpus=1 if torch.cuda.is_available() else 0, - profiler='simple' if profile else None, - ) - - # train the framework - trainer.fit(framework, dataloader) - - # visualise model -- generate - stills, animation, image = VaeLatentCycleLoggingCallback.generate_visualisations(dataset, framework, recon_mean=data_mean, recon_std=data_std, mode='minmax_interval_cycle', num_stats_samples=256) - - # visualise model -- plot image, save image, save vid - plt_imshow(image, show=True) - imageio.imsave(save_dir.joinpath('latent_cycle.png'), image) - with imageio.get_writer(save_dir.joinpath('latent_cycle.mp4'), fps=4) as writer: - for frame in animation: - writer.append_data(frame) - - # visualise data -- generate - data_batch = dataset.dataset_sample_batch(stills.shape[0]*stills.shape[1], mode='input', seed=7777, replace=True) - data_batch = torch_to_images(data_batch, in_min=vis_min, in_max=vis_max, always_rgb=True).numpy() - data_image = make_image_grid(data_batch, num_cols=stills.shape[1], pad=4) - - # visualise data -- plot image, save image - plt_imshow(data_image, show=True) - imageio.imsave(save_dir.joinpath('data_samples.png'), data_image) - - # compute metrics - get_repr = lambda x: framework.encode(x.to(framework.device)) - metrics = { - # **metric_dci.compute_fast(dataset, get_repr), - **metric_mig.compute_fast(dataset, get_repr), - **metric_factored_components.compute_fast(dataset, get_repr), - } - - # print and save the metrics - pprint(metrics) - with open(save_dir.joinpath('metrics.json'), 'w') as fp: - json.dump(metrics, fp, indent=2, sort_keys=True) - - # done! - return save_dir, metrics - - -# ========================================================================= # -# Train Multiple VAEs -- The actual experiment! # -# ========================================================================= # - - -def _load_ada_schedules(max_steps: int): - path = Path(__file__).parent.parent.parent.joinpath('config', 'schedule', 'adavae_up_all.yaml') - with open(path, 'r') as fp: - conf = DictConfig({ - 'trainer': {'max_steps': max_steps}, - 'schedule': OmegaConf.load(fp), - }) - return hydra.utils.instantiate(conf.schedule.schedule_items) - - -def run_experiments( - lr: float = 1e-4, - z_size: int = 9, - exp_dir: Optional[Union[Path, str]] = None, - train_steps: int = 10_000, - batch_size: int = 64, - num_workers: int = psutil.cpu_count(logical=False), - compute_stats: bool = False, - profile: bool = False, - ada_ratio: float = 1.25 -): - # PERMUTATIONS: - datasets = [ - ('xy8', XYSingleSquareData, dict(square_size=8, grid_spacing=8, image_size=64), [0.015625], [0.12403473458920848]), - # ('xy4', XYSingleSquareData, dict(square_size=8, grid_spacing=4, image_size=64), [0.015625], [0.12403473458920848]), - # ('xy2', XYSingleSquareData, dict(square_size=8, grid_spacing=2, image_size=64), [0.015625], [0.12403473458920848]), - # ('xy1', XYSingleSquareData, dict(square_size=8, grid_spacing=1, image_size=64), [0.015625], [0.12403473458920848]), - ] - triplet_sampler_maker_A = lambda: GroundTruthDistSampler(num_samples=3, triplet_sample_mode='manhattan_scaled', triplet_swap_chance=0.0) - triplet_sampler_maker_A1 = lambda: GroundTruthDistSampler(num_samples=3, triplet_sample_mode='manhattan_scaled', triplet_swap_chance=0.1) # actually works quite well if ada_ratio is lower, eg 1.25 instead of 1.5, but might hurt recons? check? - triplet_sampler_maker_A2 = lambda: GroundTruthDistSampler(num_samples=3, triplet_sample_mode='manhattan_scaled', triplet_swap_chance=0.2) # actually works quite well if ada_ratio is lower, eg 1.25 instead of 1.5, but might hurt recons? check? - triplet_sampler_maker_B = lambda: GroundTruthTripleSampler(p_k_range=1, n_k_range=(0, -1), n_k_sample_mode='bounded_below', n_k_is_shared=True, p_radius_range=1, n_radius_range=(0, -1), n_radius_sample_mode='bounded_below') # this one is really bad - triplet_sampler_maker_C = lambda: GroundTruthTripleSampler(p_k_range=(0, -1), n_k_range=(0, -1), n_k_sample_mode='bounded_below', n_k_is_shared=True, p_radius_range=(0, -1), n_radius_range=(0, -1), n_radius_sample_mode='bounded_below') # pretty much the same as the manhat above, except more strict... actually less real because its bounded below. Real episodes when sampled by time will usually be further away, but not necessarily bounded below like this. - triplet_sampler_maker_D64 = lambda: GroundTruthRandomWalkSampler(num_samples=3, p_dist_max=8, n_dist_max=64) - triplet_sampler_maker_D32 = lambda: GroundTruthRandomWalkSampler(num_samples=3, p_dist_max=8, n_dist_max=32) - triplet_sampler_maker_D16 = lambda: GroundTruthRandomWalkSampler(num_samples=3, p_dist_max=8, n_dist_max=16) - triplet_sampler_maker_D8 = lambda: GroundTruthRandomWalkSampler(num_samples=3, p_dist_max=8, n_dist_max=8) - - frameworks = [ - ('betavae', BetaVae, lambda: BetaVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.0001), lambda: RandomSampler(num_samples=1), lambda: {}), - ('adavae', AdaVae, lambda: AdaVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.0001), lambda: GroundTruthPairSampler(p_k_range=1, p_radius_range=(1, -1)), lambda: {}), - ('adavae_os', AdaVae, lambda: AdaVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.0001), lambda: GroundTruthPairOrigSampler(p_k=1), lambda: {}), - - ('triplet_soft_A', TripletVae, lambda: TripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_A, lambda: {}), - ('adatvae_soft_A', AdaTripletVae, lambda: AdaTripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_A, lambda: _load_ada_schedules(max_steps=int(train_steps * ada_ratio))), - - ('triplet_soft_A1', TripletVae, lambda: TripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_A1, lambda: {}), - ('triplet_soft_A2', TripletVae, lambda: TripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_A2, lambda: {}), - ('triplet_soft_B', TripletVae, lambda: TripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_B, lambda: {}), - ('triplet_soft_C', TripletVae, lambda: TripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_C, lambda: {}), - - # ('triplet_sigmoid10', TripletVae, lambda: TripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_sigmoid', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker, lambda: {}), - # ('triplet_sigmoid1', TripletVae, lambda: TripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_sigmoid', triplet_scale=1, triplet_margin_max=1, triplet_p=1), triplet_sampler_maker, lambda: {}), - # ('triplet10', TripletVae, lambda: TripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker, lambda: {}), - # ('triplet1', TripletVae, lambda: TripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet', triplet_scale=1, triplet_margin_max=1, triplet_p=1), triplet_sampler_maker, lambda: {}), - - ('adatvae_soft_A1', AdaTripletVae, lambda: AdaTripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_A1, lambda: _load_ada_schedules(max_steps=int(train_steps * ada_ratio))), - ('adatvae_soft_A2', AdaTripletVae, lambda: AdaTripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_A2, lambda: _load_ada_schedules(max_steps=int(train_steps * ada_ratio))), - ('adatvae_soft_B', AdaTripletVae, lambda: AdaTripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_B, lambda: _load_ada_schedules(max_steps=int(train_steps * ada_ratio))), - ('adatvae_soft_C', AdaTripletVae, lambda: AdaTripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_C, lambda: _load_ada_schedules(max_steps=int(train_steps * ada_ratio))), - - # ('adatvae_soft', AdaTripletVae, lambda: AdaTripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker, lambda: _load_ada_schedules(max_steps=int(train_steps*ADA_RATIO))), - # ('adatvae_sig10', AdaTripletVae, lambda: AdaTripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_sigmoid', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker, lambda: _load_ada_schedules(max_steps=int(train_steps*ADA_RATIO))), - # ('adatvae_sig1', AdaTripletVae, lambda: AdaTripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_sigmoid', triplet_scale=1, triplet_margin_max=1, triplet_p=1), triplet_sampler_maker, lambda: _load_ada_schedules(max_steps=int(train_steps*ADA_RATIO))), - # ('adatvae_trip10', AdaTripletVae, lambda: AdaTripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker, lambda: _load_ada_schedules(max_steps=int(train_steps*ADA_RATIO))), - # ('adatvae_trip1', AdaTripletVae, lambda: AdaTripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet', triplet_scale=1, triplet_margin_max=1, triplet_p=1), triplet_sampler_maker, lambda: _load_ada_schedules(max_steps=int(train_steps*ADA_RATIO))), - - ('triplet_soft_D64', TripletVae, lambda: TripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_D64, lambda: {}), - ('triplet_soft_D32', TripletVae, lambda: TripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_D32, lambda: {}), - ('triplet_soft_D16', TripletVae, lambda: TripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_D16, lambda: {}), - ('triplet_soft_D8', TripletVae, lambda: TripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_D8, lambda: {}), - - ('adatvae_soft_D64', AdaTripletVae, lambda: AdaTripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_D64, lambda: _load_ada_schedules(max_steps=int(train_steps * ada_ratio))), - ('adatvae_soft_D32', AdaTripletVae, lambda: AdaTripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_D32, lambda: _load_ada_schedules(max_steps=int(train_steps * ada_ratio))), - ('adatvae_soft_D16', AdaTripletVae, lambda: AdaTripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_D16, lambda: _load_ada_schedules(max_steps=int(train_steps * ada_ratio))), - ('adatvae_soft_D8', AdaTripletVae, lambda: AdaTripletVae.cfg(optimizer='adam', optimizer_kwargs=dict(lr=lr), beta=0.001, triplet_loss='triplet_soft', triplet_scale=1, triplet_margin_max=10, triplet_p=1), triplet_sampler_maker_D8, lambda: _load_ada_schedules(max_steps=int(train_steps * ada_ratio))), - ] - - # GET NAME: - exp_name = f'xy8_{ada_ratio}_{train_steps}' - - # NORMALIZE: - if exp_dir is None: - exp_dir = make_current_experiment_dir(str(Path(__file__).parent.joinpath('exp')), name=exp_name) - exp_dir = Path(exp_dir) - - - # COMPUTE DATASET STATS: - if compute_stats: - for i, (data_name, data_cls, data_kwargs, data_mean, data_std) in enumerate(datasets): - data = data_cls(transform=ToImgTensorF32(), **data_kwargs) - mean, std = compute_data_mean_std(data, progress=True, num_workers=num_workers, batch_size=batch_size) - print(f'{data.__class__.__name__} - {data_name}:{len(data)} - {data_kwargs}:\n mean: {mean.tolist()}\n std: {std.tolist()}') - - # TRAIN DIFFERENT FRAMEWORKS ON DATASETS - for i, (data_name, data_cls, data_kwargs, data_mean, data_std) in enumerate(datasets): - # make the dataset - data: GroundTruthData = data_cls(**data_kwargs) - - # train the framework - for j, (framework_name, framework_cls, framework_cfg_maker, sampler_maker, schedules_maker) in enumerate(frameworks): - start_time = datetime.today().strftime('%Y-%m-%d--%H-%M-%S') - - # make the data & framework - framework: Union[Ae, Vae] = framework_cls( - model=AutoEncoder( - encoder=EncoderConv64(x_shape=data.x_shape, z_size=z_size, z_multiplier=2 if issubclass(framework_cls, Vae) else 1), - decoder=DecoderConv64(x_shape=data.x_shape, z_size=z_size), - ), - cfg=framework_cfg_maker() - ) - sampler = sampler_maker() - - print('=' * 100) - print(f'[{i}x{j}] Dataset: name={data_name}, size={len(data)}, kwargs={data_kwargs}, mean={data_mean}, std={data_std}') - print(f'[{i}x{j}] Framework: name={framework_name} ({framework.__class__.__name__}), sampler={sampler.__class__.__name__}, cfg={framework.cfg.to_dict()}') - print('=' * 100) - - # register the schedules - for k, schedule in schedules_maker().items(): - framework.register_schedule(k, schedule) - # initialize weights - init_model_weights(framework, mode='xavier_normal', log_level=logging.DEBUG) - - # train the framework - run_name = f'{i}x{j}_{data_name}_{framework_name}' - save_dir = exp_dir.joinpath(run_name) - save_dir, metrics = train( - save_dir=save_dir, - data=data, - sampler=sampler, - framework=framework, - train_steps=train_steps, - batch_size=batch_size, - num_workers=num_workers, - profile=profile, - save_every_n_steps=train_steps, - ) - - # generate data for rl - with torch.no_grad(): - dataset: GroundTruthData = data_cls(**data_kwargs, transform=ToImgTensorF32(size=64, mean=data_mean, std=data_std)) - dat = { - # experiment info - 'exp_name': save_dir.parent.name, - 'run_name': save_dir.name, - 'start_time': start_time, - # dataset data - 'factor_names': data.factor_names, - 'factor_sizes': data.factor_sizes, - # sizes - 'num_factors': data.num_factors, - 'num_obs': len(data), - 'num_latents': z_size, - # image data - 'obs_indices': np.array([i for i in range(len(data))]), - 'obs_factors': np.array([data.idx_to_pos(i) for i in range(len(data))]), - 'obs': np.array([data[i] for i in range(len(data))]), - # representations - 'obs_encodings': np.array([framework.encode(dataset[i][None, ...].to(framework.device))[0].cpu().numpy() for i in range(len(data))]), - # results - 'metrics': metrics, - # descriptions - '_desc_': { - # experiment info - 'exp_name': f'The name of the group of experiments', - 'run_name': f'The name of this individual run that is part of the experiment', - 'start_time': f'The starting time of this individual run', - # dataset data - 'factor_names': f'The names of the ground truth factors in the dataset, eg. ["x", "y"]', - 'factor_sizes': f'The sizes of the ground truth factors in the dataset, eg. [8, 8]', - # sizes - 'num_factors': f'The number of different ground_truth factors in the dataset, eg. 2', - 'num_obs': f'The number of elements in the dataset, equal to the product of all the factor sizes, eg. 8x8 = 64', - 'num_latents': f'The number of latent units of the model or rather the number of encoder outputs, eg. 9', - # image data - 'obs_indices': f'The index of each observation in the dataset, eg. [0, 1, ..., 62, 63]. ' - f'The shape is (num_obs,)', - 'obs_factors': f'The ground truth factor of each observation in the dataset, eg. [[0, 0], [0, 1], ..., [7, 6], [7, 7]]. ' - f'The shape is (num_obs, num_factors)', - 'obs': f'The raw observations from the dataset, eg. [, ..., ]. ' - f'The shape is (num_obs, H, W, C)', - 'obs_encodings': f'The low dimensional encodings of each observations, eg. [, ..., ]. ' - f'The shape is (num_obs, num_latents)', - # results - 'metrics': f'Dict[str, float] of various scores from different disentanglement metrics computed over the model and the data.', - } - } - - # save the data for rl - # | rl_dat_path = save_dir.joinpath('rl_data.json') - # | with open(rl_dat_path, 'w') as fp: - # | json.dump({k: (v.tolist() if isinstance(v, np.ndarray) else v) for k, v in dat.items()}, fp) - # | print(f'[SAVED DATA]: {rl_dat_path}') - rl_npz_path = save_dir.joinpath('rl_data.npz') - np.savez_compressed(rl_npz_path, data=dat) - print(f'[SAVED DATA]: {rl_npz_path}') - - -# ========================================================================= # -# RUN! # -# ========================================================================= # - - -if __name__ == '__main__': - # make sure we can see the output! - logging.basicConfig(level=logging.INFO) - # run everything! - # run_experiments(train_steps=10000, ada_ratio=1.5) # ada not always strong enough? or is that a metric error - run_experiments(train_steps=10000, ada_ratio=1.25) # seems best? - # run_experiments(train_steps=5000, ada_ratio=1.5) - # run_experiments(train_steps=5000, ada_ratio=1.25) - - -# ========================================================================= # -# END # -# ========================================================================= # diff --git a/research/scripts/clog-batch.sh b/research/scripts/clog-batch.sh deleted file mode 100644 index 49d91cfc..00000000 --- a/research/scripts/clog-batch.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export PROJECT="N/A" -export USERNAME="N/A" -export PARTITION="batch" -export PARALLELISM=24 - -# source the helper file -source "$(dirname "$(realpath -s "$0")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 43200 "C-disent" # 12 hours diff --git a/research/scripts/clog-stampede.sh b/research/scripts/clog-stampede.sh deleted file mode 100644 index 60c73574..00000000 --- a/research/scripts/clog-stampede.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export PROJECT="N/A" -export USERNAME="N/A" -export PARTITION="stampede" -export PARALLELISM=24 - -# source the helper file -source "$(dirname "$(realpath -s "$0")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cudaless_nodes "$PARTITION" 43200 "C-disent" # 12 hours diff --git a/research/scripts/helper.sh b/research/scripts/helper.sh deleted file mode 100644 index dfb8ed59..00000000 --- a/research/scripts/helper.sh +++ /dev/null @@ -1,192 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Description # -# ========================================================================= # - -# before sourcing this script, it requires the following variables to be exported: -# - PROJECT: str -# - PARTITION: str -# - PARALLELISM: int - -# source this script from the script you use to run the experiment -# 1. gets and exports the path to the root -# 2. changes the working directory to the root -# 3. exports a helper function that runs the script in the background, with -# the correct python path and settings - -if [ -z "$PROJECT" ]; then echo "PROJECT is not set"; exit 1; fi -if [ -z "$PARTITION" ]; then echo "PARTITION is not set"; exit 1; fi -if [ -z "$PARALLELISM" ]; then echo "PARALLELISM is not set"; exit 1; fi -if [ -z "$USERNAME" ]; then echo "USERNAME is not set"; exit 1; fi -if [ -z "$PY_RUN_FILE" ]; then PY_RUN_FILE='experiment/run.py'; fi - -# ========================================================================= # -# Helper # -# ========================================================================= # - -# get the calling script dir -#CALL_SCRIPT_DIR=$(dirname "$(realpath -s "$0")") -# get the current script dir -SCRIPT_DIR="$(dirname -- "${BASH_SOURCE[0]}")" -# get the root directory for `disent` -ROOT_DIR="$(realpath -s "$SCRIPT_DIR/../..")" -# cd into the root or exit on failure, then get relative paths -cd "$ROOT_DIR" || exit 1 -PY_RUN_FILE="$(realpath "$PY_RUN_FILE")" - -# update the user -echo "working directory is: $(pwd)" -echo "main script is: $PY_RUN_FILE" -echo - -# hydra search path and plugins todo: make this configurable? -_SEARCH_PATH="${ROOT_DIR}/research/config" -_SCRIPTS_PATH="${ROOT_DIR}/research/scripts" - -function submit_sweep() { - echo "SUBMITTING SWEEP:" "$@" - PYTHONPATH="$ROOT_DIR" DISENT_CONFIGS_PREPEND="$_SEARCH_PATH" python3 "$PY_RUN_FILE" \ - -m \ - run_launcher=slurm \ - dsettings.launcher.partition="$PARTITION" \ - settings.job.project="$PROJECT" \ - settings.job.user="$USERNAME" \ - hydra.launcher.array_parallelism="$PARALLELISM" \ - "$@" \ - & # run in background -} - -function local_run() { - echo "RUNNING:" "$@" - PYTHONPATH="$ROOT_DIR" DISENT_CONFIGS_PREPEND="$_SEARCH_PATH" python3 "$PY_RUN_FILE" \ - run_launcher=local \ - settings.job.project="$PROJECT" \ - settings.job.user="$USERNAME" \ - "$@" -} - -# NOTE: this should correspond to the code in `sbatch_job.sh` -function local_sweep() { - echo "RUNNING SWEEP:" "$@" - PYTHONPATH="$ROOT_DIR" DISENT_CONFIGS_PREPEND="$_SEARCH_PATH" python3 "$PY_RUN_FILE" \ - -m \ - run_launcher=local \ - settings.job.project="$PROJECT" \ - settings.job.user="$USERNAME" \ - "$@" -} - -function gen_sbatch_args_file() { - # make sure we have a unique filename specified to store the args - if [ -z "$ARGS_START_NUM" ]; then ARGS_START_NUM=1 ; fi - if [ -z "$ARGS_FILE" ]; then echo "ARGS_FILE not specified!" ; exit 1 ; fi - # make sure the args file is absolute - case "$ARGS_FILE" in - /*) true ;; - *) echo "ARGS_FILE is not absolute, got: $ARGS_FILE" ; exit 1 ;; - esac - # generate everything - echo "[GENERATING SWEEP]:" "$@" - echo - # generate a single command for all the diferent permutations - _args="$(PYTHONPATH="$ROOT_DIR" python3 "$_SCRIPTS_PATH/permutations.py" \ - --line-numbers \ - --line-number-format="+EXTRA.sweep_num={}" \ - --line-number-start="$ARGS_START_NUM" \ - --no-color \ - --overrides \ - "$@")" - # save to file - if [ -z "$APPEND_ARGS" ]; then - echo "$_args" > "$ARGS_FILE" - echo "[SAVED ARGS]: $ARGS_FILE" - else - echo "$_args" >> "$ARGS_FILE" - echo "[ADDED ARGS]: $ARGS_FILE" - fi - echo -} - -function submit_sbatch_args_file() { - # make sure we have a unique filename specified to store the args - if [ -z "$ARGS_FILE" ]; then echo "ARGS_FILE not specified!" ; exit 1 ; fi - # make sure the args file is absolute - case "$ARGS_FILE" in - /*) true ;; - *) echo "ARGS_FILE is not absolute, got: $ARGS_FILE" ; exit 1 ;; - esac - # check - echo "[RUNNING]:" - echo " - submit script: '$SCRIPT_DIR/sbatch_submit.sh'" - echo " - ARGS_FILE='$ARGS_FILE'" - echo - # submit everything - ARGS_FILE="$ARGS_FILE" bash "$SCRIPT_DIR/sbatch_submit.sh" -} - -function submit_sbatch() { - gen_sbatch_file "$@" - submit_sbatch_file -} - -# export -export ROOT_DIR -export PY_RUN_FILE -export submit_sweep -export local_run - -# debug hydra -HYDRA_FULL_ERROR=1 -export HYDRA_FULL_ERROR - -# ========================================================================= # -# Slurm Helper # -# ========================================================================= # - - -function num_idle_nodes() { - if [ -z "$1" ]; then echo "partition (first arg) is not set"; exit 1; fi - # number of idle nodes - num=$(sinfo --partition="$1" --noheader -O Nodes,Available,StateCompact | awk '{if($2 == "up" && $3 == "idle"){print $1}}') - if [ -z "$num" ]; then num=0; fi - echo $num -} - -function clog_cudaless_nodes() { - if [ -z "$1" ]; then echo "partition is not set"; exit 1; fi - if [ -z "$2" ]; then echo wait=120; else wait="$2"; fi - if [ -z "$3" ]; then echo name="NO-CUDA"; else name="$3"; fi - # clog idle nodes - n=$(num_idle_nodes "$1") - if [ "$n" -lt "1" ]; then - echo -e "\e[93mclogging skipped! no idle nodes found on partition '$1'\e[0m"; - else - echo -e "\e[92mclogging $n nodes on partition '$1' for ${wait}s if cuda is not available!\e[0m"; - sbatch --array=1-"$n" --partition="$1" --job-name="$name" --output=/dev/null --error=/dev/null \ - --wrap='python -c "import torch; import time; cuda=torch.cuda.is_available(); print(\"CUDA:\", cuda, flush=True); print(flush=True); time.sleep(5 if cuda else '"$wait"');"' - fi -} - -function clog_cuda_nodes() { - if [ -z "$1" ]; then echo "partition is not set"; exit 1; fi - if [ -z "$2" ]; then echo wait=120; else wait="$2"; fi - if [ -z "$3" ]; then echo name="HAS-CUDA"; else name="$3"; fi - # clog idle nodes - n=$(num_idle_nodes "$1") - if [ "$n" -lt "1" ]; then - echo -e "\e[93mclogging skipped! no idle nodes found on partition '$1'\e[0m"; - else - echo -e "\e[92mclogging $n nodes on partition '$1' for ${wait}s if cuda is available!\e[0m"; - sbatch --array=1-"$n" --partition="$1" --job-name="$name" --output=/dev/null --error=/dev/null \ - --wrap='python -c "import torch; import time; cuda=torch.cuda.is_available(); print(\"CUDA:\", cuda, flush=True); print(flush=True); time.sleep(5 if not cuda else '"$wait"');"' - fi -} - -export num_idle_nodes -export clog_cudaless_nodes -export clog_cuda_nodes - -# ========================================================================= # -# End # -# ========================================================================= # diff --git a/research/scripts/permutations.py b/research/scripts/permutations.py deleted file mode 100644 index 314ee46b..00000000 --- a/research/scripts/permutations.py +++ /dev/null @@ -1,109 +0,0 @@ -import argparse -import itertools -import sys - -import numpy as np - -import disent.util.strings.colors as c -from hydra.core.override_parser.overrides_parser import OverridesParser -from hydra.core.override_parser.types import Override -from hydra.core.override_parser.types import Sweep - - -def _iter_value_strings(override: Override) -> str: - if isinstance(override.value(), Sweep): - yield from override.sweep_string_iterator() - else: - yield override.get_value_element_as_str() - - -if __name__ == '__main__': - # sys.argv.extend(['--line-numbers', '--base', 'PYTHONPATH=. python3 experiment/run.py', '--overrides', 'framework=betavae,adavae_os', 'dataset=X--xysquares,cars3d,shapes3d,dsprites,smallnorb']) - - # From: `hydra._internal.utils.get_args_parser` - parser = argparse.ArgumentParser(add_help=True, description="Hydra Permutations") - parser.add_argument("--overrides", nargs="*", default=(), help="Any key=value arguments parsed using Hydra Config. Values are treated as a sweep and all permutations are generated on the output instead.") - parser.add_argument("--base", type=str, default='', help='The base command that will be added to the output.') - parser.add_argument("--debug", action='store_true', help='If debug information should be displayed.') - parser.add_argument("--line-number-start", type=int, default=1, help='The starting number to use when labeling lines') - parser.add_argument("--line-numbers", action='store_true', help='If line numbers should be prepended as environment variables.') - parser.add_argument("--line-number-format", type=str, default=None, help='The format for prefixing line numbers, default is `CMD_NUM={}`') - parser.add_argument("--no-color", action='store_true', help='Disable ANSI colors in the output.') - parser.add_argument("--no-align", action='store_true', help='Disable alignment of different arguments in the output') - args = parser.parse_args() - - # From: ` hydra._internal.config_loader_impl.ConfigLoaderImpl._load_configuration_impl` - parser = OverridesParser.create() - parsed_overrides = parser.parse_overrides(overrides=args.overrides) - - # color object - if args.no_color: - c, _c = argparse.Namespace(), c - for k in dir(_c): - setattr(c, k, '') - else: - import disent.util.strings.colors as c - - # DEBUG: input information - if args.debug: - print(f'{c.lGRY}COMMAND:{c.RST} {c.RED}{repr(args.base)}{c.RST}') - print(f'{c.lGRY}OVERRIDES:{c.RST} {c.YLW}{repr(args.overrides)}{c.RST}') - print() - - # get permutation strings - override_orig_strings, override_permutations = [], [] - for override in parsed_overrides: - override_orig_strings.append(f'{c.lMGT}{override.input_line}{c.RST}') - prefix = override.get_key_element() - override_permutations.append([f'{c.lGRY}{prefix}{c.lGRY}={c.lYLW}{postfix}{c.RST}' for postfix in _iter_value_strings(override)]) - - # DEBUG: parsed override information - if args.debug: - for orig_str, elem_strs in zip(override_orig_strings, override_permutations): - print(f'{c.lMGT}*{c.RST} {orig_str}') - for elem_str in elem_strs: - print(f'{c.lGRY}| {elem_str}') - if parsed_overrides: - print() - - # get stats - num_override_elems = [len(elems) for elems in override_permutations] - num_permutations = int(np.prod(num_override_elems)) - - # DEBUG: separator - if args.debug: - print(f'{c.lGRY}{"="*100}{c.RST}') - print(f'{c.lGRY}GENERATED {num_permutations} COMMANDS ({"x".join(str(n) for n in num_override_elems)}){c.RST}') - print(f'{c.lGRY}{"="*100}{c.RST}') - print() - - # if we need to print the line numbers - if args.line_numbers and (not args.no_align): - num_digits = int(np.log10(max(1, num_permutations))) + 1 - else: - num_digits = 0 - # if we need to normalise the length of everything so that the output is aligned - if args.no_align: - elem_lengths = [0 for _ in override_permutations] - else: - elem_lengths = [max(len(elem) for elem in elems) for elems in override_permutations] - - # OUTPUT: print permutations - for i, elements in enumerate(itertools.product(*override_permutations)): - command_parts = [] - # prepend the line numbers - if args.line_numbers: - num = i + args.line_number_start - if args.line_number_format is None: - line_num = f'CMD_NUM={num:<{num_digits}d}' - else: - line_num = args.line_number_format.format(num) - command_parts.append(f'{c.lGRY}{line_num}{c.RST}') - # prepend the base command - if args.base: - command_parts.append(f'{c.lRED}{args.base}{c.RST}') - # append the arguments - if elements: - command_parts.append(" ".join(f'{elem:<{l}s}' for elem, l in zip(elements, elem_lengths))) - # done! - print(' '.join(command_parts)) diff --git a/research/scripts/sbatch_job.sh b/research/scripts/sbatch_job.sh deleted file mode 100644 index 17b7f5f8..00000000 --- a/research/scripts/sbatch_job.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -#SBATCH --nodes=1 --ntasks=1 - -# ========================================================================= # -# CHECKS & SETUP # -# ========================================================================= # - -# make sure that this was launched as an array -# make sure that we have an array file specified -if [ -z "$SLURM_ARRAY_JOB_ID" ]; then echo "SLURM_ARRAY_JOB_ID is not defined... Are you sure the job was launched as an array?" ; exit 1 ; fi -if [ -z "$SLURM_ARRAY_TASK_ID" ]; then echo "SLURM_ARRAY_TASK_ID is not defined... Are you sure the job was launched as an array?" ; exit 1 ; fi -if [ -z "$ARGS_FILE" ]; then echo "ARGS_FILE is not defined... Cannot obtain job settings!" ; exit 1 ; fi -if [ -z "$SCRIPT_DIR" ]; then echo "SCRIPT_DIR is not set... Cannot find helper.sh"; exit 1; fi -echo "[RUNNING JOB]: $SLURM_ARRAY_TASK_ID" - -# setup the python environment -. "$HOME/installed/pyenv/versions/miniconda3-latest/etc/profile.d/conda.sh" -conda activate disent-conda - -# save the working directory -_run_pwd="$(pwd)" -_run_dir="$_run_pwd/$SLURM_ARRAY_JOB_ID/${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID}" - -# load `local_run` -. "$SCRIPT_DIR/helper.sh" - -# set the working directory -mkdir -p "$_run_dir" -cd "$_run_dir" || exit 1 -echo " - set working directory: $(pwd)" -echo - -# ========================================================================= # -# RUN # -# ========================================================================= # - -# get the line in the file corresponding to the array number -# - sed starts indexing at 1, not zero -LINE=$(sed -n "$SLURM_ARRAY_TASK_ID"p "$ARGS_FILE") - -# run experiment with the specified arguments -local_run $LINE diff --git a/research/scripts/sbatch_submit.sh b/research/scripts/sbatch_submit.sh deleted file mode 100644 index f5f7acd9..00000000 --- a/research/scripts/sbatch_submit.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -# -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# MIT License -# -# Copyright (c) 2022 Nathan Juraj Michlo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~=~ -# - -# ========================================================================= # -# Settings # -# ========================================================================= # - -# check variables needed in the submission script -if [ -z "$PROJECT" ]; then echo "PROJECT is not set"; exit 1; fi -if [ -z "$PARTITION" ]; then echo "PARTITION is not set"; exit 1; fi -if [ -z "$PARALLELISM" ]; then echo "PARALLELISM is not set"; exit 1; fi -if [ -z "$USERNAME" ]; then echo "USERNAME is not set"; exit 1; fi -if [ -z "$ARGS_FILE" ]; then echo "ARGS_FILE is not set"; exit 1; fi - -# get the root directory -SCRIPT_DIR="$(dirname -- "${BASH_SOURCE[0]}")" -ROOT_DIR="$(realpath -s "$SCRIPT_DIR/../..")" -RUN_DIR="$ROOT_DIR/logs/sbatch_sweep" - -# export variables needed in the submission script -export PROJECT -export PARTITION -export PARALLELISM -export USERNAME -export ARGS_FILE -export SCRIPT_DIR - -# ========================================================================= # -# RUN FILE # -# ========================================================================= # - -echo "[LAUNCHING JOBS]:" - -# change the working directory, so that logs arent written randomly everywhere -mkdir -p "$RUN_DIR" -cd "$RUN_DIR" || exit 1 -echo " - working directory: $(pwd)" - -# get number of lines in a file -# - the file should end with a new line -NUM_LINES=$(wc -l < "$ARGS_FILE") -echo " - submitting $NUM_LINES jobs as an array" -echo - -# submit an array that reads each line from the file -# and starts a new job based on it! -sbatch \ - --partition="$PARTITION" \ - --job-name=bdisent \ - --array="1-$NUM_LINES%$PARALLELISM" \ - --time=24:00:00 \ - --exclude="mscluster92,mscluster94,mscluster96" \ - "$SCRIPT_DIR/sbatch_job.sh" diff --git a/research/scripts/wandb_cli.py b/research/scripts/wandb_cli.py deleted file mode 100644 index a97dce8a..00000000 --- a/research/scripts/wandb_cli.py +++ /dev/null @@ -1,31 +0,0 @@ - -""" -This file is an alias to the wandb cli that first sets the -temporary directory to a different folder `/tmp//tmp`, -in case `/tmp` has been polluted or you don't have the correct -access rights to modify files. - -- I am not sure why we need to do this, it is probably a bug with - wandb (or even tempfile) not respecting the `TMPDIR`, `TEMP` and - `TMP` environment variables which when set should do the same as - below? According to the tempdir docs: - https://docs.python.org/3/library/tempfile.html#tempfile.gettempdir -""" - -# wandb_cli.py -if __name__ == '__main__': - import os - import tempfile - - # generate the temporary directory from the user - temp_dir = f'/tmp/{os.environ["USER"]}/wandb' - print(f'[PATCHING:] tempfile.tempdir={repr(temp_dir)}') - - # we need to patch tempdir before we can import wandb - assert tempfile.tempdir is None - os.makedirs(temp_dir, exist_ok=True) - tempfile.tempdir = temp_dir - - # taken from wandb.__main__ - from wandb.cli.cli import cli - cli(prog_name="python -m wandb") diff --git a/research/scripts/working-batch.sh b/research/scripts/working-batch.sh deleted file mode 100644 index 53855652..00000000 --- a/research/scripts/working-batch.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export PROJECT="N/A" -export USERNAME="N/A" -export PARTITION="batch" -export PARALLELISM=24 - -# source the helper file -source "$(dirname "$(realpath -s "$0")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cuda_nodes "$PARTITION" 43200 "W-disent" # 12 hours diff --git a/research/scripts/working-stampede.sh b/research/scripts/working-stampede.sh deleted file mode 100644 index c4d7170b..00000000 --- a/research/scripts/working-stampede.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# ========================================================================= # -# Settings # -# ========================================================================= # - -export PROJECT="N/A" -export USERNAME="N/A" -export PARTITION="stampede" -export PARALLELISM=24 - -# source the helper file -source "$(dirname "$(realpath -s "$0")")/helper.sh" - -# ========================================================================= # -# Experiment # -# ========================================================================= # - -clog_cuda_nodes "$PARTITION" 43200 "W-disent" # 12 hours diff --git a/tests/test_data_similarity.py b/tests/test_data_similarity.py index 5ace9d4e..33169471 100644 --- a/tests/test_data_similarity.py +++ b/tests/test_data_similarity.py @@ -26,8 +26,6 @@ from disent.dataset.data import XYObjectData from disent.dataset.data import XYObjectShadedData -from research.code.dataset.data import XYSquaresData # pragma: delete-on-release -from research.code.dataset.data import XYSquaresMinimalData # pragma: delete-on-release # ========================================================================= # @@ -35,18 +33,6 @@ # ========================================================================= # -def test_xysquares_similarity(): # pragma: delete-on-release - data_org = XYSquaresData() # pragma: delete-on-release - data_min = XYSquaresMinimalData() # pragma: delete-on-release - # check lengths # pragma: delete-on-release - assert len(data_org) == len(data_min) # pragma: delete-on-release - n = len(data_min) # pragma: delete-on-release - # check items # pragma: delete-on-release - for i in np.random.randint(0, n, size=100): # pragma: delete-on-release - assert np.allclose(data_org[i], data_min[i]) # pragma: delete-on-release - # check bounds # pragma: delete-on-release - assert np.allclose(data_org[0], data_min[0]) # pragma: delete-on-release - assert np.allclose(data_org[n-1], data_min[n-1]) # pragma: delete-on-release def test_xyobject_similarity(): diff --git a/tests/test_experiment.py b/tests/test_experiment.py index e2600faf..fec35bba 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -37,9 +37,6 @@ # ========================================================================= # # TESTS # # ========================================================================= # - # pragma: delete-on-release - # pragma: delete-on-release -RESEARCH_CFG_DIR = os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'research/config')) # pragma: delete-on-release @pytest.mark.parametrize(('env', 'args'), [ @@ -47,10 +44,6 @@ (dict(), ['run_action=skip']), (dict(), ['run_action=prepare_data']), (dict(), ['run_action=train']), - # test the configs with the research components # pragma: delete-on-release - # -- we need to modify the search path # pragma: delete-on-release - # -- we need to register all the components # pragma: delete-on-release - (dict(DISENT_CONFIGS_PREPEND=RESEARCH_CFG_DIR), ['run_action=train', 'dataset=X--xysquares', 'metrics=test', 'framework=X--adaae_os', 'schedule=adanegtvae_up_all']), # pragma: delete-on-release ]) def test_experiment_run(env, args): # show full errors in hydra diff --git a/tests/test_frameworks.py b/tests/test_frameworks.py index 815516ed..f454050f 100644 --- a/tests/test_frameworks.py +++ b/tests/test_frameworks.py @@ -41,8 +41,6 @@ from disent.model.ae import DecoderLinear from disent.model.ae import EncoderLinear from disent.dataset.transform import ToImgTensorF32 -from research.code.frameworks.ae import * # pragma: delete-on-release -from research.code.frameworks.vae import * # pragma: delete-on-release # ========================================================================= # @@ -53,16 +51,10 @@ _TEST_FRAMEWORKS = [ # AE - unsupervised (Ae, dict(), XYObjectData), - # AE - unsupervised - EXP # pragma: delete-on-release - (DataOverlapTripletAe, dict(overlap_mine_triplet_mode='hard_neg'), XYObjectData), # pragma: delete-on-release # AE - weakly supervised # - # AE - weakly supervised - EXP # pragma: delete-on-release - (AdaAe, dict(), XYObjectData), # pragma: delete-on-release # AE - supervised (TripletAe, dict(), XYObjectData), - # AE - supervised - EXP # pragma: delete-on-release - (AdaNegTripletAe, dict(), XYObjectData), # pragma: delete-on-release # VAE - unsupervised (Vae, dict(), XYObjectData), (BetaVae, dict(), XYObjectData), @@ -72,34 +64,13 @@ (DfcVae, dict(), XYObjectData), (DfcVae, dict(), partial(XYObjectData, rgb=False)), (BetaTcVae, dict(), XYObjectData), - # VAE - unsupervised - EXP # pragma: delete-on-release - (DataOverlapTripletVae,dict(overlap_mine_triplet_mode='none'), XYObjectData), # pragma: delete-on-release - (DataOverlapTripletVae,dict(overlap_mine_triplet_mode='semi_hard_neg'), XYObjectData), # pragma: delete-on-release - (DataOverlapTripletVae,dict(overlap_mine_triplet_mode='hard_neg'), XYObjectData), # pragma: delete-on-release - (DataOverlapTripletVae,dict(overlap_mine_triplet_mode='hard_pos'), XYObjectData), # pragma: delete-on-release - (DataOverlapTripletVae,dict(overlap_mine_triplet_mode='easy_pos'), XYObjectData), # pragma: delete-on-release - (DataOverlapRankVae, dict(), XYObjectData), # pragma: delete-on-release # VAE - weakly supervised (AdaVae, dict(), XYObjectData), (AdaVae, dict(ada_average_mode='ml-vae'), XYObjectData), (AdaGVaeMinimal, dict(), XYObjectData), - # VAE - weakly supervised - EXP # pragma: delete-on-release - (SwappedTargetAdaVae, dict(swap_chance=1.0), XYObjectData), # pragma: delete-on-release - (SwappedTargetBetaVae, dict(swap_chance=1.0), XYObjectData), # pragma: delete-on-release - (AugPosTripletVae, dict(), XYObjectData), # pragma: delete-on-release # VAE - supervised (TripletVae, dict(), XYObjectData), (TripletVae, dict(detach_decoder=True, disable_reg_loss=True), XYObjectData), - # VAE - supervised - EXP # pragma: delete-on-release - (BoundedAdaVae, dict(), XYObjectData), # pragma: delete-on-release - (GuidedAdaVae, dict(), XYObjectData), # pragma: delete-on-release - (GuidedAdaVae, dict(gada_anchor_ave_mode='thresh'), XYObjectData), # pragma: delete-on-release - (TripletBoundedAdaVae, dict(), XYObjectData), # pragma: delete-on-release - (TripletGuidedAdaVae, dict(), XYObjectData), # pragma: delete-on-release - (AdaTripletVae, dict(), XYObjectData), # pragma: delete-on-release - (AdaAveTripletVae, dict(adat_share_mask_mode='posterior'), XYObjectData), # pragma: delete-on-release - (AdaAveTripletVae, dict(adat_share_mask_mode='sample'), XYObjectData), # pragma: delete-on-release - (AdaAveTripletVae, dict(adat_share_mask_mode='sample_each'), XYObjectData), # pragma: delete-on-release ] diff --git a/tests/test_metrics.py b/tests/test_metrics.py index d9807e94..92b0a9bc 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -30,7 +30,6 @@ from disent.metrics import * from disent.dataset.transform import ToImgTensorF32 from disent.util.function import wrapped_partial -from research.code.metrics import * # pragma: delete-on-release # ========================================================================= # @@ -44,8 +43,6 @@ wrapped_partial(metric_dci, num_train=7, num_test=7), wrapped_partial(metric_sap, num_train=7, num_test=7), wrapped_partial(metric_factor_vae, num_train=7, num_eval=7, num_variance_estimate=7), - wrapped_partial(metric_flatness, repeats=7), # pragma: delete-on-release - wrapped_partial(metric_factored_components, repeats=7), # pragma: delete-on-release ]) def test_metrics(metric_fn): z_size = 8 diff --git a/tests/test_registry.py b/tests/test_registry.py index 87868138..1bde1aa3 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -44,25 +44,10 @@ 'KERNELS': 2, } -COUNTS = { # pragma: delete-on-release - 'DATASETS': 14, # pragma: delete-on-release - 'SAMPLERS': 8, # pragma: delete-on-release - 'FRAMEWORKS': 25, # pragma: delete-on-release - 'RECON_LOSSES': 9, # pragma: delete-on-release - 'LATENT_HANDLERS': 2, # pragma: delete-on-release - 'OPTIMIZERS': 30, # pragma: delete-on-release - 'METRICS': 9, # pragma: delete-on-release - 'SCHEDULES': 5, # pragma: delete-on-release - 'MODELS': 8, # pragma: delete-on-release - 'KERNELS': 18, # pragma: delete-on-release -} # pragma: delete-on-release @pytest.mark.parametrize('registry_key', COUNTS.keys()) def test_registry_loading(registry_key): - from research.code import register_to_disent # pragma: delete-on-release - register_to_disent() # pragma: delete-on-release - register_to_disent() # must be able to call more than once! # pragma: delete-on-release # load everything and check the counts count = 0 for example in R.REGISTRIES[registry_key]:

Z4SX=Ff{GwJ__#*~if;*n{nKKrOb=Tn+Rd!`3*nji z0$6Lu)zJ>>L&A#!I3zj?UT7Eq^zQ@Jus5*VFc6aVZ-NxZQ;@&^JM2_>45{DC04_X% zuAnaXY1Imnt{-8a#Rrgj^&M2xc~K=$9A_kpVS3wBP(C1w!V-!Y!K1*9%PpY(OB?@i z`>f*BwGrRG0?9IcluG8{msh4(db$ZZ9$H{$%M!GoZ;Yl@ZP4Se9e=n*;EbW&C}hwH z)=43FR^%N1HyMcyk`kEd=Yz8TLD+0@27A9&;Jt!lsJ7P)Rf|qye+zKZJx}768*VuJ z&@qf{ufXhUF?e?UVKhGGgZ#mg=vxwr-iqhYeQgMuMZAJ#IlECyCK9Jyb;F)DT>as- z5jve%iZ)va*3WE!*Xm}d^NWM61^Q?^^%ZEzYU6?f^D$tX4hmFB#Gk3^AcFQg%2AYh9R}~J=dv z?)xylaRz4HH^rI)J*1lX@M(z!rvA0Vf>$Qkr=JU>{nqGz(hYTIBC4(BK#MBDnGRc! z9bS(o0tR5!E)Sem9*DEEV$dk50;lw!#7KugEH%A=lDq9V-?`VeIf1y}Dhl_0sK9dT zGdLa`h&Nt%pqat|1Zr)-T-PmFx5@&4DiC<9k61BxJF+8-(8wnjgy)&y)QFWRg%%hs zlLt#?=%WcY{=fNe2ClJr2-Z)P(DJ1=E`2r)@7YvB!80+uFCvdQR#R{+>LKL*6G3Li z?IC<9gkoi1;N9=Gv6!#z?BnfXm|@@qdWp7h<)#xHQmcnN9ZxW; z+y^(a&O>sqe$ee(i|h4$u-nEFpU>%sQ)@P2s8s-l=dQy-{Tw*vybI;xPvU%?lPF?qg#Aau zk#}_jit)uG$#}r+$&SVZZqD!B8jVKV7IPB0I6l?)Al^R`js5Zu@q#9)d9FByck!? zu0l~RA87pI6Rf_kk6RC#BR`pqrBz*UVTvLSR?bBKemR_9&;xT@CGiF~?tSnO!t7Q- z?CIgfqr*SA+Gj6>%a4O}T`MFuSA(BPGrZB_#|fGRtJ9Cc6Nm@P%wZUuat_?;0^oy? zH%J%O!}3ZOSby3MhOC{yEwBXMr*V6t=XSD;EmNR#y)L{e=A%bt9B8F;JMma@m(;#1 zBT;pN)b)xroL#E{f?SLr81jyljobke>H~GDZcv!E6TUUHLu$e)sC;(;?yWBbjWdGy zFY^j~?Bw$2hfAPwn>bz!?}4q6ub@eO03tgDu;=hU_;6eZbuA^(qk0O?O;SMj({gC# zrick$zu@^SZ5&%S4;QD*#@q*!km5zKc;#aJIFF*qLmtfMawnTAwxHL!wKz*~0@T#E z;o)Zo@!DojyfaM*qwo2nj`9h-BzqVO6^9{iY9xlcUO>Iy3Ap#zQhb|m3XhFO;lo#n z=(4W{?|n|dEx*FC$tDg9*VuAoF2&)&%i;KB=Q%v8Q-h{w67l=3XcUt@h5CHU@b{Ah zbcwluVtx_W=RFK>9v7?Rosd(!!~$a zgcoPlP<++62yHG{po7CCWb4hsvjc|MEvJJ_d;+#SQ$YCyS-h5|fc@?gxc{FxS|1R? zs|9}`FNs@YR(*ma^SHU=?R(h2SscT6l)$2p1UPi;3S77?fUT2-V7&1>C~YqEgK5Lx)MTLLO}KLTr);}EJE0s?i1xtc>4+&Oawl8>Z9!-h&Y93X?nl+l$5GVY2ZulO!MnQ~@YL&ls9d!gU5A^%LCY3bZds4#truf0?*JG{ znW206Qe4<-iu0#`0beUkyytF+#ZP9UtHls#ED=ZPaz)g-B8dmLir@utF`Qe;hfYP4 z@OY~L4*B=MG3yr~%&nLAKNrRY_N9;$mjZ2Nd5~`>imNp4!r+cnh+Y~C8@|1TnGcRa zYf%Wi;cETyTr3$^BnA5_WkJ=>0?ucz1TWtSV(?-k-F=Rq>XrR^zPWhNUj^U2N`)r1spujw1HFz; zLjlZ%H`fGE8YOU_9ycbYXY0L4e$aN_0zkPoN^?T{z1Q>Y9sn^(fAiDam0 zIt>{s&w;#j9q9kt2d6SDAS-qQ2pBa$YPKI}Z{$GS=0$Mct^yV|F9*>jj*u|813266 zaBF@gn3S9chB}MjPog&5I;2cJZ|TsZIs!CCSCU3J_mRDsrNk;flD*B~13vS=Ouwy_ z>D{~y%HRV@?{>njH+&?g9~ zd;_lj-@$>4<9xEYJkDYXl*^RGk1&N>XHP?$^V%q_o|YXH1vuhn6>6Fi$TC=jM6f(Z&CRqU#RJ@%zJt z5|xM+O%kD^G@j=@_fsj_MkyJ|Xh;d!SxxOJ?LkAONqb2HktV5Z8HKE5^YuHwKU~J8 zuDAC+=Q;Q1UT2(4_S+AY$Nuo2cM!U5qcr2He~iJj zTf-@ihxUL!FUd(Ihl|&ccfucOk2vi47G=Sn(hW zZ&*ieZBiDD9DR3hC^XE zUS9VHkIp6r5vS2r9f*Tl4k4vl0J_WeV5+V+5`DJgO^+Ck$k`%o(^m96uz(;DLDeRG zBzDqgh5-+Ld!|9fUISNNX<&uxJn-8}LE@DRo|wuaLw7ptBE&%w_;5t$E1~Xu$i+(F z?7`b)=tLTEqk8bHATd}8c94P6ApPBIu+}3JVtzZ!DH>@bbNVijJ(;a!Zv7kLXCRH86?LS}_$;Ol(Y^Y(|8^ms`|{!DIYSI? zu|o>QS#IAa;O%r<6lLv3{zFGZmA)h0>zq(E>m-VGs2Atr7^$o~hGD}XNO=39wPh04 z>INZrYYa*gW8hI>L$mdfIBODxmstr=o^%tZd=nveEf%_wX*eJ6z}?VJLvKzj{wOD+ z9a( zh#AtJm2>im4*D^*3!2nVf`8W0r~z|LFq zP@A+6r?sW=`|eaU&}_g{wOL4_{BV=MC>-3r645gw#C^+G^0;*-{u$gN5mX=JOlyfi zg)}~U4wFk~n#oat3?l4wlO%-N5)ajNMETPOBA=T}Y!kwG>h})tqB`|>*Cqcl`}QuQ zT@p0MnKO;Gu+w83K9(>-l^Sf9sT%uxyqL$SDG~Y2l%u^p6$-Y)q;F~$$$Wj0yySI| zw=p5)6aAiLR7aUuB!yiYTgfSdETUOeKpG`~k)Umt$-}s0!oMtw)Wo)v2(xZdruLBJ z-R&j&RT@Z>>r;}J`G*Xgct>u!f02>^Va(C}j|4l$5@#DRcoxcIzl%5&FNPA~ zY#I1!s=@fQ9Q4jvk@$5=ST=7tKKZF)$Z$2;v_T8@CrmIUgNF*%ovgJ2KBt)B#3ys~ zI({PC=NeEQ2lxs4Z?4fedT9mzjcTKy&j>wn3>Wjl2=PmGu;0N5eD=|Z^E89QS`G@q z1~@bI6Un+_1|=yo@I3&8Ub~a23wSWOxC+CQG_fwol8hCq!uzNePRS}^H_w9HYmtM) z!6kU|RR&$-VdT!!8PL#DfYp;})EgQ{z2^KVs}jbt$$v=XwL&soe3Z=F^oQhzJtfhP z8p!38`@~850h#cllk7X(My{OABC!f73a0+KJ4BpvfU2WiB$j$6hW&(b(&i{BTtAOwFT2c38FOdiOpRGL@9FHjL-W|> zvKs9BCr*rTPAK!@&s287PZ9Q`#&cd>tqU0#ng-j|pU8#_edK{lIkA}Yl^mg-Qhmd2 zvWs$f+y2dj&4(^B!|f*d7{CWNYaVp}%to}dFx)>+qnfBJjCK5qZ3~_mf7^W?>!oGA5@KQHH`292tl)IwV{shdJJy3i< z0ln!*V9ytf>~*K{qfrFXF@bR9k3>&S5ZEqBdJF}|6;PMzcE>2ifWJ%i3| z+=Qm%9#~ytfumEzAev3r-bPETpGa`SN&?w$buht04`*E$A{6xXFIPdsJgV+bk;hz7 zIi#0QgVt7IBuUM|BYT1+Gx%U(c!wNo@1|MTnJ~Wog(QY_6LGeJxS#(@PHwCs3sw6` z%HfY>pR^d1j_e^Pcu#rh>3^8(w429bZiL?IjaQgS=CmtB>*9dPO6Y7i!D}l;EPJ#L zAGNn)sL~D+nhJSSWOR1V79EIwJnY;aN>0 z!m^|BZND-+Eh4bubS!GS&p=RlHa6P$!);$M?!7;W+t(Fg&~*rd+6OSy?*ZFpdAu63 zMPP~p)K}S{uu%aUewaXSjxnBn(Z#$u^U-=@8CJd1gnH-#Y#x}4M@wa}>ZUw=YZt(4 z$O2cTKa;G#al|`pKXG#CCO_7<5hcw!61+K^#H{Zp`va^=#D`NPW#|M^4h$i@95>#2 zXHoXUl9{aOr>BhW;|Gknq5zwtoX>1sKY$7}%^0kmj_LJ#Fg()_8S{?duIogcB&TpxCJYNG-m5+#iuCj#JQ9n+`mhKD z)XKwwL}RT@0)G4#kNcaqq9ir})88fGMPNGmw%>xn^9=laosNX>IoMa`#?3Xjh#AMz zF||Ac()%d)6PgaauSs~KpMZ)#TQNa50i`Yp2$hXSgSR}a??+&Ncr?^+(DT1U6s_~l zAT2c%({1Vg-zk8!$|GnC@`LFqPlQvw&PCAyWv?ms@Ng53Z52n!X;X|1uE+UHt0>1f z6ZMk1h!|m!wsHwB(-3alMr9aQFTh?gS;(u)W(!MR23#cyXB{>EQ=>Dnz!+Z%q?S@15C(Y%074(w#&rA8UM{3lFs%p&$g zR}SN;QBKmNO`7a17!Uy+j zog}fbn;eT8B)deK3Ayryr0?a!_NEasNj9B`R`H`nbUH4%O@QyM5VDi{Vx^-LVP`fS zvJqB9>4-Sudlq1+EzMKXS@oZeBF+&l9DKO|vOWR`GKP9DYT{O%37LW`I7b8+N+vAX>}xu*1p#LcL3{H=9T$1L9w%U~#Ji66WfFqdeo8UvG)8<23L`8)00C;+_?nsOwpR z#A+ode&b<@zZu$NjquGu7VJH1tZm$m87g+j6Oe(+9T)5^K8Xi(zlYI%v3K$w{IU+B zn%ilxWm7R}W*{_{#lW#O6nf4x;b;&6JFj!ldKQD|vPJk<8jsa#>4+~$gK{eU+Kdcn z{FjBf=ku`4y$7>T<>A2XizxX~0QQ*|_tC2mYlL&b$K-)G(hdKJJUrfL(0E!p<191XAz6nh1&`Ps8I>5HbVY zVdE3X8GPaX0cEQ zb93)Gyco!UN`DistQ^PGxVwCJyW9`&N2vZ`8iZp>p^$wZfopRYpus5yYm8H{Q7{2f zi)c5-g(QgOT|oY+G+0eCg6FGj7~d_zed|Kpx#<9%_${A4qjeg_< zvOrM_%ytgUX>Gi^u8WW)3oJh^1=lwvq&Q_Uc_k%5cCML3TAW0&<5~``ZKC}j5#yL% z)C9MU88CPjhjUL9Sof*gnf24On=1v$ix;juBK7+G{5O;P- zCq6h%$JGII-imwXtcPV1)BIb3tz+cL0+Bl;LQ4o8s~2F+pdNCq)gfnHMc&nhlUn|- zR72hh&e#Ge> zOt^a(-Y=vO9dH^&(;^Y|(+65jGvVJ9hMkY&v42`LJzwWx^V~Ri_9tWew{u9HNOil{ z$#7ng1*dD7I40u>l?Rzn#^c1>WT=jug&N;H2#!S|4gkB4XFU~|kI zCoSzTWAjQ(UAzRf$8@kMjQT&8S|A~AD?}cT5o?4KgSAye=qsIdx9g#_O9l6DPR8WC z4pO70NSgjGV>zQ(X5g>_dvWC&Udj_OsvUJgZ=(=bkv@ZaAKroZ>@c{R?Zk?QMa*dN z3+C%{EmnP9D074MF@1dSoe}+ZojtQznk@V}7jJ{BDVKQzc^AA;yG5B-mX^RQ72D77 zDrnE*wdc&QeiKGk{S_0jZWd4EumKqp>LaRDce`fDBg-Yfu_t60QeG*Fsvqx(2ls+J zI@Lw04_qdh`Xi)N`Ulm!s1M`3Cs}#y4H-Kofa6{7$#xY>BKCf2XZqnrT z8)2MUtpXMEndmpT#p7AZA>gMLWG<;<1NtMDh3gOeS=A4+}O69LWlJjh;P z!PvEvm9!42`hY$PeKhecb|b0TL%kX@^#AWxM-it@EOsr#!a^86lGPZsf=6UaEEdAog0rF5XYahWJIXY$Cd!Ix~?wB?@)cBh*#{0H{^NBNMAPS7?_9KAo=yI*iY2)1WmJpRS?6gp+4O@` z*kf&?tmWbfY>AcxtG-f-x#FY2`c{auhb1-GH%p@!-ItQA!s-EL+I?|$sz(($Qn?FD zIbF2of;A~sVq%h2+3DV81KYa@IEE;1}SQ}Nhbzt#`!J17B>VIya z*kC*6-dcw`s$(9v-wyTlhw#tG3A-;UA%JS@HU(#J^WI5(YE?j4Rsi~~WAG$445iy- zu~9!5Jx5cqMUKwj2@9cdH5Oafo9`gPxgu>xs zOenj9()dF7Uns!e)FOP>Im#UiC_>Br0+hED;>zR>lxY^@adsiJzT{$~(0)vva}n7~ z^B}$JJo+toPgW1fdx2yBo3#IQc*%V2dxfSteY4H+s0^wS_VLhRfJ^K zDbzha10~-hh%{4z_zWlX+aAK=UE49fMg`w2tl@0C9b5zT1voPpGSh>`C0+F8X(Mls z5o}F_Fi!O^$C~L7*rkDQ16RoWb=A!I2N{fk@j6!0Nf+9?c0vAUHTe^%&YFGdWR#D9 zeX}T;@lugww@65_ziHP}=bvb1>Gl7ZbNzLUmy0@UD=o$L%-GC4JY3E+T`*zqakJRW zl{Xl(5k1z}@fTw!=D=u#UuTkQ8<_5{2aI~yLEK#}#)-*roC3`qd!=jPMeh0=^&xV^Jd-O3r=v{-oRRg;J)+6Yn zFXynS0qv>{_`IzVG4YQfDA|G{*=rC>t-*${ApGZ91<%}C_=jAE@u7{-{dW-qKQCfK zT?T}0DL1Ql0XE7R$hw{iy+|$mdm0bPX(_1PPH_pHg@%R0a8WrHLoRoVMYD~{`4BgPe)AB5`fXH3ZR9ww~nCgagl%p{ezFOu_iO+ z+0neI?C5v-IAOo^#a-HcaXZ^Lh0I_87k` zt3PEEGp}}#X}wM{@dHE7XQ>a@qioL=XXPMq^FHETD#G@hpFn)wRqmtcpUdgevB3k3+ia1gr82qs;617GdhnE*){Ne zSV-4VIi^o6#pg*!p(Ah^${IDuUtNRx^ezMqRKnsG{WVmDJwAS1sBtwOPOidIiAn^$ z>4No04NfRjqmkl)>_bO!qrL>nedSpDt`PHDt>7BFUJa=7EG4uV9K@S_z+-)$lHc^!&cQxQStX?lv+9nwGPB?9Vf6{`<`=2=Zr6 zKWb#uzU^ch%6>3*UksV0S+;uoe0>c2E0)>h(aCster6^bPGToMTEa%}U(1?FE3$cp zg=CSFAeT7BhFkZ9w zjjQqKw;@hTT_VSgjj=%37Pgx$Xl^^21nn_^rn)IU#_Qqur(I+#J^S{3pm>9CAyy7+ zl6Ta9_H2ael!Wr#J0!`OAW;Nun1^hBnm?$qAWOx^NWGdcgp0nAzIj2U>HaX8u}~2I zEZ>kryaHmXGC;z%j1%#i@8sRbF*196fUMWJM?NLLA#Y4`h}Y{v*1P-)<15LqCx=78 zJ!ZKwBOC7aF9GgeRuP%tCdEFwAH(KYsj>BHTC9qxCOfnG65}kKsJnBc7vphKg9%TM z*F9`^k(s}|nR&i?8T0ZKJ=a^;GAWU?1AX8u^TYB9bK~a@_Lbv)cJ9QN%=Yk8%-P+e zjP{5P`Mdo#_8Q;Cf^YxGBo$$Ht}-7h78J-+mEVSi&)pFF#Rz^H@^J5;fg5d`aq;p2 zY^rsF_IhLZn{I(FM|n<_y;%I2#pY%E(0&aQ&^kvn)lzXE~Dl@Pm7iK)MQIVamnbVpZ0V_pU2_U=&ry9!$zD)H2; z1h-7~qPU?LUE9jhpOKI0x~3RDkp+*SJWRWq08K>(uivIX)P-^gbIyWSu8vhq93q91 zQJocm$>-ECZW0WYs%Ye~f!G^OdpLZ(VImoTUElX&`ePRE7M|Gm?jRIwwxG_<7}q47 z(Ua*#@#hxYJ|m9ijdF-wLAg(|8&-5ixc2%GPwc@F#(%~iW>(fe;_>z_f|lM!|A#e1 zfOdkMoN<=+hnp=DmeBoFN}HVuf@29v@zoOWsL2}M`ls$PDZ}Tjd>@1gORQ@ zWCU+?GV^^-G9IeI%(>1vY|?&j=AZgfrn-(}KZ#1Q@(G`rIn>`Hd99TZ9WrKaU(RMv zs^4Ln8WR~2Lj{(ZHIb)6^RQ82+4#dbA-e4!5z^R6_@>z7s8=W$5`=0ICwzN#5SM-j zqb?u-nFi}IH0X~z&EeFp5Jb;W6F4o2M4f#izLZ5n+F74c2j@_xn~Bi7$>{jWVW2D( zcjn~6cX2k>tuw*nZFGJc%Y*mKBHD>;2a;We*~RtPkWoW3*gp7fQG@P;CWQWNM$E!N zoVRGi`kHzqpRR$@Nnh@;WermP)kAPaBQF0KM6GfQ{hB6h3#>uvZXYZ;Sc8pX4OsJ~ z42xIV;`GKML@vokU}+vwl})h!cNRW}<>Hh^8uid}cyEzHJ%JaXM)@D*SObLCM#CZ@ z5%<R-wEt5qjyZh)Nw}rXFFs4 zsgd;>9l^w;8)&|M6n@9f690*l*!y~vi|+QoQ2|dxT_8|4-2gSK6ZqnK7{|ppJTN_o zkJo~+_lY0ED|tvb7zD;M4*TOH@Su}oSe00){Y!;Q?>UIP)WqVw$$0wdB2*@v$AG#4 zq~B!WlwlE0k7UrfX%$}6^ZECb8YBmlfaSTNqWv=RlWI_BSOf32TeyAY3S`I$IrPWwrT}m~bl?c0*fjsvj z{JEcn`s4bzt#}>>R^{TkYBJ_1XhL{yJQ7vY(A*b;pxJuVPZEJL$2d$Lr{3QM1f3WB z;QuigAr%L(dq3rk&mDs0;S-RWV~>~1s7^f16Dv0Efy*3e*u8qm=rz&4I>DpJe|-af zEhD(+TEJebEMyj0y=MYC1z7d#KbX4`@0oJlgUqM@UNZY#7qBR+;oaM5&b#zw4qMXS z&$L(SF;mC2*kd}q%t_jz+0M>pbgCZdxqnDzEVC5YikV@I;uI6+!ZUA1O7}m#bwf*; z!xB7F{DoqixgFGh;)3>WDc;SJDNvkp0=u#$G0kp1o2VzxuI9NilWb4%uExDbJ!lUdaZD4IEwIzn&O?i zEnX?F!c?hUC*ESR1O`7OvUV-z;i?Dpyfmk}pgU?4Duii^w`ae%%mMDN$ zlj%@f{Elqh>PGro7d}AC=H!xx`FPKw`2ic*)?X0iT4JK&KHbzu2gDGz6V~Tn! z;s0HZGfG&*O%(o$w}ES6{qa1$F1&}W)de(@Ot9w0ccwm}_1L z;o~dN?RA#e2RmVcML0|neK9Xi4~qNUaiu*7UT&x1*l&Q(zi1B*%}u2E1mQ_7Ad5&S z#w26z^k{VI=;Gq%csyN~jRC<;L(hMU)K;F9KvbcZo*cBCVX4j ziXHBcP+ZpzuZG(=^tKVIZ9X_6dJRj~Q(eEL9L=TF2Q;M=o9fF@*p`neW#-6wUx+Z@ z%Mi3U4+nounkz{`$V8gwJR48Dwsf$iA&PpglkvnW65*8h{+Ftm%@Az`EkI{{yO?FffgiZi77&yvCdLsWYf!fo>>BX0de{BT zRT}rN_*Z~Ajdu|eavq-d*W$#NFUSdB#Q6!ybFb%AL3p+;%BKvk9q-1_zj?jHPxLN@>oWr{-$6$Bh0#?&} z+>syWa5`@*#bKdnT9||-F%j_evBd>N>Vxb`hP!I$n7s_udnI=3M~x!KPyT{yEk z3R_dVF_<=pWnqu#tT2YBqWw_YaF2TWdchiob3Qe_m~3+ob58cdylM)aDE1ZCFbu-FI#6bUCGJNYIacQb8Qfng+dNq-Hv_etbxD79* z<1u4qK6|5$u(F2gRJZ8+ncjyJGk;+1;eE($jVA@B;#ko2facBlxI<$9q4nBT zOdJnDUa1uK=dS}dvT_$U{8O9zMenb%uJ&f)2UD5DYw7({!oTz$8T&B#Yh>8R%F#@{ zumWrG_a1ZZQ6%%e(Sscp6=KB%9f_p#5^jnSIQhmo+<|LRIJ*+efI}t`ZVSSj)E4xs zJV)@XJ&4~p9gEF@z{x3C|HBv5F^Kz>;jsBdXWl7^koKlO_gF9j4L6~4ZYXkTK7O4< zD5O)YA)6kBp}16dHpD=VXO8dH@zBrB#1Xp$`Z*Y&sv!-oriEB>FB3tkfUo>TNC#En zG~MSNLdNubD8|~@8pORXfph;_JUH0|IkpS8v+E#fPO)BX3&j8(7%=a^o6cuQ-E|YJ zRSQluHUHmZ!8PVLBf_}_e?@M>^Vc&R+0=n^|2iNa+CuXKKDd3V4o_BfQJ?KKyw0PZ z;Zr5hI$DF?cf}BJSc%F15Ef5kD5D~xSbl0 zql)H`EQrC_Xezj0M`P<8S5 zlexcy*%4`Z_SVN7M%SU4Sv?}brjMLt;)BK65v@h+gf&G>(yPC`B{QC|5y@d}&t`8% z{&O=K{dEs zAyY!bh?R)|#=UshBQqP?3bWACA_`${lpMaIiD+LFoORa4!jd1PLV;?}QZCq&WP@Ju z&m_#r0T1L4BIE8>n3=vNX*WDDbleXSmY%4yYA2KI_oF^O2%9>NBDgu9wC8zZ%B|Dr z-M9-jS~+CVUmyHC5Jh|Pd~h;&h%Bal-mz_wXnhz7FWRAS$2t;|GvXmbxj^UAVD9VI zTc4&wnU`#pRt?=Qtp1X>i5}LnlsK=OF zKm69VW8KkCtO~8gDM24J>$jj`W)Jc{HloL3JyicI#i6CaW27mzmt6 z&c53JjIlmTn7fDP!aHaj=QkgR^VA3AQQLeXHevxb)Ekasn{cB?3ev7iabb-bEb>jD znXnr=bl!Qliq0f&oX~9U4P(l2S2Y}<9)#VfpnFx9W*S-&+#yuF7a~87L!$c_mQbCh z|8F?f{td*nt5*1-6osnY$rPK?{KOItdZh^vG0lPKND?kIbEr6$4c4v<0_Srf?n$2o&?;$y=F=Kv^t!;i>o(jB2l4f62S&C;!neN@M=AfYX4N3} zm5x9!k3P@ecEf09A0$5baN!PpAd7n-E8Pi^uo0Y>8N`C`y%^cv3E@qVSmW7&w?l&n zr)y`|%3auFRD(~4n-KVm@r+&G4X zo)+-`J&!nUctHI2AICGQ<@%Uh0uA*p7gQNyv80LXJiP4wjmt z&Ltg|?U$(Dm4PkKXrIxhJhXhN!knj<@Wljpomq^I*cL?Q)FADhDMn^iK~la28~;{A zC7t%36|}?h{Ud~?bl?H?*h|cBM~K`YPDFPhc61cgKRe*B(u=CHdpKP0&6zyCi#ZX! zxUj4fC1&q&Z@3GAn+Ku3ydBw{^y?El;N;71eIdkV*Lf zBk>lTdr^$oW&+*bOL%dl8e=!}(4n*v(pS?VxZ@J~lhYAnXNI~f2^3Fcp<*ZrWrugb z@V^**u}X$bY8-w#Mx*sr6!K!|dzKOg@2}agdY*utIpO%q7XzieSFubu3JYeO!ol}G zaQ4c@`+>RVTq+mY9M*;Y;A`ovrC*pfi4-C1B(4CwMA5fX}KQD?`tr z1l49zjpa18<~nL;Bz#t~T7k_k4H!-QES(M?AG|x$iCYVw;QNdo zgi(HKn#?W8Pwa!X{%uGW(R{$?8%RHX7blMzCdM=np6=dvLBXJI-y@zA#(5|NfK9euxxZ*6DC+sKs2cD9g(YfeZ7lm6RHu$oJ z;_u~asaGutZ}+*u(PBTg+zZ6#kTiH*I0ofPZ)iCBLr^dbzq7-!T!3Qvp;$aib%O3t z0_-d^kt~;pwtXw`n(mLabd5RSJRU3Q)AKD4QY}>&^DMx-^}yBsLJYodL{M!d!bi>V zLAL^v)Ed$D%Bc_HA!0bNV=Q5pe?xP(YnvqyV*CgxFkK0@;RtxRG@xUwCr_zYy zDix5SI92LwB^DVp(*0ZrnP8xMeF4oNRv}U`kLGyvQKWnxFW(ekW@kFS>ltCkSL!vZ z%|ynh1k~Jf#=(YIgw2V^^9$ivsvd?HrPadOaV!%ypPpBLX|pL@c&c62#0! z!l#K`=0P2(g=(|MwPo0(g_GI2uih|e){C)6^(UVA$>C^05}8K3T10OMVyCSFuC%=; ze`5;CCX+sL(>;{9yiF&v`IE8yz$tt`%0eZ9dS{oaVe*qRIMri@6kk*P9ovEVKSOXP z!3J{eYawUiM6-(Cc=OO3U!R^qi0&EWORdAaB@x)YBMGRD#f@?uG`Xi@`RhEK`J0ZL z$Cksrk?Q3$D&bUr31=qj<2PLwU#R9TQd$YkPzM+&Mt{;$N_?OXBuaV5k<8{p(x0AK1K zy1k?TIsBFAQ_6+KDsAk{PKT{xKEzI@(*AyJoV*u{BcOS}-4STXTL+iPXK;aP9TlsN zVGiX(j`r__|I)R{skFxL*CE*WV<%3Am|`H)9P{|8=24*zM;Ar7O(l?fbPAX=8NE&E zWKwGgVd(uF+15p5hr=j|mQlcuO@iq8n?{=HeSj*|56ATV!0?L2xLW?3DNh<=Xbml| z_%)qM*GQt?YXnk9l%X%zO|DPogZ)u!r1C9ClcWM>izuVQ*$!HNRq6gw#1{i&H13PT z^ALAP&bOzSb{C=@GO?`C2V!Tu@pwld%-CEk==Y`lS$??X6iTxt6z`0OV^ruYq^`t* zRPM&lltd`eJfzO?H26`U&&aYI{Gyusik=+EEoGoFs}Q>$*CAxAl-^Oh60PAy_$%H3 z%a2v~xx@>5yC~-RSB;G_#kjIB3g6!~VXjp#UKce}j*E6BZoP`IkUm(c-9u6H9jcM{ zB6-m;@}_>k^FS-EP51-GhhAgRxn3AVGluA)G^jnvg3;qdT(5J7bb2f%q@Be{%6;zaO+ai>D9TUyBAa?VT?;PKzPUg& z6&%M$e;?%j%fx}&UFaXMhmy2AeI~}?gP0MN)+^C9tBSr`cJRwkhDo0sl2SFXpkWmOZosWpn5&XVF^RhB?crAZSdC~n5Jk72t?ERPdyibcQpiocqJY&Mx(&I#X zOYdUk^)M8^jerwh1B$m!;FO$3aU=IW4jWOPsgsYpvh)!GFF(fD*&p%#m;^VwM2H)d z5#gL4N^+Z#7U) zE5(Ylo9V`_E2O_?xNu6EySd)aN4V3Ij&tH)e7J4iKAdn-Ft;}&m{YY4=lJVGIMD__ zZm*p$7a!%zUHRh6jV^ZP-Us?|WA}Wy2+^b5TRC%X=_q}jyH0Z|7rnSY)jTdd%Z>Bv z+RL3u4qTXpBX|9d9p_(Tz`b`}&E+Xsa4T-CSjDrDH;jelVs{)_pNQ~l zS+H7{hDF}SD3_)=x%rpjPV;gSZ}nhywh%{a>fm;{6n)oM;YDx}9+L)$_f}!{&;8iH zpbDF*C&xLv2zSmzA!<((z8Lo6&(3Cy(T-FhldI6O?}PixyV(Bx4z8@GTyD}Zg!MmA zjm3)ldFlfOHVva}Z!h96-NCTqJ?QE6VaDmJ2>*Q%2Mn8$uGfqCwM~$0i-P61A_&{m zpgg7$7wGIgtW-_=_#2@7tOx_uE0O=C6hCg);f8S`WJMUlk?aApoiq?QLv@_rg!UNVJGAIH;qlxI=7fX9EwCSB0sW21C$MUgs zyya=5S}hmDMwM_hET*%_L{=sl^GTcoI&DIHOOpGLI(Av`d&VTiZ&t2 zU8NC)C1Ip`Vn6w}xR6wuzb8ZX3P@0&Of%nUB!NGXY$I~$Y5#!-8H-W5;wn>bD#*IZ zwJ;W?0_>%HZESisk^67PT%7l;APb|Kh(?$QK1f<4zH2_VD^vear#STW?vtq0XQX`D zTsRHVJLotY9H2b=<_&9+N;?ex80^MMjs5To3q)Q^8vgLnPOC_IFXcaf+;s`X_*07W zjpDH)I~MQco$!ltISZ;Y@rPz*pG6p9hk5(ychYtsjCfZ{WpD+MjW{9&ge+uyK7S z%|HyGKeZ7_UG11u){Bh4ZrpoqdX`VPgR)uai=utcc7aBavieBveY;+Czz=jnI%tW=f@G6f!d+B%2R@ z_xJZ-e^ie;=e%Fzy07bbnaWpU<+}U$v-m3R8ZC$ErqdXza}l@Qi*S~?9v)&bcJQpP z7i&&jU5fgYJP7>v);VQkkG(kpW74qlzdUpum4*e8obB*VM6-T6tl0O{n7I?ZvCoQ+;n0&!U_5R(@tvM;j%TUKPE)i?`>T+X4IwbVbVMOdkM76Gyspu*46vam}K z7FNNm<|cak`k#|x5>jr+X_WN#40}Q)9hRdQlr1AY( zCAtP>i7l9ra|^5NdES1!23JluqlR^YP5s^BIP?)L3p#PW<0;=m;$Xt}y`tbZST*uJ zd#k@A@J$nJ``*Hm;u_p<+a?`%vA6S*%MpunD{`fmX+H4ns#@@$msVC_6~GrYLpEnhw#CUb)D;*cMH zM0#Om*$)bOs|fXuNm#94fs?G?1br`q+=nolVN@)b8YT%V1`MW%$UIbEpCGMzSc!xm zGoXG`4s*VUcuwKCC=J8oo4mf~CnK4A2Tn}eihj%$oBl78GsC_xvfTwa)|--Nmt(13 z6rR5gLs7p(=pH$W8=RwgyE7DiX_4@*i9|?1EIMx_p?-5TUg^8w3V&U+F#}SU1T2{` z4ObWNJvciTPyQalP#zqXAJ4|i2PKeUT~Bp7&|~Te^x=Egi~Xgjah`^Fj}j!dU&fAU zrI@3#3{#zJaPRRW?8>V}FQ-U&`rbslZ8OTc)WdS#dvtkO3-2wr(RKJ8ye#pSdMn<+ zz@4|zqF0ACi{E3_;#*Ms)P!D(Zt{#O0*O_%SbgFVu6L`!ugS~ULr{wMUoInXQ3=+B zO~Yi%Qfzd%g2`nkuyj7)&AFZo)3dl&myK4=ns1$#hOs|#acdCk7TcM@;g*2?nHfBr zOG4{y7mV!|4fj(?csQKBJ@+CpBQO$HZK3#E6pBq}kK$ZQ0_O31uz5oi3cp;y+HriQ znYbI9vU&eMl7;0nt^scpSiRo@k}%V=XK0sWRu7}bH~`5oP@JYQmAx4 zijeFxSjl;dzgdJCJ@e7;Kn3E4pXE-D8LU?nBYEj1$lPRK-9uMwI9QEEJ(?j~Qj6Cc zA}}31mzmOt`_p)BaD0Q_|FJ%=-GK9t?qHDu-w|ByVnL4vIMvp|ec2mSwKO4gbR+Ae zb!e}TfO%>yK96X|OdX!_D6GN4&8+G2nW7`M1eG6W;7Qh5RNbt=Z{_3AydZ&d8P5fG zoPnatQ5bhMN6Wb+B=tFl#~%_f%t?w+MfP$iCS$=s_Ny4og!{`tJb4|4=So57ImjEX zF+SKAxB*K;*5Y8W)FN#8wF3|5+u_oWfwZ~0nXZX^Pb~b5eC+|S?D(R&R>UwM~RdhHAb?%ue+EYJQit-3~^18L20LFi_&eq#98usf^B&M-M?>&{b4+t znm-3m+|8h4V~O;z?a;bofhB{dp`&aQmfQ}3`Jd&ucX2g7m280Yl`G=r?L+>kaKxS4 ziyoV%Bjs5b>fR?|Ilsz=?sW7hmNMckHTV|!pJ zq}vNnQ)_`(+mnzOmttweDfq=ZK;EtbX}$Tqy>|t^Jim`>IEQ(I>af1^CT82-hvnh( z*g5I~TC2*jD&AMB7*>ug<>zsE{do)wxepc7oA91i$NsT%_-*HlaTQk(cD)`ZJRfX4 z=Kx>FQ&>Ky6y0i0B6qq4_a+v=jrD8eF4=GpL^O2Iz~I;uI3ALQx>PAXA4x#|uyp7p z#NonSUVFVGaPV3Zy71?Jo7xNn=d#|Z5rN+wJkybL!ps&0JdXHTA-Vg#?&TjnW zxv1tK3(VKv0fRA?c;X{qo5CDCdl3hXkEW2+CD7Yre!{T=E%DcbAm(HrrKIx)&#7o{a~;{L54B)>i^f&GqdSe$T(g8r?e>-+QQ zpPDks1%^@ZDtYc2m%y#E5Bd$zL~4r)Vk_F{^yl%+44I0j?Iu|M=O>Nj?#sZO)zF{o zfULLqB+cyg%jTJT3Ft)gnb(y$)v&oD@;~H zE!!4l)<5apbyLLD+2FW~F>Z`$r}JN!Rdc=@#1C)l(2hxwr}M1Ez{OQXR8d4jSTiONsb=`%A%N-C_#T9w|-X zmu*{HU+F>V0rZLeCoFn(n6$qIAyVNgCVY8{#iJ**UrF%zUotkl^Misz0OHfb za7{iAzxX`+XKDf>j_*gY`yxzQ62jb-IMA37xRuOB;}Pze_DKOfi9*EzTb@hAW0Gq+ z4Bo~=^}k7Yydf2T#^l5ALOLX@W4g)ntVk?@W6uH<4z$8*i<7A5z5b(G0hT_VjXzf^ z;AeUZy`9Q1jx(_b+pZveUo9Svtc6beQ|5G3;@E$eG2`21OwQde{TFo^3;6u-bVMcc zu0DkupA%-@uf^}GD{x-vi>FTKnYVKb-;Y+nfAVZJv8UI=xfCl#oW%DFR!Ef;fIK<> zrId-8g9$F~8OUNS@=|dswwv?1Fe(B5oC)6L9*<}fTTI^-1#7<)9NZp>N5OLtI5h-6 z^W!mib_iB`EIx4vc$p1X9Nw$ef77 z^C!#DFee0##fj*z5CIeA`PjqvuuuDw@uNHn_DQx-?u^IZKz^-^xVt2N5@&D^AvEMT zeAtiw@ebelGP4nL<{Vxm6vDgPJZ%3}h=tx4c(38})r(c^zpqB$ggRKvxrKVwNOTLT z$DV{{l>T@G+00H@``^dCzl|tsYk>2tFzH(L+fY)zi)xkoIDD)VO3{xny1WTJThGS#Rlc)y`&+|PDWaF=MPT5F2|UU z0r(cBE$#n&sdU1WHB!IEiBjj6^5TZqw}g%9^98>V=HdjM1fkNcRq$%l6;oQz2m?3l z5Vz*+7HcQJ652JHxmRu?+TCat<}BtJ_0WFO5y3{%f1|%(@H~5-jUUE5J{zm;&SUNa z!QUfCnDJ+p@Ud+;0`rxnr$={Sg=s#t3M>(67)|CeE1?@0hSA=B=#^!PLz(MvGCK&N z%lDvJ$pW7zu)pt5BqGEhX7fsMXio$>rX(Z&TO`)knjpU$^DdN+qP&XzoEVS$7Kiby zt^iMJ^Wjm%Cc%Zd(7$*FFZBwT$>D&z-3#$2tr`acE6|%gz>&2V;j-o?woI-;ukq~D zop2T1UR{DmSmzrGc?tm=pLBiYZpsT#Mu z3t^u*7f)G3>fd+T91mbt zOfqt1BCw^1|6H>m+&s>AqAq^OTyBZ;1$!_kF9@>@+>v2u2DRyaNWU3|0g0=S{Ue6% zkGI4DAs^}UJFxVVvUELn!`c7KFdMTgONd@Y%zo$0up1K~`W=C~raeN{zd$v@NP416 zKk4lk#rScvtEii?N?4y?$IrB$P@@+sYQJ_D2WPwy+)Mrl<6p{(!K${x8s;$wAMylI zC0iJzW-UA@VO-;{e4ZEb*|=>L??ZZ$La!dUT;+qjS=u;Trz{R{zbI^~R1{a6?vr#F z%1Or;KgD*5A5=SK$Z-E2Vd0bkv}Dr<8nEgSt?dWO)5xZO>R%{z^DCP1!;ShJY@;%7 zHQc_z`4u8*W; zzip6YJ`XM%tYK5@O5fZZFj8|h3T^B$COnhMk2|oRat-o)mZ7m=9j>o-f&GYG=#?3X zody0<3Xg>K_g&DO>VoCJn5}zf8T^`E;q%A=WnsstWQ9F0pI(I<+gVF__ed6 zi#>&Q#VLZ9k)N9-fbcv$mf@I&H?xM%dRv|!{^PNX)gr}tsh25tWMeTcjg0@?VAkHrkn-$w2 zGyXcFx^8CQGvv=n3aqsudY zyk+L>N~Qf+c{&zdPqRPup((05qTo3_mAQrS=z4q{e5NJyj3N`~#~sFX{Yh|s$ouQ= zEnX+_;ywGa8rh$_!WvG$)9}*vIEMJL z_vV&4Dx*1zXO@M!;mK$)9gn)3ahOq&il$-Fc&}pyr*+KbGLFSPuOJL6CoEj#53lz9 zDAm~lZRRp>G~J3DAAL~0&JiBvfWxEpQ1Mxh2KLBx4mKB;?}?%zP21sd@;YwGxA9&2 zg4k--AY|kziC-sX3CpGz2m{W=3g?D53lqM+Gb?!dUHBAJCV92XTH?0DTnsL`X!dd9 zX3pI86K#LKH8Y=8VRo~7k}z`lZL`GvA%fZEi^9<5DMIPzAmPTx3&L&3i-N6(y%3Wi z6|Px@3i**bVzYXi@O_uRkbR*;@N#YuGTk%9=N=yfweSYPyZDuG`GuO~uMS2>q3u4Ex_yvA$T}B2o9Pn(N@0&RR+QES`&b&kLDtJK>#|&gd>U9Lf6hI z_#7XBf$51*KXm{ay(i(r**G{Uq+|4rWM-2AlKK>U6|(S@eSGN&HvF1qBeOUMjcblW zqjDY&@%rDj#|5lbFNK+_FZM*1Vliupt9YK@E4`UJD=%VO%6WVkU5Ya)e$oTMrC7p! z83((tA7fK9mdv^WP0vfnomz@6!}yNpP>R-y^YHOJgDh9>YbZPp$4A-7pOB5L5F6zE z%0QG&7HbhH7<*WXkva#FYn6@>kK+(`iJ4yC4`5?%B9?O=PlGwJcG^J*yAg&nE_*qf z>i|Wa0PIr>M!$+JP>Ngu1;ZdbiwJ?f+yeAF=0f?)WjHgm0=5oEAu%|Ez;H!L?>UXa zg5kbGm0_jecd$_G9@Zo@Ms)}e<39UVl1i@$DlTbIrj_FP{jF{ZE*s8W+h;mRW^b`+5hx(D&z`skhh`` z+2tn?WViu&O(*g74C_CeO7QV4>#m&X7*=#h%>jTccs| zY&2?q#=y=k2`!Vuv3>0*EaY|Uq(&6_?G8k6xG@5@>|o~LUZhrf;^!$54mKO0y>1I! z<}ZQu8}8RUx)O=f&2Xq%f(04sIQvx}2FtAAHDE2KJ#fbM$N-6aOSmw=NLGa2KdSC* zMBdjbBr46JBF@L{@Ch~(eS3*LpZrgs`5?GH7KCYL0fPRS8X-!OBHWr1ETr!77tT$I z5)O9L6F=BQnC-oNMtJ6{CUzMYBy@cKBRsvKFYZzvFV4?U7SsOh72|_Pim`o9kfw@) zv}xm9sdu_4t(+4KBUc^q`O-WgarjlSk9ds6CBC7hW_M{&dlS8T@R>G$`_3JEUr6!q zeQH~HoFphEN*XqK6T%{w;_^LPh@R^(K*1J+pHIi) z6P9oZ$)RH}CNsBp9&GRO&y9(sRNG0=eQd*g2UDEf?M9>fn6MvK${eX-uroKN{fPrH z^y6sE%kG7DftsW~Koy3@gAhDj8It6wG|-tl{j`;E-Si8YX$Dg#=kF&8@_3g2hJG#1 zC!L)8bZycX3Y_&!?@(hb zW(Ef$=i+j=wfw9sfqwWn=)G{mzrA~~|BMIhq(*SRw+kC}nVBc-!+5)4SX>dpe!K)! z=pR7pxZ$vMk41}k1WI#au;sJ~o_9;Z#hMIkQ9H~U`wUq0KF(Z+A{0+4fa}ohSQ}b^ zId@9Xvbq%EFK*%OhawmqDaHn#yI=6}m%3Y=#RG*B+>bBDt|Paw{7NZ~j692vLr-DL z$L%=CKW}-D^H++;5ipZ;m->eh6mS%;>JFm*-9#MP7Xt^sBWO7kiwDD*i*YFuKaVBA z*n)jZ#Y3??D-id2Piy(dK4()S=uhOHEn{E2|GWnCoyK8De9$)w(Y%4k{Ol^x5v{G&g zNfJ5E>30aW-c!ZGB74!dga)#TyN}n zxe42%CFrNP8)Mkl*?fE#giK@18Ww~Yk7$H9g}{2IG51zQLCgFgYpU^>mu8H%fE0Y% zd0nr;SK+m88TfJM5!{3)ZvjQ&RrI;8Qgtz@mP;&bm)=oc# z;i?Pq7+I(soP`fF(qL^WMRVE_IF31lRTe3*@HR&KuXs3jKZxXIQ4slVBAXL}U0b4& zzAgw4TZ}O&!UsE_2IA(T-Pkigf>o^f?Wx&;o_?HjFqjF~BQ|(6Q_4EB6sEuCL#2y3 z>?%$0-xo`qa21hd?oX>+28r62^~JMoWTC>&ZL zluS<+J_j}moARaNqk>LhtM3n?byRn8!Gcnu;7E6~sX_I^q`S)Evvo$|@jL2b%d&UE znz;i+<>RA7eS0M_^!`slt&fS=(oreYhozY%CKc19ZLa*Ba7NbINQ@K$g_Alzh0CR< zY1sc}3A$>SJLV;={jVGTi<=A~d?FNsOz}iVz_iU~P@m`s+5eW~hUFZD=np|-x(lL9 zy^!&J1+EP+M04#nWOeRC#qV8c7~LOF{F%`(guUrip-6GngH{6f)m=!zjmdGmh75)8 z$7r~oWxoe!T>~8?I2wHzmHOEjJ}ncMN;w}Y&BhG&_QoAJfx(G2nAi0p(j{^Q34YM~Qwkvxu_&h1U>@a4&l)z^hd*Inm zx;8i(F1#*jWyC_B-*=a|{aEIyhchQaA;a3;?b8A1@~=Nsy6wicp+V4L&hCF}+0%My z1#E)1GWTmaE}k6>13i0Ya4pC5-H!0p70`T4z{jP`aanAFq`Ax|zTS=ZznA2(R~=f7 zqw#$EX)2ZW7o~?T3jR6}KYw7~ZOj_hPD{vhLND=jXq~Wf_ZOi+;fc_DwV~KBK~?nm zZ7*I?_7XH!_7P9SX^FkQd=^X!eT4ktcH#R)HL-8@8bNK%9r51{U5Z>X5DyoW;_{`;XjCeu9{Cc{oa*DQ{dPAR#zf=6jddiJiPXjIad{Dsspj$rBidZ`; zTG$m!hUwz`OL^?E(59v+B{VRL?U`~fXl=MEY0A;Vq}Suo(lQKHvg2v-!_jy(NQ!G+ z$D?TY0%~J!%Sz5b7JCwAJJryC%S6muPMB#0Tuk;wjx}(Q^D!ekraFm*`TT z$F-vUH!qVp&N5=&%_qXYR};l|H%5xP9o`Entb2GL;OA>qEw1~iOB?5Qm#$n^hq6sQ zaWTtW((lPUakgb2UQcXAV+DOm%%~$Pi|#1AFa&??1bEFIg=Xio%%eeyf2zOG_YtzC@ji2OZgxYe)k4qZEFmBSCg@0L_GU|`2CrC5Zn77 zMc0;e+~4T|wOJX^&C6mxV=lIfb=V)x9_k0VXlGwt!8Ok13_OMY{qwljG7n>2>d@$H9XR;?AiUlujFno5wh(smht63hrUM zr4IR(oU3mQM@-5d*ger^KiW3*|LxCSFL%5=sgFV}XP#B8!;9|B)!eVgY<=zyXrG6o zW7c@Y=N;+hi8yF&fzCU2Sf}fRfv3LE#L!669aus?8%N=-jDSOb24m!o?s&vq7Oz|T z(`wybVopM?@HuL!cq79}vP^?nd`@-PwyC>xMV`9!)z(@N=Y0NlEXSg;{|T~dPYQc% zW{AZm1%hmog7|Bjn&{g-jyZ^FFuipNjh zNqWzn8S$-l?1M1F5*ZUL$N|iAnWx+L0nKbJrUzeSvDj!C^Q3IBVvH5vv8BwUH^J}4 zGm-Rf8JfBGZ0DEJ&{@110VBNOF=idECXIsiuw8K9xgWAx{?PCr1_}*=QfCM{lp}ET z_Fz2f76r%S2ho$~L$-^?aGr-BSe}vQ?o7plqm!{K^(aD3Pawr09|;Nm_$4`kzr|i)?n4<3X53Cz>w8Os^#C~RXttIMTuW;}%;b#)Nd zFng;b%pZ4k_hZ}roqWF-i9OA0Va0R$Prp}VcFQPy^AR8ayEMcJHwh}dY$0@E4bD^_I#*}o zmZl4IKF-GuQ(fFY;EIiNc43g42SU^|(C)U5&t^f`mAMDjhRQJh9so0!DEzey!Q{ie z@c2a-3h&1u^fvFYNk)j)V~_i|Luh`P3SaH@Xj+qonX`Bv(3XuN-3D~@&W5#B7IsSV zVOGCM8mp9#2Rf|FF*|FcYXel0vSI&)xyeCkNK)qxjI>nDyOoNqAL5bcW`q?A(Ky5N zm20WtaO>9#sqXAi_;din3Igy6N^t&f4;E|;!hCmkteB~RziJ*>qrQuoE32{Ny$)xQ z+3UQ?1^-%S!Q6#=(RoiltZIj4nyk%#GeG($X2Q3Q!_3no;JSY@K8C1bTGKnq$&fK;c`{xsk5Ll; z+I|rFV~j|Oal+f(XUN@8A2iz*{q-i`ymA{AdDclPJl!N8l-AMizTAU#qziggw$hNf z%!FMigQa`|=<3u!o^PAz%*so&!0HW6%I|`qW8P4kUK)3Fcfrgd>Ns{<7IBM1xx-Br z=LQ;LbE!H!TBlQqZZAlG55?8)eehv!UkX`n09gxTjBXl=l!w|BB*R^F!IN+<^sG{f6H}!)GGYoGPKKj|`S~h}6LIv38TP!ANE@upkQg%& zO5Wy}H!%XqW$d$NzmUH)O+e+ry6&MEb4`hCvzybL=F0nxi811C-QEE(3i2YD7@Gc%j~-jcLrJhs+ML&bAFydQanwjBr%?worm{FqxWXt#C>e@uppPLoHA_ne!A z>-~#_w%`+jd!Vb}cIvF)qNpWqF8e2BeCjIx)Gij3=N%D-A8ip9KKvtWc&H%Q->hU>x2&Nq#gmGeD^q(*iAzei@ubasJiC*YB-WmqnBN`_+jWg%!2>r1T`@*&& zXyIb~m!gKS_v@j?-i@-PZTMp#k1gkS!HxZ_Y2){yKlg~=`xgYwg9ngV##}DPez3ZJ z0Ogli570`$+;i;Vwn{)MXI@9P976V#Dmcm?#+oySu!wU%>#w;~~~Rzco*3Z0je|w5=tR?qw8QyjbG;bFO4`#By=mJ_GTaubgP{r%AY(moE69ekWvH z|0E=6eigjOe-Zlr>?v-`aT7ay-wMOK-VhqjDT_Kj$AvD9u9E44f=T239=Z{{o3hv2 z(+2JP)HnSo-S8@0LV;=yu06|x!zj+U{LDq< z9%hyLrQ=)3X6S86M}OyZyw6ES&70Bi&5VWB`9!ST6oq7$LD=#*jP>Mb&gQY-?m`~~ z816;4HT$9RZx@1#xeGgRD z^Dj{pH|Gjz_umVSbeIPO@0_sIUxO+hhGJ_&OO3e3J8;|=+PHTUo;O0vntn)D z)kkifDzZG45Oh};L-tDG)cKE8eQPKqx}Hu7n((^*i4G0m4xKNBq_O)mN$QH}e*8Uh zP3xeC<~?!U(Ty~Rw@3`ud=P>()kRs$SHfnQ)51fCDgnX6M5C<@!h92X@!iQALQeZJ z$#{do)Njmo8rw}7`yF4C#@}0{{4k5k%InBcF@%O^X44152i(J~gEbSb&@b+5JDQqD zD-?gx4$A^sJwBPD8je!V-)fS-Ur!G`9@2whcS$Gr0x55MO0#NzlS!}lROXyV)wchU z_IzbDeEv&61nzN^?}>fo{jgTlfHXLmRyFj&!nLE&+gu;^t7gz3XG0v!H%6xX2s|j6 zNyB%JXV1M2qIwY`lsxG5E1>u5Y52a>7Tdpf(4o9Z7;0vYL7LMr!{z|0?^wcXi3v0o zTEQq%knYR3LQgeQSUFi?lHvjAsZ4{#b1BsM3~?i@gGvtBLiXr1Xc?KKBF>${SU(EZ zut7%qcue)0Nuj@ncutcKJYLZ&%HuxY~Ij_IPOi6zd-lu(&;Dr6pO#~!p#?7 zC@S;~>7D#W3fq-oqppJ1g_@{t)P_NbA{1(MaN1%h4t*GiA!FomP8`iSRw;%C4M*R% z|7h+kOZ=QTAH^f5qH0}NjI^DPh`6=zPI1Q44ezPv#Ffmk;`Ov_0|qL7rt}bRq)CJD z^fJ%?&&eTYRS+t|qR=fe3aZRQTv{4|BSw+(y{we zapZS2y3FNgJ>xo@Hzr`EVI)>1L_lFU^L36!;o-dl$aQA6`(SxyX#3;dcxEdFdBZHN zlLnZ1;OSQ%?BLAV%ysX`Vc0TEXqWZZEq~^nspw8#Y z%Y0Xl<6h)01L3JY3{B5Z+ zwCBTN3Q{^xUgP_~)8#b{R=q}kgVV_P#x+{;&XGC~Igo4@N4mB>i#F$mNS;HT0ku!wd9x56J2KJlUH*pi94$3d#o0ll4N=Q_>E>I$Z$@K zGhAQPaJR2MZanXUm%X*I_xeDXeI3I+CS#B=P8E+5MRfiZdEGWge3=p!pR~oLTqjgt zc7WHBo~TP+4toP{*gCC8;}Uu7o#F-c?fX#rY!9sDxSL{Q05f31a7QZ`|GKEdpfD1H zl9^vuoq`{KR%5155^{d=SuZ66dmS1vcUC&4)g@u@=?on9ah4X9WuWs>5|+$Mhi)1B z_#b6pu|_hwB`2YDwks^Za4$nmGQvXm_g?9Vtl0e+lO2Yu>H&!LRD{w{=5Gz!hvGt> zb#%(3{>3`9&)kWlEz6OAlh5{ij``^Bgg(3N(08^Hyxp0(n7rUjA$Fy8(u~C5)M}jH!bI{ty8Yio|aQ4U^YLi!E zQj;Tc_1{pE=z{9(-Kel$i|_tlX{FLeOkTYg=Y6-}*UYzMp~P8#nFus9Z@;$KUDgl8qpgDxqNRn~k&4{GsDZt^ zR5*tuhg11IP=2a|u3c-P?q6P0&;OLs-M^f4)e=Z?QZ7B%rhrfFXGsi5qrJNi(1+^{ zG?oQyug&Wy>FP$ZIC_Sb&Uqp!3;!eeZ7f5ZrtFh+wr2^M(c0qBlKH~rqHDsCDTBq` z32NdEpFuS4#Uv_J98Ir_w5hYKklM9YP@ab)Z7ZKfzb>64=h4nTvDZf6#^7(=qq$bZlDUf*yt$ zxcO-%u0`9Unpt3{&rg!}_`Mc;TJ2#f!wl178R+8Uf>9r)Bd_NSWSRV=%nx%Q1kHn8 zp#_vP*V2g1QnblB;DL|GjME@muz4&#jFmvsa0oL&){^WNJ$#!wnty*^^f{YKGmCrS zmahRG6l>w+-(1S-E{|WER3RNMi?gruC|bLN&aVAM+gzX1>aYs>cA=6M3)kpAmC&xm z7f3rVkq+Jpr~23g+FejZGv<1dV{cP}!9sd8;TEl!zL5g8K|4oT)7iu4DE{_z^1VEt z5)@X^MCC%#pRGrVl9BY{+9Zm&G>8IjlW6j+m-u~dw(z=bzEJa!`wui4glVdFtYcQEgAGukHi65i029u3Ks$J!{!&I;G}EaJXVJ9NBhpw8Xg8g*(DmijD)6ZZ)& z{pNwZbpa^e=Y_ReugPtkFPa;}v2|z&!Yxdpe|InRn*;I2Fd8#juQE3`1{uBfVHRhh z1@oCwhmdGy&Fn+KimwJU8#5e3_imWr2W*DFp)+2K5pTB%HnMY|+VwV746wt|dy8OiWrc&Wd?pHGeg2OP)Hayl!pnDb zanL}F<^9fr`xFlU`9S&?l~D9bhj}lo-wo@I%exd|-cuGvBY)F56FF$NJ*P3H4di$9 z0a;w?hAtONschRp3YdR_hHveGAnkIx!*g_V&cG0bTUusp}M;^{sNZnSQ zHr!UHU7O6PQPGZaYSINO4?%oXttSr8Z4^{C+!K;UUl-24v7nLH45|FuCyBw8my(5( zO3CTtda_D!A&2Si^w;tMnavENH**tcX6zZN?=O$AlUXFQE|EeHo~0|iUt4NFqw1C> zQmAaBp1tL{dYERAsnYsUXK(2ca`Gv0F|TCdWCKb9)HZ_Z*AquZH5ptUol` z-<117tx#VtV#V?Kg9>d ztVch;9E7bWm}l}d4my`@(0Mux$sfY-DupwD1FE2F8ILuKf|!XDgRkbyyO zQ;+yQ5KXnP`?WR-T~#sEUzwR}3J9+IM}bS_uw18&_MK>=vf$y}KcPbDuUMfAD%9P%%kPKplm z>By{F8kXcvgAaRBa9jd?oz+R}md28uR{#}H%%G%kvQV|WLszd>(%`-KNnhm$`Bc87 zS8iYEZg>|Yj8(wRNeXB$(ZPs2s?6AbL2gsDFhrKSazk}+=U)qH+#H6(zf57=I0B0j z+h{k>R3j2+A^*5Fp7&^@{BMrX?d5?M>0FE`ZKI5rOVF&g1&>zFM`mIs4J=!SONJrL zo(jQ`vBp^H83?&4d=GGph4awM_?{bv;jTNdpFb1z`PR}EE@2qh?2W${!ywoDGOVm( zaWEwa%7KA!>^mOQbVIn4I0Tz_x}%UYU;aBC(XGE1&T1@yXvm-v( zyrSMxYs8$GiLrKsQv=$VV=xjFYzi6W;g~k(2~CvYj_B8eu%=xTM}!xYDyU+}Q*BJO z>jwQQIb3S%0&}}gTGZ4^PRG8Hxa=N9C{;42wvpa@bcKGmbkZB=PsWd9Y2nw;6k(S@ zx5sRvD&w{EDyf$Aycf{ZtE`1v&!bh^#dO1=O)^aVk>shUN--uw=(Y1`@$+6~aqhPB zLW=4>VMoYwA#QJr;Ic-Yk_AmFijgO8B~6;WuA2JwSx?!67tmFO<+L`rp4vhel3n&Z zayqe}wDLcY!>m|(leL#_Ps^Z@xw6Q(a+|K~t)d6(Z`0E&??@c`fKdw6R{7TCR)w?pLU9LVxBAO+@m^ z5m&5AK}gHfDl|ccpeMuA~D{_8{2c&z-RYc z%F0;8o~vazplgP{N+6YZg3N&x=<&`9Rp;|)?Wzfw`+z%?m=BY^wSXF&hQo8wL_BUW zfQsWK$_v(Id22ALJ+#mz>O771RYZ(JZ=5UYfg=$WR1(n*FAk_+M-8uGy+4rA*iQCB zyrSE;ACRQ>9R=1mkVRxAIn*~&-xyh>?n$SeUH6inSqz=f{76v|`{@Mt*yP?^Kyt_H zssB`Gny2nWNxj#TzJ3)=FV~^#yG2LD5 zLpKziNOCQLT339b-2<*rUh6qZ>v4@7_dKI_MUC{Z`~`VCJ)^O)opiTJ2B%6@P@$)Y zZnBm1)=&d+^L1fgto^?Za*CZg6svo4x3K(RyzljZ(jJUN*Yi@yw@if7>3WJfVFsgp ztmO#CILmtD2rp(=3<^SJhX<H#gEwH~{zgLrnf0!P)GsHlZ|$`;t5|4!~~D!xTa7EZ)K6)9#1j>KJ;Ci-4C2)V|r zXPdKLH|{POC~L#5iFp9Jnqv;cW$-PnJ>?&Nd6xA`9CH3 z{#R-BigRSxeu=JJ?WFL9;dEu@LQ?4AL#|1WiGjOR|78Su9U4G;3=2ssW-O(6kEKVi z&FNloAvI?|mo%PuB-yj^k7Sjy9M$d_D3-rpF4&*D%6?2a(b|tYv8tX67r%X$>^@yB zv9Ao5tSvq)nV^NY%^n%0fFr7KdiZ92Vs*Nv*3)u=FK7Hu1ImD1kNpeqgIDbHya zNn&2nC5_WGt7|sZZaGWg8TV-4wJJLE={iYoy`cLhugN(4GfA&0K>30!g7|;GGF=t_ zHRxf6R1+pIAJW)2M%+(5p64<{5d5=&K1?!3vYr)kH(KB=_dy?jWD4(O&Th$?;703L z>M?jT0;YuE>*n>imHUyFb})0GPY!pQCLlMh*Z*f;J8x`YeIOKC1I{92D0lNU%*M(Z zW>Y9J|LZ?L^xH5Sqc(bD2eVQ}<%c3;tS7?u^WA=WFATO!U>{jFnlFTb9)F;QOY7M) z5CV^Xo8Wx>3kj1Z;`#%?a*`=rhCZW54=m82VFjg{aWI-&PZmW(FrkTOZPSe4yW}AS zzm9(_FD>MofuzFi3MQE}+eQqnY?7mL_*V&iHWATRl+V?0+lr2kT z%Py|_zV1h*U-oE|_JtM{qE(9}OG?Tv6{09At%xF2wo)iUnx zo$(p-%*Wi%jCr1!x$p1wU1o~ZV3%quM!6NCLBlyzesl$o4|d}h!4S0nW{lrk?XmYO zhqYBZAoR7uzWVCH;F zva>*q@SFt6mCz(Oe9DllwfREd?YK?<%0$ttyJF~q++^H8eLPOg7Qrx8F|4UQhcEQx z@%TkWT(H3wBlnh~@#t*ql{3M<83DK|uM?MgUcg%h8Q3+e0Dsf5Z02LJ@BiRsy%G$~7h0qcwY#ZO*!aM)05Vm2B-6clOJb!^3uIVB$VcW@>qc-L&BP zdr_0waKl=*Q`mnj|Y62_Li(B+x9gt5Dg3)yG6Aof%2CbNBW zguM&$VOzV_aeTwcY%hoH>l{17`JQ^Rlg(NfucXLc=PYFxf>tt(XFpJb&w`1q;C%k~ zaL=VRU$CC5O`PlIu)JmZ%sb@?m;b85Qn>z|RizqJ@Ogx0-=)|iuD&N!F2pX*Dn?%A zZ{$-P!|J~G=w;P`rC*xyVqOi7TXhw4k2m67auN3@XW*$B`56214O;y1!{)b!xU^y} zj#=G8#a>73*fbG6HDz&qMh?nf5J%ow5p3g^L4VC8^!^x1#XBOXLslt$Al*l!JKe}~ zk0Cqhj84~J`vmg1{kF3-C|mVMbF#qLe1N2M*QtSMNF(>bYeehg!{LcoT( zMsk{j^_(8RX&kE?v0?k>FJ#0_zuFhfRfYn(L-OFe9u zV!agmM`$(Y^|OXK`P(qLU3_fxFPCxVsKx9LtFn(b>M`h`6x%aJj!9*6{hY$v7;{OG zEw>QjFvB1Cy7x0?CceQDmFKv*r4i!}b>P&X$0(avhGkfV(*AsGk#shSe2hkS)p%TE z(1!_s6H(;G9-Jh#6Bi7&qGtC7OnNVQtFce_JrrpaUiLrE}i;>FC^(jrjQynrmG_YF~uTZy%tx zY%{Xa7F5{$35A}#Ljm6t=sRuzLtpVRQ<3kOm70acYh>7-{gasfIeBKH_!iAZ$1$h# z+*s=HHc@?NphN>KH!BjKe#nf9TF1bJ$RI5^H}H!|rmp znOSroldKP83g0-~F1UmZ`bV;>K^|UNH;2Ye!}M( zt+;weBgzil$Ddk-IC;uteAnE7eL-i?RVM>8&SauWOBmRhmyUF$Xe?2?}yv)0W!S&fhn1t~aYq zvu9^gW3VAQoH?z#$u6f}WG~`E*}Sd0*u;|_EaTo>uJ890->;g*_6N^o8mjNH&u^3n zHpJi;yPN0{_z4w)-{IG{T-~>Zo~zYQ0a zKR}~~8a!5Y3!QeI!$~5U$iMFne$`LM{fS}NzA_pk1uBu>%N9j8%i-`m18lpLi&v!0 z&>-g*{qW^I-RByQ;;farPA{W3zm`(Jd#h3M@>Ke>B7jcPEu$HyKhbX>PMSy=RJ{EJ zVSYbgZixuFQ>RTT;;QMNgqt+X`5={u;?c!HYjMxUeCoBilxF%jQG2Us^w4@rZzy%q z$}$7AOv%Pv8v|TuKY=^P)~NlY1f4xYaP0nJRJostCNC@T`no*y&%c6;Y%9^*t{Fp0 zn{m;^w^+8d13QCLa9Z&Ptl|^kcr-&eQho(Ps<^S8jSgF4uggrM1ejxwAj=o+Kw@(f z+v_aw<(%=HhQ)@>FLYp@XFp&^vlnYgJInf>Jz$^GJDK6HudMd*Gq$)ilqHNx;Wr^J zOMc#VF6$?b9odq@WCgvMSd;_XyC;OD%@F79%tmwiA5Ju3Cp3uwCtJoC@=gGLT+>oiCGREA4h>~Z$HBGeCHK4;dMy;cVG8D}cH-aQ+vt&0f#zFVaUtIrx|nof-qapkDZh5n(eTYA{!I9CyD}U?-*IS!C=_oNYwes_%|Wj?HDuY~{I}h?(qlohh3g|5yDY&M7O|L~P!Ch2vkEhow@;d9AKRGL(ZCsuZ2#J&?aG$|C@zQ>>~ZN_g`4`Gho zBJ{Cz#>Gnt5xgDnuL+{^bu+xQI2RSm2kAZGZ}e312Wl8~ojzC2hUm{)pzTu*2W!tj zX+u90gm(d-gftQCUP?xn6cWR|4~S!s0p6XvggVH3(y{6&`sJE4b{ik2>8H=o)|208 zIN6QWw?9%Z%%(|7uc+<36jb0*lzC)_Yps1yad9@vXZfMa@?gx$jKP@FOpLskjd7Yq zXgHI@r6+91Bi?uLfERU%_d;gV^6Vu z-Zk7k7>YsB0=Pjtg)G}1N_{*g(t#!ABOS0ka}Kk97{nsY zxx6#3W@ne|%6NK)OxnYXd5HD1tqq%5lA#g%n(M;Oa@ucYwPS3bMiARzH-|lb=8Ye% zk>!{8v;IqZtT{83Ue3Nm^j$@%(t`bTw%!#|vsC~$e+8oPWaE-w2^4Ue9r60Z)@ z*XNy3DE%NcJh6=OB5Y{J&FSc4{h6%VP(^M>E+Jt73&@ZDiR5>z71^PZ3!>)7AnBtp zss5P?!dgI)bG<+|ObUd$NU=RxI{sI#zS| z&gW+-IH7bSd|$E>)Q;7G9{mKBFFAhW$_XT5g%XKS79`I11jvP)IoKdKomr1ru+|F_ z%(0f*&CdA1b|21lNJ7Po1hh&#huVXe zu{L!D*1zR)&b%(-yN|c==-_?Kp8f;}#_zxtCWBNWsgY{f$ugl^R;=o)7<)TYp3;Cq z$T;Oky1V?j@v9TD5c>de%d6pC!algor%s+LJb-meCa~9EIFIUoI8ET~1#ENtR(8ZY zgw?A?Fm`z{d+AHrKR>+LcZnA)ywjWgt)=Xk+hW!{ErNw)gs_<x(JI(%JpeC)u0h;tf4sE9ns{3@LrSVBoy1|)m-#K(DNdW^ zeXxn1eK$l&Q6OG_*?^z3@8j0lCFo8r;Yij6bl$iEtK}}EiEj@2nkS-oUIO+<1Yz2+ z9qwCYgX4e1VSpz~R+^z>N;~W| zu_f{t)sYIP|9o0PTTO&9PCt~`L~D@uQb=|e*Ms#oMrQQoLxsshP+uue-U$`~e@Hc` zhl`V=X2p;i?@AJ%ijbU1AL!MF6ZGc0dMcvxjB4!D$GEcjR6r?){!;o%htyQ?`x|rQ zdrq<8-X6TYq!}w}vaxk(F3Mpc#(4H3=L(9mU)@LN$d{<4@CDuFzhT@pVRqL;oS7ee zgndrJ>=4H(%y3d-Pd+KJo8|JH-^3H#p?eOGUG2pzH+SY9@5B;h8}LJ{EE$qFAeNU+ zS+YnDbNn04=7o)6^!NKPt1%CD9ndAavyqkWYG)avZW&ZTl5ubU3A*m=j;pPNUn#EHQXcg}yM z2hT0c!K&$x@t;z8F1MM>uAQvJ<|N24BaW|dwEQ8i*eJ$6W(qU=gTvS+@)dU~zCu^- zd?_6%#(;l5RK71?y`&QTo6QEH7x1FiMy$Mxj$_RXra%JWRaa)uaF`CKziN0kIGMM zqbW}-pBr$Rt*PeNl;DgvhY~UWyFHHM@ZaGEf1DRw29CA;Y0%_cVN zVFj^)%p*pF>4kE6FDq=A-z+X4gBvrrScq}815NgNV<1z1y@%Nx_GV|NTQNn|K^npN zYAR0Sk{zQVYGI$Yv6fUBRj;@jOlDCS>>UFx@R1_Yp+br~k`a~v7&yi6NCgsppg zP}Rx>|Cl=9RqsU9C-ZUlQ&T)6JOiySM&afBVY+4hJ?fhDggV)MqDwdIqynl%gx@cg z9Fb|FR@ZjW=nX;Cy4H&3Nkq|K2dznMdIQXy(*kKHA5Svpw;NU%T zVvx#dcGb?qi9!LA))x-hwe@hT_&rIRcZ@hjekY2;H;AY#p)(BJJF78pCQgEKfEHoXF@?3qE7VwA`h z_s#6LJdbzRAcU9ay@+S?xt6th`aqhgBV4xG$GrKUGVRX4Eb~khTXwPw_%9@3mxV2p zJGGPL9S>j+IG*s2p$t6zW(o_oXk~?aHP}7749tD_8g~T+vg)@xnA&n%=36kfcqUwQiuhZ zBgthMw%f7e9EbXZPzk1u%PGD4pZ4~XZ*3bgQoo&q;`P-t$P}_s&E?VZ@$Qj ziNX0x;<2UC76tZl8dITEEOtACJJh1l^Xhd}kIF*(-;eNkq$=w<)PqkKnBciXO`s9W zM~0hi3qM+lrY zgWcM$LdM?% zv(#+t&nRP-FFE~vr8rx%R)9SG;|8vC7T~x55uR-NY+i7z^*R!pacr$b5D+k2#LbPsv(B94IH3!u5qq?#0g{ILhHUQ&W+g>*ux{VCAunM&OD z&LN*}XM>w-1K7$~fRNf4Y}5UR$evcCU){85LTMZ=&bdqln)y({%YdG}Zc0alTWI(p z1H4kCi9_>LP-uzE|HR+sWkq8}Lp;j5oJWzERxv+3S zEhyXX;CMBh@1Of+%o0^1j}y#^**0ZTQKn8kB3)T`Q8?3@GYZE1Z(;A5Rw(l|W-n)^ zF{exUtTkSa{bTbC-%N7j{B8w!@#RZ6Z}?~UjH}scCFFBiU1`kI$cW<)$P$G$*Fb69 zMJRp|&MF36*j>>nG$OVEK5i?5yW==+Yw|@@9KOg*H|}6DZUW@q^b8m{JCEe~%HsLB zL|iR(6T7~ZB47M7Y;qgM?|UEOoc;$W9bScZWS`)Xvn6<7@+8kT@hY>uG7wIVdNaz^t$Wt5+!fl<;1Si0^BEzdHg+I~~$ z%$@wWWNgn$O5U)fxqUkF{E-j|uyjGWp zZNCMdnkB(JBoFGa1NyIMl4r|0$+6>AgJKQ=|Q;`=mzt z_s&r&xZj3~wM0^(K09o6eN3xww9sx>2@E~50oCq{;_8bM*fG@zXZWo{>5uNHnY#rI zmWH5mo&&z&IE8!*2&Tm?M(wg(Jk1I*@znzKUb%)=Exrv)OZ=!vo-mH4J5mOk#N^^9 zIBk;w?Xqo=I03Ia;A)Q%yJ|EjZ4^ zn-zH*JH9d3B^#N_#zdIY6$}3)TmVas_x6dUSryk+!*s7FaFxrFXN#B7&MkacmcNsF zM%lyJP}UGsN7B zV)(sM3=3v$#z$46DACzO`R6~OPlN68$;*S(&0#sc_$z@b?60SM>-=bv%{B6R>^PCQ zdy~q#%%Rt8r&6c!&BU*>iWD9iPuA>FAk!>5f#>25Uk=KXy$9vUmvPs?V>~4#!g6H( zWCfzDZcJW3?u4pDNg|;wPwoe`k?!T*Bjdg zpB#q6Vmd_pyb}%Bv=8|=Jwq#eg0q@9olAx(zB#)I4YJ(uyL>2a7F>_NRGl$un>$A8 zS)jvQVZ1&pg7pX8P$%IhwHGR-a))nIu7rl`!+fcU#tfP=8bSreDrrg1b{gN_M*ir1 zAnr*iw5M2|t~obMT8qL+dii5ww^N2_xk!@k;uI*}O3B~Zk3s#k2vL4yLqg0}lHgPA zaP@32{OLRcbz|oNBR|39^64bt!c;P2b_BU)7)RoIT1nkwLt1ttg2V>o5I47}bgA@7 zy1Ck&&U`6P*-TfO(qx7@d>3h3QxSEt8K8%pm!WQjG~)Zos3T>KS=W8J+Rg&h4fMsD z^WoTKy8+KcZ$mq-f72?m413eNadB`zO7R8Z&Qy2u$?!59=e{5F#^*qKcRx5+p_RX41Zy=by z4rjbob2sy*sH*X*b1tzvd5-k>@qCc_5kVU6N|GCr8su`K7>S&(K`J!^S?HYx=4qY7 zR`V^x;IkXyVEqcxcP|(P?7m{v@H^BlT8v-Pb&;0%;ii?ISTHXfeU^HoROur8FuVn2 zHRfRPX9es(DveWNDIVEBOgEd~rh8;A(z1LLv{YU|XPlLx3YiOOR{ANbp>0O7E}VE} zFCd!}BzpjVagH6Em$OM;TAux1e1`$7}MQ({q zChty7B-33q$xWNZB=dwJd40``RJQFPFaKdgcf$j6U%i=V?wUj!JWrA2p%>)WrUUfP zx=uRYTmZ$Vjl(xhx_DuIF5aHI7X3|Dpqxo4e&28dKkrV#M84BlA#)R-MATwIVG&kw zoF1^hhY)@p4QJfMW9{+icQ6;9l}j*J85!m@(m3z zYkmTGyh;cD*(XaAItF<%WWjj{l%Pqz%+TD?L zmr-_c<_LPUOLBe~qRcRI5yQM@b~|+evlSBIG8-k?EYqK;Mv=?ObYz{q#!M=p3M;(~ zc%4@)cnU^E%!7}i9@nq99e4^PE<2Fh+vLcLSr@Eodwan8j}Un!{eZnQ732AsHgh@b ziHI8fB;M;JSQbTLU4#rgo zS5To_j|$WC?&YK_!i~ILu#Y6PY#>oSeK0p<64AWlN?!C?l3W{gvZ(SQ_<8++_qrFs zcl08F#}KsT%^`7vDkNrV7?D`$Mfhj!AnqfQL`z4Oe3%48)~Fad`rqo;#&VT_b&{T;dQlYClBhR-B_zM|)`NwTT!{x@hO%Tl)63 z6y|MILX+A?>Y2(%9fw1y`QzpAV$#l$FWaY8g`F_yODfhFE0y1m5@IVkV?5 zM0^j2!R7^};I&SR`}|Yk#?de!_1B=iuNr)FkAv#Tt1$9yJqqz9g7)RLL`iQNjsyyG z8FZ4YJ+2a4YSM8=$rxtVda`COK34TQ1268Xz>XdXE|*@2h01E-;XB)jy7p60;`hW& zqh)a8lOV~dxB`kqj!4NhfZ>=j1dZ5}0ignDedh?GA6sDONEXg znneSW!l=llFSKcJH&t`cq(vzPbhqgzTJOz=SM6AVs!tpYh)HA;foPELqhgL&{psNvLTz%sk!<3*^j5t@=I4dLl@AjCnAB z>2&gXZ3-l>ojZDeXCc=M4;N4j%lGfG?Huo$Df2AO49BPGhenawm zDd)|RJCE!)t0r!re^MX+LsW!2?g-0c^zg@Cx_4b9O`n%S%VxAut#VPU*zlH0jQ>ED zC(5D427RVjWi#WZi;`c4k>C@;X^E@8!gPlwt49%)5P5M9Ic8}|E0;az zYO&6&kMqx{8prj$mvUN{<&8MmS&He3G~%-Tsdy=~5T8$;#io@yvy;7T*pX&NZK{w+ zyK50~HDOZlv>t@(@4!X=^Dq$d4JxWmLeb?PRRFQ`YMHX4O(qR#Dvx=kOUMHU9w|63NXJo!My=}S6$o&<`h zKBKu066rj-DC)W51MR#TL-`(h(_Xrtt``46*IU())p73R5T~uH`SuCUwXXoZl|Lci z^*Hjgu^z^aT!4GJrO;l|0Jq|*K<|SVS=7@8eFM4Bn9Wa4cQG>4>l~Q8;c^zv*$^2W zITHNB7V=+ek!|)iMC|c+vUjO1QHz^T0?+G`z^ZmASuH}u^`*(LQ&mv>S(uzu7bCCl zc0l4K7g9~{Lwx33V!q=OJs?y@Pwl--_qyDto(|*C@xxsz7XFCxJN}?=M-=c*oCNN6 z)<73UMDw?EP$_PVy6AXP+uuk9-{sLOS7gcBK?zcS)fvwBheF`;A>8Ad!P?`Gu+@Sp zs2tP?5l>#j&{zr>&e{WdC5>QhqXqcWPW}ZXgJIxYGt<4rUw4s>0Hv1*6%C;Qh^8Pq~#ifJQY}~+Jw&h1Eu4t(z z?bMdk1q)g3xo{ht_x3|@hB9fr(*R$NhC`)#GWf{ZfZD+~uwr{7NNk#n{&h##o;evT zGI$u1ZZyN`!<8`Oc0FuY;*o(p7pU2LUHW{7}>qgluWoVf#_XuAZr#jz+e0ZK7Z38lf(XAYNSE) zhdelG)B>k7r;<4X7Q|h%m>#aHp}#^uQsaeVwEv|V#xMU$%gp-eD>G5dcp;7VXK3L~ z*?HKZiceVr4evL=w|0UdMdx zYly*-Zpf)ih35syQ21Mk?3>^Z8!mqVNih$|BL|^!<7>F^g99MnhQN~Vzp3ER7Itdn z8+)8(&Rh9x4zG+)i02jjgB|nwz?$bea{lfgm;mo5+de4B%Q3X!u^MyUR;3}4B1AOwB!UOKv zAlmy2&SpiBdUgR4kA#Bxz<1!E{t&325?OumIeaqAfh}c}h%UGY{xY>t&^3V+-pd8e z@>8I*&X^=`PKVy*lgO$m4?$mR2HC8@c`|<-ft1z>#2{Ro=pK+DK1#W;q3tQ?{8c6A zZEL_I$`;~|&Vm`a>g3wfXpqViA-KCi}jEvCB)4(@cd|i_XD_WFPpHc)`gN z-jKekg9mQ#D{?Rt;B6>Y->bpTYvi(;4O@9Cxv*qE^$ynHRMS=*7o&=#-gr2vT;p!I| zGRZdt{^q>`fpZP;RiXvPM$f|8z3Rm4Gmix4mcXU~K62N02=+a7gKFQKFnIMZY|Swu zibuzwX38`&uA&R-0xZaDH(63TCPQjJsu1tAayS+)LUsi00KL0b#7XomxKBO|T^H}d zp*J^S4X+rcNI$m9_q`0HHXiQjO($`SQlMai8JV9_2076up@l^dalUD!uu_U_oB9;m z%f7?GJ)&g!`Kj8uTGfxvB zbCp8iaosV9E0Z9HS4fhUXY+{cEiJMtUIjjsbb?r zJO`48r{W16j$_2v2i6;eiNNUzROeU^Y!5J`5~umewDXh5-Y+q@Xuc+{_^eJ^O2!kJ z`w2=z zlS$<2x#T2wea+``OcvN>gBX(|d-ikQx#d@(K<^Js*>?{1*-H__;0N$BPM5f3H9**k zB-jp3&}l14Rt`ImSv?;?B4;u&@|PrgHcus!bEL_ERD^RTzriw0oAe}B!JGr)^o+`qbmOg%9r_F; zIc)C7kz5$exB$EAUO~0QIk@bqPrk05Lkxd@goTAIa9Kl?&}ES@lAQwQbF|3O;W79b z{TiCpd6pGX&%nU=1?WBygvEpF;IwQQ`1KrxFE_Zc|D;x^jtqlrO1o2&&7!_%S@JR^$Lf_I(=5x+_%G{uvgII(F2_W z7vM;hBym|gh15IB!t+fDkYDfv?2jbDZgB(Bud@M4zB@sQ!d+0lF@fX-ZzfiQ6r_HJ zL1FSs5LxsF6rCTz)sw13dyfX0bfO%BJ_r&qv%?VO{SFfS-+`lFIcWZQ0n9N5R<0K% zicb2(?KyY;4_tv|rjjJ9?l?GEDwA;*$2i>Z1(Y5cgKn8ot2g5dVC^n3(tdq1nQE#; zqQzCn>e0ur&S?sA#xGs*B-^Q{Z;*zC1)gVD|n0fAnAzVqf#zUBY_lbOW7dv8qN zv_yd0+rP7`BmcjP|Gj^e$t4cC-1@)s_<#TK-=_aO8qamx*BSczZ27-i|JVOZ`oFat z;nx0ZYa`?TyX}8HDw+S*cJbf;)Am2kgR7&6h{1nsq$W7ye{U}MAFuv%S@a+4P25}j jJ8|tq|6N9p{`;GckDr@e9fd{yeJ#lS{JZ`C+V+0{;G9N5 diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_abs_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt b/research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_abs_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt deleted file mode 100644 index 40975fc7a9d6638dedccabffe1c03c06f805f867..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 65259 zcmZ^Kd00(f7 z?%3@uHdol*$6I{gnhgv!`H6e?&Y<9tVv=-Xj)x%UHV3mXy-gyf^EL;MD)~<@*1<8ROK;vX-Ihom6r^FXFG+EFM4~;9k%PnEh|#s5 zWb@CxGdABLV%fBz5pC8O~Wq!gh@%l8sA=duT)^{C!Qn7fErA=K{!shDLHlUW{vA zFp7JWH-hU^S0PzL>o|Fj3YT{V+@qW|-0%98+(9`LZjD_$Nlmll zZmsa-D!<$D8qtqk(wQZH>3WNA^rdMEeM%Il!{-zlCTog{rU)!wd=<@i({cS`G8)KQ zMD1$huiUY~-YY&3(L0R|MF}|5e-h&zJn{FL45n|)q47fU@VhV{v#qvaW8Z9i*)9$% z6Dw+SPK1;FeTeVZ%c8e8f24uqGpVhWEC1hO3C_NIJU3-!Iq^#DB9D(0kSq30WPYCl z=hgI*^tdIH4ozup#?CqtE4h%&nR(alS6bf0ohcP$%Q8K#xBMpA60gP?eh4J}M?=ZR zO&9FqgU*o&<~d~IRts)vTOhf-w~PESzeZlWgpoAoZaZ%oo?JblKo;<=9;lHoRqVW?O>8~AG zba&?-K1J7ttFE%5hFPNcDK`iAo3|n9HdlTu^{h1^W)BV<74(MjVX5)`?~a8BC!Qe81B54KGEeT`+f*B6XtgN>LKBZ(DT21tolHMywTLEcRI!}}}5(AMoc z=>^LK67XR@_p;zODLVRuXmk#dD{r=uaxR*TzM4*y$T6Zmbv=on5<)@)ZV-#C*(7Ct z5AnWooxD0!Ml89-WRl~5q|#iU^m&XUFNRV`nAcGvo*qf2YF3eTuWyn)qrZ|}-@3__ zE3ss5{zco5?pyitiQCAT6lE^orI1+fsv_yX3rY09Hd0&>Lk^h!BH!ZGi2lJ^yJJso zk-`6r$@Shhq$;V9yw3_C7*mF+?kBg+`y9Cgcp{?$AS%5@F5ru zT1%nytda(bTGMMk#qlLO05dypBdECwyQ>Qjb0-Wsk13S@_(6L_F$}G{AhTr{Uj;Ez zGH!(2j#O-Z@{kTx6(dq{5MJ?;%;Cil7F~XTEq5Z&CWyPb^j|79?<(Y~htM%3$Ey1b zSfGs>vEhE3o#^4^W?W7=xdt;;dIP)LwQVa=Ij= zc3s#boCChY1^Fi*(DVnB@Vz1(Ip5!+NbNbEWJkmNsR*PBpOUyDDSrRi<=FG*Djc(J z!{+y4ybv>^m!)r!=FvW+**~9@FRCQDzDh8walqSwm(8k%qW^%h74C!dKS15#=&bbehND%)bhZw|!Ex;_=**R5;}&*cc6KseesKcUABJ$-2Y9L|uuF-W%&tO%Et(_2 z(uRs~s!@yjD%N7FygK{&*nkZg%d;=a!+3x2DpJh{sN?w_`mZD%c`=eKqQ-!Eo*uOJ=Zw74?lOivwJ}b>|tLOruUX zj{+;tu7%^XaC|(chrV}5P`Y~n#kE>&x`rmJTh#~Qo>=6mN}x5&5{?lyaLm(W<0LHD zmv~KfAh`rv!=BO9MfqgP_6%IsxJCysf9W5xMbbk34_)K5eO$NyF1 zhq@f|4wPr!c0Z6LcMusL_S3=kYkW$x271G8<5FJKn}!uj9za9rHhMZW@bORC*~y=Qe0TLiz8 z6C%%c?T};UPSyA-xDIpOc~tJiYx-03C`Qcf!f}<6%x|?6+i|B8f6n}+HlFHS)%|i> zyru-U+ceq0fH_-mT9ZvXUjy!hCAN2nV{>XOBoz8!?5WA>MYP%3kprmM5{~R%J^0HX z#o~-wd>yaI?uY9!e|u@B@+=KDYKCZ^y%F<7Ut-f)1$J(fKAWVg!}`6%*|I;$FjAU9 z)qRQ~cTa;&iyp;P#u&2vmy&F*Njmll_0l!6h1B|3GH&P&;*-5P`@LI({k>C%5u4l~ zdHE@IQ?x+7RUQ7Glws%6Ranz35%x(t9s8^l5m{167f(pUGHWIFA7{?y+UT;r#18zO z9gn}a81#=_M&cVyW^>tz-P`BH=tN_7*|Q#f3;nRg<}?Y`%#-5UxMw&cI|1qOTVVdH8T0yVnbzlhOksK;i#W7`&3sx0(g!&0i-lN{VEnU| zF^@1WmVMrpwSH1z(aTRGZMq)@ZR)`L-&wXgI+0D><UUXulYa4d!f5Z8k)7WY2Zzw(6hLlNN zNZare4Hv~(+BeGXj|ycCn=Z1&CO*v5M230g`a`AfI3~BNF%s|34o4(1-QQ=~-YRAG z^}=}EkspoP(ckd>*ebSsdL)a~k6@qOH?hjCc?f$gf&j;0g!O7MrsT=OP4}=<8{ODw zHCeW(d=DCxGHyu@}gV;1mmWj`K zk89o0*hm;wi-<$|lPaP}IR1)fKyghBDs~we>))$1r_e$I)mNG5Thpl`mW=frpz@VR)$$ z_x!W)RKgZb*N4aqn~B^Z1!Jzh--q64eu`U$GA!+9D=fNXVX)SMG(AirANxeP=#|gm zFh!TyRv56W)wj?!v50uCbK>$ITXMcL$8g%dmhAhHNo-GIg&>csaji$Xh=0g^yT;2Q zWcu}Jyf{Ca?Pia#Zfzoc|6e{SOv@tr8&(kAgMZ1*6;;G(<~geDUr)B|D5+oj6EduB-qHnZy*fTC%rCGRcNr zn%tifGj7+{G%~*R7FIPGuncWo)-mci-ux^h9x*<|Z_yDVQYeQ}QLR`wTZVbNJ;M(5 zJ@ik68u$5%6?dR>E;sRw4JNG5g5SPMtRJ6>j*k+EQ2WKZT8h&qd5$(Y3ZdbL2Xf^O zAy9(CZb~I}eQ|_7vOPhKB2UvDTLNjwz&85S;Q&>%zd#r6J4BuIN5JXTQ+h_pnT{b^ zG-&ce+P1EhUs?E``&pgIiC?uqR?lU~>lR~{bP8Mx%`oYtIdp!c(4PG*wAR=F-Mf=; z!!ZMX8v~F$BZkV~JWTKSchb%&u1HnP!lu?1?2@_-;{qcze-Pu=WXWB;-0i#BSwDsA=z{7cWQATCvgm zrypAU+1DLZxJH11_RFx3k7ZGzdYaTlZ6qH!DbD26?zgG(hQkE$pg2@#b6yIUY>8m2cM*r=2nMrTBAnIEXOGF)yHMUq>Bp z$B=lpa&qF(Dc-{43>>y~pl{<}*u6`J#DWl7y`hstZ!qJQ>@mW;>9_GIvIV_0nW+0V z4?Cp0sP3T@Hvq(T9^?E1fTMff`RV$C!FENarOZIlLIMdsA^S z!U75Eshq6JSuX0M501Bf!Vymi_U?f=TWVW}iI!&IwjQE$bq(-y!#f-h;4MWR6&9ZT z2j;6!f-D!JWlB+0@tQYWVuaWWMHOc9wgU?n1!MZ1e*V*_403WyC>MU}8a#^r;&@Lh zs#-(g+~v%LIOW(ai5$T_sGLJ%JksFNTZ=JSm!RT0o)00Y=wLBmB$@I$0B3U-2KZt?NBW7|i7P zBkwRq^Ei}>3TULeD2)yN#!r&)q0Zjr^uz%kt!@o?xR#>L?FPHGFs}@ROX;^JDo9b*+;96s@ z5cgk8xh}mhEGuutGTVN9iM$5Ayj`55b0WE`I>b-555|wxov3kAVTHGaSpFSvls$=} z9UFz|>hhELUj7FXGAiuh2W8gz;w{|m^fAz|pL%zhqQ$-rYj=yYYhxss*ycw3)7gz9 z!O@(C(HK4^?=&8|4Z}Q3gYE0oW{+cJnCYSzJi51oEWzFhJpTjrjH=YngvSjr9q6HMd{2OGJQ`8sZ}T^SLR&f zmh}R{hQ&xMxdF{vLU7o3hZOm(BmJfC$*iqO{Nvy&*fpa8%@*m1*>jR+6}}>Cv)oAU zgL@nw6ov7PXVV7i0F7?5$N<(X=n#l{TU+?Wja8gVAqWXRTdj%3IFq{6zPm`ZIH z$A8*I$e%cpRf?LkBqd$u-l)uyEid56MsI3+I0j+oW!N-h9riI-gM|+aVPX1d90{`J zALaey6EhOfWIPNPL4D{stjB!IRamJ&8`}ov%MAatYu{$uHRIIi)=orxvPxIOYN9AsKaiLQf2arPvM^2 zM>V7q$#L;OvZieVZqNRQ#eH6IV3VQ~5`D%|?gDl+eRC|q2A!aGWYy_^Vu|4JUHdK9%s=^#xA(%~|K6Pxw>!kXn75d2$hBKD*>i&5Pa`< z9R{R=sCwxF?sv&u@+|TVeVck6MwgnA(f0*`VU?J3+LrsYVm{IByH1RDoTjOs_i;Bz zge6S_;cY$r2kCZamb<-dF=?GnPqSRL5BEXVHi zYV7+40~VlY&JJ3sFsF5mI= zBz8%3L1lsh3z(w7Ow2`Cp-DK}vQN>xH34vF{DWP6%51-<8VhmghV=T&c%^ca9=#qw z`~O~oo!dy(hB0i@Vi{IvcMG`>M|c&|86&xt z&1ppOZy`Ru9LLUg1VdqV86+o7VT+HHA%9yi@_fRu%{>T7>jkw*^-Uz-9K?^W8Z0zz z6Fg;Sap{>8D2d;VLG^#gYd2s*g$7I~O_hn8&gRbbD-p*A70!EPIQr!jSgGP9R`J1< z#a;T1j#bx*cw;xoTPn-7UAl)oD@<8H^%SP~ZZzB6R|vz>QzUJF8F?k(d0x%#fsMvE z##vi3vFQ>lcGPqjWT%s~u#xC?zJ;4-Wtj1dF|0ssBy)Xs3DG|f(jbu>s%3Q0K7aNMK6CAa7l3@Og2n`87L~S(N|Tjm`50WU6JxjP{IQ3sta{>9R`z}tE8DHbo@B(~bq^u; z4+(LaOCHjOmw)$@1fA| z9>FyAOj*=-bvF5QJb2$4GFkLG`B2?OUNJFN=sAgPmE~C{m1TB0&UkS0E-`Z3#@!u! zP36Qa*^Bv%y(m#)JJ#I8*XBOTc|IbFzx7}w_Zn|Yr?Q|Y2F$PTHDd4kA$0@?izW+b zzBPlTMG$815NE1VL1?Nm!;Wv}XuQl}=b+%Y_%~>AGGuqMrZVXjuh98T4+*(7WYE5s zhRnW)wg*bAbR%PPWi8pXcdu#Lhg00dWv57ayfeyFWLf4A$Bs>##N<|rvGX|0Iz_&`Ukbq*{d06P6h?57S=t)w*pMPY}n}lXlg=!rnuMo#W<$nG= zJxm8!I8F5y$DJ|-eC=P4^032rEa{KiCUPiWIEFvaG>unYa+5xqDd3*Er{c#HLv%f? zq^kwks@k=ZrXA^|Z?mIlX8L`qW-SCqlR)}QDVRF^I7#iir10bWYV55&g20+xXj-U) z%F9~Z&=XUBQll=fx?K({l5gPr^81jByn-`hjZwCzpR63dLY`dABANdAv}b%VT-WwP z=JtCmvlB-DbB^07BuyL_7m^k7>5$ZrW-@{tS7-7FZ)Xa0u1sG}*m5@+`|t@lbyS9} zpE8PRWWK_fT4QWc(c&7cE|5);wftE{KYUhFVnG}0A!WJ-uGt>6J3O5KlKhg-9X`u{ z<@fP71zh-;%zb=US2Tap?*%_LGlaK1yPj&kT7$WB>#=r{66?3yi^}%*ymRDj@}g)K z_h*3-bXs1abj2v97%Ic|&X(pjv=L&r`XBl2=*kn}N7&k_%(|qdSqT+zVPoEq(A#f_ ze8~!Ku7xnRI()#kh<->66hUoT5!F#yOqlOgVs9Ej=odq5>%9W)*!u!rI32H-%7Ln= z@Cx_z`H43saPfDw;qiVax;KPl%APeaJ|l(Qd!ndjYao3ze2|(C4AHA%B{abOG~GJC zi&me0PU#TfFIz>o5Ch0vcfu|aU`zi^TC7EQ+kvfooRU0@Lc?I-pNz1(X()bs9`@QX zct4&;=3yVgY#PmCck7EXBE){MbN(roP9X7v9M zg-uKuxggR(Vy!-rflUYSc1$Dw%kPH1N+ObuPN4RZQe5GhS7c$`JTm)yA*3y8a40wt zuMYf&H_9$lNc0?+tN(<%{V$rX{reu%&s;&3`b6}$jS^_jLv(7lIttc}L~q_dnzgS4 za%7mMhKz(nof_U{|D!2SHPH0L8atD&;N!mch;)dia}O19-v^^O_eN*>bLMuu(@(^i zi8VN$RES?q6~sqZ;3e?LCSwXF(t@`0_?*&>30oV{Z*u^ubMndK8zZ=DC%%zlJ2ABF zXvS=j5o~EzGvWfA;41ruNFI1aKDA7y^Z#8!h3ybNeimcqryk;_VEoQq8%eKaiBtJ+ zbCDPQ2*~Y4(5q(%4+w$Us@2#$MG?y^jo|k55{ACsLxE{3>Q3y$$C<9^dB8(>P7Kbq zoQHFD8l3kgW2bi*yd~r?a)Aeb&NrSfS6hG`YYV{jv}3lnG;=dlV$%lSK}_KgdepM; zu0)7YpOH+oS)2XWq0DlY)??8mL7vYzf(F5L==>A#{3h1S^;=vZc>rp#O{qLREL5P=EukY!1hgURf3*z-QU}He>0W z9k71vfr2doD04lF*JoehPLL3Ln{13}UirMl6c_rUX)?ALJ;eH!pHLT8VD{sL*`?bb zxKFo|hzgy{{m~eURlj}+)-4^jeWeo9|C|rKW;HI+bO}lCROeof3B-REitJ+g7*?bp z&&H|7Vb!(&xJ3$Yh+*#p7+bbM>6Za3)-_=}Z%MKCPxFyAQ<8eu*kFs_Q!Lr7#s;0N z*~HPR>~-Zcn3o>G>lzm%e5uEM*OBb<6KxhdsLbYv3$ax`nHX(;2ujP|^jr$zj=A71c%Q3ZV?NoF`*k1ac9&JulOS<74xEP2y{6b}_P zeA6nj#%6{jl|IQyvy7f_~jXZp^<4gv$Y#%R0U%^Z9f#8KG2mSM<{1BADLyZAhJ@P?WrBdiY95X%@@;Y zmzy$os=tKL$gTJ~S)e-_S}=o=My&0_M`Rt#CljV0B}ujwWLx?L9RH!i4t}&|4*N#2 z)l)KHKed#&FRdq$4$IJdVE~41W-L|TmgS|ZGwZ!a;8a1V!fk)7p86G|8;sfKLTlD- zZq2lw^&)=LE)3HITnH6sV-}5J;-Ae}*BwKaGDC$m?r*`@4Ia>tk!AA*@hoF{?Dwz) zD=n?U$A#PQ-_%F^h?lm!cGX<$wz`d$rCnHQs>n`k*JVFm45EC>5OvD9jhCxCaAuo0 zbL*F9Ijd#ZdjZbe-?stFnzV6My9|ng+M+Sjm^B?1V^;Phm^sA;u8aH0M2C5}xK)Bx zyNqTV=4mm%kOpWzGsC-yYsse*8%fLSI5Z5&v(SfQ*?qzFW{f)xafy1n`?7b5T3|Ub znDhdX(=^!b1^VpT$=?`TH5=YpAIYoHrks107ad!z%D$d8WGe#|*^be-;I+X3nDv;S zv-%Yc<$NH$Mu~m!`-YI`=dnA~0T-S0k!xjyvA(lWwJ`%f9f5F%F=_B^%Ey(RvWD~0m+1eIWHf8cwdTYr) zGB@!ES>ZPuEA6|HJVl>9iPB*051zwF`3MR7Qb~eT-;nj^qp+4!VYRJe8TVeES~m z9q4?YgZ2TQ*IKxd+hHY!aY+FvSIL3Pyjo~VbRyz(IwWd9X_*n6B!i)GKN%Kl^DukY zEhG&G!eHG38X!WrB(=H7nDrFDWxopQ(-KIldZ5)ti}p@(=ZGOO(l`DX{>BL2R{(!$iC9RK(Gpr!o$7L4_j@ zgo-iU>+f;pK_q^zG(uv&3<{%V;OxucM0^xB-06X0gn(-l<)CUKi?h#Va6xi3Mz=&l zdcg<0IVi?f{&a+LwgWBj+RM+o-bPovISbp|e^?Z*#OnP-SwNCB_ub8xSTu)`tpcn& z&bJ!rj>=4Imkjf$y$SWY(_|H5P5~qJ}6g?HR8n!AWm|y%ALN}ixIB70Yjchrs zTvs~zdmvIGZ=xzA4?|7K7#AOezG5TnxXII;z3C7?-T~JmwQwJogMY;V`0tJwcC_y2 z^zO~(YML$R97#ziMY+KEQaBR0cr^U8h667_f2!MZM_s%5>q+T!gs>7y7ph>(*45Oh z^(42-dMw|xLWExZG6~B={Bb?R28)mDP|+y@@6!`4&MF{x5@HU` zcc7tugC2`bBC6k4+u5r(k{Kzb==A>#tGqi{@G%k(-n7vlZ+7#||Gnfknkc}+;Txn+ zKE%@hg7B(w4AwizU|w$@J$#~$9(I|9h$m(6wV#C>o9pR@Q~h*V(Fl}Dk44v;AXxW3 z#AW^))}NBc>W8nn+#|mH#~Uqlhj%0veY}J7TR!8%@lwR?|3DsJxJ)uE9}tbGYt&~} z9zq+0m=^nvNwr6DO;w+`8W@oBwS)xBwZis84e0dyhaj^jn3FIP(<98d(|_c+DVGem z;R|k99CQmF#rYU5y&V&7p64eY8z%DgdYq4}0aX#qe;56IaPFCegfcNYX{-gWy8IyL z7vw=T$B%@Kx&Ze{W>5*89zHN#luNoQN?VewQFtRB@14W(wbTWRx&_+%RVy0Y=*)?~ zI7K|xQSMB&6`d#XfQl~D!=SG713>5t?_-BnV@-jUrN z`?z(E%DkoUNSb(Fl3ICi)cJV~(YS5RO)HV-#?zx*Oq>*ro7zNwpL9mDwG*7o{OLB` z3#4U>AD3QIKmw#L@h+Yr0zKd^^z<&EXJH6Uw}>P+W*3q}Ps#}YRs*h@=~&wP1R^XL z(LK?0rvFv4S$u$$s~sc9e!1a|{z?2=6$SPp2)^%Hsob<7eqvHEFSAmIF37!(`(9Cy zp5}-PfqD?L>!78_9#YG9VpR8O3wOO~F7n?uaXuYlbcf^vdhts;9dOje83{+&Wk(4x z!FBA<)~Cxi`tg_V{^rL%Yo#YsgW!-8g~NMKLVVs*oNN4Rr#)$yOtKikh2%t2SwW8E zJf9%XI2{)88t`wvPqK_tNM}(F8M`Hf-qXB**FJXe@y z>rGF~JHa7n7CO2=(<9rZsdJVu_cdFG+h*;^xr=IYjc0A?zSWY{t45hBD@yY*y~{Z? zZQvdr>>*-Nf5>78YUlU_68`iF3ET0V&s_G09=OL)E~O7$QbJ|oEUCh_Y20AyeDb@J zaibRm@(0#cQhQq)%<>7x0ReYcv?vNGDYEU|GA9_R@-M4;_nWsxg#=SJM$$2>(&sRY5*EbA5zY4vL zMbvPG8JFTpI3>RXqPgS(MsKPB;x6HJS^$n2KcWl8#&9+LKgh^at+ajgGc=scf_?oW zR9wWPig@ zMFlQ>T8^)s$>jZ-N5nw>1-TU`33L4tYDw z!-+*X2-k~2gIWhYs3b)v=q2$^^N-TV`>xnnc?3S4^P#`QMo=%x!PxL0HN2vPgeoWa zG%92EC=a@Qnlo3e7e=*YCDAoH3Lo7sBjb2D3W{y;VpAo3JZmp6Ya-3{I~#J{@mX|c z^%BS^XTTwI0FiQ9Oj1{Y9kEZvebZ#*kGO_iAB*AD(2h7J!ECy^v8n13BEQ{$ZP-It z4M?%z!|Lq7EeU4s{~SZ{+Yuf5nI_DQ!Q~hcmYHwPE*vywp{;|sA?F7f;b4A4d?7#5 zH4nM|8q8#h6^kFE$F2nzVs(@_b-H0s4j&%HWnHg^)gdkBdt@x@-zvgPT6Q3Q?FjnH zYd@zlONF=e7Giobsw{HCKS2!2Q2N{h&Y_2?SHJ=qoVg0#T7&p1`VF$*(s5+*DO4m! zV%3HSbT9Kp?BEG#y%~h^jtC5$-2u;&5y<@+iD^GiV!mMAy𝔏^Kpd+OZ0|qvunn zpb&c3+!Je8mZ9DF9}X#~va2hF*nuZ<{7Bu+T-aQBs*vdol|>?K=9#hV?hS!Q(Xkpa z4@YwG0$;i05fN(qrvPDQdaQDt6??KngAH2cVak8G{LU$%e2bnR?t2bFV}mjKcG;Z0 z8z;g(+r=Pw=V!XtbUQ9hcn-ZG2{!q*I#Yis#a>Q(i2OS@VEz6Q#t3wP50GHRUhS9^ zQ;ex9*I+8p{c1&1F)XXVP7Dh)oWXvWT~3FV!cwe%n@y*sJ8`${H*&jmKhlkv!8r5v zEdmF%m`Jn}%RazZPjw}P7W_rZ@xS=QNwAxvblH)ymMr$U5p(*}g{)2YG1)+z-SaSE z1>bF$H$R%?c8IezvHAFM#2FEXb0A!0&gyu=L@U8!x2ZGDqv@#K-%dBE4ANZxW~>=) z%~t6;vxTE4v*C{&IFz7(HN7v$#6>+cK}m*fInA?hZ)esfrO)JN~?`KD()66kA4SEDzWJcHQ76t3|I# z{N)a~?Y3YFgNy|)GGTFfwkSL~mtVE^9u0EIN3^9rTmHbAop<3`>*WzFS@#cJoD@S; z9Tbq!*#TXFZt-@kGrO!{&0L+DA-rgiF3SH&t3)%fVYWIe%mtGkN7%6|=4_5+4o+>K z35$q)IKB{PMPo*@!xLIb{AgO8nJzLmh7mQF3VPyV6^lv?$N&} zyjO`rSH@mGbz(yTzZjkU7Ia4lR?0WhWAgUgQx{9l?Q15Tdvzl=D?Z0nAx##x%!zpj z{B>q`vtX*-D=FnHGt`X*X4h$|jygujm+0F}6XJu&d59*}r~$X1FvDTT0rgkHvIe!`>cOYfRanG6&}F zY0VC(i!jxjClNY%9BkzmV@QB43$~A9Iz>R3CP#f;LEm~Ah@Tvl4M zLvapl_X|^YAan|9o1J<8N$qspiaeC}>NAfiGg*7D9kW~~#(sQ!MWa%pNcYlisv6da zj@#qe(*`FtYl#^Xm8^vB?Q&}3R!Cb-lW;{#fhpUJWq$;gFm97E%bX-wC%-x3)5cq{ z<%du>M~jW{F=L0{3p{K!g93j13hqg@V_?4?b5R_}tVC6rVOuZyjM`ASrW0mL*%&M8 z$i6LgV#1sz8>*=gXs#Q><;j$ zpk1oLPbk0w>_Ma4<#JS@lJhR3jo4VrBDcr~8e zuY|8FTY(F7rayLNIouBpaz_$to6Tvz4Y5n72QNG-`#D`f%uVb##z)yN(5UK!l@F%TyV=s5$rX;SJ8linjHR$sHiw_$4Jv-olAG#ZL8_~#@L@H3 zkz$yEF`bDJt)7Lio@L~xK?ylMc`do?8%f;@Gw}1=7fgvOLiGi4EPANG$sDUDHAxNR z^el?^g%!9}--&4+DF_>BN%<0EE;~ttn`UG~b_-p`{lHJiQ-21d(p^}v%tPS)oX$O( zGeo);R#Cm^tEkmZg!~+LtTb$;OH&hQkVG_Hy*{6|E&WUV-*3Pw`@i(>pL;a%V>E4P zOrQaRI&ON6I~K|(B4gQA$frN2d1W8TJPk)Ke(6e@z0(IaLr+nf`3X^-mr&$pPL4jU zCmR=jAo=bSsAFUbESGgcUA+Rn|M8enf0uj@sU$0ma!J0q06R++!Trz|EHTVP;*#TZ z^q2E?9Zi4jlxBain|OK_!WEKmpe!9d0$p{nwiDk!?Gq6e#9@R%2JKyDhPE@yKzt@( zLX!=hZBkFvKQlVg$_cOKmSg?LUb=nw92d3Pgq!Oe#jBk%=6YlvGJuOpKGd-%iH<5=Zo%z0kQm9;*W-xQidOxjwp@JU+CP zJnL;Hn@*&W7`a9g_&S3)4A_wQCEv(drpERE=_COe?PT0?OY$Zsk653a$;E8FNe=uB ztR4coR8>`&LuO)pnW9Tb?cUc}M`E;01 z%2lAs0UT9%@q#);5cJvDB67)Da^#i=ckO`|*DWj04@}OYqu$wIeewbr#wF1HGwNJ; zN-oKC8qbLui*paU2kEF~CsC2^0r_2v_%PRF#O9bT_hf7q@m~0v^!iT4^xYx&_;e(^ zwXL{4Z+yt6O|fLgD^1c>nnd`ig>>wyc6#mDG5)V=J%ONje)wLP%W*$L)^tB4JKq(M zh7Zq3$l7b<jU|LOPH{2i%K_w(0_AN0xpvXskut4td)0lB`70xDpA@0`?l81UP z$(zNONU7c#Tg%7$$cD$8xS3Y9#2~nd1gwf72RA!%V``reg@{WeA@#dmy1;uV9Pxu( z30CDIzvuH&TmMlH`6smLMxcQ6xlD|hI_JK_oD(fdAVLB^rrex^ut~7Rj2>;sVUWVZ;i5pr01rA&MzCh)mT3yW%Js6wmB+MO|YYbX58Ixb#r>_+P=`E&D2baC$TT@Pur7s?kx3O!flNG>jEi}eoqwMzNc%X4`Xu20w}1caj)~Nh{pUQq_bx;=eqqP300U3 z&io+83iR(iLZ^tG-CQoip@hiX+CzdPu8_#3Pv}*>A=;pp!#@*$PP|ld$&Em9u6&n0 z@tFCD1UDTgZkpq{iOSc>ec>a-;MsUCG2fTF$j6X@m0QTcrdyNI3d9$4zVg3xM*|^;eESMzCZ=) zhp+In6Y{B{Tpqj(1^qOsFR(cO6!KFe==g~Z#NFv8am;;5$6qRiXUiY#(X2q;2?Gq; zYID)w&yb%tCGE~;&xC>RYj{fjfx%EJ4wX%#8V|)dw>D+&i-C^7#~pz5<4>X2S%O7* z0q~Ejr$%1JoCY_G%>J*GI{DbaV*e8JPx|O_ses4p zKgeBL3ZaxL`o1ZLD7vQ+rg(y?2cCjaNF`z)G~*%iuy0!d_4H2S=QNGPVB$`Jw?7Vz zh3RM=ABThjC-`+2&}9l?*t6gfE))p#xkDTA-R=av_R*Q1Sx`r#C;CzuK^%=&3=NjVIe3F zPQ|e@4dys7fjK*xv75z$Uch~s$e*i<>2i`dcd-=v!t_{Z$qd%%GJz%TX@UMI4S38f z>LfF4Ju7i~pU1kJuVC~+Jsz@V4FB+PD8JdK4Z*|i zv+}YWYB@+3HLL3JVBc`wdEJrMEjQxhJs)CYxgxZ;XhJFW8Zw2xV>i!&kI=B;T0uH| zXvh`JH4nhg(FHgi`y1D))p$jkF89|}<4>>D;XjA_xbaz;7YsD#v0u#ijEdh_;Fg4q zqvCPMwiJtSa86d68$KJuHRPOlk(2ONM*o6KsuI7QHIUDZ z7dmd%AReXNmp7XK!QydualQP7(1(VJ+1;5xY#h%!Duhmd=Ob|3A74i8!PVQ}F+0kU z&#`jj?9McvdsBn^Ukbs?y^7d-einMa=)=#SpTfzm#a!|U+;HPpoEx|gs^b*lr@T&R z-`2dq!;Qc0F`gf8)8#$wpCGF!5Q*j?2=u7JJdy8LUS|T>25~?FVyj=?Q$^TPOa>br@gj`U0mq z?qI{FKPdL>%b%Pa#A_RDxMAx+-s_qYH=6nj(H7?X%6cchp>Pb>Hq_=1!r!1gXumKZ2pcrd(63d$|$ZF-j_cd{T$=I*5T^lUfj56KYm}# zX`U;^Zm04MitEn7X2JtVGb0|YXva-cta;zTn!IRa7R-B1z`b8~*txd^7Z(oalai+K zV^?f=X0#k{qf2n}d>q@OtqbSHvizpkIDYloOg=r*mQRzaMXlN*yeNLf^w)ZzPumdA zU%2q!3Zr>=PG5d}dkIp12B2>3L6mJ4I@kdXo-j?1Hwzz3ZNhs<&!~b!K{*}MXP%aA%ewy4mz?y$=7{%qN5tq|>fP0V5A@HNf zB^{RMLH?TjXM-Vk?@;F!!Y4kj{244B^yP-V?0CRxb6&aYI}DD*!b0W}MvEoEjA@@R zx!sdbJwAso-Kodj6oR1N{~_6!sYu_?ctXC~T_DyAtBA*mrS#iLYxaJ|11Lu8@^6{? zJjhQKHUSarL*Pis@8K6Do5F`-{PTNQ67^j8*3V$sFd0~A%3-goFN%F)aQs#n0=;)a zF?}d{E#J?Uc`Cv|%rx)6x8cgLcUV!90L!C2P^{=fhD)C$2C2q49a)d!jT(Hv(EgK8 zh2Vm94f$^Lo9vcSrfU`*!OQiEe3p_i51I5Elh-W5e40V!qDsiNM}_3vzYkDat-~J; zQ{^E&%CIDHJ`B%>vnwX&S?QCR2u;Yv>HQUG^NPnmmmR2SoQ=_&frd$au;r~bY_`ur z_eGJh*6)LWP!8+z*?4|&2UK%p#c2*?H*nEQ7b2DtFng z!)=8BYI$7~shXBXW*)yrtIHRnee7?%H#Fwwk12AS&?7i~$(rhi29xb_!|AaNA<*70 z$5-Fc;Hu&Ea5go8!TxXLbI%|`_V+;7^#tVkzr)WTZO9mti9yne=>6KC?a;Kw3y+;p zPYM&-dJNXp?ZZh~Iiz$;!?E==epEfjx_$TH<30?B2A!A4@3ocOJsKf$Xg!e>qtE{x z(BO66NEbX?~~)t^Q93pZXleWMC7F$S)WlwfmURrLcVy;C zWUp$_%IPw6`7AlQWSO_*oBa^>EAkt=9`A?=V+Nqf@gs4V@qy5XxnytDRkC@ErpPx1 zC~jpUx=E7I$p2RZ#mita`exw8BXkvnlka>*p0T#7NLlV+;Y1803mRAv_G zy`+TvZ1lsJ0Vj~#uY?V={z#;&Mo?|bCgSy^fT(J#(=$)&i2iORnqq5852qa`OD6%j z+OjaBZ^#B*@-T9c|ba44->myH%RE+NFq(wvC?&!%p&T#MAycZTo1WHe8t=p>@Aoc z`Im|2!YGpHpF!#~a)?XoClaYrNPd(`(ak@SB}>0%vLz;K*x?o5NL^4oS$!KMVURXW zyfK2lyZM-D>sB*Ig+@{t^_`e0Nz*w0RC48AfxS{%u4H7dVJIlP$%f5{C49aNt&ZqN z=lWi;A7s-^W?00t#at1CmL^bb*A6nKwTxW*U_yUnca#6FuOPpqGs#n(Yoy@yR`TSo zlHL8WHst%Bbi2Qj>tx@|>!e4B8g0BjN_bUc$^B^tWcAleWarZ*q-ePub?-K(C;E&h zMb|@#%0w&rn7Z2;MM@)Z8lYfNM7(-`wJ(2YNn7Su5jlrcGSZ@zKX%)T4DknY zsPQMec{(2t3h%&Tg(Xei&`$nqvnLJCbI5eLW>QqTlr*XtNqp|~pb6gD#LG~DhX1ms z-Tn5ED{j|GMo|EP#%j{=AeS6ht|AHjRcKuIcZQe>c7ha>`aPk>~R)%?F6CMLEfwCq;j3)5V7=b8uj17F*Nwh4h~1OFY-Ll4;R#M0%S)i5>ownWcVb ze77uZ|9F7RIuJ<0=C_h%5xa=d)o~>9O*NU@Q=9hF%q6E2<*9mw9ko4WCaLM0M6Ud| zk7Rq_B4gaG$;joxBQ{KfE|rXVYp+sO5ENp63Un8V#D*&_R4!OlV@0Him3_4*8cN`?RqO16$_ccfuX= z8->JHL6>HGA3@m$S)O;N58v$E2AgkA7`s)4>NQP}T+w4N_bEY=@Z{}kQszGcYcRNC zE)I!3(xqjGaP3?gY^+P6AXkeTqYS+HxEo`3C&6!~98W)_#XqcS$GEM5I5Ain_0bCG zcg_ob!b31+i8uefe;zOXAiR37C(vt4L9$FolK79O>}=EgNQUnf!ZjO6`z8(a_YqqB zMqhr>?g*E2?Z+=Uyu_ofWUP5{6^@5nAbr|gWK|~c@BSmWhOQDvWi1Rw_2!e6`tuF5 zb$I$*75>|=88J#p*t)O;vE%jmw*wMxY(V(+^#=TGx)7<>dciYhI_T;OJWCqNhf1dK zS0Us0jZKQYm-!aF3r}NlGykyHv$}A9;ZR;c-TA*?!}#{KRY=yH2KBrH>}vf(cK)?2 zFO0L|Q_orQ$)7d&e-6bW3*d#V&ZFTp>V)uMw&O&C8o$>q%~MXcqR)~RC~JN~lH>`F z9?3)b^j}yxJ`Zk1Wq4ij310M{n@D|82NE%tA_>3|IFZ!3`fz;~UCdw9 z&E$zv#yoa)99&;mV{F=F?5+%WM!5Is+CL2lCby zSMK_s6TflElDDn?2fjZMI)+hL9a{j!={@;sQzJemU5hUgnXV^kDtztp9$aHUD?+4e zk-Jct>;HR;=!LC<f?{ZiD$NF2eYZ3JlkyT`k7qAb1}bgXAIx^rxWSs2e2&Q81}sVf}u)7dHPl-e(RDmPmCVKyYw~L#^$B+^H>pdCX*Xlk@-rd4!;rWfI zH{r&XM|sTGo!sW323JuXhhuH`$;#KOi2s2R^z!MWL^b6kDe7lP^!GnywoYO>A!o<+ zqZaavx-{WaU@%MR37r~%w%rk-+95Sa z-gX;(Z2a+}HV(}TWq6#cF@L3K!VP5tV!iqN%A_U1Y+_S~)9 zme>#4WH4zS%yCs?z6Bab)W`+|3Nc=Flu; zdF;o*)N?2vScs%2FLBDf7GL7qv2>aYx0}@ht>w{(@b-qc6{*t%3#@1jSy(Ga*Dd%=z8b}n#z13Qbuvz{Y7A)O#$af!7W-t!CYW8YB(pcB zli~qSh^BD_?VJ(H){A*+PUR`67du1yx{Kt*W@|P-br3qH`QV4C3!DoK#LhW^%}I%5 zPLB6konb9Znik@sLJTP@ zl_K+=xZ8i9+80LKtKhm{g>S8V2X@vR4)Z6_%LiM?=GVW8jB+9-cmBn$rQgvv=N`H? zOoYCYGrMG>#Wvqi#m9y-SZ)&sPaMXGc5j?p>4BMD`sg@l%5=-0N{*>^FpK|6B)=Pt z*zlwJIJwImY1h3G+!=zQgW|BT;tb|{sbT(A9roJGnROm?K*O7R7&YcQwDknTpm!pc zT8t-mx4j_oa~IJw@`C4ZP<&1w72cmzVea9+_<2-#PT$lKjnw<3>(xRuZK%aypRf3M z<_Z+O99YhYcj5Raa_ zX<4~6JEpy-q{G^Q++O&BbzcsFga+Z`UL&N{ z*AqR%%S1B8mpq*zX2e4_h%O(8{bN3`MK8i6s{fiv&ze0XZS?`t9dwBpENvz2gUX5L zdpX+TYD3jmblBf&J}j9rrjB{akH@VfTP(cVNjB{`M7GQ+B)v|1lk@$I@m47aQWrx| z*Z6^%sZS;9LPJ}BW*!|==|umgxeK=7Ma){{2FtU(X+l6cS-s4ZNUVR5$zC_;+V6{y zbAK;Rf6Qb1OejtMQ%pWiaHpAK4h;SiE16%|!lIAdWwr@DSx|bV;QYpt&&J{8p~p$` z>HaD5JGhx7ZV#Y*0w>dcjiMPd%!x!wgEU;MC)3-C$#VB+Vze#P^Wdh=&=j$$REEcEUeEk%xN2ny?son zbhkQvRy3A+oSsEewPq8ijTgy_#(U%+PSV*UPLgQXd*o?J6KT=7O>E9c(>>i4_Am7c z2=ll>8kh8^PqcT@DaBJIV>`x3oV2P*s?kXLnrcGN{yjpT zQ5zawIgpqxG^Q7NQ@Y}tEKMr?L0r~;BfF3Kv*fG@maGN4nwoFYqY=dvZ0Hn?CA#P)saK~3kZqlpF5RAcfB zQdq1``Q{hQ&`bea6r&{f|NbMJ)MAL4R|T2cUxS9f6}g3f&XQqMTqGv<#?rd;1L(iG zMl|*EE>dq_O3DL^hCfL0g>3j@Tx$=X@}Ef7qB_HJ7G~ z+tg`48=*Q68-{BA;h4V7g^dnaMf`?5BZ)@6=$O(c#8kx0ifSg)eoOn%TW^(U`Jp~| z9w)rbs(pACdqn`cOHb$IvTw^O*uoh{N#nlz$`=)ALAe~wP1sD@w1$zbR(&MtJ4Zr!{TL(^9c77mR^-oCDXOB^i#pV~lHV&& zu)8bPVde`DG$y|!W~)zD%>`gL#^coV}@RKO` z7|`wgv&rFQyI9iTH!S7wCN@X;8`6d9B(3dR}ARP`SXZ_)CXd?UEKd#_aw{uO0oW_XPBaPCR<>U%)%@5X+@4M zkuK{dA2;444H|3M{=(O+VD&yGQ=&|l57wY|2{Lrz?&ajKT{@X8NoHmzW?+g=A)Y~x3=EpZ6#)2>NxSY5V{9Cwtpsg1$b zOCxc4pPJ-dLOW?XWk)8QZ6`y{YS95E>)4&%NlbQQ0eyCJBWbYQPE_wdWf#J;nT4Yk z^<6W9YW5Ex-8RPbpM5ad>tIKQi0o|T%VI=q>Vw&nrD$+$8RUXl7VeIXLE_zM5R*|_og3<2Li3_wBSH7djkFU)|!f#6yv@T_r+pAdRpX+d8wb-s8IHw|C zXScrzr!`+;ziS$M=S6uIlDUOYyx6HopILo1?-)E^OE%uDZ?<>mwwimaC0!hw9DwjAFx z#Sd4viam_fEaG(G8;crQhkJjA@M$^L{MJhao-lPEuH=1?OqV{(ELL4co5&1pw;ZJgf-9LqHe2l7dGY9Z$}19!)p zBRaGQPCJKiQ@HVfW)~j1Mu&f@4~NCoO7<+}KHCys2h;mj{I%>9erb?BU%cWU{6>f1 zR`OlufE{eQOD9wZ_T{o&*1V6d2`@7Hj^lz+r~A|iD@-@S>84;J#YpkMP&qzkv^+OB zsl;y>NOR|B)!-UQxIOAM-t9<2)q!d}71#6d8YON*<@uN>Io=T{a#fe>;Il+z29G%+ zVs3?tB2=s%WkUHIaaZv55mA^fbL;7XPafn&pT+zzS5Or=5GCTuKsYT|sj zk`~vFFTwQ0W2jv49(SC3^Ug=6d~DfZ-cMv=D~w-Yv7Ih&7gA!O}K2rM_e{f zz=Y!&m^AY*zV8^p-<*i%_j>H)Zz5EAOw?KCk~M+s*Z)i0BP$5I+h%{J_7NE!wVN27 zDr6th)%d5+UcBSL4n8Wd1Wq+R7^ba+J-3C%>2{+45J{zr*?A0eBlf5d+K{ z)*+t1hb`7T{kFrH@wz5j8MP# zNYB>gos*6E*zeZ7-KH0}zxamD8d1jlY=SU+^mokdF@mQEo=)aLOWt!!3oO@{v!<`& zv=SPOa4A*ZeSvcmrSUvL)`-iR<$wz`(Qbe|1cMw;9V#1@I)j?t0ZcH*!#(e`H zNb*WBX#NgFenp5PRVl|oiGJ#0n`$qzAMv)FS}2Co{< zOG>qQy2&~6u~nMwk`~&9SjqpgO@xE^Of_OpHuv&9EZaE%Qod`^>79$pn@{m8APx1F zr?BPNQB14#L(9Q^_*tKa+fUkXXOs;8EAs-Ow>F{2tm`bNO&Ob#_d$kqVCXupLKx45zg$8&HrQ-kX|TJp-LeRz54JwzWg65L{SiN~T1Y)G9VKca8K z%if6lzq<*upI?K{a0ir~H$X4dy(oG02~u(&VON(8%ke3=ZgW%cTce@;Y9gkwDQG`_ z4sze8pwr6{?+%>Bf{iy}aVQ1);n^57{3DXfK4J6RJ&+n;fS5xQ(D(f{xHdPzqf?jf z7QZvA^A&j-Tf@%e%1d-c8sqEvdngwCNyD|4{B(-gyHw`jhJh>Vo^MNaSM(8C?ovn+ z`tpr+!uKy`^$lb8Vem#}q<>0b&F41bm&ougl#$`F*V-^EB@d2H`>RZ<4wkII37sEGtp^~t5eGMo5`}S%EyEc=*#`u zhx5O4J|jmWVOy3O(n6^kA`LC7Fe{tr%MYc)cAAjg>2h?A$sUXn`MA-CLJ+|(5x2Fv zRQ0MEHT>oy2}T8LQ`ri`QvqO?x|pk#IX$`4mGP`mm@~i)k=7;{*{hKSxizru?w?rJ z$vk$kmnYo9BcL`i6hR*aWby~#sK;_9^Uj|^c&?9MxrRIW@1Z6qp78;>P;*w;^UaolPr(hWWln}`);kIevB2(2xLVWCG_UA@8qUsFNvR1lw^E|q2$x; z9MbUm71gUhz|PBsvZ=jW*`T=-@WjIv)6&e@PL+#fLewsrHX(q$QJsJq&%=1=8jgh~ zf-gTkh1?tDPR9TKK&Dx`u$yCi;e9$0_M?_yf?I(^Q$L#&j;$w46uXFHWdxi5I{?Ez z2SR%7SN5{DH|=uNpv6g#3A36*R|Y4u4dH`PlKGK!e&5eJRmagGem^9JE*m6W{e2~? zju%Oc;)2;TgXt{0Ig~oSx=O}$O40ofRdd)dIR;xkZ2Rstq@N_b97*NeYIZ8ToO!I8z}jxs z)7w#dC7<(h#czolo7&7FiSj^+tQ5B0dQ4_-wxh2M-jU1m^2xd=d0gBZf^&)oaWpN4 zy`NY@E=^4#(!F|9pNUem>>S5$`6%=sxDqo|HCeH-53syvYY+r9nU_cCb6g0Rcw#c31;fR>6PWFRCe`j zwyrgt_86{8@4XIZ^PW{OW6fmd^*Moknk|K9Duqqmv+3jFVf2FgSYqU`lATaD0DJ0& zTSp!7$bAv>qm)$a52Ci&Q;F=KzU+^tH;(8;;m8M$4T^&M?PNi<-|iwcckRi`Nii(w z#6iJ(2*KH#^04ZcK~hs9$;~f|=#fc|RH;tniK^XUU^*IRN2jx(Uroe3Wt*fWu85k4 zjAgxKpR(o))lBANu4L?@O1g7HwPgBoV}y>VUNT^~XV+>7ms=Z>T;EQd@h`$hf>`%HdUogtdZ_T)m64!VX)_|<=# zc!7NccE1&z3adED=TI{`D8iqOHdu+Is0Uc~{wZ8rw;^qLI4he!0KqZ4@KJFy4h|iM zoC|93`Ok!fg*C8?*&@e!KN*u3q`{%>5ajPXX3Fhlq_)oxYPjej8#XTz!*^F<=BmeV zsG5oAF&5M;H<8HITp{P$y4Yp+6CgthaUmlb>2p=_L+k}NM6IPgB{uZw{3=#^Ya{x- z4#wooyU{$=468RTPhdi{i4H8;ZrVs=4tAQw|@)VtNh6 z^_iSy6+ULGdsyN3j%jdE5m}lyHt4n274aI*i2Pu{d?E#EN7Is3pEiTesXcJ!2ayxB z5y_;I4Q`UB_W3S!Z+A6`I-!R0fM{I3Qh@!5Cvc+n2ebQKN-ll7O2P#MQfk;t?08&) z$vIUx$|LdPNDjNVse$NxdPb5*m9ojd58-6J$Ps=J9HOTSv2w~MHsEeQwoPy(o=T6y zL(@$n^S>Liie~t*ErJa@xE-wtPf>O40UA?R!aK{4?R+FfZPk(`W+PN_SiC!S$!_L# z^Ce;q(}Cs+Uvf0bj@vDHQ1eb6R;G6!KaXuAcW4bYf>GE6CHF%xbL%a_g zfXTJ%+4al{mLc@%ccAhe+@4lKbHWpJ?G~B6g9Z2(n}}0>R}lKFH(%;?1^s(m zMMp#dlu~Q3RP_l))K?-Ht;ooI2A5W6v2*NZca&VwzB3h737;UH-YNJab#VVL1s8uN zvB_GItVZ4zhcBPOBEbVw9MY4|*wT)yoFp6>Qo+L2*R!R&2f%S_F75>hJ#M)!-;gEP z)oH2l{m>ujk)yE3Dh}Er|L411m%BOW@nbLlz_qCeUF*bkbZWxmV{%+-O>Z7HK#RLC zl;V4KJ;Utby?Ot<34E{c=muBl@ZoD;LFq;sf?}(2TJU$IW6Zeeh>Ki%!#?F?8j^TNsdM2+o!_k(}DlqHQ|l$JXC$oMYrJD^ci;@35HE5 zJJ6dOyjAB*wzgrqOf+syBiI&ghmt+_5p-OUn>d;B^H=rw@RX;b6UPw-(=yr55enG1 z;u-!Ep8lhM`td`DW%!GcaqvGo4h9Hd-5TjEb7U>1o{-|T!QHrbL+DQ}@%X*X0Zutm zf~(|;%ja4lIq?{S9_Qji&kQv8C`6^%eY7k(5AREX@GQ;79C6Mt*n1vdhu_EP&jqM- z%)qROTzrWx72Mrc#GARob(j>a7C0bgY68k_>Tuby8@8EJTv_N(kFRC0H+=({)1PtJ z790!Ni!$7^rXQ~k7a7yCXZRGT0Go;|w*RD~@QGBw$VZRsdz$it`xJSz;MtFhw8Md6 z1e>*@k)PUzYsDfryt_BwW6^|#A}`z+8ixIUb78*aIX1*Kp)#cd!EsM9@=`2ZMfUZh zv?;%|#)J=2_y_CzBFj2>7hd%Y!HnP}%$S{r0rG)7H_wX?G|quq$a`X;{D@p_*-tuC zH<5~zJ>udXZao1v&p3E2JB#E#$1vicABI z;kV{7d;Tk%EyzEL(gzK|Mm4U}A;+H$jYpPqJ6l+EO>(d@jE%Z4_5{Oa_{BmkK0vzz z{=(zltX9YNnI#d4Pc-w;`T`%pEzQqp!=K05_|>@+|4BD7Umnj4SJ)zYb|u#L7CPtN zlQ4LG9HF1Xga#Ri;+Sc8bXsWAK@m_*2Xb7dVf(s3%u_#(N0Yii;^{;pq^Er#(-iYr8GVuPuhm%L?#UA0e|WO)$OY zLnpF<70YBWovt?4I5?V>{bM4}ZNc|Gj)%tRRASA_$n}0xiOPoaWWfq6!keRrvwQ++ zn=p|0#QI{DzciPRmgjj(^GMKXeaYjQN-TAoJM&Ug#nGN&_|lnx>{B~n{?m&+KB0?# zTKn*=ZZ8)19}7c=0niK@h~6{$p}E))la)oz$RrKPA5&na?T^yD2IffxdvD!7=Cyhx z9G<45T4b0GoP7^%$DMdCZ7J9{18Kaal;rCBNJRc^6`dVDd9O9K=oRmcWobtw8702t zNgpY;F1QIcg8gtn>}`5lq`*LHA~c2^V^6G-+2ExT)QDNCEIu8EUf~!QI}Q75r@nEz)f^1mw!^T20zWt`BG=MQNC~J`|d} z4AEk3gnYL_Pz)Xb=Ots&!S-RkzAq%6y7*h_$$C!M0d3<%y!;l5J!4d{V~#tkQdDA- zJBLWZHWrec0(ou~D9s1|^M%K(O2M=^OMc}(Bdh&|)+RrgZkwP$w;Z}bTCCp^nMgm; ztI>fw;{95z@{!z7G?JL?_$=wOT9NV1uh{j$ei$mAqeLHyC((hDYL&l?87_v+!UZ^( zW`w09RKS<1U_@>k`&8M&eq;;&)q*7ac^C;LA6JZyIKy7eQepekO_+5r6{O%As$Mr^ zsCyM=KbKuEk^uvgr+JdXuD*VNUS5WP8$Jk6p3|1W@_ZQre$Zu#u zbf6MnGy5kNe@TPVmQh$8ae$Q-Rk6qVZb(e45xBhqNL7Ym(K=_OOz44?InPdr9cqOzcdNk{i5oDdie>iP&ocGwFkR^; z-tYN|e-oAXn;T7Nwa=Ci`(~1yJCej1c;IV>VD>){T~+ZdIOcE|vqjHpv3odqFvCY; zwtNnx+N&V@uNhx|B_l^s1-%9uv)G|(OeX9!!*5rS--*OA!6xaOa1;wJwK47TRyJLt zf;SE-u=X~_!;uRxFxV53pSoF*^+8E0lVJIplW=nu{PM~g_RU*`$(A-se5LwHCbb6< zk3XN`Jh%hu(*AIfenCEGO4F#9QgqUuel$5{H%WH@dDaj>lwJ*`+UvuSyh@tCR{RR> zy~oKlL}Fm~jAZLx5T0j$?9KZG!_>{MCDOg5*{D&GY+0xW1_}0N_oie>)Q`YydkWjfbC`zkPA@D_ zhK;K!q(?46<<)htztI=<)=wp;ms#OXSR~SH_9N|4Gh9Y-N@jO?veO zf$OFDns>+X3et4V*il5U?4JGS!D;qOBKO%p+N(=8)T+|>gJ;MnbI}{vQ-S|j+W^%& zyNG&g8#8)Sz&;*a#azBMu_f02NNYWg%l_u*yd5dYu+>D4!%FDPTnsD0y|k$#7`Wv% z8`4{r#ZJ;cr@tCF zq%Kd%@MfZBCAtVaJ>fL$2@;INnN9etj+FId8E7X{-dB)rt#@RxLmHNjtA^#5RGhmz z5f4v~VaB;n$qlvVBx+SD+x{&PKJw?$ z-fP`w_xAb9-j6`(*8=x4a!~D<6FvaNCZ2m^UcTaM` zwJD3Sq0<|$`e~wi?J3F5BNmu9IvmDV{PD4?iS=(>!KCC0SdxDmTkjM=J_R;FZGZx= zymSH!l=4ZGXK!kHyO`*?+L249oy5sOht^oHB#z?~S?%#UOdFxi%M_b1b9XU`=_7hg zS20LWGlzNiR@A=D!y4bmSpU%%nW)7Evg>o>izfZ!k*b7)Da#{B;O~ReK9n^Hx zZ<0L!2ws&{{ktBKX`;*Ndea>^rMKd2aULR zlgUG!N#(35P#u$p1#NFZo(AIOmZy@B`!5U4el$s49zrI+UW{Wc!apn*fq5QNkdy4q z^lqq1VoqP78S6NEF4*5Wy8_|m!7*;B3cQv`!J$qc=6^Ktak)G$%pD1t40)(3X+hWC z0B0vkq3=Rv4Brb#f&!t?b_E0Pjb;umvGhTpEr^y z7hWRzZat{Vf+|T5@j0*9*W&%bJh;4bh5WiyvUl_+^16uAkmr3dRAR^gA1kz{NGV}1hDlKCa~!7EFW^^m5_Hmn zpm*sX>mR*O;_l~*zHdw5J2)TqceY~KTni{BGb~;-0`Gk;6U8x2SQn(tSGLyS&dPY^ zP&Je++05w0kZyAQwHtLk97rDDTq9|@EV9mH5Ai04gM3xK19vryVzIWlcxL+spNhX= z@5jD;p77O9>Pz^{ee%3@-+t`rn*$jgRsKhG6MB^P;Ll4tu<=elmiL^3BPs+<2mYc` z=!;6q>iovj0`O9A)Cc}x;|CeA_1O@bpA3K3WW;UXN^{Q+kmCg&RAeZBo zL2=M8{JE#hZ>wZsoX0ojJMjit)MZN5BPBG#B3UrfUL&sR1uA{d;aR;rax$VM4VPxJ zup@)9P(_314$Q_i3(TUr((mteEi(gDbAifxV>#(9whkRM>TR# z;(h>`yXE-+FTyXc9K(;b_2p;tMZV(B8;E8?+?W`}CX_nx*>etZQ+|kN9T~$TdWh$B zN*GZy2&4)@a&*0>4Xxbs!G6lac;;MP2i0lX+{d~Jbu&ai;fxWu-UC?v+Z+x5iGCsd zd~q!wqdUVFle)G`EJOaXrtlzW%O_*x#tV47EeyUZ1gEs}E#0^OCehJ40&{cG6*A`` znq|a$xAYgg=us!Bn)HXrc-6B}H}0UXMC?(V@~~m34n}XQAiX~wBvKkuv@l`{Oo!#+ z!J)TU*lRz!N{c0CnNx_R(0UY>tso}57GciJBy_Zf>{GrW1(o<-o7=%24s@W)uGEV-B=XcuO9<-zX&Hi0?B&6TegpME7?$Yn`G35ktq! z@<5cPUO~EekDMQQndYz3l7vaGWdHU}5goV@SnrsGWiJsz$A=V9uoDH!N+kZcHRCWX~D^lH8izV6M#q|2?C8+-@%-Rha;A!$1BZIh&8 z%TMOKBLJ6@A7cEOYPdxl5qmioI`3>doi8{%JMzMi_2>d_%uhze&mc7Wbu)9prE>o4 z3)`j=REqCK@4{AWNEssfsU+}UKLWKI&JrI3(fetw&DA#5Vf30rrZ_p-F8P`lP4IY4 z#`KOOw+(iaZ(oba6W7P&_LCF%p{~Fe+BIOhXEA9X{DtjwDr9f|tzdaEjqFsYKkQeA zV{NGg)?N*lNX^w0#SGq7U1(hNiHV&!3s>@oA_X zcOD!6B%<}rV`g$ulvZQYsx)XUFCdK*lxc0B9(0>ee^&nS z9NsFlLTBO=)Qb+waucRgmi)VjsSa&pin#%#AfN%|`U?E8=mgm1@s=>fLh_~GX7b#4C&|cH zCaXS&ksKv?l71|i99UJZ~*A*%!(#|JcIXV+OOW$<1u<@codz9*HGY zmZIzCrQ}bGHu~)r+{Y>}7_T)I44R(!H>8U#d6&bozooDrUPr)Iiv9G77+fjvfb`Bt z<~j0@B($Phaz{fAQr~W(-yy+8tSX1h1dfprQuO<+B*JUP5TCREqv*T?x$OEdPRL#% zqd}z>pdtxvO{LP4hISdD6ltPDN`(qhB#9IavlK-ti3XZdD(XG&pU*!% zxAnWv`F^kK^Vtm<^LOYaEzh5hdLX#bec@&FUi2R?ke6Ai^uyzN>|oMdV_pl)UkewH z^m{h#$V>Xm_!x^EIs);>?~4v%0d|SMVMApjEPnBv{kV6Y&AU*{rmpRaK({SWlIe?| z(~6n%s&lN@v065$sy_}K4MAb5V2b>^kCG9?A#)^(4I2NJW{r8r&gjKM?rICVtfcu3 zl^3{w+Dfot-;vsmKJ=Pk~a}vM>mn<*h0A?My$}075v%S8*n475zOzQ_? zn|LR!U*e1IS-Q}YeIgm5GXi=Kq9B(Pio}RkmK!@v><9hWkyD46)3<%3!u=~MmP>Ps z`$ym__mvFZ`kOphbDwP1J3u_%^rZuDOH-#OFNm6pHeGXiHwI1njx!5BV%H(TI(uRu z(RlS$a_N%-b6;P_CMfL2y=gI6H?9zV(6 zJ?Drgk)n5vB2ZEOQLqP;`Mc11c>Q+6`~l)Rt&>0$huxHDE3_iRUy($NMMD$wlcaV>{jM4>OaCuBLWxn`wB;+K0UL}N3wtI9K0M~jq{7z;8vH0 zO|O(tsW*VlU8KxDM4e!*VNUQD-zWD8sc4poMDO39+0|8_Sht@tqkk!M>G2NR-qwdv^t9wzo}UZZSa8Yhn&rY5x)a{l8Ui9}ZX? z!l{{85y3N2BN)>^bKkL3uO%#!iMie@YiQrQ0Z)ex%s<-*H+ivZem{h%MdXuT?|VqR zMHg_f_?`@UmxtwES3?tUXgc668DAzbPd-m9z>ef{QZ!M>6)tx3jdS%aREj4>S4TYj|lZJdMh| zdBwhVI0l9z_}Uw0mnBE?=pHtP=3(c7aiB2>OH_!ENWG(94g(Wv4qB zt5l6`heyGD(FGPN=D>RvZ$s6~TS$2H8LHy{`$yv|8e(G^56vaTwr`k`LIhMlw?lD` z4!07Wt~E8+pzC#ny|zh{XkPoy;E{ro*RovkxEgC#eFu?6hlL~gGDhpR;JqT&S z$rub+Tp--m3Os7+Kt6D%0k3;gh4+dM=xvzCnpesrQSKSWM5=MKH~qPndk;RLPclgD zcq}&!W+SZfn2T=%#w?NI;m`gerEe29*k8c<@Cn%IBaNzRC$#8(!j!?~*cEvPQIGTR zwDLZN78hXQ`qRj13c;GZTUdEG1TSMxqi$0HN*~R(%H*Mdc{;`-BBAreWVC zX<*_6X!f}Poq#66^8bfNks|+V*MQ{cJXU=(khxdbp=(ew^uv1aKZ5(^&@K2*6UBb} zp*+T_<+F&^$(R)U7%grFeEJaK4vtaaapMZ0bJ`Z&Nn>zZF&>?fpJALT{FR$D`GK?# zFn7KT$3$^&s<{L2l&5$ZDY~t;-;w{a3ZpNI{prjXuxNch^sP#EgHQAJ`v)fP1ibKe^@_OQ^@WWQV=T!Q+w>YaB zW`k@G;Gf(Rx%!d8JW}{^%5K(x{JJU}GxgAWrpH%EkLRQFCh#M!n%rm4Q+S6;@lN5p zojy&Imxsx6Z>1LaiQH1zpi;#D=)!Mj!TH&(&pnPy^QQ}vQMS?se@7bQsMuS~pYsoI z4D`8LVlOUp?vD6uHb#NDFZ0~-n5A4x#gftOcs5dn-#grfg@vLYr!@*5(~hy5%*=S9|7a`QX_6)&Mc%x{-z)nU28LoDC@1cu^nF)jHi3P(D^#Zd~&E1lq-lL?>m z?@;iyLu7XZW6JL`W))p#Mw5;)Ggm9*tdB)$M;q+-tMK{%wPW+;RHXH++N#@PZ4>ARONrDRmp^TvM~nVu|f6NWXu*jwCI)kd<_hEKus6aUzMP3(?u-x zXo1Uek@pvD?{6wP+()qEA8mXJ|3RAks_;LWJ{reu%JsPHl6v%)&q7#O4Wi|hxn1&L z{>dbfPg@hfuXtr+rWT>;TJK5s_Q~Yo%FAS2O$)g$sV2YYx{#5^Pl=zZ9ulWWc;dee zd{p%@@EjEsF1Re|Z)-{A>~^y?o4uiNxeyD)?_N;&X1onM#wNswT%_X;;c{G$CMi2O zPf>?#FC%7S@ty@<_r#chG`!cz!kkgz7|~F{tYh*?$^3ixE!fHG32$W=gP4b_V|2GXlf@R zCRiA0&#npwq@mIX*bkq9`5_XV`(cG=8=bHr+YwWm`v^D5OnM`AI8${UjFo41VR?QS z#y74PiB~p5-K`1>#Qj*k(Hjn;e-`^jh8AU~OBRn)!H$W?#QR|* ze`P4)r@C~Z`unf#Qy-z{rVYE5pLRdZE@sVtS=pZEm@+Fa|Jn@m3dmy zTC#aymhFjpR}ymiKhk}093A%9iO#97Vj822koe)D#9=@J4eKnHY>a!#R%kt9macYe zf;haW>ur-5I{zhK3}>(*Vt$o0)E$OX{xJ_-BU*UXgD(7hgrrWKL#C9;f+vI{cgsGE zJDR{)<~lNZuN2w%r#F4mTt|Gfftc|ppm|!%13Pt@XSF5?y0DgZKHO&8^V|~lEO4>_ zXSqPJPhTipU&;atGKj$pcbcKIfX@3Vg-I8sVA`C-tW49{o{LY|wB%#VtI3ouophZB zbj)Ib8}q1s%5QQx>L}Z!U(IMrDhsekV)rzqAu*H2;75z8RroAg-&#e&4855}NguTT zngKS_0bhD8Vg=#rNanT(`gzSJ5}c~XhWmTt-?$S99LDkQR3Z8Cx;M=%T}>JXT(-Ru zdYW0x+lOykMHWW8C-$6pL-PFhkoLGy)V92t3{a6m(>Hg)4jzl2gQv5Fx@XC_(jZAy z#cgW*!j3Ka_>jHp|BU^yxFu1szD{?%ewpmP?+kl$KX?5fdv)~a)sY*G+ip0E_EXgZzkm=nTu zA3u=TyjvzAkNqVktG`NY#P9m2>mlr-VKcjJHxzb1(%9Gw{kQ`lIR# zd$S}23+II(DJzOK*zYHSw(rUL`&C44a*<@S;!^ywzlbYMVtFl zvD!5pR$|xQ6%+xNbrW!>Hkf&@@t|g3+euuC1zY`W8j|Kr#INu+mOk|$8~-nq$&`Jj zXQmiaw?JjdPKQLvrB*#jOkD|?IPVp$c(Iq=^FGK@U$nB$VH07Hpkv52e5Sj}brdZ!E9%r8&i^Z26g?c!{^{(X(KY zOSPqshy0XWoV!XAe9%|&{_QDw_rPz*+7QNX=+i|dFE>7ugXL^HuNWwo$!PKPW)O8y=Ju;VM9-B&+KDa@~ zNADyFH+RBmLmf=Snfh^d8+lw`O%~`3qYq40NNnPtu)A0N;3ypeosXR?CUzA4^=3L7 zG=B^-{boRMtr5m;f6Mw+yk(PLyk%1l-eC?MVz+O340rVop*DRUu6&xyCXUsR986PU zqs?D2{|)<5Y4{M=Ov(_|F%M2B%gMy-SAs1RMYWs5SQ_s$rjx2#~0 z1hYI@37+i{{g0AHEMM{iZq|43eY+a1KBrHY`3(~9&Pg!Z`Uv_P|DutWBBjU`rGe_S z*ZEJR@Rx*SN6*8Mk!AQX>mmGlh;CW?OSbT@0$X}SRidWj$ub|z!{vm7xaxKcpMEdK z3-^00P-OEz^%Q-F_zx^(;sEqsI|C!1jltq+)7ZlKD)d0%5BC09Fiy7nVTrBy`w!eI z*;eLDVe^cnj%^^WOT#hGuMSgMc4EJ*Aq|#jlUDN~RP|N?8L;jrDX7{_5)Vn!`PE}s zyJs7M)U0^7M3;AYy&wjOGHlKC3O3nJg25-mPE@T6udIX|OU z-c#U@$B1)ojXo!lt*BJE4Eq-op!BB?qQ!f;NBcYB9cse6>$mV;OU#bjuOjS09HyI3 zhEHR;q!M`x?mSFg2aw5bOKy1ro*wN z1!jV2J5*QrwbO;GGr^Z_cK%2AYZ~ETS}v*{YVj|w!b5pioL!r53-^~3yEmsN4G?a* znd2+5rc{>)KNe@b@!w&wB>-;o#NN;|LuBhW;I5<&@4iX%U1^_Df3g_Gz5ycV6NIM8 zk@&SK1|RK&|1q`-pGD>~A>lu?SsKG@MJ`&`^x(2T|6ufyix|Fvp=DD!`w{p@Vs=ZF zb`8|!#jPW`>(Dmr%Cuq2@0=mSkC>Cw^OUJ-fd@HW>_bZao+I0r-DjWT)OeZEQf`^C zjT@XTgCq69&b3Ndc5eqxo^FEo^8q|x_~sR_X!02G^R~Dcj}3!9iHwUHpT9tZ@5$+c z&xdR1d+sncT{$au>wl0DJA&tUTJdv%27HQjDW1NtMxbB|M`+%L=3qm9@}C_KyvO+p zbya@(wBRJhma_HIRcx>4M|{0FoHH*6-mW*AcRl%mb^Z4uD5{dB8NFa*B^ta}KTAIL zh!Jm85DXL1S)}6gfAL0F?9(ruKN3-sxt%o&qiQMZX?Ps>GSz}jJWk1 z3x2{(gJ-UK!BnyzGWSh;@xI~*-v1oMPxv_SYl6!&!22V9OcY)H%rf@6b1yn;8>}!n#NhvJ08t`7Tta$d~5xg<;55CckiyrPZc-`v4 zmJJ%b(npQUi&IIU6#2M9lna#*xv`ePLyHo*=>Az z%M#vW`~N%8_nBr;3NdR@p^r`mlF}Weq?G5A8qq>lQ}Q4(dSZ@b@(f<&cCG&<;NQba;tfo{NUh5Tn#yh zM87hqi|qCYtFb(P_jsPsIE2d|Yk&_{Py@4Fh2$toOc1rQjqDQNHay?|N9Pm8-D2Uw;Ocb9pvmaUTkJEMe35^ci zcihV6j?Q7jM~36z^1o=dvgI+qocNkP=6rZmlgQ5AX6Ls)VIiB2;(N3nKYDE%KecQI z|2AhB@3*A{a~BQ6hrO;CbhZI9w}Tn&KGEA6%5D`Ng;TfvQH)#yw z#>qqZDA9ZFTu}%2srvlK49>N6sp#wu;FnG|h76wmJH){*&zyNHszHtlmRn6?GT5|`s~B()}%My#wA932m?@+pS%b>_VG&rjUi zk^u{)82Fy7#DS+eJfO^sf7dkU$4)A9|E8Px?IF#f)Q3NPqrQC1^@@7J=0<1W^U7EF<@^O{$6CaWxK*%e>d|)UD%1uoLeGsH;D;?|HFaQ@AM&Z;@>xxYu5z-vSEBrf(1`D zRNyZVfXe%?CFifkGKXJh@#%*u4}D|Gf3FzEx6F~^m$oI~#4it=@-M>LjWS&BmvA9< z8SqLaX`Z$ACMNGv=Fdw<@MWcD;=grxsavJk^~B)Z&%_dd=KMi>3^%Lv;7Th5 z1FPRLdh7Ew(#8VG_gP-lQ#+B&ax$geBL$aaZy5PkqYc~7rNTShi=R1FPEv}aiTACa zgkejirq_?4tSkfm=8q{w=& z9@KERl4R6mYizkc7WJ|R*_SzHr>I}-IldDVD5p2Po5;ySj?Pn zs?p#jgK5Z({&dFXdu&ab$Ur8hvAaqdBzNE;G9XI0g}x4?fIy*jTPvcsf$T-;8UXG zCNg;ikL; zzmlgn=`b8>T5=s7Ej3>7@JYlcq-)80})9J6F)^z-_#iaeU8a0h}BC{$^ z5mRX`UXWgjZ`Rsqy7ZBBv|T0pif@t6C;ySR0}RNeEtSOSbrPA=R8HbnIkS5?mRNeu z0*+s$X-P~f(f58u7Nka#v6B*sLhl{K(PcTYd*MyLmfRz&Rf>sMuUyh!?Kp|w+L!j% zIzgT*X;a&)FUW*Q1*%>gNnK_+N!AuRN}{$tA@}!wAZ4DrMTaw)*d)9lT~cQx#{1-v zkn)_Rr^->+bOW;D*I;_X-=ChBZzhu$?O{XaSP5^?Y!=;?FP>Y69*PeqGv$IwSy||nxW=m*J zMI31hFr?9E%g0m;E>6Kb!6P^jMK*qjCBJ_EA=~`|+4`Z!nD%N1dgezURXuZq%t{nI z(oQ9btjN>LExyhU?ye_3F?9s~J?M19{#4(ioY+6~V2dVLVd0QHta+O{-FI<1-EibN zQJHtpcIV?-^6=qvrj#U)YY$FI=IvIeefzv2VbM=W+C)p*PrZ{kxJ;5{-JT@rXSIP^ z2+r=GoEv2S+c9Lcj5Hnky@nJ%j3xruM@y$4BM(z@h|%jX+I#;+Vtb;HP@Tsl&rFpX z=8dP5UJ8fNenYyZ-5l>KEif=( zOJ6)@3G%BDd&U@t6EtYqvg^b-asnMz*n?J&xI-FRi^#w`@wTP`{m7LlcPhViBY7L) zO5bKAkgaR`kPFjyl2gkkkr{8oh}DU^B+Xcrot4dDJM@KXRxeTFh6&Vn*$f)>@gI>n zGLSk26fs=)NBW$F$nh$mC2x&s{KS*QTsUZ!PL!tW3|vGmH4)=>xIr&>2wgdM0+qTt zj{1u`UeTB)ma{kV8+D+^>h8!iDGLgh~b{aX~^N>U)wuy~(vBUb)R&Y(eO5Cse zle7g7iDcFz+qftrj56Dc9a-Tx9w{7yij;Kx7)PaKt!VFBMkn-i!zq#owUX($G9--* zx?@lJUeqUx4p)+C>i6iY1q*R#h4^}X@3Q#U%_K_cF6n!J935`)g7~Qj)T#FbD!VC=Z1`^qZHX9+wpXhV)vJny`UVnx;r)qq{XlwDzap8VVE^?V=$MH4S5i44pB)-2b z7`&G2dQKBqL9>GuJndB|2jUvNn&txEtn<8-Khps9($j>HW>1XpYCwOPZz#1+f_AP0J3OwI6ejE^nxZjtF`mNopjkaQBeRa-M>hfyXhhXfNap4zu%j z&e4{^=_FJ1Y&T4DC*5u>*tM`pxRCdu-ljKIxn)j6nTf9If4-y1m6wGFrU?f9~)!DLlp%tvY$N9U2+Ck_?P)Ds$rYwCm}x5 zfXn{`kGnUDuh9MjsYp9aw8)n%tz&G%u4iy4vgF_MTzReV3Cgc|jb!P}ij^EF9RctAbnzQ23%!X4jH8rhF~N{!&YK}LL} zq?gd{O7VB6@55QlK|B@-ht4ubZn)Bc_cJx;!T;|H2%c_=(;oOd?7&&Sar}bt;UpG% z@EgL9(G{GGN3$%k(q|g_oK)wb32ywxy#+k&fh!*trp#AtN<_HjC{6f;YOmvp+vA-J356RND(5V)fav&L9 zjfOnost4EJyNCxkkK-|Q9dLKtgN9=VVV?2@!_ z{7TzOTXB9vDw3v`VgHpreAsyjcWkxg8$aFm_bLscOvvi;YJ(es*FTuT5PLD>de>i3Q^i^Qx?f)&rbhUw9TNW5qz}&y6CU-&sTh2% z6);!kZeDskUuqCPYOBfnylKGd5sKV&?`WRfYXa{mG3B}9`4vB0!i%BlNOFD-OUwRz zs>uQVDI}1GZByX;<213N;tH`&2_?qsG>Nt3j_r-9l+3v=M;@zBr2iDZ!Y6PZH(lY) zbvBmcy4w=mla+^Iw-mOCZpFILcH9tL>Ws;<++8IVyO-`n&==tq6IrF~n_@rN`zn-X z?Gv3qeP~O~gJ!JQnRhDjo*}Y)$c8dxiwsmy$UI5l#ewvUwjKE-1EQ# zm}#d%YxLifEjyl*_V8J#w&_HF!OtI{@eEsmFcvysgkQ=$1FR#4D)Bp^0~q{F!w<%Zr+~*MSnf`uiXc)V!_Us zS%JR;8_=@shVU=%#5s|J+_zSi%XRDU)HUre-Y2>*3#E`(+!HJyTD#l7w@h;w~P=H8nIX^iP=K>F^Tlmv<-rs5xvMZUy3-{8a zTMcA>d@k|%ZB2bP{a{ljKW1~EeJ2GOKS*nFf4cL06tR1*L$!;l!s&UVK(x(yktXs8kzkng!pf~MXm&^&;c>} zG*NhAL*INRDmQN&4V+y-^ z?;(@IZRS9S(FN6eh(=p5HSJwy+fkcN(k4-s<}c2?^?%q|m1ksrYAab*bD2bGf3ltE z&1uImeI#kmLgbeM79+f%Tf|JGW?%*x@Jo)KSQ<_&#eUp|{bcWZO3~=GH;K=wL~>O( zmi%cPMw~T{kx{$8kPGrNX>MW|sqU{!yB_}`!5j1>d5Tpe?Z!hg!8e9R5^OUWER0u zFEJlpPIf7Vkb6EYw%O4GsprdZ(kc8s?^Q383k@k`V7<7%_wG$L&Wa$P@}E$%fr==V zN@io4-;%ilf7{+pQJ~+o=LojDGL1Nw#a6XjK)0hrvgKC*QC%599ecLRXemq9GNOt4=WWmWgp!th1*AJEhAcYa!y;tz+0OZC5}np=GW7iea$0aqGxP&U z;p`)1@p&7{k3F(|GQN^{UG7C67-o?*8;gl+`4!0sr99TP_CFR|6h<_Q+DQ4cK=Re5 zoJ`+hL3f-kXSF$km$=Y_JkT?sH$=Z|`#Hh?>MkHwiiMK6G7AJh$zlgCjiB@2P9-Dr z32ptCN!HqTkQlRMmi9{lUYiQ(tc#|kepe%zU86`Rv7U5${%0adJw%2WyO4Dgortp6 zeX?pp8HwMMK@wX=lOw-xlYvR~jc1a~RZLy|PIHqw?#ExSwxe#j@+r#~RwPt|B=*=(x#W+(|f zT*8c3L?fp=2tS^gl9LOqiFV9P5;V4g)pMPHOW(dgK?8U+SPPn)2y-2685|zBYK;n*v zlIo9NS(E6Ad^Qua-Pu~WlyQ*oEAlkZXC*PO7GB&R#u%0}87`UaOs;DvoBi-ViQWrk znmbdOo-BP%Dyo-B!Uf}~Xn`IL7QE+CcOQR}n{v;BW6q#-|cLZ0ov24z5z6 zUK4A`mXCi)mWC8H&|iX{ZsAxY{{AIuUF2b=J6)I4MV{XfoZI0$B=g(8GpCPg*f&Fg zZL}&UGrbNI)4PXlul0AJGYoH&^VvCM;o2N>@o_kLV`)cS9I{B~gR_$5YTo35rX%%S zB2C}yJ43cEmmvvr^GK!UZq#Q*fov;b=L8Mf?bC%dqAkP9T#67y&RyYCjPU2FnlyHd2Vt1L? zHFhSTIPNh14BL**U6$w!31$O#D4^Q!CT0|T#+b>k5u|((C$99w)F2spHSG$SCNlsh zT;5{-@m}0raC+oPIQp0WBD15Wkfb4BiGRQW{0J52KSLwlujMx`E?o%jT1q{H%U7-< zS+J*@aQ(I(Z&Xy}Ee#c@5dOM~6;aGn?KG2)_rM2{ah%}z7)LEquwvqNR2`Uu%Y8WF z_Vj|EuO3{C=Ahw`E=GRW!qh5`rv`J-|H(Gg2c^J5c=Z+*iv7S14{W{>&nD#_W%h^W zK}(?$sdlRTb+{gn`t=6$j(#Ua8=Hvp;#|6D?LwS+_Xirneb6>ZfmgX6z|pM(=$vJS zq<3Zo*>rwCYBtL9kD=8COx<|>rS3R+?^8&`zG(b^&zqGpZPI97b79jZTvXGUNM%d$%VE3M3z0M7sakj+z-)AJ_f~+Kq z?W5Q=7gfys*oU)7eLi|eA7%--WQ9yF8Aw}5k4dFurqxU0G-EF&x*G7J zlXG~d!V;dg{~f#w%5cQ#EZ)pbg4(JdxHiI+8}}K*RnF>i*M84Xf2#@qdTQ_p)&5*& zxi(kwmE#7pUm?#j0gc9m&@nOOQ(xKe5&JB7>?Cbouqz+cOC};}wk`7dmto$fk^FtK z3;(#zijNmL&uf`WaA;f4+ZoM6s6}zQF`$zFy(+2$a(-SuP>)A-=fu!7F zJgjmWcPZg~QkgDKKUIbe)*-O;+=t8~;Jv zUA`lm^8mrcz5aU`Z;=Rg+_*aC+2n~lF+-@?F^a#A@Zd{U+i~?_GJHmx9>(V+vBC3} zp#F?3&vf%o}ovLF1IeX;s2(a@>#onpnFjgY`xE7Tvi#H zHoe0x=_TAXZ4Q4gc(#XA_8@ev0!<$&LvPOeK>Ce+LmDn0A@nHqHLwfN3b3iv!>HjaGL;$*Wkmz}zZ`#k|~p8XZ+7yZ%rQ69nj*MekOaH}OV z_>lqjJXzI%znc6Io+%+%FSs|(X4Pn1AhN;2hpRG8@O(CQAgihs%|)M)UHl%o!kP2v zjv?pTUUtzu$n!)nub$ORy|9nM82e_5%B7Qr^nJ<4Ko?l}Q95WP<9~+LYgkF4D zq6=SUI)kr!I+?qD{DFU!I|Xk#7!x-&W9}sre!112ry4r)T1`vd{HYFmCtk|R$bFan7JJTN+;*P>uX#O=U;C@W8}__}+pXuQn%09CZCB>? zJvZ}rYkau2oM1b*WuvfkJuQ&s)Rpi2DHhA!L zA4*U*ssj2gDR8t+g*@v*YvW*EsB6l5)G6`5@dcQ3T)4sv)i@XLiIGp#_?0-3Imo#W zED07_#PfKNsK+1mwcS(AC)wvpU<>02x* z)Pu2K4ef}#$vRei#-r(71W&h^71f_d??(naNX)DBh5w6Ke#XG9w^$e{qm>)|0V8*aiz?AEEqm3_?kUXinzk)+4FU!8}IOE8uTF5LHXf6#0w?v4@q z{Ou?;K3Q(pJB?qz7OW7dUoMZaVlmPrC@SmInM4V#b8$tK6|ALUwX!X z`+iDB(z__87Tk+IKed}ws#?+w8hz>UEu9kIWpj~ny$&O8Xmi_MDqQpO1{M-p$D+Nb zvP)r+%+7BGo_@HAmCuT>+$0nx;w&uju*dT97&u%=#h#DH0oM&U=%t0DTM{ILigMW^ zGvRR`F1TciD&W;}2=fvSu*ZvT6WzEpva_O+Rca5pH_y;JB;A(qe>sGen1wC zJVMrOcR<|f$57M!ft_|qP_&%Hc5gGJwJBFgp3X_4?-~HDf)eEE+(yxe06f%r#bTDF zlJ0TovEWObJ=n7|E*N9ugeT3iM29!c*bRS$Z#*xwW5_M8DBO;;8k|oPHc%e9}bk?l`=H z>~XK_22=PDg4G|2AiXFbPS00hk8dRFZRjlW!gVY>Y!e$|uEMt+(&p;Q^vD zNS*AkT|zbtT0rmlr;$r*E|6`X9+LK?LG%T!B>z4=CAV`F$f)n%iQI)ovP-s$*nhYp ziM8xzT_O+WlYF0y7`>PL9THD&y*^BXM+$dltvtTD?nIC6W8i+{Bys)bOw9{3Np#{R z^3-3Ja`%~towf(Ve?Mifv&8Q9Pc^AoVor9RGNdo|HAptBGe!O+Jq$mig!lu>O#NX` z`dhId@xI?dLT!!VzSb78g$3-Pc>o(C|aLuO^&yHBU-|R;WTS7jdA%ybW0wP-E}#nd}0r3eyh_q!9<@%S9(J7{;=qC zjX=qAHOa986Rk*gyPlb`#v=)%BW^eultY7*sOc4iCyjWkE&TWhL&=Mvdh z_sO=*IG&96+fOb9^`w8&C|P&(Ac65I@=xn7ac3s!q&{9L%L}t*g!^_D{4I8+55`36a!3r)@rX8oqo==xz4OXvdgUlz1?c(387j2V# z_OGR`9^H}xO%-OioUpD9qCawlVD4UP%saENMOzb^- zl4YhjQ_H*)q*=)lLsetjOx`rgLE&+L^{XcZ8t{<~@V*GiJG)SE>8 z7fo2P8dcx=n5_PgLS*H~ioNn?a_K-W39P$H)NOANP1TRKCig3B$ILuRR?40raUJG@ z{k#>!@5o`ywbf+l0ey-WPE=tRGlO8#}2YGt#piTXvWM{`B-*94mFu- z^vdlUq$Ke=vA1T(*;j^3W4|DO(-|mASuxXVo#gcg75Xx^!FI=(a|jeWQ3Wn?j(j7u zrh2kPhU;lhJsEm^My1%(Cd1M(9x`qVu&tw+$xEMPC;LUSFSMVviP4&Z-NZk@3!3>4VPs>}|?M6t$LMZJ2PA zUOJ0CnG&L3d5S2HNFXI?=B$6o8C-qx1^?Cw2F?|RZ*JnQ?K*?d#yV1YswV~o<>S+; zX7pNdxrh_$MBzDC+H$$UTNE(r(LzicblWI z?+StUo5KiTC73s(1Fv_y!{DuPcojVeMX6^|-2D!-egEKz)+_80+<<#&9*AxBW6hiF zQM3CmZcNq{J0EpkB)`9n)6?i`|?h4&-&inkNBQ?N(P0jlQ@a4q^Y>)O8qh66BE_= z>Ys=3O8GDuZ=OQVwN8O?qS!GFQ0M(;>hjb0i(MjjQh#+DYqRjcP_;B%9P|=lYlMd+ z`W^i3&*4wWAoQz=#fO4-&>heLt3f4DzIs?>%R?~6@hI|CY%qO%f01)G=JF<;FpN5e z7)5y;4Vox{f-#lS*Q2q;=`3&53bfahVY#LOZ!NUvyGqCNNpY94?8RrKo_h!F+MkHA z(&5b`hw_cH3_0!h34c!}VZs2>LlC*`ZR5>&NdG=O^pMD^iY#RAOgqRqreJlbF&~); z-fy%Cw;!&~r^}o|?e3e*WTnXA-L1d@FEb(2vFGXO7JToQ2E3U6on<*I(Ae@v?ASJ0 zp3-Q`Cv0=zk3|l5@4OuGd)CIR^6VuuQpVx1xf%CP9M1!Mt@y?9J-CR1Jw>0}z87B7S#7Wh$;7dcC@2Lb!20fGWJVNWvZ^|-5%Zh&s>>)io`4lOQP^XZ ziIrWgFuteG_pR3FQ*K@r>=}1V(;5odlLxWvqcl%*vf>AY|7TyVDK{814l#cmC0a8+ zvs-ts30A!ke?HBD&s88?Az7BMd-Ir$?pq=9@9k{doYxp5?(fPS_I#%}&mS=1sTUk{<=66&+UrnBz{R!=Mxj5J}17AEc zMTSlx>|HBiY}1Qh>{p2sg0CVk{Ez;*8F;Wd7x!&HqkOL>Pq*#CKU8KSQ|L_V^lO>V z?1`JjIV1ulI zn9XS!jtLza(Ej}&$D}`uZ(}-Oriah2~3-Vtd!-<>esG3oN#uG}sQ+zkBe^Qa3 z>U$4k{gw*-@>_Oz@^18AFY;(t+VUk!&3WOe4)lJu2H_tgB&!AIq;ATWFj4eIkNoJ% zeV8#nR(}OOzbLQ+1vg#9NsbnmJxAbT1HN;SB`=vL$46!^LvXArOI<&m+P8F*i0~r6 zw9@7AU;dzD+YK0x_rvLZQ7rDQ7wi3R2C6Uq#e+NbIGvh__l`TUX-Yhny2L?23Pz4+ z9D-N2qrX<17#;*e!#hsY#qoH%dndyB2@dGxdfd44SMUX9V&0zd?9$yRR`k&ilkeTY z{D4156~2R?hh(_P1|7+$csacu+Lh^4E<=ULuE`3sB6=fh{-9DV6OkJyp+c-zx6iPm8@HI#y)w%sH^ql+V6LO^dB+M?*>MEV zPC$&{kAID>ML=liJvmN2*}moCV)%9R8HW;CrhVez?T4 zNeZdb{g5s!r{5QLTfF~|r)Q(HU{0)Va|3c99chRl(LoAy|JTcGL-`4e5I%$Zi;JLj zP~=7GpC>t^+eza0VsgB2DrPUdiz%v~u)5$RvRew+_rIO#l^BuZIDHf;R!_l!m3I;K z@EF2IhT&7k4R)XtrBAEgl0v(Bw&g}C5}s#a?5+v;eccF?t(us;UJm<|t;crX)ur=Q zDEhxur2QiF*~yhTEJL}84WNeb+A|)eGUqV#Q7In3sAYpgN=f9rVbrJO28-Vu2A4C( z5EXnEv+EDT(QP3~tgxYn?(HHebFZjL4I(q@=L!6nn9ufCZzDy5<2577RXX2y2*%i*f^tAME{)p_%}t+~Z@nrr+;mK8 zao@tWv#}^Y9E1I#*J73fC8w7w8OBX2sx%Yiw!Ut?<;j8U^zv3(wDG_~(ono#XX z*S4&pCsmc%!f7?^WZnq;+vj!S>XML||358nQVG3|Q< z`z9Mi26!JMZwz9|$T8iKtDk{G@wc%|io_NRL7lYv;j&&)p5Af`6m!~1=sax)KCVrG z*O3UUkom}F&iW<&W2>aGGQF6yV64q;O%VQUH)N<7qu8>EIa!sk8R*1PZoZ=Gt31)M z`W@BKm1p<;O4#>5pV`yl#`t`|4ffv>;CKEE;%A$(;$IQc+Sx5qrDGl2RS|*dZxawD zcqs+#lkhcp6X`KXk>=HYB^Eo9*zqqBxV`8$#@l4T?tsu;%v7W=HFlBXVNGQFu_%@z zmx9h-d06Kjg`${)(%ZumiRDR0Y8ISM-o9JIl-{^ufd5FS{j6aw6O>uN`mJ=|zrJ*% zn>Ssk&_R|-E^NAn0uxX~R{To(r<3vbxGNIq3^oXQo8cj6RY5EA{EyOP{6F zkAAT2YJ;G6%^2qjikN$Pe|Avh^p2krOH^EjP?=NxS;c(?DDCjXxx2A2Td^Nc@*L1? zt;jMGs|mGhk+g-+;mL3fOx~^o%hxlRpZ69zwwsIOSR>CAL>-;4xDNi@Suk^rnP*Zz zI{Dxr`eAa46sNir{$=^dSec8YH$GUQ+Jja%cA?io)o5=W9r(PxjZ=^RB24`al#2U8 z1^s=k&*i?mE=9YMaC|@jxCdFglF#} zRxgN$lR_QqK!CK^=_wtc-xVH?4QT&Rf@OOnFm|IQDsHO6&G9$$UiOl8**^-VU2h?@ zZWJDGtY(X}ezUk{72KF)iO}N_IGA!BI&@B$g0Bp5^cUv9y=12WWO%t->1(c~<%M{3<` zEPU`6$_sD9Hbqfz;zha!x@%a<(R;7Nri@~)N->Q3;w7DuQwW~rWwrX#ZT$tMJ<|pNuHJ14S;N? z{lcdli?CUqs1!LpC$HGEYq1iwJGP$c8pva5eHfC0v$1bh6+VhJNo010=yy}LaFIC< zHb&s_rc}{`zXZR|xybny4m|f_M?`<6@ZT8ZiN0!eL_Grf7U7KPEcCT9WUIT(p7jxq@dBK!Ue96yj zq9yWw7L4po#vU@nz-2;f)vn1m^its$GIH>G+a^+VVWUg!1v@$`YAsI3DezxYyYZJv zf6#l%ay)QyCS_5N$;XiyZ0@5BSSNMh!e<2@a^fz&2gI`AP1ETk$%Ngnnu*OliqK!X zUhrdukMF{0l;wHSvw;(t|Mdve%*jPV_9duArefi~2o&m@K`iN5)`<+9SA3f3IF?_)D50U9J5hsgfDtbo(~ib@|Ja%t7jP@C31dnX z`0-cjJn)mI$%?yPSbk5D zFLtru4>}w0HA`RP<{=wI77Qo*wEvJD9ebf@*p(-|w&X{}{Bq>H1e_U?>C#>40;!LS zB~3l=L%~CztBW=MF#8s$AMk+s%qJo{U5)JfvXrUZ*X5^w_Tp<-X>f1F~`wZYKadJrxF4LB?EX_n+I$0t(379GAV{eUSRi+wz7j|PAEtQVI`{^MyaV*V)= zT*=yZ|Hce25hp06SGH{k1hE%@sU3HM?(J(Zn=8+=7nt+dJ$gLM z<|cadHAKIo^AMc=3PU%k@YRj#e1W$DzaltbeTpyO)y)o;y`vqny~O(8ScC8V{1xQP z1vH)?Eowg($!=7U7uxc!imH52tRLomt3{~44&PbYi?_VA z;hEig^Riq8uDJ35T5Xn#^Ij`PtZ?8vV;p!!nmNx&uZPT^wV3WBv25WNsNN=WAiq2D zx1ORdJ@E&(O{c){f*KW~S(nZ4GVryqJ6CIR;H6?NU3@MDUW-(yx|b~N^z0x}j1WG` zC%yPQJv*LmqQq-QEW+fMw@Kf1YshP@3>F+>z(23B;`OUU21!{3Rw|7H^V`I-_Ej>A zc*KcP!==0-+J7+A_Ku!XQI_ru=Zy8;zwO5 zPDh-?rX*3z2!7P-!u6Q(@tNRlY-Z2;3%*2Vg_zT*^L_bN+(6_KC-zKd3X8rFy$>(R zwZ|fdw2u-GaQiByCGmwR@cC)To5~&dyU^}@s%0kfa)ywq zqMo!+XA0Wl|G;{Z4cD`9=K0~>c;AIFklo4H=-F%Gt5uJyQRe(?hyzcwbKvPw1G}8}o6KI`gGdAK*;- z66~-jmMoi-q=OT^!3(dzZ&DN1I4Sczs+v4aqY|@TY(ziHERY--eu-)Gxmt#Nf}7CP ziXP#&L1W=;9gCi~%Wz?}E;kjv@#dX+{NCkKs19<%+I5Od{i8P;?kI5uqdwgGnkiq> zR)>m`{@7AlOp1ohrtQvq;jE_4Ykv3UmYp>CRJ(BOQ_m;Cs#A$$+i>bql8?AintW7< zAs8I4)MDtq=*=rn*2wQRdeeKQ1I;!)Xs(Z0un<;I()Jm$1 z9Z($n5pzG7awE?^yy>M9PcHSw4h>}%F~bS^%S&-NSC2oS4*Z$AF7KCFhKq}1(9-M) z6RTGk6ACo=r@>$9j%3&l>WXBcfEUdvcz?!2N2gT`IKKTB^Yd`y&{ z*3Oqc{}j1%FQf7Mc`8cZox-oKhp@^})RvhN0*`xPN@x%)cAP+{qsX?|bPO)Xdf~`) z4NQxe1!K1;Y!JMZ%J;1Zb`rhgE}n=SC&#>A>#!0FCyY)mfr;QI_8QxP=V`~GUhd33 zpYKI07oH$NbB4hErkL*@Q07T_?=f#xC-Ht&klZ>AT9L7eO!nRYn@`m!Drmv;hJ(W3 z;=r1h9_i5zZioSoXvEM$ujQTmS!x77jSx?w?02V%1_iy9A?oyYV{fAU0>7!o+m3W~++E09`d4&z~b{w-1s+-siA4 zOMny~ckG^PiXW*DnNBRQ!|gsBwY8bWPEBA6i3RMP&p&p5To}ur6Tu7?Ca@u|m0+(p z3nqdo;q9~>#+5x$;y9DW$68CPY_+A3W>u`oPKWR6Yp{Hk2Jaj*ye`Ql!5(E~_oE$R zJ#du;HWgyS_}>UFsf4|itjK2YpzcmBq^(eq%C5=4EmdW1BB#x@^lsqK&tZ~`6Q=}2no3iVIv!lgNH7jwa z!gRPKy&(NPb4gjpY?^vo26}^@VQSysu=`bjmm9Az&$pY%uvw#spL-^e>2HRjlWEYJ zdkrgU(vkXF4N1$?q&gE*X^zhTs(8OA@{~jHr#c1|2WKH|w-Q1R?qb?0;cUkJFm^sz z1`ZR8*i-pL_GIffM$5~YZyE5W{yytSG{vV>ci7GY7KY}sr}qX*n`^D46l*onBUyv6 zpv~B3G!M3B8hHCeXkT84eAsTA8967%!uBcPd!`|>y|UT73UBFf-Bn4kB!NYRXkcE* zSnNA+05OLHFnN+UJ8&t2X1Fb)F3)~5orq%?^ST*thc|-DAB55~S7tQ4A6X*uPM1xL zM)~h9{Hf3Z`QGWqRb>vq>hWVz^+k=e$!XDfOGH0^nIms*rhHgeRlawb;2f$oleRTU zG`*&R4ftutHTrSh(IL(Y(=Q`)|1XwX_nf$XFvgF(3e=Xm^9O=EsdnTs64TcRJqg9R z&UPrzbZ$)xHOV*PGE{uFfMw`=)*R@9hqpsS#hBH>Kn+#S?(d1a;h*oXGYs%$Ix%21)N@ve^<{BOE-c zD7-#0%pqzsGuMh`9!KQGd#Zt6adRQJVKWMi)?hkOMP8htv}(dQ>D^L)oL4Qxo8!+g zsM|wKJdqCJ)Pk!@JS+XSo2}fv11`1&Sa|pj4Er8O+8;GMFz}L`o`$ot;`8RfwpTd2 zLV^D|`buQ@i25&fCi~f~mCS30Oab>I+uw-`zC`WVs=GB=<( z$eOo~AHlD8AIRmhpW)KZH0fFW8DdawAl*Nw%Y*8MaCT-CpSjY2x6a%P+5InFyl-35 zf`!G*>7fnZ@QrX)3o-u~Ajb!vj>6HvK&G+hAzO5%2q#Xt^6Vp~{7_Xb?Ayc)En*KA z9o>x>%Uy6#xrd`Fgx`(62XozBFy62mJGbn?;KI|`IkOfUR-1}*uE+@KQHQu1 zAhx(T3U+3pI=QBC`VJ~|68}Rp6MsXkN&hrGLr1Gz} zG@@i78K!YsI(_IF)_ig1i^4{5S66GU-Sq}kp1IKhKMRTVG?A_7`Vn__7;|1~#+Tdu zz@kx_sD7nJric5nkxK<9yXX~a#k%f*-76dzVTP&k5v-tfvJ~#ChC$>w)JEKapKk#? ze#fKXQ8L@f;)HKS7kVOBh8aA>g2T^n{$2qJocxh=+(#;UoFmEH-GOcgnz35U*#=$c z%8mVmezhnDUydfQZZcsAt^J8X=Gr`ZzaGCdt_3S*9LFoCY{o4@StoIh+ADm1tBiZ| z_#-OZ;(HFfmuRCVt%&Sv7>c2yRvpdlxqGc8uOUB>r0cY5ZG zF5i05l>g2$<>4}av1Z*$xK6WYH3L0poXG9H(4@~xj#!Kud>e(uCGhVm&t&mBJ5eI?IuFere5*R(*%ALls%Fy$`*j zc89F=$j6H1)_i2CEBC!?!RP$AiU*I6k$X>WkjQ0IaB1)l3~{#N?Y8#3$WW2{o^yxZ z8d)l9s4cQNbFn@^g>SI#&EIs<=JHF9Vb8XWtnyVBTkdxpgC2dygH3vT>=144IN>Ke zeZz3><4)Efdh|7$<+#heuKdV<^>k1(UWM&Itj<+w=9uqNc-8?Vh#6sgQV+h!#e}C{ z(BV6T2k>fR1p7HG14C_exV@q+e;^pj=jZA1rMK@uImHlPPyb`S_0g#veVu_k~ z%`1MQTA?4ZcKB1vdVQL7If%68Y{J>mUvM-zglcfC-Zvx78RWS;qS`iHm@eLk?gr(o5oaMR@7 z*i;h4CX70a+ZkHC>AoTVF~pWfQf)qGasi4o6>#=pAhcarynphkTRU!N%)lpkLB!==Mni zcRwWM?N_0{enu19U580mds*~co{k?fV&?i1=&QM#MpYLPuSiC0rxucc)qAi()FmIT zuEXcqz>HOaQq|x!B&yp+GEdWt)~cSz=0EH3&Epdb&m7148mY1;@61_{&jPk0#0EsW z8GXN4qki;acJ!1L>*=n-bn+*#(WIX3idYZz=v;IkWJEu2%OFQjzb0ite$wSn411Ps zK;}BZY&^XOSig%bySaqC>5@a*hn}JAraCAx{$EQChCnu>gu7M~)Mf>n8eEN1_1 zga~i=WKo}|9nj}Dm;A%HE{WJ;Whq%ZCDSY6A-LGY{CV3J!fRveY#-IfR~Xk8Pc6C>btyF+^S z`y0t?nNO8;Pb0LT4KF^`VY*7Rm`8ik*b71APO2Oo`fCIma8KkLAFaZUt;cZO*jCC; zKSN}@93rb%C6o9)o#DJT6Bf@d!0EdR9MqyoXtoNy*r$U8aV7dIio2Zs(nqc3pXj2AOT> z5Gw9(F<-+bh1E+vDim4FzANl<#a4u*Jca6t)6$V~m8AKp;0`_LM&*O=vC1i31b^cS zZudBc=SB{6(q~O-wbqkXCgzew(^c?Y_9DzrW@1pk&TurhC92(TlL6@$Nf!lY>0{6_ zd@HWPlsTvHe#2Q{dP1X|T)&&GpR<`g3i-;)c6i~)y0bzTQ;Vie7jSv48SI@^Xe6qW}3MHYRo_ z=`y5+bUo@%-}ae}>eE%I-t!stGGO4}#BQxnlgKzVY2w`kHvVoi+rQosf29%(FLQ+c z^RKLXaUwhCY%Jg-MpFIZW@g(Va$dL!Z~dWy-HR`{T=_Jdv`!Ix6?HlGJ$VsGR6APF z{(@K3HkTM3S$e;16&Y+IOFua>D2)@n*)?J=Qtk(xV@)JPwVI4o{p#Y?lEe~<@50<( zjt@$?hZ75*vpvh65ak?Mx{St46SQ{0_C*u+)cwMTwp1MNvYF)z@1Vvtp#ya{N4@d| zxL>G+(WkRm@nS4Kqzgad{!OguZ4#R|EewTj8K{_a7#X1^IPUp@CC)g)67JdJ?>rmf zH9vy-afP^=a{`;I?GWj3loqY3qAk)F$@An5rkdEDT{|?KEi=B$40?CL${FFT>SijN zF?S<-X%@_?CP%P0f5iIXL>%+?j%7atZ^}zwLU*-LY^j@tWhNcWa0+9GedO4=NgpNp zQW5jX2^_dwj!`1-F*U&nb}xF+!rAKd^QeI|e(P%XL8T0L{Z;sUp)<+|P(w?vP_lnR zEFmp-iP?mEFz_|vM*6*Ye7n%oC|xGG#Eh058cyvLOG)B*dtQ^`$Qug;*T~~1b>ytxR`TO`G0B?UKq`#(u)n&`$dS|W zG#{w24%-4l8QN4|NP zJ^v9`Lw?`;LoU9vr#4|BBvk%9bRU}X&x4G(f#p3kj<`$=c8nuGzIzi#Pc>Lxs)PSx zaZWuY<}uMBEO_#LqCI#R9bZyMJ`8oj-oh+ANG%s$pcB|=sfdyBpC!!}CAQLvvHY(8 z*rn)Mko~d|>vAOY`E;MfUXNj)jJ7km5sB=<&oGvCJ(x}P-N>%YN?~UcBiW->%Ge^_ zPe+Fl?DLE6ET{B3n^^WqQh)S-YR;*aew?3;eldku-M0z{>k6=Hmk(?j$KcpmWmwwU zqVb>&ri%VhU`Pf=_l(5?d0m{)R)D?}@cnxdA~Y*;Ww0uLawK76tfw;gZ`i@Wv3+a)s)XsF7sdiwC;gI00#e}269qXAFu8i}m# zayTG-;|7!G^EQn*{@Wm)FS|2?k6zUP{@xt!1%1)=p6H(lZs78;Lp*%xT7LhZ9)Eg$ z8?a_Q#`&u7W0oQO96ieYwM72A7I>b}l65ip0;8EnhdO9?~^z~5g{jVO5JwD=0 z(O1+h`~v%T3Vgphc$V5G{%gchK4MS^?>bJ*5`EV}rgR(DDR<*@_pIfa+Yj*zo%iyp z7A?N*qL|GNw1CUx26&{-;JeJ?c{i6heq_>oZnpRmzWU0+rqRT6ObZwpJ=H30Pm83>r$4kwYr zq&dG5mwb1l{T*;KTOM;fi-FTK5dCsD;#;iI<%X@8O(`Q%%=_A=`M}IZuvnhG zfNOps-oNn2^WAk)!>cPSw<;BT{Ms?BN%V`|_4(Z!FVUy3;9|z#WplsU2p^ir91c|B zURgT)Qm7nHzMPKlMH*sWSKV zhwv?>L%7vYa~`et3d2;^z#||LD+_ga*%L3GaBndWh?v5wRtrXJ&TiEH2FmB&g0xKV zKm5YEP4+teQuq^XG~Q#?{e`%y(I4Sm?gE#5xL)mAo-krI*Z9|;2S>fY!knpyi_JlD zPl?wo4dY)=`SYY%Hr!yZET0%HW`R>>dAez59yeQ$xB744io^7Hfo^9W5hcqvi25~F zZT$8j=&nt($|i~|?wIXL zd~u@|FWuIRImcr$ROnuJrg^YSwc2>=@EjFq)OcfzI{#+=8CwT$z`F0DEUoUWlr>o& zrsDjl$Ehu~qE?G~M;F#p{gpVDUPk30RNKuRkF$SY#b}s_~3o&#}~82Nm*TSnHBU>~W3I zy&P*szo%NfcA66Z^ZqMxj<&J;=bw|qAMdc@p+&g9qX%~|w&Mk7dT^IbFVMBs4ncMD zXlhPInfotvSoYw~Uk$jm?;n)(iAAc&@A@sX1w9g~@L;M2Kf1=42P{$M8TqF$c6BdU zZJ3MZuMco=OIPlxYQigB40+&o;X9sp1hemqV4))m5x!nu@NB#Dz9Ofm|FzD1x=RL> zAOB>|6WUmCw8*fL`zKgA>imLcFCK0#c;+rI@GGc0d+Ae%+2a(2mO!7gD?NCP&=Fsn z`V995ZN}#fLy-M16}<*Di9E2*yoX#fI*AONN5?&J+oFiQxh~dGRT+3${sj9p@4>&@ zZpg3ggk2}!xzy&XNTbPA7*w5v?cNJmb$v6G##*zhCq@5Lej|Be`<0BLgXogOnuzdq zgO+mz%ULj=4ye6K2Dti)Jo($?>y2a@7=D1+jJ?C|WDaC+YL=34` zl1&Gz$n*Ub?DA(<;Q`jfBpcy@$+}4@6>7*#!8X}A#D@KRv>Cds7Z5f46xRNkitEn$ z()1UZr0|OzD&v!IuJ=6z89YT_?@XNd=7@f0idjmBC)V0#Lw!aw*4lUGL#8z0;Ota5 z?;3*GVVf~k=u-}Abmiruet!|8&tG{c@?oZhFn!*gc{aU3pm#TJ;VgW`&g#6(`5)$$ zohc((DMKo)i8Hv`&{{93FD&s=9tBTm5S*js!)-huC<|KZo# zX56ePz)-ptE@J$ZeY~rMc%%f@&It-(b5fp$K zdpqOEZe?0k*qf65CZxzqmSz7-Vn#QE*xrsJqO{@=ZPfFir5R;p!7p2?)8b5K>Fp-f zFRqdDExSm>rYq#GLj@@w-kFlYlgTq<1L|`rh3KujPF(wF(@}pS$;-1Z2ku+&&t;$0 z6H?)+LdQQnOYGIU)6(m8X@{|9)*`nd2j} zZ}=9Xa8{RAI87k^-HOP=1JQ)9BV_Bh0Af-bNmf2QNiG9c~2U!P7&HZ3Jfl66SKWJhYxP3f%%W2m9*bUOLQ1LBmtj8#wA#ax40 zrKy)JsLgh3s_vad&TsCFma74HH7p9r_b0(Kp`4j7v|}A>6tKIKnE%>e#O!tA78zyXpYG zU*CazQ-9Q4RK=L{xva5RO=QY?<3{CTxYdkCQJXxJlx^AOv97f7@Ny}&ZzfA`c*}Gf zvzXugVbWkkZzk}$Sl+Ov~PYm+1Ij_bQ1lw z;Z+^Pzl#r<!4^4?r zCw1LUkyS38=+571r1Rq>m!-ESke>O=NP9~SkvXP9!vbVz)roLY-CdrV&od-FPOTLuZbC?t z_jqIA4>deonal3FtAbqjN0)Cqps-WuI1VJ@yTS?tZn49M`Jbek^x2s7I~AebGx1ii z8vpKyLzTHT5^f)51*g^7`>-Pn9v|6m+Qy#S{bcipUS(ycYoyz1s$+oft*xLBZu>&iA*J-*?2*$7Q7-842#J~KRvoOzLxAMJV)wM z&X5l@&t>IdYqG0QiH_f;MH?c%l7UMbge-q6Ngj8F_za#&^g2c=l9Ph$OJ$gZu^ zsqKgu@@7yb8Q19wak1J&Rtjd8<0=#S%O#NP+tlL1IvR-i>gR;!T_p#G6cDA6@-)C! zjlSK`jWRh-e=WaE5_&|ComZBVm?Q@}qM4kA_x2b|7Adz|DOHfp@4~U;@SSYdU?qHck{oVzh{rj zqte9v|E?PU{lWin`Jb<5j$eqM{jx=i|KD5x@Bc3K|6|KmaqoZMx^KV#_qPB0tLplH zY&)U-|Jn9`><2FoMMc~H@gSp4cK_?)o&Lv9|MT|Z|F~Zv{)+!DV!O`&-NttO_gzLt WR$RS26cqpaxs&+%@A3clWB)HL&Sez< diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_none_r15-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt b/research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_none_r15-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt deleted file mode 100644 index 7b5fd3bdc200162df6c0d88f92342ea2276ddac1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4587 zcmZ`-2UJwa(ngXn1Q7#K#k8Oz3kD3gtMS)Glv$J@CJc*$2?Pg50YTIO1r%k)T@#WF z1ThB?rpgQw3@9QhU_wMhMp043@ZNBC-yV1Wv(>jx-R`dH?y7UEZ-3<~R#R6|(b7`+ ze`KIysNx;8aLLkDzCZeSOTBGY`1wy9udg!uZ?sNzp)^n_Wix-Jbg6%^Xnv6If|X1C zedbGj1A+sC=C2wilBif&&Frk4KUgX0z1UaM=?@jrME^iL1K;&_vP z82z0G7SA5%08##2y0!(9G_&B@rDTZkEdlvuJ6_=T2`-s7a0nIibvf!he@-pDDp$aw z;~d(@`tzr~HxUNLQs8M9+Emwv=1VKbuyG>SkTJc zeQ0`h23}bw!IRqgIO;ns!%lR>xKPI_v4nE7GzPB z3?J5)z-HM=#JM|AJ1Q0Le9Xt%mLnJ$(!dUMS&v#@Z$Q9^bUbR2fR}yN<1MRn%nVSY zRBd-=U7Q4cEj*Zif*Lj1rJ~!bb!d~l6(elZ@&4xdh3Cf~VzhoCbjfi1{rX7TNM@ZyvPUlu!0F6t8q znWGz+^T&H^^8KgGD^vtAqpV=$OFO<{Xd6^cT9q#!I9RX{>kE%!H01XL8G1eUk()P4 z{^jEH9{oyJG;E`tsE z!^^E8x!VShE{nLIS5Mxa?8%dh9AV;hrTuL;plw$nOny)a(c87TYm+k{G_ZkR(Ghdw zHQz99s1wyTIg;5`b82l$!$JprDA$$ZsZ~9>&(t_!VD$&Qo8>@Ht~yik3o9}>QO}YB z?0LbcXH4gU6`c=rCdp?9dKS`(hu3XT`mZOym=%PdXXt_B?lkOk)SS!|jxEL$)TvU9X$77a$75Krf|)!sXTh2}yk_u6eEDh^EqykYY;GD+ zth)jw6I@x-7X#kibt;R`e}&P-Mig!&Q0?g9)Y#gJwmBkRyxxR;7^7fgYqz4Y$!+ZG zuS$9+tjXg<09DOa=d*@9gsF?d>09^Cr1@(SP8|}Bdpu;Au_>N4Y|LU8BimqP8>grG zV`=MrHPU@tit9!lz>N37f?=f!Upjud&>QxnkXVe~QZ=fp8B0N39B6L;Pmp~jo9*%0 z!iEV^Se_J(*)MjZwR>j*i*Rx+t_59R1*|TLAj6#^db>6ai=x-#^NZ=2@3l)X?Ayq6 zT|U5(U@Pszg@9tSbCsswvHHKHK~VybO# z1;0Lx%xm{{VamjG3^SLZR@W4iS&OLlU<9QkDL~!LhCk`XsGvfJmMpBrHColUElZt> z@~+8$=&c74s)oEzu@yP1EAZ~)1Gp+J4}0WvrpP0sY1Tt$ZgOD^txM3PGoSLXCgcDb zhcx4i94nH2=*p$qdT?qHcLVXR zYfP&-l69B6v0&2>_#x{steqFd2hXc#W;*Wd|5UsMA#xdZ_ZUMl_F+V>g*+ca>E)_1 zl$RHP@Avr#2_EjOb^3Frs{HzV>|PV*`Hli3(4@t;dJvIP*i6 zh5XT#E#vZB3i*n6&fF|dkDGi};YTVA_?pS~{LH5+z9zTtxnW*Ku=0Eo4A+T=VO9wc zKdTT*o9%h4zoo;)9y$EW{YCKhYImq=D`UCtGFEEj&RiCRz@yf~us|8dr+GblSmDO7 z&_^(-U52Y1N7L~8p_FD{$X&;Tk(=un@{X0Dlld~C%WHQweZe!PIV*}6wH<~D%^{Ft z=E@?p*0Yflt}{hdH_&ff1Y7NMcn^sUFR)>hwNr;utZLA<{620vq)yF`Z^fB+lmqzQPl2b#528_> z4EZm`y6~88$Zsj6U#8KD`x%$B(7s z@#x*cyVZ@%p~nX}(OpcFQ;ccx^HS7y%ED;NJj}hf48!i|@sL}7=%9WUCl+Pk@{7fI zW{EL z0P2aZVt#iux>qTX!#)SPP~Qe+dox*iL_F)?D8mxt4OqA#2~S_rqy^_!laKO#HOWt% zCshQ{z-`uKKU0+LOQcj}n$BTG6GPlhjm9 znb)kSth%0Qx!CizoO9jG2YV*9>m;dNk7uC_QB9mDlt=&4j3KgXQ1 zPfVn`m&$cUe?^NSV%|KsfsZ=t%(o_L^Yhu2ke*lw-^y;l@DdGv=b#fz-{r|~hxOzW zD@FX{g*I4J*9zH#4Eac9{GR$)#b3Ha@#6Zf{CQ+6kkZem22O_IR7CR?{{g`aV zxP34VPH}>@ua?Wvc)pypWw>wu`+{;CBjLoGeu7WytbFy;aX9=iXl@%XY8_VC)-!+Y6Bt=ufX%6Ccxvqq ztZh1j`g2OaD=8UHr(2NCoK$?G@WO#(LeS|@0X}_J#m+|S%BOEGgh6G!XxjJ+tkB$m z+goI~c4ak&4O+(vdd7ngxt@)iSB*L;G7PPV#*DSMG3%>2-L5EvWzXIg{OW#>xoQ^R zKCN)fS?GnCn$3Q~_S$I)ZeuH9 zMSvqM@>8R=5BK2g$I19^UJVuxy2joHB|ysCYfP^C2&Gq2ux#O8lv=9M&Nq(aK$UR3 zpOMh2U&nejq@i1vnK)J52H!j7Vz%EDI&mx$zWmw(#rd%$%+M$OWx2S7??=DKzhmG_ zZ7>m82o(j|{Ib;~y0p)lOh}Uw-)qrMi7riY?*V%1h2SCU4o%N=NaMIR^~lnslwfQ6 zq2DBG57p-L-VGH_P7^_P)$jPQE*%eF$iu4Fdh|dVOFDO2AT+_B&w3X}13$)6RL~Ua zJ*qDq+?b2H2A0s|7MNdm)0q!G@C}Wg2GGYJ!YOD@1nqDZXylbeAVGWHuW1qZXA9&J zx`sYBuBQ2KmeWrwzhY^lGv5#%ogaS15}XpxV4JiLZP+}8Y?OGc^hO+=boJ*me;Mbn zdX7I2a(oMS2V_A=LNZ)cl)!>|JHE?M8GmLC+@zD3pPQ@Bdz(IlaW|Fcu%8WY>z%`o z_*pt^Q}RF@u&nEbCY5Ilq_hb7U=8m z&&%eIbLarq*K96$s&`RQ>8Sq>uDhr>`~|L8`340mLHoqI?|}Bdybc$2tbQwH zv8SpM)^}7_ai!+Wu4b`&}vB@*Pc6 zZsdQufBL~ytf^`Jr$}$Lp??W?Bttsre?0GxrpoOqKt2U2N)=Q@DQYr1 z*c*zVf?bMOKmmJKZocRI@BjYJxzE|jWM-1RcVojJ&+aDcGc-hDwt{RO zuzGEX;-pxOun?tb6XYx=bdx=tCOCzKC|k(hl|5y9-NHgtb_N8jTCpN1AZ+QX;83+G zYXXDT1O>V;4G2>UQML36=^hrM<`vSzZt4UT)lFx`p#`$3|6Qj3uVVL->-*ot9x~L! z)$o4^{U2vwsmYK^Kb=tN)~{yl&$JYL8l=g~tup5O%+G@onou3C%B<)VecYXeri(rK zYIkRTdfp^HDb0-6bxuOZ>Mbbxnt=$bDs(Dq@;?$C_}hOy_}@)leAfGT-m>K|&INkl zn9l}~a0wsU<@hI$mHE189sXEdxKru@9ll?=GJj328=v^`5|X}e!0k^S_;Kto`ZdS% zpR&F9$?YEeyO$1p;tvhJcgs!uZb-))`7JoKJrU09&G__5pG$=gu2XD23|u!BNVmFT6NDhr#QC%j9U zB+*GMCH19dnEF}+E4^CC;Ey`LE>N5O>8HY0uNWnC_4TLJ(mV7(E}g5`WWyG8Q*v@) zHtcz?BrbS}dx%h`lHQt!BJ3j)j>D>T`e;XpW)f&U}bm5otP|~IJLeNee&Mds1aBHgKY0Tvo zYID%X?bCm$=B^=CUU?`s=}=0B_2P24Nq5t2(4;}+9BwJ3>&@Jy)gIfh+P zzlCmjBLz*thY~h6Q^456B=z*+`Yd!6l8Tkt(pOK!4=VeRr-~e$3-vK@>kIO;i>K!; z&$#Zb!`Z^v7QrUvp(IvC7uz3NqsrC@>4*A5AuABy%FJ2S+5M0=OvSSMMtJ|ek&2G| zp#GogsLp4pO>wAFx}H%{Z* zLiE|7MV{n3rk?sn<&#?4cTWF*>(`?{w{J z6aAWQ1RMEOOp;5%-fJ5Ca4LnvQEOPA{!S|r+Nf5(iDqZ0P|>k>!U;Vk=-Ju8a-<6S z$t&QWc|G8wSVO~$gSkQ7w{R)bpNn;O#L&w6vovGUFPeK>52G!f&_Z2a$bYsOXEmFJ zy{~e)rN7gtVaZ#%*uwzlKWQN1^=NVYmIOZGCW#mNX`ndG0Ml>0qoFzJq_-i9o3O80 za5ve4FI!!OfH_Y{saYTKF+XYVDlzSKjiX2DZQ?_h;<$5hOF3n64aI2nz=R?NIDb`! z_Fo$u+@yp+w|2qTTajL@YNk8G+NiMacM8h4MxbUXxK<5*S=^|E1hBc+X}(+*AQ*!+FPN-K^Ma19*Fd`$AjBJ@Eo^ZnC~g2-hZEyLjPvE9$!aK2IwH+XeMrX z8S{Zl^KfyAF9p7QBC#XRh4pF+wVm1stUOKGL1GquBX9E1|m1Z z0~v!o5b$s#Cam0t@7e*-`sE6d$7~!Ak3qj5n=n_T3x%Rzs4kxa3#&tzGd>3=2A@Ph z<~8Wm-ov~#EAe}?20!z|O6Z&2$HuH{*ubBJ)rB0Ku{?w!F>|qF#V`CU(Lsq^40Kk- zV8Mr3=pN?+J&y;P=ANlnW*>`5!&l$oY}NdiV&wp6l6eDPv_}b(#}-4WLmDqtNeh>r99b!~c%fNoKde$}@=v&QV=6Co z5e%ih^4nR#nWL*O@nzBY+H)HG-7qvc?H(2}kHsmaYcvr!m-r9*IEuFeb#nuUfeZH)8yB#Y{jib4Y*A;7}3~(X`U*4 z`TW8BA}b3%Q{xbyGQom>7&(}?8L7fcwHr~rsRn0VpJVU$H}Epk;@6w}#@sSBey2kh zPAN7*d&zwqmhV7gZ%y9HcRP0LJjUaxP1xjm6FWolp>`%2-E_nl;nA1(weNBUq5k{^>1NVJlvjlHQ*q!n5WE?(pOedO6VsNjvQkxIrGmaWiIm z>>SQkgwv+#0PbvD66Jo@!iNHPc&AvSGIrFknzy~jTd^5AffG1dN-C$nU7DCe@|MgA^9C1&yXq=reNv2=(=4_Fi?q5b9V-(1BtRD6+si%Ow7Noe#UGn*s zns7ey0{2K5C)1CSY~!EaINzp$h3m_S^qaZGOnZHt5`}%yG*qTUF ztJ}C0VxsCPM@i$IGHef6!f0+d+mY>$spX?-#NIAPv-w4w^{$1q$EAwCkL`=hXX`jt zUrkdItt6}e7z&>(RV8#-n-mPn=}4D4;{W*Y=f>pOWLc)Mk%{$W)To30GZY{<_$f_~Z=qcMAyB&7lRrA#nOAb(hK#>@a8&$8 z>%?kEnPq_qnWng$wUss6-^7u6EjSJ;C%c@Z)TUiU|J3w`LDv*m(Iob>D;9$ij?nFT zciOBNM_2Eckj1Z$bp7BcY#PzQMM>4r-tm^Zx3qw(`4B`U++7;0IT#HcOOcg1n-?}L zf&An_X!W~8|70$w@k{eK$A#~?kKcQspz1SsVT?VB`+X&!wWsJnpIAD$+LN3w9j0{$ zVi7!WGAkQB1r16%c(&*qiWD*v2yQOmWH95-(=;z`WS6G|EK} zYnN<=@(O1@V#g%D-FqjL>$<4jzJx~qyhCDpnQU!ip!IeF>%OrGQMK-{{i%uR>r{}i zQwJZlhQQr69v>&R3UO<)@W%KPIjdbK#U-6I+EfFn7lxr%bT}eEH_*dNK}g#Bm8>^^ zBdwjLkZMiC-Ss=M*`OE!O&w(;_taN&N z&I5}-_Fx9}kFjK-6&6g^f#NAExGbB3E5`A7lz9o0Q%b3=q7#dDhvA>&zSuA)3VJ^K zFzS0D-mH0mDeEP;=G2Buk|He6%7L-&3Cuil7BkP^#LCQC^siRpE39iVE~5%^K4bOeK;zvyAi>?yP8kd`!-^uO^nd3V;gpF8_0%zjAZ3A!dY$NMAmz3t8n4g zX&PEy8(C$DS0t`yFL@Ki)C#={i!w zD0}Id`@Yi7ioMbh%?@d7(_U%WD_?12puP0=79DBu@OCy>^(dLc5$EdX3_o|iKrR01-jh{96hWyo7TQ?T+%pWGMaTqc)3k~->af^FTddbTCXQ2^qTiXM>=u; zekP|D>L?d!l0o1+vq=sc&b?m$M?he>P6CwiD3 ziix$sEa_zic4UT-|C`O6+;%f+o|H|!7b_v4H3+*K4vBAnvBw^<9%1%kPJ4h63BOAz z;oVRS4^M-lg%)r9H5J^Fp?KAHnsUud=wbD2PV}ue>GyHKUWH>~{keV&T z<6v@FyNk=JTt^dUW`MI;#e(OAA~5eW8K)m6jp9-Cb+(vWEQ>Ri1$5y3?g;kz!9sNI zeoC7=BFU`RcCKN&yD(@?fnatq9p=5N@Yg*+vghPbc8(!^5qIsP+MK4b) z3acN?5Z=G4lC+#w#ql-gg#jN#I24;qeru|@YewOc>1nkRzu}uWZ`t>**igj$&icW; z`5GNLJBIkAexl#SDuQ@@95+4iK4y()tos;OI2Uw~r>U51Q=-YeuRC`+eYV6f>?R&w z(csG}PSc3yEOP5xOARl6lj&qbxRlRC-l#g3x~vgtPL_1pb(p03)-kSPZ!DQj=%meI zQ8nTLpkBMk7<_OPU6$PhulRa6J8t626|M)GOTLqd4F6!~obfI<8mSxa zvMI;AaC&YteF?B5(;*9KdG!g}w3;g; z)ho*|ZGN60>2$&4dVh|ExrwYUxYIuKE428g9^Pz>M#4YC`NEztxbaIDRg%jz+1ZWM zBrYQ1(KxRBqcbLk<_aaBmO+1D1%=v1)2MoD`WjoqxgN`*QF}_@Jh+5eH|#^HLod9{ zKSgtleW|O`nnuX{-`S|fFRF+*9t1E2HTgTL5E z1H+#BQqh;qG(!6^sc_cFkwn98)@7D)SB>vyEW(A$_4M=BO?oD?M3~E+L zrZG}Q%%W?^?Vrf1`V=88aTp%2`9zjwRdhM?K24r4!k_PI{Gkb#*($XdWPh}RNBR?b zsK14#7cQjjFM8qJ1rOe;(u*G((@aGx?1iJ^eZrP{PN;vonKP=+6}5(l_I zBUlf=E|_4slRd6Dg+pk3fXp>rLe%UitG^t#EGA&WAP&PuF2a??-!L!fuV85Y5M>9$F+5-tTDSMX#Uwqv zSmuI)l;3!HQOYJx)Wx{7G2B!iXQA9GNjM@uN?5V{1qEI8;s;%i=Z#%@@N7aX+5U0(apLb`Pt87?DN3*ci|CyMc&A|ct7{+P0qBTJlPd1j( zw)3|sV`&eW{ays?s#XjPJtZvU-(rc@X!ta$qEEd({&9A}n6rs^GgzK?u60B0{RjB) z%pYmD%y7fo7E{-aL;u_aw4b<#9e>I(J^CIhZX`f_eJti^$*%j&X2==hkGmru;^751 z44x&=CzvE5d9({mEDf;TLKU-TIAGJFx7c>LL|C+_4S|(GuwJZ&jgPBo_PsJ1|5+ZJ zKfi(8T6 z{3((Qm^+)({CSi17A4`spcjI-noN!vMO@OQ2NLi8u4jkGcu~O!BMhCkA2r9;;#YPu z>=zng(;07ierm7dg>m;Jed}_#ZFc^+_^w&VxRZo;6;-75$)B@6ct)aUb5~*#VL&5) zm6>&T;KjL(Xek~k{C>GZqV)3?3+u*xr^^?fIbVk?FUj>F~%mQUw!jZv}~YFs3J1+zIs|;ML22D7_WAE}40Th%>tinDde2&7w@^VS zZ84{YlPOk7m(WnTUXoRRhYHJ96moB0HlX(oL+KU0lQ@3G2@8h5 zrE!t@bh>yE#oe7x^WrYy**iJ@&9+*ad%A!ML+{h0L<2l8429Rvt8hEaOXEkV@IyzP zpgy;&MTd^`rq{CE(U-!huwQu=z1P@FyI0(Z8(O?b<&cb+SJxRMzm$> zR7!ClizT0)u&8^B@qVfuroX&MLq5sV^Y%BQ$o5lY({>YK1@_XNerNH>bsF}h*V3NV z8nnjnwkZBhG5s)8;hpdB(jxn7C^#32BW?!h=~Yh`_UBRF$UAiIo*X~znlkUGe~3Hn zaYS4_YyurJ)W8#`T__!R51)L)r9J}&^Ur#Y!l}@BY7DBNXNH5Jd}Jrq&#QvvH(zPO zl+OrB9*VVY$uxFJIZ58?LSDlgCA%Wv&b6~%dlL{b$OxWG+v$sgDU=t4qiWnf3^p79 z%bQWC)f<4cHW}Wmunrw_O+hN1WGgmA=i&ssS=`PRHbmgvV{aUa(SvE7OpkddQ~1Ln z81Obq9qu=Y;OuN;WfrDb%#R{|Zfw8!PygZbl8;Zh6j`&eeQ3(sXXU2o|G zlBN5^dHDyqx@=`$?ye61PofR;(p|-hALz5drh#I=ICI$EUkR;&mC~#~=6u58HCX=m z4Hbu}#?oQuf9S(%?HbDp98%pJFl)8KMijURhuuXK5OEt19*5)=v{ z*n;1!RVkhQbOoZ^%`s!+a&b>r1J=B96&qx1%kFR2;iujUccR~lPW-oX{BgxJK6ym~ zFQ>4c4^PqHuQzE{s&_lY$418T&O?LvL4SStZDVHhOBZ_$?G)!ZE~_{%c6cj)jVj@qyzL91A5g8_SZKVIDXfEgZ^twcB1O6ezE3qDwu zFX}3PBa4B1;_k>UEVA8%U0vN{-k2uH8*%)7z^JI(I}rM^HY;5;C!@! z&D?VjPt%i8w`w_FUs{f5#>-$Q9f18A`s|XjGv300wdN1Nrg_T{tGyh)kt>jtnv9zv z_YkwYfeq3-178^~CjKbP^>mHJg=}~DTuFs_g&H4wZ?BYpUyI`kc*ZQo4Nt!uy6T-#6?&`C1*Mr&U3~P3-Qir$Srp)h7Na4Ev?h%_! zoJ97!G%)LZ0+#;02ZiWxseRyJK4#-6{Jj-V<|oRiD02`(G~?l9Sp}`0ep2%_pRq1} zD9Up8ldE<)@qT*9KjwvJ^CFPx(ayZ`5^$;15ZKa5{dk!^{uK^W-F;A=Hvq97QRwrn zKbXZnY+e=)m1a|1%zQ_F5<@(mz6(kh+nKUf1je~~Bj%DUFRpfmI^QQz%ekRwTm2b= zhM&}8b`=VD>_mLe!DxD1PTLObAlf?;=Ozv2CkBU0T^8NL_QnLrLj!SdJgM8oLUE|U ze(w53Wj?c0j(>l>hBj#xQ0S{V>NCm!SCT?;d&4!1%;2S3f0g;F&=d4rr%m*-wKt8i zuch_b(=esSS%g*COZP_KMDw;&w4&*vNach(5)&y{8^l;TUmqDTINt>px%_W^D*ZYT-MY6sAoU$^N3&)n0UA)mZG+eae~^ zErEBT9RmMcB>9<2^f~o~=-IZ@lrG#veww}1Bl0YaGpAvd-d#GhMw`^MrJ^N8Cn!i} zpZaX&r58f3!DnD73eFgyqqC0WtP04avW7xF%kjAEP+S9A=&3E3ZrZL zOJ+ZHqi_cm&cXpx8ELLpvjgHU1iSQ2>74148c z*~ONX7@SZ}r&7j>*r1_8(UI9g*nxRmu4fhs%-Y$pwjB6reWgDBIb3s!k&rP)hOZi! zi%gpD;PcsbHfU}RRQegibZ#2mtN+5CH;t8~MK}my4>rJlx(Aj#R%MPB4Z@(&PQuo! z8QhnOR_a`v24U(^=C{8ABaa8tn3Zvo>8iGpg78bJ+a58Ertdd|Z(K6YZ(fV5 zo0BoV&=4`2UiAI2s>8(ErxH^kgERTK0LHVL1;vCUc#phA;+r$L_w2ZYF5Qv1ZZx1~ zzkE1dE?_1R$MD(a3z-aVmF0j$lB^lKC6itU(CjG<=*m0FvS%E^`!%gpeK3tP-s>i` zN}mg{^HrJuU{7f8IgC!b-t74KG3@9h8@6;ni*VwBfkf4FE-Hq_u%M4`a4B9MbGDqO z#-F#TaIG4y)Cb{tO)FN=DIv$?E#&Xp!*_%#*kS{`y5NjoxrudCL(>H3)=kkp>|Fc-BviDn+(TYFPs*}Xtv?!#~?&R_rT9HRTL8~%dP6lV}G|+ zjMR!@D%x{ls?#jCd-g&w{$Rthv)tL2%i8Spx5JRHjOV#~>im|nDukc5 zw9YR$zwVDWUp4vQq>rRD{0~=Fy@R{d(9Y$)Zlq5SBv>|SqR=x}jOLUh;$z<`h4+@V z!a0{b;cw|7$EuECL=MW~`eeDoKCDRykBen0-%qid{8MIe zW=<4#+?M4(I-2O#v^FkfRt)#;yF9&p{ek9oYVtLzrpWR61A^v1HsR)KBgU_Cy!?4_%Ahs=L-$iiHG2Y$|uEcB9iA=_^V zKGdq<7*|2shPUXIeiBUH_ZJ0j$V7&w9CpgMdmr^ONvk%6l-DXE`|DMNuTErz>P0y7 zawyDheWH<1s_3{v9UcAXD8nw*`1T!_S%6UtJc6yTYWP!nHz|&~8~Rc9zFyd)>dD)r zI`BtK%wR3oM6Qw_^dMIc@f)1bW0;KN5_^}argovM^f_JHZACpo7touq6BPQm7w(1c zgG$XwmN%^g^S|ZLmWVr?%Y186{2oiyA*D1RV;KUcaISq0CPmofM5`D?J=JcZr5?bfPPb6GfjEcyXMe3%YLR3Qk9sAzrhL)O&BH ziHB@x-niS`ln1%=!?6T;<4V}_hx@Q}lLk(Co+P~s3&`NJB^}Oeq??<%aChci_OUD) zkLEg~V3t077k#H$)I#ou2jl)Z2mVLB2A^VBO8A{c4_fb1CiLQcPI*H1Lp|RsvD9?ajN)ZbY7s798wSMr*6M5 zaV?>1B_T^5NLKEM<2KFPgCi%3*dMX4ET4Ft=)F62)3+79d;mr& zMzC|%9Wb%Yr-(U~+&u?JiV4Z0`xc*Q@2F6`{j-wQNHeg+E{rbDPvBbIt>}hkE@k#n z!mz7BSTgdscz<69Yzxt$#&HWdr=&hK?%63yvL6bY>QvND*5WT;PsMZ1p$LsACDVky z)FWX5=X}I~LhKyS^z)oJGGQ?mMkrzWraU?&|>yakEG*lqU>FZ{2;?@|eN^v;0l zi#>Q2oq@7f%KQ)S_q6(HC*nj|*uNt!NF^Qd&whG+wGV<}XWL_^iN$J~4rXVe1@!!f>LcbEWz&(-8oBLO= z%a|xE>U&y*O-DLD*It^_<}2NEX0Oy!yF+^P>R##Q-hR>%+4j=AfjZK6 z=Qfi55=B|^`gL9?%61Tml;C#eRY+Z91 zOU{?&O5O>}@0KY+k%dcBex4j65d*(vP-~=K+p2624 zRcOww#nc;0e0X*(4EoD*^3t>D-}?kE$K_zoEt$XU(uQLfB$zF;-&}nmlArFwy=_q_ ze&~x`ouP0((TU?D%E)rvCG2pCN6XhKNYk`}MW_x!b*zyUC*$4y?ZMuQJCo)N&CbV_9 zL+-vN_?yZIU91E5JGMAjy%YYStwM@83&zHuX@T?_xs7~Bn@09R{F9-G_E?Ma-J57c z_aHb(ztFd=Z{#pzAdUx2!^M!DcoX z5l-%CBI&!euxJ~Gc}=}gKD2{MLvGL+_KEBkW+AGzRhY7GCp^`LK+mf;bj?+9(Mk&@ zAKj5Xxd{<=6Ita$57qB4R2bou? z;rLezio+u9(fwG+c(B zVKO}A>0K&z38E*9PH;odwsXEW)NpNRCl_RAkL%+<(~s*X=}f;kdj8XmewJp^y~JUJTMuNk9Dw(zd@RkqZF4X<1jqXL`u|6)Qh(=)4QfPO}!-kLX}kuLm^mY#MX+$j6|DWLjC?!u@{ZNXIS| zkkuDuOvtvxf!*P(FZ^*r&yf~fZF3a=J;r65E+kP`6}^h@3qErlC%Ji>rq?@2F3&I$ zQpWX>)U;~Rs{>^;Gf*A1r9OO*N9vecT}I{Yy~*}oPsx)VeT042hD(eCZ_%6XQJhI` zUxfPIrn9yIMEnWv(cPbpF#+~8KiVH3($}(yxt2H|rvl~uN6FgDiKz>f4+86Znk1bRe^yI^pv#|1;J1sf0lXAP)P;t02Trc;4 z+`Kf(nUKWNGh{iV6?3`j?HS@@Hyb(sgQavtqJeO6Z%hu1VkfJ{;bgX$>p8T$;Dw2j zs?TOLZ1;T{^HUG0Yvd{SWf`eiB}hW&MhUimY=x6X_eGuu;_2LpH?(nZ5U=vRm29sk z(7{V>qEAEYgdyExgoTkYl5xc{-&jwPbYAFUT4)_b#M+S4_3@HRTa<;T6&JaFTH_HA z6Uhe7)B__5InZ$}Pd*|%3?)IWK&-YtsTyl61_wlq<&)Dg}LWHIN}Ph@1-4W9?fezUs{ zqP=}Hxp6N}km`w1III$_w5DDN;t>==a&DkrI9RR*^}xxXmI)}Pv%aC!1*Bus`TJRm7M8g8SIhn2@1wqEA zEiq$vcm=r49|g_*ws6$wB%8g@IlHE8;mNX6ay+ielAq+W53R%5xXSb5%qa?ZV;h8k zc{1B4<59QS_T`_;>~w5FGDc3x$M(xNP@T|(I~N|KCk`yt6R~tw-#?@^BZ~+HNLQa7uvcv;z_}MM6TF#WL%&`BpwLDw@yWzdR-uXMZ=lAQ2{Gb?9N8a zZGv%21b4orSy;YHowc;Q6)yDKC^Y&16fKx<#bU#{!7;*&%|4XGX1S!X??1LO+bNTn z+`AsE$~Z!}6FFb7jf-GyR~ZxEtYb?&?y^dU%gkqdB@2*Y(a(;@ue%>b#$tjZn7nAN(6w&8U?e-{@%tyTy5Cz_mvB?K=t~j7_edt#$|ipQTqn;QQU_TpM@w)aDna9nYd;f zgObio(3`7^1KoeYN-_u6x*x(yw;Wjxy%>)IuEFr-JzP>+i9X9U`2Octz{vO>#>ZSk zFPubTMh=`C(vX!p2iC)Xq2rY<0?tIkyEPgqg|nd1%M~MK{I@&H_hF;)MqKOTi4&$C z*!|rUs*&}wTBd{a;FUZI&L^UF(O1!XEoIc`Noh@&5*5`2N{(-SB>6Samu?QrLzkN| z|ME^I4t$a6_80f5V_pL-=y*&$Qv^yKDf5fA2SIeg9#_qKAaTAfc0^m^@hELPbk3mc z%lY){+IhM?>luCc*-lqZ8)8|2GaBNouwu`8?xhU>$$Fq6dAq1h#^dRO3%Wc89bbbj zV|U@?dUO1#TE_?P9E>${ccCq14R#7VqCXfz{dyZ2#HvaL1SR3ulFi)JW^07pbHZ*} zJx0BEte7dFy8btO7HyIt`9qkyPv z8>Dq8<4BeQ3gvp>Z zL!;@H(5pI?8)0{l3iR5@!A~Ct%X^`?ki=Oh68Q3(FtP2xUa-Hbk6G8=(C(#&Xc$Z7 zhK_zIL@nP8X_k|qI`auZW;Z&yU-TqELXWyf(JC~HS9Zp5vvpT-#_MY-J3tNV^Au3J zN*T+NY|#8gR@3FxE|eWmBK}1q?H|@khiCjC^`ka2PDLpescZ6T@+l}YNX4QrMkxBz zM9u{=9{IqBl$skT8P=xGe)-jikIS`F$;KM`G36fsua{7d$$4i7Y~C#nJf19M z2$9@{o-uUAu$D^he3#)9Td2QpqA<-h9cB}4SWh!gL1Dlb&iPv&MGv@7trtAWU*C|e z?vutpa5iKY*0_=L+IngVEuhHVo!p39u7b$hh9$?#Vk0C9D^<7BoIBr0KC_mR)<)CV zcM05#RVjj%ai+La?fB^RH6=9i{8#E(_=E<1siticI%v#6iMXYEPgXK>x@5A zC+$IvWF)&b%g-Og$H)7Tq-f53`UK(U%mGj~Hp1p3*4Ue*Bjb%VNT$zg5yqV#&P4m3 zbA_opsVT0RW)ITG&!RtcFVvV0)pnCCbya5jKRFB9T2nbS%fs{>O?2=6JfgRwgsojw zu<{(k9CV%wE$8ATPJI?r*Y{jn9C=54a@{J{Q$fs6{;`Us7vzicGxJCc1oRamKEFZAu2OI= zJxd`zcMxkw27$Z9(dm()(z%91D%D4YO0BvEv&g%$I$;G(e$p^w{-jeLYN{GBExtQD zW^jsf-Lf!gX;1!xhckaRX%ZhMt3M8Ym5ARpn=z^^9W}RZBCSrQw?iHH4tY=B@17U$ zS0B%pUN{VkogVP}xB(8e7hzW=$NLs3^M|u__(xa6ohFp(@JW}I`MV`@{3-8Ch)msx z(xskwD|ZA2G9K6f**sF&d=39N@Lts#{8z^+43+U%BilElSu+VLC(L-alu3Ng$Y=B| z%gOz_{O`r(8gfqmx41k!XiccBXx+n=``@d}|D(|Sf7u+N_V2gXW-gu~FDq;RcbPh& z+rO*-ThsraW$&8*e(>Mg|E%!;pQHa%u3rCbHT?|GgQ&zwQ5BpZTv{xx2C{{%x{x|K3|bZm09VuX1w#-b7&P S1SREvw-scMfBXMe-~R`R^x?<= diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_none_r47-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt b/research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_none_r47-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_8x8.pt deleted file mode 100644 index 1ca63e52818375e8e8d6bc621d9233098a286b2e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36843 zcmZ^Kc{EjD7`Ay%GAlz$=FFmd-uD)wK|&&P1L{{%5eg;q7}1G84-EV#We8c+Ay=UEZ&N^qWyZ71qeV_Mvp5x*uD8$DnCdT*wY%+YZeCxsj z{@WB8WE{9Ie4WwuEx``9(tO_k#}+3L5WbCfmC>4r@J+!xB-Vrlt&i9g9JnStXzPw` zVQV7IBtrRgbv;FS?eE}Sxo%@nDF6SuMZzI?+q!VK|8-I#RKSy;ca2G?pr;`3H(+~2 zsF0J*?C?-wS4TcwM}FRCjpLf|P!V0;S>Bg-#2+3injR3aEjTzRAbiudtve)~!vcfC zf&v$93J8}771LW8Djpsxu`*QB$i-1qEO@CyTsg1l|GB&Tf4eQsr}lqz+t|dxR^|Wd z^#3*jC#O_21}JbFn=*bKe>$0VD!b(SaOVfz%HnyO3CYhAkt=?#TR+%22Rfr+0 zjkwv;LEI9DAg)`NDOdh?9qxoUDygOfg66w$V+@O;>t175Q!n0{--R*Wk8tA2YOYk6 zH8&-70oUsDQm&82O78Tq)m;C+K<-rOWN!PpW-dEBnY-d+AlLZ#YVPeTuH1&xOSv0r z7jW}zt+__uR&#?h9-(A)7oI=Ti;>@7p|S|U^>24#hMFR{Y>cD!^>rxQX3Fhb8N}_G z6U6;5*ob>-S0V0fR-&)7)Eh@_GpX`tbIPO{QH#sVXlwa;dhn?qt;;c>jZ2)!&c0H z(P*f~ZOqk}OMAlx*c*Wv%$`a+SQ>u@`n!f;`ltZz`VvnydXXBc3DC-xB66v*o?UXy z6udrlfKK6K7`ghI9UaY0&a&)w<=1nIu5J1Hxk!0LZ9mZ>YpFM;#Iob1U&<8`Vc=?|F8#+ zzS5+%2@;LHMhf)fp6}Q-hXP6Jf_PGH-NPpCcnlA;J7A%L6{NqcWdjpW zk?d4{deNRsg?tj|bxQ&46dQs?lBMuvBNrUAa+t&E1MGKCLmIt|+xRH*C-wgNh5h-Q z4~8^2AhXB=m_LVLLsL6+C5!{fT}Rb*zmePMMTTD`u!VOnG5XJvU_tw1czX6KEZ$~8 zn)7xO7imSZty_dMb@&tW;J7vu@~ww-E)r@CbaJNKFE_)c$O?RM{|k1^=*J6lkKyKh z%i-MPK6*P)h2}QaG37BGV5$5ER(W3q8w(HSqiHe8xy7L4a^;5dy8oCz)rL?u$bcFX z$z0lgi_K8`$pqUpu)8gGLU}o`AeDt=D_YA(^OT`MT`qm6G=U32v1);{% zZ+ggK*J;du#|9XkC$l-884ASbb~M>nX+iYQ)q|nh6Idmc2v#r8G9vMDY~4eDvNm*> zB+0L*dzB`jLgx|mKRN(!dpuxn_+#mGA!$4YlAdoThT!E@ad1&pIi*I$+u`g;3+1()2D3okN z2Mm|7Y#c9~KTaQsOE+%r!bhJG$K`vDBarVFCf}Y(v7(G(O>=PCwg}a>^ z8pN*SOmU9}g?vGB^mQGRo4p>?2CQJ#zq|9>YQ+h+M+J+uipXK_nam=UQ(&C+6e{o} z7%z2Y=Qi3>?{cBW2R>`bRzWrN*_Mlo%<@ohvlCXcAx|)8^N~W&UY~f6>2S`olg9N_&%z-`A;d5d@>v0eREu!egU@=A|D zYxWCPCAytP_j32)V*b?zWR-Ezgd*Di1Dd^ss2cGvRyPw3dbM0^8#NoYcLT^25 z?!p$h8IufK}1a+;}x=qjNDpCg(Yi&-#i}fE>eYt%o|KX^Ka(Z zMHhH#Zv#g)Q>f?Ch4j1QSqMFK2xMaI;9%Nx5M7`T6Ro>p*MU+nG%};HZui)dSOd6p zCC9;CEOz54Jr|Gp09gxbKfcMsaVZx#mMxVqn15O@9X1tax{G|px!Yl-f*8$h0 z2<}`=fjc*iA+N5Djk#({k6bARv9Dne9%=;J<7U8)8@6C>bP#?imx7M*LYglkL_H;j z;J(0PP}!XaiJ{T(&t?z29E}Gx-g?nX)TJjsZDmDzmP4KXG5D~t25x=63xj9tA=H*- zevQ_VggX-C>#P-!Dk6Zf4=ph2^+N0r{R^@Q&a^M*4JmnZob?M1f_H6gpeOMJ3SGBA z=q_!N!aj(LLB&Wrs0_WOtc&8gYmV6OvJ?+GXBL7-jp>G@zz5PMSdqiqy7aP zZcB!=jDzf}4^zo!-)3|lno5fP9A#5{Q{YqY3$QULgf_L}2E{$siI9^HT&u4ok&^k$ z(mQ={+k7T&ZI;2xfW0vN6H88%KcJmObI6~H5SXdk0j@)>Fnd`L?Bt$cM@nB3^2UV* z9s3J*xeHPFg9Y{^@T1mJM_Ah_MXaOiNT?oR#^5Swuek@F$1lUd!DFye#u;|CMY0-i zb?Lg>g)rqy0yG|shG>X~IkkoG<9Itrr4GQWr$SV=@(szF6u^}Q<4}I49ZEuL;K|xj zIO&}a-5=9IBuTPyM_)CVFmjO<~&!Cq61BrrnFmAjSise%2g(bpJv)u|u zdJ^#KnKYF4aYvcO523>M4BKIy3x9KrNl028Y!6e#q)tQ?V+9=Q-2oSGNP@|)-+119 z3f#Q012m2*;w)h<8f;XKygH$M+ddFU=wgIi8`(9E2Z{XLjkxe=7Cy0wM}OhP_)D(? zZgt&hXr5n9+Si#g`LrD7OsN5lxKd3qkkrxsaU2B?K@~j#Vqw`tCkL(45^?)r?%4W zNh7*or#N*w`<&1x<-{-S2zmQ#3iV8lqZ{2A+V`i8aE9Kniy!@EL==-@Z@fDs%&?-r z#2d-TIb}vI<{`9SGevfuH#Ti`MPZc2U$dR*tjS3D)3Y4qcjaQwi&6~xQixK*N$9A$ z3Rk?liicH?;%xd93yeg#cYI{IBBfKf;mj!Z{~5qg>3FW#o&n6)AH{hWr*I2;rg5EZ zM7S5WKE+@AkKnxP4fyqy8>%ZL;Q@n__->#SogU}nIbA0VnGA=8=lud!=0X|p>FW8sDtwea3Lw{e2qxCh?R73PA zd9|XPG?YCf^r9FguZ^f1-!{rAt)Q;Ib?CBGU24^DN>^RAq1%7E&<6W;^q74D#b&|A zv0o#U&zVcTBfpW?v4uolXCb*x9=(`%ss2ehpf4ZbNCm2F`W1z>KY{I1f9!ae%@8Uv%y;-NOqTlp5Pi7w@6^*Efh%C)%(QpkqY$@Y+;LUO@~0yzs%~C3eLjBN34d? z2BI;gg_IxIN|WUZ*z(M+Y^D8OcEH@0{M~huZ0r9*qI!Xj-5#MnLeHss3_q>?@E zQRLC5(S`#t;!xp~2<{8dgW~N}nD0N8E_2EzzfQho0zUqPAqzX~4)RCYhUM6rwe~db9ix~qy;ryc>rt)1j84I6ECuCFMuIMYawY*KDb?S0NE?2n5h$jY{%=he;Ia zJspnF7^)9F_W4ZuTPybMGk?-xEI>uHpVNP07`FQQQD&vJ6_Df0S<7F`_lH0;=OfF}BuG4@uk~!f2{04|dzJ;vA%ita_{@nhkMiRp07$pgW$2X3^ z8O#`exRB`>xC==o+PJEI0j3=^KyKG9Sj!hf=j}O6YNd>r2)7(4 zk|D6oA{`1Z&t*)Szp?XYZvq{qK33?4HN16s0CF-PUGsNkV3y}Gsjayt=V9z9nIZ~KUjwOjv>1~-H=TZyLdDv824?sjtgIRf9 zmC&d&X@E)vEi!@aG&j5POu_Bi7D-jTED93zA`*;8AevPk~iK*03@9 z2(#p0GW+-FNwTTxGCi^>jXKV&W`D95nBFLJ2s{=Gk5U%|bK#%l%s9hXK8eWC5hR#zUoR=G`#O^|?#akGke+4ctd_|5+ zCz2N{*E51kdB+I~=yp{fPqeGxkL(MO6cR>zy%Nc`Hzmxj;UbW1e+@^ju<-1d5%gsx zapE8lrhd(06Dobds`m{%d-EG|C#&Fn>p17MgF4ym^8wdu&m^5nKR6wOmB9QShuF$D z@O#`Fmfbnbx+exgt-J}RTX!y;_HBg3qE|5ZuNVrJmog$GiL|(d(bIb>VIWooB^iBe zR93|2uj-(7&uS**Oaj@Y_>u%q*1;cc57?}{146m`!Sap_@O{|9ritw#eTzdlJ5iT?P}r5-3xSgWFrpAz%$$lKU5IV0y_%IB{74%bLW{|9ut2KhuG;eAe*a?PNBgVH?Ce z7=SVtVeCBn0Lae~CS;!sJ9I;kYo#E=-rG0G^zLp2aTLLQD~I4=Q81*~9ARJoWT0w_ z7Th^i0X=nM__Is_SK0N$&gl-Yu3L*p-uI>B>nlKOjSLzen28_uO5?O_7D{~w7||uF zWa-u{@=R$Qp0Affp->@w8}tMe<|o7Mm8%#XO9kTK(FKdIsiJT+AI>>61c`@kL0)n` zWT^;%s_I$t;aN2V7~O$$O;=&co3mhgCKoaq5}|&>JP`h$J?xmWT(IbY6+t;)7uN=(0 zUI%kU1Ti{d7WVBo#btL?(M?;Hygr)2`q;_A*XNf&_{;>1Bun8YzQ52i0 ztJn|K2jRK&G_=XGM89D@Tqr#RV_&xc?A2r91NBHiyc)Q+-G_;@zo6;SU1+~M6FdXs znXUPfHw;5x=rr!sHe~R$_jvleJGGyfEhC)c+Ae?B>L*WQZoVrXF zxBSWhX{So|UY8q9EzgD8j=!+6NDAkjn}Ar&8i@NV3kvsB*;U0dBv4rm&ls6uHFp-u z`wL?7k1ODlDGw@H{_I@`aa6O^#FWEoC~GZHz<8O4+s0yvFgMkd>9QFttyM^U&K+o0l8P|#ZmBpEN)On z%&oJ-uM_I1r*sbDn#vk(pH+d%vJsHpZ-Uon_@UXTFW$dzh#%e0!~WrOZ1bg!)S-KV zQMh>kOt+Oo&%J}NG1dfr@3gVz9#kf_CCkX8Dls(Y&cMSpqG%%i05+#=hHFXZIsM}c z$!Yl;V0TIk+Z0}c;=dD6WVr~0-rr&z-n6oEd!(t`UTa2o)m7&8rI|n$IYXV?6roaEchr zE@9U;>O;L}HQZ)D@Z#|xbT^&`m$mchf4a9AQ73*}E9`-dqsh3>BpXZ4BqI|Tk7c6X zn9I&XBd>J)C0~W+vNv#ZGDV?_rFeBP55Knm$CVAv!;RIYcy0kjo4q$s6RS|YJ{{v8 z%)`z8Ubw(19+hQM@aw@W{2`fw{xdu`PyPZVg>6f5JaiYgIt4P&c>T^PwYB~VOC z;p~%8q@mr{Nnc?w;oA|)?vwt==>JTH;s@E_novLv2$>MCH5QD#V>WzyJ_rKSB=G!V zDa?IF!BLW-zQ41sDmxs3Zw*2?bEgh|nWKm0N^&UIGX>AawZK}-2oO-6f~{4{@xaZE zC^qSZiVO5GELsd>osGHQ%EWO?jxK7S^~T8V4JaqF9F^jvF#dH6M2S8C(GAjg{HiQY zI_P2eXKn0v6-L=BhavUe*Q+z$&!_j!@G$E}7RXJE~y_7kiYC>I z?vvlm43+*@-QXW8P2jCNi5E;I=Y29s<)gi1pgMtcm;3QBRsikj4CsMw{N2j%jD!3!3HTf`!^m$X4C^XZw5zFN1jCnlMLfc z?0uam#kJGGzkpc{9bm!3;>w#UV6Ck#lpGi0Xr}E0jbneH`KdfU4G_j<56ht3k(Zwt zS#D&DEv(Eu15qH1HbU~a-{B8zTCop=pYn0CB@N+}<9P^m%9W%wF&T4OcH)(sWZoR_cmB=P$tsltwh_t1BSAlVfa@x zb7|O|ZJUFLut*^F{IiVukGx?8#DC1czon4*@1ixFsVjxsZ}`#D-2qRf zyou=DLa;@2zfUPK&rF_y0UM9 z@?<`|eOCp`52--<{$2C+pY8_f-VfkYErt)1#vst}B+T_O1IL%sxI!1^LWxNsn7l*}KDFW&INv7(HKD$d#jXcus1b(d^*fp&fqW0)OS$qs5d?$*Hl+Yrw zk6U092;z-vkD$0O2lN}ckZmLek-<9|sY|a}r#J_knaGbNmr6iIbw2E5N|<*gWAl%O zePDmqInrz6M)c~TvWA|KKE~0=84j&H10p%&Aoj=tGfm=g&&>%Y-ef(@HA{egm*XI2 z+YDz`2x9zCO&nb%w4W{ljYBN}?NWG?ilD?d1D`n)96z z{#*%S%SADhhp$%ekmtqvXCV6eFnse+=N6kC205|k@Q{_qc^}nKyigP^6)(cK&|xM= zI0OoL?exUFgx4t(pg-piRPndL#eg{2dOnNM+agZ~oWxLhKo>o{9b)QDGdbEDM3ITrMDkG(6K(Ine%+(+ zGbs@y#R4EV;}^rv%Qenv7ARQs7mi1rg%{B-FiT^Ix!6e> z?3IUw=0`yJ(O+mmo)-%zR6UQh@8$*C>9>m}~IFc=o5r>Sh#RG8S-VF51_)I)s zAsp7Y3f{4TC@iLmH%_V}^GXiy$Irp;H~CPn=>>)&t)LP;4Oa@z!o=Ik_&@7gMW~%mk2% zu&`%3!cnuYg#Yyn{3Z@4?_q?sb_ytP?H!1`JO+0w4B(T-PNs2*Cf;{f!5igMFf6_s zw7*w_U0MoUTdWUu-=<@Oq!}`La(F(w2mU*78M^MB0}d$#$|gbLt1O&&ybziAN#JN4 z1qpvyXz7{Fph^cjF~M7Nx|uYwU4c%O=p`S!^T`~ME>@>#Hh3zI!(X|LxYEgumlHFf zWGoeaCuYFImsg;@><`=)R7Ep)F|3+x1cvkWLwIm4yg&OCXo)EP)73nm=rrI6DN9=HoCguL69Y<-ih?f6UD@d*^RIB*Tp=o0#vsWRhupoC8U_E8!fE z!zrjmupL~+D9sRo>1_&F-6x2}=cVw)K4F~k>j|)yiLf^EJhNuI9KE%C7@9IwFj{UZ zPQ3U6Zy(k`wAv0h`9cg#Q=K_aWrT52l_us$is0R-rywtR88Yu?gV}9ESXCy9eIBa# zGHx8y%`U_FEtz21kjmo)(t&upgIw(f^lAAE{@-T9WPb;nf24qf?RrO!s>stR@|kpO zt9oNasYIiTwNPX5z9006cRTg)KTTZ)UFi$GLnM>mik^I@NJn+2(wi=#G`#B-8N8T5 z98O9Tf&I(KEQgO|a#bOD5E4l0{;80?k3Ct@niyu=pJ0wa&=>Nfj3uIFsbtlM>Ew@w z0c$Vh&xq>%X5=OxbKK^IQ|;a;Ger z@lB2Riamzw@x>s2uoPme6Tnz-nAtmLKU@1Eg)AoF)Hu34&OB z{dfxbJG+6LocV?5U*E#XB1z2S7%w)`OO3S5+CXM4E+qDWgCxGpiPXzYBTv8WBR&gW zlA8}jY01D;I&w;p#(%V=XB7^R!9ExI`O9hQDBVum_y3@s(n5{NCnXw>uUBu3oT5zq z@@LY$7gx}5-9TFS#fx6bHKF${-jH1vud?Fig!sKlB}Os_$jqMwBta#KWPMd7p&?1^ z=f^69Yj%=s-)=&Jb6>J6M$Oqrf`JV8sSz{@OF{RutEA56CaX=?FkgH9;O?3_c>3lO zD2j?=PwNv{5F|zYtL8(pW*g*<&A|G#OK@mW0Jisp;_qwOsJ}}Vbw16*BkuxnNmVk2 z#O=qPshQ}UxdUJ6d7|$=PwwIJ)o5+L1NE&kQLHu_n~x;p`m2F>yv7Rq?B#IB&TQlc zhGFQO^*Ha%Vhs2_1LG~*0B0-!Uu`KG7TO6RuS9T(*B5AMsfU<`wNU-to6*(1$ND~O zAT?VhVgFMDIPSED$t}@k{Wd>mXZD#8=3yZTkeW`sTT@wsN;P7mm`KX56p)$+S!6Ih zgX}D!WUbsS*0k~!`Sizxe$w%#9>zgb+eQr1jw;c+=7m@*yBd==-+}X=HZUb} zlWduc0%;OD$_mbZ&AfCJVrq9y2f2l>7{09>#>;XQQ)}{?F+VL2J1f$y7Yr1zOKr@V z$68HH#0Nfz+2;or?&X1^+je+)B!$=ytYWsU3WYF+#|gX_M$>*}+?J<`ido-bYcxN- z_bvd~)He8YX)2y&jIn>~0-Qqau+t?7pIl6bIg@8qfc4}?J(2=buKh*~#MNH8&ogX#)(_l^12E6RM2<@(MF?HM+B@|?E z-m6w9n6Vz-8}rjrweK)ltcLj-D%chxf&w}%;I%vqD$hGH2dh%ZkJ;g{<-&2Eufq?P z@w^5CRMzSTS$NE9T!?75Gvl$w<`nFcot7 ztVW9hkrf?hwPL-QlUB{ZT=&4Ch(bKg^I<)H$j_DBSAhz%opANveh}$R1@q&9VC>1X;ymJQVENg&chyH=i;vWQdr9teqE?7NN8dG}bp!)|y zo>xEz(^o9O?JLtbzTH(IGAM=bE3EJwWGUVt?|I@lCC7p>K`@iLDuyZ7)p2>r`|3uPnZgd-o$ zKdFx|_zlrHeI_0~z>hQE+=q-Aez0Vk3Yjic3t5leL7wh)I9zoKCJpkTFd+*bF?tZ> zF&CoF)_`^e3lLuiVHaDV%xDaT{lxHjTPv(twGS*_uY=j0X|Vs$0IVWT=x1MngLnD4 zyA~R8BQc1(Si^w3qWTL;spg@3kR6I|`UeM;A3~AobC9wzMZJ%1xUj7eA8HC}o-AG>X*;-9L=#P>P@Zy`ZcJ3bTt3|gSvS8Lq=$p|COZ1HTbI{P}Z9xSwyb?-8>JKeP^PrybE{D`I$IoV}*-PxuEvyRp>;x_%TclnH_a-u1<|z zW@?LKe~hr6wMO3{3ycV!iQ-KH_#~tOjCZ#a{h2bTyLmniFPo440@`?bzXpnEiJ<(I zdRT9$%^Ljs0>9$qvEv*MJCuJ1vl(B&+i?iG+^S)xz(EM!HVMh8yzwYkz}E}3(SD;X zTD)+>qk23(@p>n;n6|=_v`IMVWrt@z9LMhsU-0a71MXg>Ag;4n5Le)YF8AR3J`6pS ziy1tRo%xMHXgb*e&fEDgbF(*2t=@qHF>7$u-~trdua9S6^K>@>DKzR^iD? z#9iCHxO;0D6rXfMnOVV@zbF`IJ@&w@-UfJscc0fCLq;Xs2mNL{;PlHb`0$4lUj4UDDtg-6zd`ze}z{~fPc^K|99RKLU2wj)Pf^;LS{HB52 zSXq30NCtDVCGcg#d+0rO7OXi^xcBQ!9PH4?U4dMD^T{6%@rUDQb8qzi{0}5U&(`eI7P zDiqC+z%33IP`W_@!^9apTj-Aueuv_4!gd@z;DMgn)@ZHZ!Cn5;3XPgPaJ}kwtZfOy z#Si>3>CSvy%ddzHM~y&yYXla@yP?Cl4^A)l!hU;4yqRl-8~-Zf8?$usZpS=)J<9>v zZby7_a6Z1{X*&YxsyM@46my(^Fc*52@#O}<_$ECRC{o7#8~zMS?i1c1|KqPPcOvSeuHvj z5iE$8MBVk1a5C*ONXamunyAmckja4+K3Bk5ZvrlFk;IfJQ9RT86|xV%2etPPn6}Hl zxcR&e&h5~~&1*HWXt^?)zLLb;ytlB&;VX$AVsWHi-E52M$0{b+538*SXYk&fP-PaPj| zXlOE0?O1dALs_2MZx|-lX|iB}HGG@zYSx5u$gzgV<~-BgI*1WcGnCHPV$+u-Bwy%o~`;^zPW`7iXd7?p+yEl+mOVWwCUJB{Em_v$FE|P=6 zgQQnRlHPmp7ljr|QjL}Ya_Me0QEkW}&K_yxRaz=JRvkzbhBfG@P!wq8L^W(5?PQ&k z9m(9BJOb}qNEI`MT1(YXv4GE{Fe8mDxn53OM%zi&EnfbA%SXd|C8--5PH!m5Q^WXS zA}l9QMcPbh?rWr1)-&{y(0uCKzkxP8xzV^bKRS0!7!Bt}Q=QQ*RJqifO2|3W|IVb) ze7grUW$6TUs1j?GE*5Gm((0$Yzf8R%_R2a^{W_-QH(QUp6hL%VYKE87-C= zl?ekEB4OWMvxdzob|molVIr84Oip#rA$0+z>`>w*R^}gNdZygHYBV#O9c!4!Ug*qb z)*eIXomS1XBUQVmlyUcDMe zE`;Fp*btPtx*pwvJkY_;9UsiDf}GL)_&(PKOFvs+j8XD0ejZ7 zqaJ#WF<4>?4Lq%_wY~z3V)#*H$zia&paJzGDlj!T0S<4hhr00uIG2{op0IBNiJCTO zH+TUbc=%%b79QV_ZiQC%Qs}qpKCzydfOWwVsBlRUtziz<9kE1HK1W=ck%U*Cd}o5m z0+gJ!7(daa30h~}hGmPFW4(zLezVvE%VVuLKlbIbF0Z)kZ=-r<>4D>Log0O=1uXI z$Do{104)Z`;P&E1aE_UVhG7ZV!}E*ndZ2+tVU3V~I0yE{6~l+3K6orV4PTZSVpW+t zYKaM>rCm9!E6j%^)%%c`B#6w3$FR)xE#q@p71zw1hCCJ+t-NMpViN~Xy>Ub(86}*_ zJx-23ltdR7MU?R4d7_MGp~75KoH=NRU(&-cLsXKi+As$#CQb2N6$iai9q@9zGm1&O zpn{tXngp10jknq0!u}Qbc)v4xCoDz|kE-(I@qBivb8uJZR1&v#2aeU-;l0%+IJ!;~ zW4N-I^iL5N|C2&FzxmDT$fAa)U zv85FfAD@8FMdk2PO$Za!Jy0fmF7mld!>clVV79ObPW;P(v%wV1c;nTUmy7@YOTg=D z(=gMr5ymzh0afz?_-EV!VN=!d@D3IaYVZP>w-Mex@*A4AKZS|^UVy@%F^IUUfnF1- z7&o&NciLNE`rB@p9!f#`zt@odLjf%U=i#edU-Vy_f-N(2kdxE}^>7cm$HyUVUA7wN3P-wxh*r9cy1I7>jyC(TPDaAUVCS~mc;*JGTNAU%m>Y-U2*NOH7;K<7bjkGaly&}Jb$|w z{&h=YuemykMNGi4Q#_m}`4qM*>fuzwY`i1jhm&V{HT69)tmUQfz_>IxXTZ}*woF5( z#V2v|bazyL*AJZfI&j-s3QIqog2<^?!Qz<&E;`lt}(ZAO&e5De-sKC46(oxW8KvM- zbOMq!UO}Ui95yBkV&2>EOB@CmMPly5Qvc zWmvM#4mIbk#AT9OF{~gOHL5bO=3Y9H@b$x0v!n2|Pz0V*4o6!i8jbT3@xK%4c;~qm z*D)m>6^|t1l#wVLQI5a^#Sy4$6NTTt`JwskG@`pK1JmT9(4=%LN|mp~{RMXT;nh+! z-{Fj2V}%XJN_BALsvAzcw8uZO#&}&q2geQVaO^@T`h*pOZo447tTjcr)+>0xqGQbp;YFRLYuh%hG%<|yjheRAGehB;%MU0bV0`p~AgW0n@l$nU#4!xVZaG$~>yf}A)4JbNI4yF|m*=Z-pF4t4!TucpVHm+dTZ(oJB zk?G8}K{+CvKg>FRpU>GEqy?-@2pG2ifY;IS_*;Go3RIp4xvVO<_4N-Z56{5@qJSGV zJ%oOxZ1idiL=T?@XsBU>epbG?H7f!Q_xfT^<}UDmupgg^C1XNX7#g)i;LqL!yu;1F znWvIaNa$UIdO|E(8uGZy?-}T)ydP6^k0O6WF%pAQD48qCJ+ko>K5{C?&dQ@$b}1Wo ztkFerk{TSq~7G$I?HI6j1&If@u;?UzbbZYnk_Jx}v zw@(W$r~Jn$Gy2Kexyh2Rkq4NFe{T42&so;~axEFiJ4NEB9Vb~0Ma0s%h&-j^Y|QjW z=<#AT_vV@X=o-)9_cJQ!7{!l#cQ3=(UPCx2kPB+E4H$rmxGnS;Wc=C!gQEd(^Pwf2 zA2DYhrK*5o!D_5edH|tm`k;MF1wI6Zz(JV;phZI9z@1L)CS@^xRvN^uO5sFi&SB4N zQDK6L44|?1Idj!_CS7%|3hq4|XT-}TiTuG_Qt{{{S(BMcKAii|P`$dHEZUjHo-evb zzRBrOSH6{WB6$ZL897WXw!f#HT9%DZ7rv)2J|3pEyz@66tfb9N+SJ4G9%+Ca_G9W3 zV*7NU;d@;g8Mt3Wz$KUXR81uXr}^Q={fi(`r9n4mwKL)y=R#kpCbM&?4*Pwgnp62N z71Ww#F(q7;__zsySVket?A-#_i?0cdHwfps5pmHuiosJuVQo(uyV0eW^}2k7ZQ7c`hP^i+k-MuK_&ev} z!ro|5Kb1B=jxT8b^Wo3Tb8#(UYS16Cu@TmQ~gop^&Zsh$Yp;-rxmB#^n+DKIX>xIEYc~pM)6`s4)gXEGbsNTYdQ-ZC~^~N+(eqPHO_Wiz^?$iNHsx^1>-UVE>+)gm>2xVrxZJX~kQ-{+f z63Tf$a=KwrY&;yl_!OW0P{F(WJAr6g!>c7epk@~WR#D23`sEz7d``#rN2E|WYahIH zii3pb7r-F46&BrJ2+x!nIOf{}ap;a27Bo#kNqISRy<>=hl7MSGzrdogRFW&{f$r*y z@sYn7?hatkH_8*|(p|{u&OlYZdqinwBJR5!gI=AH=qI)VLof~v>$5QUSvIddDK7IR z8&Aq*MMT{@&+~ zp1CQcPVpPmT;yQg3j@rsmO}&YDR{-e4Er2|kp8&IIsSuzv8k<~?^p$aXZJ#4?mig( zD1~CvGVrW>1!#OwhOHhU;F#wzfCL8u712lf6MsNGSC%$((TPW}&U5blGoWg-|cERE0Z=D}`$9SF%y z#^!5Yn9bvP-IKZaQfM{q-sz92n?>+KXbO>w3&8B`Rp`;m!FPrW@U^rDRw_qg2{#iT zPH0oFFFP?Ycqfik_@ZamTKvbWU)|;sjcX{cO6O!`QVn_ zk*GFf7Z#3dQzy$z4B_Qqmx~^_^br@0ZZlY??~WVY*P{(@KAyNHf>nR~k)yd9-O9P> zuVaPVzIx%I@?^|ap92Z6kMlTt8C>8Yie=wFK#lYt&>38gd$(2L%?5S6?Rp=2^@m`` zr*1g?_&z+E{}WoZgD}BfgzI@h6(f~S!R@T0FuHdy>>Wlh@HT|kXQNqC$I%(&yawi6Wy(X_15m(oj|u z?Y+s$h>}qm4b^#03E!-evLYhcdz6gi_x%2J<+@y*tdzE+OH?X|ubExiKg z{ENmV%CR_m!(!S+%Ei56G5A8x2RUCy+&tA0CpxXdMD1|gQIk(D(`@egCmT>%FANPf zdtjTu3Ezl^qi0(#o(PD7s>UiX@1Bgyt0$sdCher`P(<015LBJlif&yF*ef#(;sK&K z;p8uH*fjuGcSvB(;drz%RA6QIAVx($f@hDfg2C7_fQhTXXRjU1m3Gh|7HZ@NV`qjVD(Smpy~;ZLxF!;1 zl}F%E;dzqf7KUS}PQYGp!xViNRQuzP*=M)n%=kUn@jaUV(w~AB**np)Wj%(3V)owvbmnE);Yk5l=9z|W^ETnF`;oY69(|_%JJB;P1#b>U^Qjf|0j%GOJ`erz zqmnBYA8PFs0?^t{Ad)_>7w4qei`P}~ zHO&)t*lmQjA8jCcN)01x5XWr1a1mV3d$VF+RA~s+^X0^oJMFd z7c8^^cFq|>cUA;b%6)VMJgjkjz6Z{jK|kNL?|NYNJEG;|jsDUr@dUb| zV}S=wunWfL>dC0?c?@&b?dDs2i}3BFTzpoVjG4!h@P$Yk<}1Zx?FR?^Ca%cdKSg^) z3*%6sClwFmC*hab$#_L97sIX>VWIGDUTwuOy!kK*pB07RpTWg=X{{TcIJ**ULVd8f z=Pjw*RtGj$y>Z1lPxN_ajXUNWV$(iH>{%0oOUJ)p?il`KP77Yb0>=k{eurTAZ3Pr{ z8{qyU<+!nQ0;+;OY#V&b+|YO>Ncppfo6Ecxc!h}~^JWNZUTvTpf;?_*6YSwFf3hR<)fmzN!nI%gXHwn*A<H9UX5mef>pH zf2~8dz1dFER%?-0(?q!8^C^spfdqCY?#GK$$5a20h01?s;CF5}NNSfu^qMB?DIaZ^ zIdgq5wB7*6rN|;2*1~z-hZc;KvKmdB!deulFb5I?pKV@>+q!ZVujZ8jq(OblA#F3H+=!2Sw5Y(E4X2)r}Hy z&zd}}J(hu$Jp=p`FFOCnX5e+}Fnm+*j*^e4{*&*BW9aXESCi_{RRXx1WrIgD0RJw~ z!oRAr$bT}xIAb5|i&JA>4Q4is9zO=gw96#D&H~NG91vMn6v>X8z5f&%Z_A-I8#`MxMBMcW{palerBI-c27phncxnmUvJkg^Pg~kB=jVSox%C%H2qm!hPRX!>>Q4N5;0Erz_YxzwjAKtd zP+-S+AuInTl&za(&UW?7vo%@rZ2SC4Z2Re{?0~BrTTrIX3g`QwWo52TWnFekvRa>~H(NfrkK^^F z*amqywsx^ByG2ZvJ^pb58?39tCRxw2TeL!lZP8O?+lFM=v6-@Lr?VWpZ-Er6a{WFw z8_l4#Vo7$fh%OtFXv&%uTd*hmt=Oe)z@Fdez>>zlxbV0UtJS=Oja%-^D$aCguh`FL zwR4=={LMaWXQn!Pk(Fb|w@+mud=mTPraUWWuE6?fnX?U#L)rDd$j&NNU@yNL#}2!_ zL>rl#Xq?cA#f>s--o~W*!Ol}lNaH;n3%!MHd zB$4~^=X;v19JYs1X%l#!vKullmBXFj*|yj;n^EAradV*o_D+$*r^mE$@n%4U6*IBa zSO6>jjOWv9E${)&j5_(x#^-%*_tj@*p*c{@3542Zcd~_5xODAH(^IUY+&A=n82lyPNBe-Z$27Y)M zhW6*(P&REgKK7f3gGrY7ZN+##X9+@T>rAAQ7aD!nMpGqOeA;h-n#3EgRa-NK(+}E8 z7|?TaLoqzRZvmg0XF*Vi1TL!Dk3loWjy=z?U$wbayLaC`)=m)cwe&)>oCbNn^RbyZH~&$FS#l66#h3<3r!Y^qslk{<|wN z$;1bxJ>LC*7 z%CDdvrXk+3IF4ll3K-iv8PZkXGKN;41ObvB+<)~y1ocZraPQy{zCXAT#24)19%&rp zs_c30OWZi(H2Vs7f7uDJd$F43>b%*mqKhy&DS^2O^~}$9OZfIE8a$|Hene9l=WLZ` zhvNM3`Wpg@`0^}|?;rc&St5)N*AK&IZ$+99RYRTkc`!$A zC8&mEp}S%THk-~vy{GQ@z$px$JdQwnk-u<&KS%msg`u|3T2xebM=ftxObuC%M#p3E zV#FR?3Nid-g;YEou@m2|2**B|4JdEA37y}j;uDPkywxPbUQk|vaeq_skIp8PXph2R z49Dd7o#-W-ii@5`^QyUfaQTz1D7$7k`u%Z1!GErJENv|cUaZ6FuJbfEHv-|)BJiL0 zIy^Dzj@Hy8$TthY9XqnnVI&Zioy~)`ELAifnt;x+zhP;S2u7{(L&-oMmy{s3Z@LbZ zUN<4-+F6*GT?jTEJUmYE#mGWwmKRsT)`3)T#b_{RY{4SpER+9i2NSH-3Rm;I+5Ufv zSU;=xAUuhM^zuO99LnJ6sWwPc>w*0{5T)c4SeLALoH$t=hqv`Z`q^K!k0^qh{|!Tx zy8~9qwqoAyP(1T|0_JWL$Ef?7n0=0F^RZR1{NV;zd;_Cy=GFpNly z#$|J*Q0z?s5&Im5VKu8T%F_{#-I|Lw!oGNH#YPMrmxu0W7W38-^gY;bz^o%H@Li%0 zE?5$UrcFus;CLinagt@TlB2Mj&OaGPqOf3q-VJ62Vh-)DE53+DAMwR}@sC^-O^%^j zy$|}x&Bfv^bcTGn3TM-I-uW?~#AZn1a=BG(!N-2;|VH(<7S891&Agsf5872fp`q74_b8}=_?d&bioK&CN#-Dd}9Z|;OZ#RE|J z>JX@GnuuF0MA^3%tFg0f2r~WJL1gPw=)V0HS~IUh>H#g(7dwNWPC4P8A3wqW(o2}N zObl(@WpPT#elYRXh3{)q(AD1$KP%2eKQ%kdRP@F&rvR+oOYdiq>Ez+=Rd|#0M2BJl zb_}w3d8RiWrZaqaPbMx?H{k7@w_umYW}MpXhtuz_Kt5wVUQA2IY`;WI5y-I-uM_Zr zV=@};pnh#*AgZKMPhs+AoV8*LhCb5cJsPqw{^UlS_uUKkRQ8v#+!Q-BpKF6n{XW=AQgBNU`c{{Tb&eUPu+4k`Y_aOP+b2CNiix6qEF?UhnkUcMJXARca=VBq$4 z6L{GD87ACbz+OzWV^@pbr&?PmbGBr_wtb>*R2`&RPQm9fndmVrg`Y#R!2TL#qdcjG69+rMDGFip+8TjFRS+7= zPs0{o8vkCNK+ig3^aBBE)qes@k5r;RX9@0=n}-&qmgrDP{ltIX*!N-!URj!jDQ-7N zTTLPges9Apnawz+d;@AI?Zg-K`(H^T52tBJv0Zm_(by>$*Cy=5o&PqVgvMs9KC~U> zE+yho;TvT7FRJG#QjM+K8;6Y<+%s(&ww60$%Y`Kf^CoNAntBE^zJze_q-Bd*N1G_7A1w1Co@oU-)Vq8Ww_BB4$o-U z>EU(;GUX_?t%ILKT&;0sSa&SN^i%8`v7Hh8FN3_e+O28L`ez#D~c zP#vO#g5Z}RIyZzA*LdPH8#}Zr*2O(Hj8G+LHh#M6k6Ta2e;$fXnS$C#J~*j0;S6hKys&9Mh#Q51z2#~!v|(Utu@!S7Neyc2d@y|ICVc*8 z0v(k~z-Na;t==K1zcm)3AFGp`V+wdHF&+MDWeTRvG2$`b7 zs{vlcYDTk){Qy7cy_;Vl8pHn%Uc~?Xt;a`HTqEB!lezVxU8KA2VN)|@nd}ldMlw2c zN&j*w5)<`{8AoS^no%u2&bgZ@nPU!pNxDqaQe7_M)NX-rSQ=d2J^^2dYZ4c85%_hj z1ZEh7!&OZcSiIkuazomo;h_({W!2$T?-b_9-I-uryaJ@m!(j^T*>;L5W7*Osj1v?> zIx&HHbv5v2SO{0CsbdH2xEP(zLw>-Ul{j@9HFo^v92OlV7xRu0Vb=qs>e@jvc|k4t znRlGi75Bie8M(~cABrUL%@EgTA0u#))q~z+>p?N?8{F_oz$9lG%(#CVy!6h3t=KR4 zAgYI_t}CN&{w+|<&BaT8LC7VuctOt|3ob52X6z<>JAN73UWVl|}TwBPTWLY~Iqn+KUFy&((BZ1XT6bPwhg9Haf`!x$ML$!=;sj0$g# zVk2e!X=vqPkwq5ny^@JH%;K@^?87Fxp=8|FkckV_lW?)?Mzq)!g}b*U<2CtwhMZiQq4=tcN2z$V^7&cV;Z+G5vt)26 zE&&6bzrpsw^-zCtGN>$CCYXEW2j_BHmMHJbVJ1$ZxyI&`oX@#h5KBZzqwuEMm6*fU4gZDAfxKgvwF}n*T?OAQPW*tcdVYY5YdL_IMNoQaBVo|OpC=o3G}Sh z${-rcmg8}SttckD5ij<~V8yst9HJg_{Fw~=-8q%*QqG`Rl4R=NZpC~qhH9c4X?Ai8 zerfl|zq8Xx&9+Re*Na7I_XrGmwivA@&cOwf7vj1IcU;1jHyPE^j75V7T6xaJnhh3c zyO-t%J?!zsqA1+deiVXU(_Y{D2^gIC2->9&0Q;>Rp6bz=j(RvHdKP$E_yf#wJOffQ zOehP+MpeNMjq*za5yfI+QUG7s(8eC>VSD5oSJrfwE7= zW7wVsh~0Pyaz)D_%JBkx`uG}x$1XwJKZkH(kUV}#I1V`tX`ryR8Lod2$BlCg(ST+h z7n$Z_ku=S}Cfo%#_bcG?=nrfhqlJ&cB(a$*gCdbt`0_oArkQ3~DrbwzX6`8TW*G|6 z?_D*OUT)078R+dX1GU^O@uMQ;l{PFyKZgh$QBA_Rqow4efju5+@j+uHZ+y7am!6Zf zo9`ZiL@ErcBTQLSmoS`uI2gZ}FUN(VJ~a2@h3;M6m?deCu|Cu%T$F^h+au6**8;RS z1K9uB0`=ObWB1SLDAe+dQ(Z!RtcGP6xWOGWgKY6pi3uiCpW)EIm1uhO7(~cPqwD=i zI4k)#eD}Em`P1*gzYs&bLjBzT0+-=qJ0l!lC4r--2)u7khbj8ybcRqs5DFx-H7a+4z0v8;4fx9j|#%-rfQS|vQ=+U?jn=U<~cP7g3nWT+1 zr_*tQZ#h1Ww!#sGeu%W;z)QUkW*%0;c#9dx(9i4Zd8v3%UmtrO^@5elb&%cq8;YVd z@v)}}dWa{(hpu^;-D{4jE;Qp5HXXlqyWmzoUtIBIGEQlkPb|%7PDV@@ZLaHJsPj~$ zkui+uT7;X|Zp7PT^GU@SBUH^5V8T;7RJW(usDmzeGTH+@vfXgOehc=&7gwCFv=~Ea zethKmTpYc|;y%LQ^{qyzwkV$ziqSmt>&4hZHN2QvQ}N?&+C}u5ii5pcSiEK)IYJo~ zxuQ!krq~%v-5pKK?Mwg2ks7*Po@@sFvCeO!^_d*YEQBFW! zp+5@V&p?d>N_gqfBZ!($pdkMlNK{V8Uhi_WU6+owZ?*6n6?R&q8;hXwxCkKd~1omNtW4xg5Sdnux=1XeM7$8y}u+206+N`u(&N zWGB3aks&!eN3$$BoENH{62|wFDf{L2en|gA`xdt2u(RYIoE-efTvVdDqN8%ybyFH` zMYOSc9)m-Bobb?8dRJ`QLuS!@Sop9CX6~DS<&SkRDB1#_Z*{;(nkN{0N`{1N)x)aj zX=p45c%#V?t4_IL*0({#%4VA9y(j12* z*|>cJTK;iBIbBOk=$MQ(uJV|2Sp^N!B=O+pedIx(3i?zz;rwL`CP!=IpUcvC=bjw) zglV8h%^Sw6=MIPl3Zv`ecDUbI0vjSL;O_xp40yT(LmN!+sM*kw?=b*d0l(p-PVJUHvT?<;w1cVox{IITd}HoAB_0aFpn=U z=FSuy<(_WW5PbU_2Qb|RpQ$>de$f?pyMXwSQnet&cFbfg?J<;8FO!pfl_+knRnU|C**nJt&CMT=j&QbEm?_$ zlu1ah6YN-lBhI9HeDR(&_$72Dc0OBz3#tEmUfd4*?Z&{J&Sc!Zav{dNn}G*orlLb3 zohL;k(1&)YkMwnr41X!?wV8?UHFfafzp>b2{SX*B&$qdEfav66Zlo&-)}=JT_nHJ~ zIiv=?655c|wF|UmDL2);5|qjc!D;qvm^S}7vpva#v6oSW&F=q!=OZV)O7$V`)egAn z9U@5Ibb@ocGKce6SHoO-un#JCZNZFxs-SJjlfXJ9)2OE1p=_bm@+D=)%l#xpSZxfafClA<%&UsxH|2oIOhocGJ; zu<^1QnqB-0;c^^kuFL>_|7sX27l31K28r8S%k0ZTSS_BzEDJH?Zk6S8FF)UIG&$VF z%=bOPO}SIb=~_=FTC`^v+r67OjM$TAZ|A1C*iA5BTm(KnY9v8*hWt>HGcR?@o^RDr z;}^B^WNU>A1m^DKvtAQEZ}b(f^-Q=~^@w=0P520J7~0M+YDwkqM7r?_-F|#`_d0%^ zc`Q%bqWHf`%lU#>5B`RDDBq|k;P;ydc#reQ-;kZgf7qtTvm3t>N11pH`oeZ*9@+0xfJkOilmExC<-Xw_AHiB0vl zH^XFoZT|PBAhMRE68m`>WOheBv6eqiUK+h4+m?*yJ8J*pyh)OLS?gO;^7cIK`W6s3 zm2~p-S}G|W4j~zy+I+a~Cit$B*3{D6$0^>LNBY<9BPTT5$kz!KM7Wdv5&mzp4C`jqUtv*AZTY=I>7ni8SxM^@@)e zCH%pm9sHq=3pmaIDX+=}sK!C&#PyQhQ#**?Bt4=xx*FWJ%izQx^7!A@AMnZkG6?Uk zf}gZ^a{Y5Y%=!L@F9NR+q=+_`)#}kOYEQZ_SG;sCquORTcA$aI}7moCQfqz%V zV4u1yE)tuHzs^`=f(gwEP54M2#yyASM*ZcVfT2H+V&T0Mril%w*Zb`G-7q!5Zb)!;m5VVAZH|r&PPNsN|<)7YH9vQ zr<1Ws^v83u`Y6?*i(VBvSYM)o>8qtMNAEctclb;)Zj3=~XB)K3o{CSSWikHj7;H2D z2EA5yVb_XA!K?VM(5awFbysn0zwsE7mR^H|#VycrvJ`qZ{AR373n1F%G}P5rQufXz zkkJ1DzdY#~pg?^9)j)7IG$B)CI?0>7UJ{nIjI2p_g}wU_!|E=eDBT0_IdTwVFYLjd zjgzn-i}tPZlwp$gI(ScYvz_LP@YI7qlt^{P(P0y`3Zgx#r*c>!J_7SE=R>6KJMit6 zMAIHwOqrsJ#md05T#hw8|I$gE zz8Wf)()`qGCsdu|hx4ud(J*%j2GaYNk%>73sw?BDiU6})m*bt&8?Y!Y5>Ni|Miuv& zG$Z207Sglh9?hL>I~IZc$D@#6upEU41SqShgue?dA-5-n>XM$AuzDGa3jEMH$Qciv zwn4B{!-BMQlC5QfIUDBTnu7~*O{4%{JU2p%jEVT>KT(`(|D9QRK@FSL098Aw|FcgO zZ@-nLnPf>+GJ6kg*9*ZzfqT;x!4(ZIs1XZ{;!hhcC~dHoe;JvBy4YV=O{8HR=SYw;j>BI5-3E6SH`H$uGQ<%yXY>#%eQ?ICJ= zV|$}9s;Pg4m5rKA^^>I-r|XEZ|J>0t&=oJ}GB~QBgNr5qf@|e*Ql~Z*=L9UoiVRyE zmo*dF@5ZP+R~4mpeuUBk>zLKGGAQb1jz6@uG3AXsZoVRmp>xLLf5tC>`*{M6tdzo8 z&RY1^!U*@gwnLr#dCxoehM%Ce@GD4k3*)I8MZ8aS(l67@IM3n+c(_3X3oU66_Pib@Znni$ zK>}1!l0kQ)J0xWn2gY~D;=ycf>uBwIB4_pE z6AYRt;YtY+EV}j;h66vrqD${!(vEW=iMwH7&OewGC5>q`S4gUL@!TSNoG``%?*>?) z;*Wc9Kj1b*s!^WBUk6NL_oH{>Cu|!qVhvP-*sYt**rUBd?83E`Xddr^m%Cm=m|G^S zRt$!Z57J?*)onN~D}k$=hM|^nJInccaGB1*h{SpLDr*Wp+f896Ffb+kUdJ!pa!ntE8aRR(*6dO*7>1NsLC zN&TiV_+W!EIu9D-shg9Ks+Xuft{q^PKPbznlhk+h5Zu)dJ!4y8s={Ho>9QY6e`SOB zVSPwZHwC#Z=b#@r@Ux*^$2T`%PQfrJIZL2>R6D48WJ2q)Ab7i*W_0A=KuM}A2&yV zQ3yR=Sz=JQA2$CE!{zHjuz$HDYJNAs$`L1ah~`hO+@syA*kEiu8HV1=eDS2%bmXUv z#r2VCpx+dPFBu24S}+Swe=Wu^VjsM^;0r!ZV_``y&p0p)b27Y)8!!{H9r0Fx&XInm)yg*FZVm8}BbXgahMQCt57ERq~@B6@y)YUg_~WDedqTc=;Gh+ zDdjc$-T9}Hg(OeSnqROM4YIz&h?BU!ks0hCWUQN71uk=9_-4CQ zvcWEgyVd%hc`KGggF0nk)_D{LHr)cjdo5ymRh}Gr$qU|F>A~d0NVv8l0Q8EUF$em9 zam`m}m)lKdUYWmSLbU>+Z&eJ)2pPhd#ESyaLM0OVWFn~^=zypg`dzs67&sdxfveKDIYPbi?0eC0O)eDsm$q+RzQ(X{H}g68Yd-4=f! zYgRB0j`xJ-Z5ePrE*13uWkH5dJ%szxpHUX=F>5hgcG8q)7HNJ#w+@=W--GZhF}z!? zjZ3vfup-G#@cWq*v>XhF!0{L0Lj^sz7#UnDp@!>Zzk=RT1zy*snxU;JP;j^h6DTj@ z)p{u`P8f$7CzLR2vM?OTn#bfGMvxWkg;Q45uuCN$riPCJDX%5AmDzHvm{yK$%ReEg zN-l(3KdT}BzrAqmI19>Iif0tP;#0GGJ1C@OFIebUso|D8z%5q zTj^b(NtODdGB_jfBg~;ag@QSeAU}OPgm%snC2SxT^3Zx@u*{e_;OS@^9w2bVggf*sX~qQ0GmV!D4M zZ?q4bVimCF4D~d3tK-iH#?T&+3g)BdA+_r%IK@a~9lfL9qG6+#1qqB)h6w~y@7*)| zIXqgTh;DM`NbxE7&q)&xXMG}TM*u|Zn_&MoVYCU?#M^!UKMN+{f{VI%Oa2g~-132t zq+9TMP!^?irr^gWHB4Od0$L5qff=649{NDp(hFXK!UCF037m}5#AwoHTGw|d|sqJf`Grs2h96Yyo-GjJ}egCpxt z!;R4#0_vTg_`!L;xKJcGkE9k!$#+{qBG0?>jKg}0I!)Fut zRZ)AH!oAdkuDk{QNB#iipFw3oFF1$(2GPg}$h`HLx$&?X)={28@r8*fWvGZ(@gd-% zU104niLI*M4VJGS!Y4Y{dX>=}$4^nLU0n_P(?2l}-C{t@{11pYd7 zfWv44j0Wa2FCR_d7nX?Q-y}nHchJJ2i6Zz`uLYjj`M`dkWlY&&Pi|kB82(%&jbb(J zAUx$56pDI5P3UvxeaLP>^=C2kxiA@h=*&7}OB<}R+Y6mK$?&CoCEOhtW-}t<|_`?{%1SJ@c>hG)dqe&E`x`rLRisG z`x&ASxgV-Km{+Gnsdg`bA5Zfs>+TNd$&bSZ`F0pm)geEYZ{}XO?PPYYp9)bb#~`)s z84P^A56iEsk+e-_Txq}pcubyN#i)Ri2%iHx1r; z9D$C^U$D1M9M@F7gz`-ZaPR(E+wXF`~b2NzC=6f)I_ET8scodq&3?O^#7Dgp! zJGa4iGLawdgaONO*t4e_e7y?5>WKhCmBm2+NCGoP^*&c?NwbbK$D+uga*#M^4cE5p zWn4@C*xq&R<3iIH^3G>h@v;|Jk(LBC^7rg(uD;fm+mn&Uv<)~xj(ZpMYj==8R!XGo z_3Nf1wW>^mi!A*3Og#lU=YOhFMw>);=Yyz9?GZqLps(6f_%-sB%c>yG;{Oa3@0 zR#ZZ(k{Rfl)Z6CD=fLQ-KQR5MA|~$?#x2z+AanIBurgL)*;bm5)IJXXP6(s&Ohr5s z_#2!k$Lh=2iGucK0~qK#4Jlh`KGgLt-2e6%GOLfn4x!0#_iYAea})Ws4%grr)dY@f zP?nd*XK4I$01PZHGyUC9xHorKkRyH`n7yX?Kt>kcr^Ime~hYC6LF@%if zt>V?!`}14fx0AQo+exxd8cBMULW;$_$k=izV(oaoshDB-RpcsBNZLleSZa_-D>rZl zS2{6)W{NPqvq4b2UWxCvX(pHNt|1b?qq$vEUogMj)8KPnKAbfiXkiglET1z-79!RF`juf{e{D ztDJI@Dx5InE@h(4Uy3E$4Dgn<1nP&Hvun>t;E16io?GaLfnC9PZlyC8TTA1Y?Xlpz z_9hgX%AmK9BK@-=S_kQ4!Jsg{c~k^lwHF(`wHQ7@iiebiQfQw`msZ?=4@!&kpdo24 zGe*UVG$kG)KT7kUCq5Ojho3Q;p^@A&#R#$vn@NGL5??GGFNjl8gfWX3GMz3vxUZ2K z#7J%%*&}+BJab@pf}Kt3aS~+Q123}bYAQLrE|XkI-bPMl>?9v${dtb=HN08b&y9v# zk+ZHDWVy+4(q>vq@~_krvCHR4eb`Q(=~v_PW$wc8ux-S?TR`_KT9s9BbEE05De3dj{c`|Sa~c3@2^h4gkb8=p0>s%VHxjtGEVRCE_`2JF%O2nuwY966ql|zCLm% z-y>N|0&#K+(Hr;s!Aq_`q*ZEGKCw1MPMv?8vdJ51bZ$B!M;l&d<|4{$1JMB zAj%YHKk~;m>c9UwWr5RNe}J6dL|CwzZ@MIO3Kkrlgctv#JMVG;g`^&iL6o+q;6 zG>3g6UI9%^5YOZ|pyLuP983RxMacQ4DJB|lSZ@%z!z?LJY8lE5E$kT|LzR7czC;`H;_PW{_3 z$Uj&h&|2k49zJP-?{mcQU}`VTBb7k-lDV+XwuPDZ=o+WhEY1JAKb<+awSgJgqz!RS zZXj)+1pz~}=UUVSvg)p6VsIh3=hqGVwQi6~y9Asc2PeInKx=UWOo~1MTD5yf^fWa% zBvA+FF7$(Q`dEw@)Wo!89qft_!{oUuxCNprux@e#s2m?h_mtB82&XLZvkT>P4atxz z>^4q1T@J|j8ptjm0Rt~O1C1GhAd>>H>+|5R*j93Fl>N_8Mzy@9HEum+h~GQkL*e%I zFhRtWX?SWx{)<+H+e$ZJ?5<&0Km8_*JwFBN9MYNi%?j-4q7+R5zP$;iJ|upw#f+BNm`>oMIL{B&xeY^3NB=mColJ3A(TXo!i^D0 z{98Q&8pY?~w~stzbfch>_j#q=_amVbbEi^t&k zQB6E!{s?}XtB{Xr&P-EW5{Mn_0?tAh>vAXHd-cB{FwG)1^gcWAqL10uNWhAdL9;}< z9pJDmDp3Z&_iTSY`r8yDzu6AFHok#9p-RYYkif<#Rq)oD=1>L>yiuanhDrXFOX}<^mnU`TC<^bqw z3PXL^2~tZr^Z^pr;d?U8-vm^^?&N&f6OjzA$EQJ&ssZn~KAj)7BAl&AH8Wpq28;+N zfbXrN;O$ih^)-hfp!6X*J-Le?O)gUKUV)q1OnXELXx|dOz*+U(Bp1+(~9Hc}(Vgs)u*B{jmA`6{t)p zfW>`MV5b_zo!hjH_zWBXQTuVIapfc2`&R)MyK}+t;XF9C_mJR$!XuLBUkWAeb&yRp z4UzeA5VOPz-eVQB>S8tbAS|6PJzvMa{a8vIlrp%!%lnzlE!J@9S1MejEQ}@dQ^C(p zoG)nY=4Nf($0#h(0;2_aVAgdBuE+F4UEe&I+Hr!+x-pUTW+*V`!2s9y?1dFiYhWZT z3rZxE`&Cqvdv39bkfuF_)mSI+kBEwV0f*HDFwNav?~GgXrpX zIP9#+h^v0*V#-3H(*7A&e|RRyM&E>4PW@0)UI?4aT)2sPJ4hyf1M^O8BjU4Nxx=yr z5LPh&_T?RvKRW|VY+iF#$Jc`H#?RcM%LYt&Pa6E|;~;=CBIwx4%+=gYRH9>fMYBuL zL(hQB3_Eo2)x*abl%x8e74tgz5b;SB;~SS{0z0D)gzfjhf~O2vEzo5a#?!gKe!t_we;@V|kTv{^a%q;JUj?m|Y6yknidO>TOfN=cF0${Lr7|abr2Hea9L5 zISw#k@qWm#I}aPGTA9%PCnQ!fmh7_p(v+_%4jr}0utTO2TtqYACFL?}73Gi!*T+nL z?GJj!*rO`#l{^h}MzhU7KyUUs{=U|1(iy1=-l;r%zCrKh-^w6L5YDVulP6I(_S65v zIo!YXn;>`55KQ$D$HYz^e(Qb_oK~Moq%u3PedT20Q!dPe%%tZY?Ht+!`~WoH2*DD$ z+!lU6?3^-zFxe{@k;mmQ_1;sM7ugJc4^&}&qXXHhu#UebKMcLJKhV38=Gv+p&^zo0 zWDGxLRGOh>sm-vD%M#trFtDS@iHn{~P2L z)sc@A{Yqwb93d9Uvq+3VCpX|3&b{A$ihK7ZoUdNcN!B^9Acxo9;eId35X^QJ0+%aV zkae?%vzJrn9mLy-(o+?(X?+r77{40+EIAJHt1_S&RrpJSbh0){6)YRYv1647-Z~MD zfsTPFlPZq~rI+z(!j4Uuwf{j%-z0RsWra_wGb}kP!|uD<0!|%t-$uSKe*74Zf7^nn53Y{-)9D$@QjYuPLdJR0 z0TBErkIxTS;o`?yIAU@c3=X+!L2mAvo72>!$~GoI9o^Wo3qj`~z$H z-lE%wo$TYO@32nz2j%rhvz`hHY#r^%Y}LPySW}EyG`~K+#RI2UBxC8c!}!Fu9LMz( zpvwU#G&{Tra(Fl1IY1gsN*CdLdp}H9rTMot?KBKOmf=ST`TNV74@vfc@!ctKyo_>Y zih8)_kWLc&SHKI$?BrV|Wcjsci-}-e1&J4aNcOgi^8w$@_{$@7|K`<7e%xO@e$7Qg zUWeWVP6yfZ4hD3OXGS33CP?I~O2;*4Sv@1+4&%_uWdx)pZi7c6-66KI3@)uL0k7Un zn3AT)7kw<@Ht(1SxgIHyWpD)cOsN60+Z8bEY02CQK1Wvkt!fgZJ5Z)e&ZBIt1l(7f zgCh;Wm~T)-rj<5wPoMNLq8GCvN%;XZwf=)wE&X7(ZxyW8Oyjp69|yfUHuyRw5l26! zqsTMLT!`<4P25RN*02!H|1>6QH9H~4oa&niEIyQ1#<3#*H32*9!!4#Eq2);$bQW9GELm>&O##85sU+QImpn=_hThTEZEZpAv4Z2-T!%> zOer>H+!mjN!G|?q`}#24ibw@j%_*>S)iEw!Q=flmkO|YX4?;y+860vv57E9|5H*hS z(k_g}RHly{>>A6jSoRKjFLXo6vIEfDwUhR!wnC+PGCVqX5LzP*_$$$CIP1(s(7^A7 z`DV57Qw#-CQ=zrEnCpy^BYvfASo}ejloc0panDjAcIY9j{c{)?i;SlB z-eyvK-4v?&YKV;5F~;rJEBHR;f1O=>JXhHk|KS&nqM9g79x0`~!Z7^K+Q*|Z6L~d+ zN)!DgGBM>!g?`E-kMfM768+lALq$ZO`qHFR>{c7&E-d;$Ly%HW$UZq;qmCTi)PTDaxe0zu(b z0DCe($6GfmH zn+&CKn<2Pb84iYMLZ$yn=m{=>XRq{$a@k#m9NqvkYolR#Og`lQ@Ev?IUI+TpYgpXh zDm=T`8P7&VP$khtP|_rV>_Z|*>QIM6v+Vwt+$Q2(DvuvbpM{~$_h3`;0tBP-z^!k? zxqVk0}nh)7fFeLStF2(Hv9xQ~h8Om;NX$I0V7S$$I3mPb@iX@EBW z9Zdczf|13=@Qv#kYUY3&j-0%S>#-VapJ7n+p6aC77Y-}k2 zW1|oVG1&k~+gCxs9@eiAr9#^2V#u^HC#GwoNZrR~ZpM;x>6G>j02>7Xcn_9>YdHfsoep5Ls|fde5{w=j;6 zFHm}hh7jMJ1cMR{kk=gx?$)t5DJ~W_o*rj*?#`teLL9(UUmj%CbMTF#Nm429MSA#r zsxQtK1?;?p)+Zz(2^M=iP1+L|Jm3nmT`bA_-R6v{Z4Ru}cmSo`0K4Ocit2SQTadLsvU{fo_Tt#PDoENn35hMc zgnaq7D6-g{dVY*ge!eqF+^Vy1R+Su_`@$a5U*%KWya7f>@;sh%jmMGf`q$Tr4KQ~j z3AAQ5!AWj9^)TLuDKg8!0ZI-;x4i>~Zn9WK23x@Ei8yd)UQt-q6bx1-Q-xubMCRx; zncnf7@eh{*<3tl^P;+K&#}vrUYzMc*Hz09$9|=Pf_$=Oo-G>ECetj8LZyy04DR)8f zwOaUm$Pl~7hvG#+E3moyM&3@*FlE1R9Tjt80IR8Rg%{H8i8#AP9y(Zt##>&p^<0Nh zb6+|#_^PE22-o0)aqO+D5PeX zj|Z=KQ(J=Sz&79wjI#MLsTMb=j4=hG*fm5fORZrxKOJrCA41Q*dWr5ym7$iyEucJZ zM}CNy-{;$>56l>MVYO4dz}mtnF3r*Q?i&)W!FzpsNTsZPkwiGT(2H>n(t6HKG# zF6>%8fmN1!lD1FZ!b^#E=wq!9&Iyd|K;XM~Z=n@2vy~?q3;V*lbNlZz{ql0exJFu7JGYuF-O1MBdM-vYvRd?$R86|VV+uJ|0ZRFl zkT1IlmYKW=4r9v5s81XH^x0yxTqXdljNdauZ2iLS-gbEUyc;fNO+ZM=DBMF0(34UM z``8?zf?-K|?}<%lOz#TVXVt?PdI39iC&1#f9(c;u7&c8wK*9cRn7}KC;g?&|$nz^* zbbpaDN@~9dKku%`8P6~g<{@NxW(u9j*QCqeZ=`P#F*@DY1sR5^lO5Nkg;vYu$>M|` zm}Zx%x^=EknIsn$=O;~ocgqek^}ud3Nb%lzs{`?7TiCR+LQOSb;W*Nx3| zqiiG{HN>Zf`7X4b$dOJFZlmL4&1kt~Hh*=%g+83yjs_!pkwX15H0A!xhEKT z)vkc3xl?4#a1ELO1N!W8FM3eTi|#(EOAl$}pl>o2h;^unaGO#bsc$kQZ=3Xp+hc2@ zwR=DDncG8>c{(J>)ea-oWb6{Yp2QqKOU}Dglh6$h$(@g<$t@xw9GmOMBk>aCkgPeO z;v>mblS;C^e42<DN}2O9}4hqG`Y=YMDk$Bg4~WdA{* zATPZjuOml&{101MQk-4?KN(8TkKU2v?JUlYHqQ<=V)M-(ob`?I#*roZd{c~(pQqU&`2zq%j3gM*Evr1l3P6$za` s#3epl{odAm=y$S@m?u_GYQBw0m_KtkVk|q@ERdYPmSB&0`TxlN8-V@^-T(jq diff --git a/research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_none_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_1x1.pt b/research/part03_learnt_overlap/e01_learn_to_disentangle/data/MSC_none_r63-1_s28800_b512_adam_lr0.001_wd0.0_xysquares_1x1.pt deleted file mode 100644 index eeaa54d7e61aefc00b9fbf7fe0d7805462bf2bfb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 65259 zcmZ^Kc{o*H7`1r}mFA>TD2*E3voDPrr6?7R;#bmOD2*~CGL(ptWF9jQ6?dZ413Aa&FiQB zUxWVN?!Z2^{n$1>0b%Fgple$dE-YY<1N{{pdh9DK2sJ+70?{%1G~C~ZLUt#pKbE?{+? z8xea#mCaVkV&4_%vmO#(=!+%=D)?=HzGgnpx~`QjiD2npy^VDFW0vwK|D?0sH}O6t z8)D_D33T`3FSMmlpG}&Z#a5nCWm66{qV`7td*DMlzBe}@vtTCbmL;=#@oU+~I-^MT z(`DrfMcM4j^VvXFp1qW4z!r4Kv#MD$+0y6A?5qo7Y^+r>(=&xK`K^<*h_ zs+knK$x@nKqAtbGxFE&e*)GWn7f7(GQWESNVeY<7m1O7LmSpc)NwHgkrP%5OX*RoA zhSho{%MO~1Vc(x0!=C;q%kG~p%dRt$W+(lSWIJNSS+^1qmap^=(({La2i^Fu=_lTW zHR9%(Dp*9l!JfSdnEIa)et9Jzf6jZ%)%u3_o7#}6+X1C zjKU#TlKt>&JX=^iiM2NOL*z?kcFKXh?1^{-_SIe=Y}vIwygq(_E&SDuZD)Mg z*swKtu3$#A0~eD{j}7F#z#r>-0;#CiIP|<676i(iqT>xcXikDJt-be{UY&BET2Aw% zCRP?yZO@1xwQejDoq}kDvOiA7Zy+>&F}du2f{3@R#h7j5Z6?4AZ;_S#xeHnUEQ z6|a_HpU6qEJ%v*2z65U0Y?ES_+>&I+h)S@zx5ZegM?I?{;!P7NsX{)&cgU-9hpS79uoALa5Ko=M-De+?cc&S6-VOz=n{+xO!{#ZCr&Y7(uofy?*ctr+ ztn9Q#(lkby=&u>2v2~J&l=?%rySC6Xu~}5gIhmS`4W%}(qG@1gC|$fIi3TNP(vPHt z?!7unt#?VnhaI6_!OFyAWh41f5x|xY=&}RM9JcO)BAsd`!^U;G&^31LsC?cG^TK_l0WG*Px>ycnD2E6NTsqU^b^!tA6yLafx;QH%x-LXgPKE9D-9Cv>7K|2O21{lF*v zZ`l9p4R&0qf^r!*AEtNUdE9SI5$S>k+lA-c+%#X=4@@7#dBG@-xBbP@Ekf+{&q8eC zVPQ7YT!@WJ{)e3M5eVdlpzhTRm$MzXn%joYx4y&fQ!`#wHelVHuQ05yfoEhjs=1tV zcV8t$qbm@;^#ddZ-b4TCJA`Px!!(h1kXCvN$Ir!hWLkniwNgAi@C<^9Ke1WP1Jd{6 z!K9UA-?$O{Ea)J6R$&!sQd+w zE$&!8_b0OVJi|S^QXr=UI=XMMCGjo3S97oR-r@42cUUS`jyp;paDV#}E z+4LDhinUO^_7$R44M9{SOnTrv+y@KZAfEdTqb*?+*IthzP;(Tkj7D&%eh9{HeK=a) zg$IXzfk=GEEz@RfKKK=0U0?8M!)Kg$Sb?B1AE30X46CMvWiWPew*{CJc-}!lGS{eOi`-FRP{5MAxTi z34Flkoyvge*U9XUAMO~OD~psb3ouo40=btf&wQy`j);(LNSG;(^-VwM9pN|hN@f(j zZhMEeIR`kgdGX+$@N1D1X5hJI)_6uFxFp4SOOrB2jIcH!W> zZd`cYgN6zHm=Q1tMtcZ;bBB)L1r5=U z_}ozjm*f&$P%eV>BJ8f!tN|nxUT0>_ooH64rYrD4sxRCLTp!@lYabd@|u>YGgH z{t`gt5DV{r*;po>hZ5yi5VLuMz4k?zv9kojePysts=(}TRp@r9!PfZ=5SrbJohREd zX+bCE*Y?1^nVSR5AY7{k(a}1H>a~L??(2iHP%mufccW!o7jUElpOZVWueb|#lY3ye zs|P#hwqeG!ZuspPfPPXR2Dx?aw5|*H9JqC#-vQI5UD(pzjhS+ND81hgUS>c1mHQzd z*N00L-LQPviBlWfkg}x(%bOZtxTpqa&Q~ITW;vvyOEB=D7&}7?u%|c=p3`z*mkCU@ z5@76s7ts5afinr|h&`E%{IVpdizZ`yCl3` zvK*=WIyhzB;w=$f11G6-G&8`C3Akj#YRW&wnswv&`yI!#A1*TFdy^Y7cCTkL)FknG z-UAwFdW*Ik+(<)jDbcj8zXT_3Ao!b6B?zl!1z+WU2|h1Uro1h>G-$wuCJg)2?^2Q& z@4S&|Ep)^CAs*2=F`hlPeH=feI1ZH>HthE$cFYS2Yx=WwHLmbocs+aTaAe_fc+@6A zdx}03!$NWBUL+!(Mj>=G3OCspz~m{`?vF>jN)r0cCByGxIvVvdAZzgg{;piEBY@q5 z9320g2k*WDbcz(C=V%GcJKv*5>=Q-*H#;8e08n5aVL| za9pnkUqrjnxw8wmZMv}PVK;8x?ZvMuF1NS#LA|{fsqQ^^KdB2VzqVs@)OS=xHo^W> z9h%gt5G4Kq8jht{_^uE$3SZ&6b1qI>urT>8fXJH{s7d5FI%VJ^^ zF3wuN3qXytDu03S7U{O zg6~?2#Zl!@v8}?;^*Xc!G+}Vbclh?TD#!!hr40g(1&x4gIK4V<)9(+IB4S$1f zR3GX=Pkb->{`Fxy7kf41UTorU=X1FZw3M_#g~RXHa_do_T@4qLk8mt3#VRiDw@wsb za6v9sZUTC~XX4-CXQ=&~ipmqou(628)xB}B42nXePy~v-f>9e4grGA4X!!jAuj}sP zhT0vxpW=akDc+cKWDPzQUx(4cP)sYzqYLa>At|;F{_Fiw^2DBO)E33i0XN)GccsD9 zfj`(~fyhR0+|a3~;u?yyuC`1tro~?{vQtjba?wB2vt$A9^=wyO;L3Hp4g2hPuAh~7 zE#a=2LCs2n?8Cl-q?96oZ=E8YmsCgVUA*yk>S_3RIq(ag0+5syguGoLXqX&< zCEWP8c*J2|S3JD9{P%HJ8mjg`LyBc4Ozr{-%W|>i`zw?O6+(Z;JIKDSz|D`sYTwhT5a|G#|1~Fw?KbpAyJ9!3oy?ed5VBCjs)&01=WB|Hn z`tYQn2PZ#wAl14ZWs_P!-0QLO+h=rWe}KZLx0vGp8ro)gD2`z9Xmb{dbe`dIcPb2z zBw<5LJa(wX;JHC0djEw%Wp^-Aj2^>AJODeZ??XA#8%29PaQ5eIl$bjpbGkLQj5VZI zbG5O3T@t-vJd@q?bQ+ttKo4at%DCshh>bCqB8p-|f)j^&5kKuW^Dx+l%AS)XW3`Sj zqFbGr#M^*7avnx2YGw%e(C|cq*KsvN{VdrTEu3iDdU96E=)3*#vVO9 zll|~Ji8e&);E}Ka&G=x0z;_O?iMfrNdLCG&b`LF|@8hRv03H?x;@W~>oYfDjV6LeTJ8QJE5NOfj}>NSO6(_>0>O!UQxZ%gG?^= zP33LyKmQGJMqe@Ap$dDd%dibah>d21EKl| zWORmLislnkT@Jt@#Rup~@_~%jU1%0~AS1;UogbZ`KI1=}E^+2+)hRra%BNDJZWwg@ z#}w#oUhSjit%+m5N5Xvs{sy25NdyOU`v(qIzCPGB}~8sHi9`0!-^5ynPK zj4lXMqYK~2^6h65%;5L~X6PW9buEQ9%&EZ0&hJ`hQwiJVwvlgFEJW6gxs8SoFKJG} zDQFyWhLp{JAoWg&^>M}FX`axZa2N3Pa@f0t4 ziTLJ{3SrM@7+);FCXF0aHRZ!~QZdBFzK6iG3T@-Q;!0XGdgitx(YFJNY!8OHdaZP4 z0L#yC@qg8irHlJf6V(TMCoUJv9>j?SBS@V755f~f*sN$V)_yQy9W1x5g)N`D2}JA7+dhz~@x%a2UXj)qU8L)s3Km-&k<N8uf2TfaeaVJ;TFqEOxY069UQ3Abg8_gfBz> z*uK^mA8&c%R2s~DSHsjEY%5TQY`KEuP;%moVtqwVu~Yx%l#jg){n z^HPU@{`5_v^?DdjrGC~743o< z+D_R2+7;60JkYYw3pL8#_Wgg;*M9Pe{b5UL4*HL?Q1-D3oI+?qsANebv^NPo-0!G zoN#{eb%fltg-O9N^u0O_p-G0gHuex!tvQ2oaSN(+@ifhE492QCW-#+kg3hM1{M;F? zz%b#D?RbEB4#`Z<{580hvl%DPtYS95(qML~YBSXQ5R)Q$p9%jN#EhLUV1m9@F;}g= zFoV)n%-Nd)My@WHk*K-HxV=8WG>Pgk6XF&yl6DNUGEf)!wQG=}k<1vDJV5?i!k=a4 zivAO4`AZ6tFd@ebu1pBFq+8I3mKM}O^$bpqIfO4;4RGm>CHni1K``e6gtYA8)8vHj z$*u^<_Q1&--f*YBkoo%nAvTX7;}VR=8KID@k3!UvIK(|l#IE^im~Nekpb?PX45*_));-HXK7Avs}o%bQWu=MG}jqz|17dQowt8-s2= zP+ZfCa#B#flR zA(kJ7tV^LNDh_fl5##5hE;SJzRuAZsKO3<0h8*j2U?IA{U1#(YgK+4zB-$FI(d;D7e_NMJ-Nzas zQ0@-J{jns?_A}$sTg*6{6*2eyi&Gf+VW{3qor5HKwnB0RrOW z*dzHHa8W6MHdrU3b?!Wr;LQ;oYWCjK30vj|ZNj^S>l$C8r^GP8R+ZW#hui0-R7Og{DO%?0(mv z)U6rYT7Dq0qaDxpcR^dF2hm)t>X{xau;Y3>mmYZ8_u-q-Ao`3(alTiGov>SsHJTvB zew!@I+C!F&43uK$WQnqcr-az8&76jy*bj?8-LM< q@5XvKD*#5Ip|z0ok$6GwKrA@cf7{HVB&hiP`0oN9@u$B#jDcrOZXZH0rvJoNUd!IAEO@zh4D zFnh~d`%E90K6R&3*s9&UoTj-T^*25`gprIF9+dak2b=cnS=$uz zl`0+EiP5dK%EF>GF( z6f5&llsz(5hz)5S!i0ey>>2Jr)Z|}?xZQ@GTrXkttPQ%uKd{=b1^QvlII2;HTEEX& zck(@^q!q#FZXWgz2%tNfj{O&s@U%XbtNY=YxFs0FIf1C|@yG0XA53<>i^KCg5$4I^ zwoA^~6XAfT=Px2X_8iX6If-|#OpwWGseU255gw-rg>yQ1(ab~s4kbj{@Q~)0f=l9_ zWWL4ZFnFG?jmdhN zn47x?ZXXOWN)Exy{Ul6|oX3*#s98^ZW!|DbR~lzk8`#rC<5VQ+YhVRy+&vCXqYSP$n>2yO3! zbRm_%EHu z;Pk*_Y!y{!TNU+i_2?REV35vJTPH=2AFO3o6@Fn%=8BN_2c(GVhw&uOcrF<`dkqP{ zx|94|aELS&>?GQ6SCfd9^N8!Q@#L5F1LYn+)n2Eb;nPCMf+N+nrTgh8ZOXqU^ z+^8DsKRF29wqtNsa{@oiBACeR;BxcDG%Zz?bgKU{h~5!laVIMtGV&? z>f+|{eWVJ+Jv*?0A0*au=`SnUQ5?}rPqiW7(MM6Da1mhEU!Y(d@Y zCe;0{$G)_Byx8&;>mPG?erq*m?5ISyei^cs7UAlYeAL7Pe59kZD-qZET;1IlhP?}d zpqcs*WjB1G>~xpY!#q&7(G`88P7uqy4(;i-IPvl<@?Fg!(rtntsRnS}umhK=4$SAR z1~pKHh^h(_CCA}H;79tj*p{xFG(_*W#^UtjDeS4`dhlA5g}x{$B6;pF*||cOG`d`( zzaIv`=$AVG`HI`L@|z#Cr!1XuscK?uC-*aU2WODe!V8J8^?EY&c+^0wNJTgIxXD&%8U{$gFyo#{7UE)A-npe#ubhI|>Eh_Kk})%TSlt7XBrt9Hq#X z`b;Re?1b>DDXf8FESe{b&`Bn?)M|VcP3L?7d#DN)I?cz!bE{ElsDtnu+i_Um06$k8 zLfC0D+_yT9-Ro^&lxvUvHYd!A<9ge>9x#`^i-=3U9RK|gbqj-VGb9Y5Sy9jvBqIG& zI>KTp#-7T9q);(t9Vi1EUy0vI)p!_Ci$#;aBHE%Je8UFZa%jSVrLEAo+zx(K7i5Hn z(Cs0_zI-adJ}i)B+c*zNn6NmzVB;T5DeA>uy>=82wQ%^W0Zs{Z_&8jRR<&w~wpO7q z_#=ilmm`bg7$2W1gx-QY{5Zuys*!Y@c%BIP$I)0*5{kPif!zP{!+1v@D3^M{rs+1M zy<8xy?g*)rYlzrngZ-iwaEmfU8s8YZ!}nnJZ(S4}-iX8B*Md=93K=<7Nb;BAQt=er zD_VwzJuy^i#bsu4R3al8(nQtMo0qCcWpQ$?<=y zh}Hcb-bd4XX2CK!vg(l}k@cTR{HO3ppr#SIGIW_N%5o=5QZAESH};X8ueHej3T1M~ zN0KNfk0FCPZIEun$K}u^WxAiV1~p5}VNZ=Dasd;K708Hm-{q1c-ojpuU`@yS0ORF%R~ zDG!Erg&43YMe>ny7%uGBPxy&tXi-F_L1|jaoTjuyF3L^1+viGEzM!yRw|ftkt%En zrG^RbsL8RVq$cYpPh!!2LOJwf}H%x&qPI{{np>^5nlNBg~!ki+3_%DG9SFrBf2asdD-yy29cc z-F!d_f5YUEkTn^jSCw#QjtbW&uRw$mfv)u?xTx)hv;021e}4p51I!?~<{VtdUc|Hh z>qt#>##C3X7gYAd>1+33vd0gDo{wO6G#D}GqtP0b2&1p*kZa($pQ;=vtSErJR1qfj zmtthydl*0bfR!B;h|2tgKL#}zX|2b!+Eyss>444kAy{tUd>$3j>|!S=&Qm7L<{s_? ze^on*@|zIsT!RA%mDs@blFvVsB1ydjs%1rZ_Vx`5^Yajuoee{;Ot{;oqvCM_J{m{E z{B|g=MLfd%CH}a+=Pq_ux#QZATTnK4!sB1|IQ8TLq_>^H=>sPqyY?_<=?Cni_)!7I*ZHl7M#411?T-OP&?{jH&zWHJh zxA0$(JGT4Y#g-92e0lT;Vw1z5pdAgKY$EE7(owuU6N^f+5qm2SwL9KGiPQde^}j`2 zQzJiv61<#QxxXa}Kw^L+9jIuy3j$ zP0CTvc?n)VFGQ@%YiRi7BTDKe*MoC<^N&oFTc;uCRuWP?qH(+;3~%EC@ougk9@D$f z%W%if#ct@o>xiXxoCafI1HGA+xN_7S!&eUB?}z=+xTS};wz`OgHs`ZjjkShL5VBtt zH|1twC~7jYyT{>+#2AoSLo`&Woi6hif~S`f<{jS;r_3!d(Q2kGf{S#EMk+DTO%trz zzk>;NI8XZ%I_QSqLZt0Z95csLl2nB&leNa{N&JhQC#CH4@dP3(L<;~cFlzscrcuWb#@xm|&Z>Jt!gY@!*F|e*2hZVh3aA@gl{N=`{ zJhT`QKUSeDhL4?jTR5Fu56vY;uv&f?ya98_C7#BzOj|6PXAkf9j;IQ8MMI%G?&#gc zW z=fPAn2OYr_Yeh0~@?jc2PD;YAo>3n&KoS$M$Chp%rR*a|;x!&K5%#a^r-Vc$0C=Vh-+2Qp4&uAe*M)VB{JONbkFT^4w*~-n5Y?A9JcG~|!xpX70o%js9Y42gKSj?@xSBP%S!SezZ$#Vtp&3J)N z98Y_}A_*sM#v;%+97cITkWzYxZ9zUby5AEDnYYkS9dRb^DpogJ7y%$f6(FVDjLlz zqvL~eY3h$u`aVt^L0MjuJT0g8%9p7?ZVVlS2-ALEjVB(qil$ZRWnSDvnI%hpF+!Ov zv#)F{v65X!R$MnF%O*RJM_Rr_v^0Tq1(uMcm%qsk!xD1xcLJ%=@g)We9Efl2ablUZ zj{N>Rmi&s%;kcl7#=%g)1l-Kee7t!L6_Q`VTW%`K4DOJo<`vf{GxGy|lkP=FFN%X0 zrqKbXJZf?89X(l4Mfd0bq)pQYXg*IE16SqXe`^ZPPn`jgaq2kxcnN~@S0P!MkJV`# zV6U|klTPg6u(L59^G{%c$!UCuvxed7tMD*)M2Yn+1h4jlUAPZCtRHe%JP6a(BOv7$ z%jxe)P;KUXK~^td#S`GzeHPM+xu8|Au%x^Q_imOUchhIc*Eb-?`4^-(AG9{-onNmY z$TzqxOVj!&a-KV7?T9M3^=!P;)|Wz;!fzvcc!urycC!*+2TVfv^}LL&|JDx}WqGN|QHGL89{N|zbB(WPn$ z)bCCdy{~La?R5`OpZ2MA()M`4?0?R@7N^;apIQ=A*SLb&YI>U)`#ps@^k@Rv(!nQL za>q!+7-!-W7C;W&e?|h;tI6cJzvOjrHBlCMPG(;WAQ?-YiEQIhve$`E%%c>E_>N>o zO~`|pHE$*3m7UB?{yvMT@^j_gG*1=;#80R7(FdvCha)sUDvF+Ri=$Uv+^GEQ6naoL ziKa4Xv>@gU-Ko<^>pt{TPhD{sUzJCH`a}eYD&s+k3c_z`;K-sCc%IEbv26o9?r($X zbOVfwIDqb&Be0lz8aErPAya)Fr(~V5j?)54s27I3d=WqV5F0c(e^_G#mc5R{Vf92T z@=8O1=QDh8%EINHfcL^&7;Gp&)1wkBa<9NuF1}jrKcTNbf*o%p*ejkAtfST-LbiOz z75Q37F8qL)n?-1E&BM3bEaabOqI=~t{C=H^(_V?IUGOV7L2frK$Pd*=R7-i zaY@DkZ;PQNi zGRF2z!jTp72ve8DqZNOs@s~DACV!zd*K+7uemot!AcoF8=1;%IyHVxczBF;16Lpr> zp_*3P>3X6@-`v~68?C72c`G>ZYI0{VQ6vJKTomw7U=w3H-MhxHheN!hm}*-LM2BNJ6|ruc&T|fynO~j!X`oWxf~oz zL@}Z;Naqf<(vgS?T4|C`kMv~FsFpAqk#dKowb;|N11IQW)7|v%E^Vr~t3=>m+91%7 zjTbzd=FJ=ObK>0_H^dW29%UMi`Y`{pVj05=RZLrqBC)MnLu_S_k=Ajpq`NSLobSvf z(qX@e{TBuPxSZca?9xl}JRyXLOSqEO14l`!#ad#rYy!!n)r|P67{;aS9^)i*l$n^; z&0DklCeQVygW#T5vEW0%I(kmYfbMBOPPZuCpb5+!8X_G=S8u$c z`+rr^l=VO8^2-BsrH?3f8OT9u{Ulslr-URK6*S(|fTD^fJ}=aQ^jjUc3vEL|v;n4H zG)C@9Gc2{VL`|73D!KLVHr)lyM?A1xmD5CO{qaWcF+M*Dh4^q3$Y31YbP}=3A{FzZ zINyY)01Xy7(Bt|un_zA~hF&#xI<#WJg8`_ONU*VMCE1)i{ixJxhG%vaK39}L;m|95 z{Q~IZzCiTPRGh3!LZW;;F7smWuQ3$AJ_fRw48*%W07G~#b;$Exmg>f1Yt={DD<^spgS)fCprI5VOuJ`@5_W$ z9Pml{6+V40flqN2c-$V4c<}*LZINWV!lc;ct2nLxY%{{oR$|5Ux7eqWhvmx!*!3y{ z7ndeOI4GX;2*tqMI+F8S1!M7rN1Ru{7sAGOA$^h4WOAHPGxr(_iq2vBJ5v;?97gJN zBgh`yiMd-g;zSCAEtfQ*czF>dGUws>RwdX^pNKoAvIsdZhP%px)ZkAGy<%EP4Kxbq zp7LZWvL~2&thh^!7hRxtarRg zt-G?Ew`>v=2@C~}=@o*dCUfWuVO1JsxQh;6KTcCpFVPumz3KNaLDVIj^Z8uKry9R2 zX>LO^&5s|VXDh`ZD?bL;#!tW>dnNel&PD&UMfj$}@mpUQ*e=?L&VU^V=Je>MTZeG} z)d@8Dp2bVC>sau@38@!24RGaML^=9`H|r7D+rgL+9tjP<7{u7d!~JwJdaX0?_fHn$ zD_$a8=q+ZdR6?n-87eFKF~>lPy{IO|hJE66`j3q`Q(u9pnT5D8n2pbOUckdU6{QLB zcyllYNgpGiY94|Y*B_y32RJ%VG4EFsL_;O8_C^t8)rmm-VP8iYl6bv7h7U z4k1<33`yzM&{<>;H^ZCIob7=sfj1nv{@OF}5yFx~;Gh$Uq0=#_bdN`BLMou=jZ{f)7Zs+B%$X=M3lPLj#f~en9iD*Z9d}G1fT)r<0R0e{~$k z9Yg^`ArLA7QASd%cYinDz5R;XcN}N$&w0Wt1Tb8lj@5AqAo@`_ zx;z{Tr=DPb{Uext@_5F zn{qVp>(6``RVZPats<8BkHMZ>9IiXiOB=R-qt7bK=<)SA^z^GFIvo9ks?NJZSMIt( zujd}7KgTfiX8d^iXL7gT@sXE;S+^|(rlSi4i${XjO2$s&wZ?ey3dO(k;^!`64DXmT zzlZNLwP&K3CF@F<)v;|%?e8(fOk0C|6WLCFUbZIfOT39vaXiugR6#mLCHX7pLjL