From 804c1853c6fef3184abbb5b77b1dbef5bf13ad14 Mon Sep 17 00:00:00 2001 From: Romi Polaczek <168196934+RomiPolaczek@users.noreply.github.com> Date: Wed, 18 Dec 2024 12:59:23 +0200 Subject: [PATCH] Integrate `isort` (#385) --- .github/workflows/lint.yaml | 10 ++++ .pre-commit-config.yaml | 12 +++-- fuse/data/__init__.py | 29 +++-------- fuse/data/datasets/__init__.py | 2 +- .../caching/object_caching_handlers.py | 6 ++- fuse/data/datasets/caching/samples_cacher.py | 42 +++++++-------- .../caching/tests/test_sample_caching.py | 16 +++--- fuse/data/datasets/dataset_default.py | 29 ++++++----- .../data/datasets/dataset_wrap_seq_to_dict.py | 8 +-- fuse/data/datasets/sample_caching_audit.py | 6 ++- .../datasets/tests/test_dataset_default.py | 19 +++---- .../test_dataset_default_audit_feature.py | 17 ++++--- .../tests/test_dataset_wrap_seq_to_dict.py | 9 ++-- fuse/data/key_types.py | 3 +- fuse/data/key_types_for_testing.py | 1 + fuse/data/ops/caching_tools.py | 11 ++-- fuse/data/ops/hashable_class.py | 3 +- fuse/data/ops/op_base.py | 9 ++-- fuse/data/ops/ops_aug_common.py | 5 +- fuse/data/ops/ops_aug_tabular.py | 4 +- fuse/data/ops/ops_cast.py | 7 +-- fuse/data/ops/ops_common.py | 13 +++-- fuse/data/ops/ops_common_for_testing.py | 5 +- fuse/data/ops/ops_debug.py | 10 ++-- fuse/data/ops/ops_read.py | 8 +-- fuse/data/ops/tests/test_op_base.py | 9 ++-- fuse/data/ops/tests/test_ops_aug_common.py | 13 +++-- fuse/data/ops/tests/test_ops_cast.py | 9 ++-- fuse/data/ops/tests/test_ops_common.py | 9 ++-- fuse/data/ops/tests/test_ops_read.py | 5 +- fuse/data/patterns.py | 2 +- fuse/data/pipelines/pipeline_default.py | 11 ++-- .../pipelines/tests/test_pipeline_default.py | 7 ++- fuse/data/tests/test_version.py | 3 +- .../modular_tokenizer/inject_utils.py | 12 +++-- .../modular_tokenizer/modular_tokenizer.py | 25 ++++----- fuse/data/tokenizers/modular_tokenizer/op.py | 24 ++++----- .../add_multi_tokenizer_special_tokens.py | 4 +- .../add_tokenizer_to_multi_tokenizer.py | 4 +- .../scripts/create_multi_tokenizer.py | 4 +- fuse/data/utils/collates.py | 8 +-- fuse/data/utils/export.py | 4 +- fuse/data/utils/sample.py | 3 +- fuse/data/utils/samplers.py | 6 ++- fuse/data/utils/split.py | 10 ++-- fuse/data/utils/tests/test_collates.py | 12 ++--- fuse/data/utils/tests/test_dataset_export.py | 9 ++-- fuse/data/utils/tests/test_samplers.py | 14 ++--- fuse/dl/cross_validation/pipeline.py | 16 +++--- fuse/dl/lightning/pl_epoch_summary.py | 12 ++--- fuse/dl/lightning/pl_funcs.py | 21 ++++---- fuse/dl/lightning/pl_module.py | 7 +-- .../dl/losses/contrastive/contrastive_loss.py | 6 +-- fuse/dl/losses/loss_base.py | 4 +- fuse/dl/losses/loss_warm_up.py | 3 +- fuse/dl/losses/loss_wrap_to_dict.py | 2 + fuse/dl/losses/segmentation/loss_dice.py | 3 +- fuse/dl/models/__init__.py | 2 +- fuse/dl/models/backbones/__init__.py | 2 +- .../backbones/backbone_inception_resnet_v2.py | 8 +-- fuse/dl/models/backbones/backbone_mvit.py | 8 +-- fuse/dl/models/backbones/backbone_resnet.py | 6 +-- .../dl/models/backbones/backbone_resnet_3d.py | 3 +- .../models/backbones/backbone_transformer.py | 5 +- fuse/dl/models/backbones/backbone_unet3d.py | 5 +- fuse/dl/models/backbones/backbone_vit.py | 4 +- fuse/dl/models/heads/__init__.py | 4 +- fuse/dl/models/heads/common.py | 5 +- .../models/heads/head_dense_segmentation.py | 2 +- .../heads/head_global_pooling_classifier.py | 2 +- fuse/dl/models/heads/heads_1D.py | 3 +- fuse/dl/models/heads/heads_3D.py | 4 +- fuse/dl/models/model_ensemble.py | 2 +- fuse/dl/models/model_multihead.py | 2 +- fuse/dl/models/model_multistream.py | 3 +- fuse/dl/models/model_siamese.py | 2 +- fuse/dl/models/model_wrapper.py | 2 +- fuse/dl/templates/walkthrough_template.py | 17 +++---- fuse/dl/tests/test_cat.py | 6 ++- fuse/dl/tests/test_vit.py | 1 + fuse/eval/__init__.py | 3 +- fuse/eval/evaluator.py | 10 ++-- fuse/eval/examples/examples.py | 38 +++++++------- fuse/eval/examples/examples_segmentation.py | 27 +++++----- fuse/eval/examples/examples_seq_gen.py | 18 +++---- fuse/eval/examples/examples_stats.py | 6 ++- .../metrics_calibration_common.py | 4 +- .../metrics_classification_common.py | 8 +-- .../metrics_ensembling_common.py | 11 ++-- .../metrics_model_comparison_common.py | 3 +- .../metrics_thresholding_common.py | 7 +-- .../detection/metrics_detection_common.py | 2 +- fuse/eval/metrics/libs/calibration.py | 9 ++-- fuse/eval/metrics/libs/classification.py | 9 ++-- fuse/eval/metrics/libs/ensembling.py | 1 + .../metrics/libs/instance_segmentation.py | 1 - fuse/eval/metrics/libs/segmentation.py | 1 - fuse/eval/metrics/libs/stat.py | 3 +- fuse/eval/metrics/libs/survival.py | 5 +- fuse/eval/metrics/libs/thresholding.py | 4 +- fuse/eval/metrics/metrics_common.py | 14 ++--- fuse/eval/metrics/metrics_model_comparison.py | 4 +- fuse/eval/metrics/regression/metrics.py | 15 ++---- .../metrics_instance_segmentation_common.py | 2 +- .../metrics_segmentation_common.py | 4 +- .../sequence_gen/metrics_seq_gen_common.py | 8 +-- fuse/eval/metrics/stat/metrics_stat_common.py | 3 +- fuse/eval/metrics/utils.py | 5 +- fuse/eval/tests/test_eval.py | 10 ++-- fuse/utils/__init__.py | 14 +++-- fuse/utils/add_preamble.py | 4 +- fuse/utils/config_tools/config_tools.py | 2 +- .../tests/some_conf_internal_include.py | 1 + fuse/utils/config_tools/tests/test_config.py | 2 +- .../cpu_profiling/tests/test_profiler.py | 1 + fuse/utils/cpu_profiling/tests/test_timer.py | 1 + fuse/utils/data/collate.py | 8 +-- fuse/utils/dl/torch_utils.py | 3 +- fuse/utils/file_io/__init__.py | 22 ++++---- fuse/utils/file_io/compressed.py | 3 +- fuse/utils/file_io/file_io.py | 19 +++---- fuse/utils/file_io/path.py | 2 +- fuse/utils/file_io/tests/test_hdf5.py | 9 ++-- fuse/utils/file_io/tests/test_path.py | 10 ++-- fuse/utils/interface_validator.py | 4 +- fuse/utils/misc/misc.py | 5 +- fuse/utils/multiprocessing/__init__.py | 2 +- fuse/utils/multiprocessing/helpers.py | 5 +- .../multiprocessing/linux_shared_memory.py | 7 +-- .../multiprocessing/run_multiprocessed.py | 12 +++-- fuse/utils/ndict.py | 13 ++--- fuse/utils/rand/param_sampler.py | 5 +- fuse/utils/rand/seed.py | 7 +-- fuse/utils/rand/tests/test_param_sampler.py | 10 ++-- fuse/utils/remote_execution/__init__.py | 4 +- .../remote_execution/remote_execution.py | 11 ++-- fuse/utils/remote_execution/shell_handler.py | 5 +- fuse/utils/tests/test_interface_validator.py | 3 +- fuse/utils/tests/test_ndict.py | 4 +- fuse/utils/utils_hierarchical_dict.py | 4 +- fuse/utils/utils_logger.py | 4 +- .../classification/bright/eval/eval.py | 20 ++++---- .../imaging/classification/cmmd/runner.py | 51 +++++++++---------- .../classification/isic/isic_runner.py | 49 ++++++++---------- .../knight/baseline/fuse_baseline.py | 42 +++++++-------- .../classification/knight/baseline/utils.py | 7 +-- .../classification/knight/eval/eval.py | 16 +++--- .../knight/make_predictions_file.py | 19 ++++--- .../knight/make_targets_file.py | 4 +- .../mnist/run_mnist_custom_pl_imp.py | 33 +++++------- .../mnist/simple_mnist_starter.py | 29 +++++------ .../imaging/classification/stoic21/dataset.py | 5 +- .../classification/stoic21/runner_stoic21.py | 42 +++++++-------- .../imaging/hello_world/hello_world_utils.py | 3 +- .../imaging/ml_pipelines/mnist/funcs.py | 6 +-- .../ml_pipelines/mnist/run_mnist_pipeline.py | 16 +++--- .../stoic21/run_stoic_pipeline.py | 17 +++---- .../imaging/oai_example/data/data_ops.py | 15 +++--- .../imaging/oai_example/data/oai_ds.py | 33 ++++++------ .../imaging/oai_example/data/seg_ds.py | 21 ++++---- .../oai_example/downstream/classification.py | 37 +++++++------- .../oai_example/downstream/segmentation.py | 38 +++++++------- .../oai_example/self_supervised/dino.py | 30 +++++------ .../imaging/segmentation/picai/runner.py | 50 +++++++++--------- .../imaging/segmentation/picai/unet.py | 5 +- .../multimodality/ehr_transformer/dataset.py | 23 ++++----- .../ehr_transformer/main_train.py | 21 ++++---- .../multimodality/ehr_transformer/model.py | 3 +- .../multimodality/ehr_transformer/utils.py | 5 +- .../multimodality/image_clinical/dataset.py | 6 +-- .../tests/test_classification_cmmd.py | 10 ++-- .../tests/test_classification_isic.py | 21 ++++---- .../tests/test_classification_knight.py | 10 ++-- .../tests/test_classification_mnist.py | 6 +-- .../tests/test_classification_stoic21.py | 17 +++---- fuse_examples/tests/test_ehr_transformer.py | 9 ++-- .../tests/test_notebook_hello_world.py | 2 + .../tests/test_segmentation_picai.py | 14 ++--- fuseimg/data/ops/aug/color.py | 8 +-- fuseimg/data/ops/aug/geometry.py | 14 +++-- fuseimg/data/ops/color.py | 8 +-- fuseimg/data/ops/debug_ops.py | 5 +- fuseimg/data/ops/image_loader.py | 13 ++--- fuseimg/data/ops/ops_common_imaging.py | 3 +- fuseimg/data/ops/ops_debug.py | 20 ++++---- fuseimg/data/ops/shape_ops.py | 12 ++--- fuseimg/data/ops/tests/test_ops.py | 9 ++-- .../data/ops/tests/test_pipeline_caching.py | 6 +-- fuseimg/datasets/cmmd.py | 48 ++++++++--------- fuseimg/datasets/isic.py | 41 +++++++-------- fuseimg/datasets/kits21.py | 24 ++++----- fuseimg/datasets/kits21_example.ipynb | 6 +-- fuseimg/datasets/knight.py | 25 ++++----- fuseimg/datasets/mnist.py | 7 +-- fuseimg/datasets/picai.py | 45 ++++++++-------- fuseimg/datasets/stoic21.py | 24 ++++----- fuseimg/datasets/tests/test_datasets.py | 19 ++++--- fuseimg/eval/metrics_vis_2d.py | 1 + fuseimg/utils/align/utils_align_base.py | 1 + fuseimg/utils/align/utils_align_ecc.py | 5 +- fuseimg/utils/image_processing.py | 5 +- fuseimg/utils/typing/key_types_imaging.py | 1 + fuseimg/utils/typing/typed_element.py | 1 + run_all_unit_tests.py | 3 +- setup.cfg | 3 ++ setup.py | 2 +- 206 files changed, 1111 insertions(+), 1087 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index a76608c29..53ddab41d 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -42,3 +42,13 @@ jobs: - run: pip install --upgrade pip - run: pip install nb-clean - run: nb-clean check . + isort: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: "3.9" + - uses: isort/isort-action@master + with: + isort-version: 5.13.2 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 594e171ce..6aa485315 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,10 +6,6 @@ repos: - id: end-of-file-fixer - id: mixed-line-ending - id: trailing-whitespace - - repo: https://github.com/psf/black - rev: 22.10.0 - hooks: - - id: black - repo: https://github.com/PyCQA/flake8 rev: 5.0.4 hooks: @@ -25,3 +21,11 @@ repos: args: - --remove-empty-cells - --preserve-cell-outputs + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + - repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black diff --git a/fuse/data/__init__.py b/fuse/data/__init__.py index 4e795a060..040c2d9f4 100644 --- a/fuse/data/__init__.py +++ b/fuse/data/__init__.py @@ -1,27 +1,14 @@ # version -from fuse.version import __version__ - -# import shortcuts -from fuse.data.utils.sample import get_sample_id, set_sample_id, get_sample_id_key -from fuse.data.utils.sample import ( - create_initial_sample, - get_initial_sample_id, - get_initial_sample_id_key, - get_specific_sample_from_potentially_morphed, -) +from fuse.data.datasets.dataset_default import DatasetBase, DatasetDefault from fuse.data.ops.op_base import OpBase # DataTypeForTesting, -from fuse.data.ops.ops_common import ( - OpApplyPatterns, - OpLambda, - OpFunc, - OpRepeat, - OpKeepKeypaths, - OpSet, -) from fuse.data.ops.ops_aug_common import OpRandApply, OpSample, OpSampleAndRepeat +from fuse.data.ops.ops_cast import OpToNumpy, OpToTensor +from fuse.data.ops.ops_common import OpApplyPatterns, OpFunc, OpKeepKeypaths, OpSet from fuse.data.ops.ops_read import OpReadDataframe -from fuse.data.ops.ops_cast import OpToTensor, OpToNumpy +from fuse.data.pipelines.pipeline_default import PipelineDefault from fuse.data.utils.collates import CollateDefault from fuse.data.utils.export import ExportDataset -from fuse.data.pipelines.pipeline_default import PipelineDefault -from fuse.data.datasets.dataset_default import DatasetBase, DatasetDefault +from fuse.utils.ndict import NDict # Fix circular import +from fuse.version import __version__ + +# import shortcuts diff --git a/fuse/data/datasets/__init__.py b/fuse/data/datasets/__init__.py index 27041c85e..b1ec45b8d 100644 --- a/fuse/data/datasets/__init__.py +++ b/fuse/data/datasets/__init__.py @@ -1 +1 @@ -from .dataset_default import DatasetDefault +from fuse.data.datasets.dataset_default import DatasetDefault diff --git a/fuse/data/datasets/caching/object_caching_handlers.py b/fuse/data/datasets/caching/object_caching_handlers.py index cb60afffc..7e29cc10e 100644 --- a/fuse/data/datasets/caching/object_caching_handlers.py +++ b/fuse/data/datasets/caching/object_caching_handlers.py @@ -1,8 +1,10 @@ -from typing import List, Any +from typing import Any, List + import numpy as np -from fuse.utils.ndict import NDict import torch +from fuse.utils.ndict import NDict + # TODO: support custom _object_requires_hdf5_single # maybe even more flexible (knowing key name etc., patterns, explicit name, regular expr.) diff --git a/fuse/data/datasets/caching/samples_cacher.py b/fuse/data/datasets/caching/samples_cacher.py index 6d949d946..9b9eb05be 100644 --- a/fuse/data/datasets/caching/samples_cacher.py +++ b/fuse/data/datasets/caching/samples_cacher.py @@ -11,39 +11,41 @@ limitations under the License. Created on June 30, 2021 """ +import hashlib +import os +from collections import OrderedDict from functools import partial -from typing import Hashable, List, Optional, Sequence, Union, Callable, Any, Tuple, Dict +from glob import glob +from typing import Any, Callable, Dict, Hashable, List, Optional, Sequence, Tuple, Union +from warnings import warn + +import numpy as np +import psutil -from fuse.data.pipelines.pipeline_default import PipelineDefault -from collections import OrderedDict from fuse.data.datasets.caching.object_caching_handlers import ( _object_requires_hdf5_recurse, ) -from fuse.utils.ndict import NDict -import os -import psutil +from fuse.data.datasets.sample_caching_audit import SampleCachingAudit +from fuse.data.pipelines.pipeline_default import PipelineDefault +from fuse.data.utils.sample import ( + create_initial_sample, + get_initial_sample_id, + get_sample_id, + get_specific_sample_from_potentially_morphed, + set_initial_sample_id, +) +from fuse.utils.file_io import delete_directory_tree from fuse.utils.file_io.file_io import ( load_hdf5, - save_hdf5_safe, load_pickle, + save_hdf5_safe, save_pickle_safe, ) -from fuse.data import ( - get_sample_id, - create_initial_sample, - get_specific_sample_from_potentially_morphed, -) -import hashlib -from fuse.utils.file_io import delete_directory_tree -from glob import glob from fuse.utils.multiprocessing.run_multiprocessed import ( - run_multiprocessed, get_from_global_storage, + run_multiprocessed, ) -from fuse.data.datasets.sample_caching_audit import SampleCachingAudit -from fuse.data.utils.sample import get_initial_sample_id, set_initial_sample_id -from warnings import warn -import numpy as np +from fuse.utils.ndict import NDict class SamplesCacher: diff --git a/fuse/data/datasets/caching/tests/test_sample_caching.py b/fuse/data/datasets/caching/tests/test_sample_caching.py index 71747c837..9b23b8e8b 100644 --- a/fuse/data/datasets/caching/tests/test_sample_caching.py +++ b/fuse/data/datasets/caching/tests/test_sample_caching.py @@ -15,19 +15,19 @@ """ +import os +import tempfile import unittest +from typing import List, Union -from fuse.utils.rand.seed import Seed -from fuse.data.pipelines.pipeline_default import PipelineDefault -from fuse.data import get_sample_id, create_initial_sample import numpy as np -import tempfile -import os -from fuse.data.ops.op_base import OpBase -from typing import List, Union -from fuse.data.datasets.caching.samples_cacher import SamplesCacher +from fuse.data.datasets.caching.samples_cacher import SamplesCacher +from fuse.data.ops.op_base import OpBase +from fuse.data.pipelines.pipeline_default import PipelineDefault +from fuse.data.utils.sample import create_initial_sample, get_sample_id from fuse.utils.ndict import NDict +from fuse.utils.rand.seed import Seed class OpFakeLoad(OpBase): diff --git a/fuse/data/datasets/dataset_default.py b/fuse/data/datasets/dataset_default.py index 67daa9257..38ad051c8 100644 --- a/fuse/data/datasets/dataset_default.py +++ b/fuse/data/datasets/dataset_default.py @@ -16,27 +16,28 @@ """ -from typing import Dict, Hashable, List, Optional, Sequence, Union, Any - +import copy +from collections import OrderedDict +from operator import itemgetter +from typing import Any, Dict, Hashable, List, Optional, Sequence, Union from warnings import warn + +import numpy as np + +from fuse.data.datasets.caching.samples_cacher import SamplesCacher from fuse.data.datasets.dataset_base import DatasetBase from fuse.data.ops.ops_common import OpCollectMarker from fuse.data.pipelines.pipeline_default import PipelineDefault -from fuse.data.datasets.caching.samples_cacher import SamplesCacher -from fuse.utils.ndict import NDict -from fuse.utils.multiprocessing.run_multiprocessed import ( - run_multiprocessed, - get_from_global_storage, -) -from fuse.data import ( - get_sample_id, +from fuse.data.utils.sample import ( create_initial_sample, + get_sample_id, get_specific_sample_from_potentially_morphed, ) -import copy -from collections import OrderedDict -import numpy as np -from operator import itemgetter +from fuse.utils.multiprocessing.run_multiprocessed import ( + get_from_global_storage, + run_multiprocessed, +) +from fuse.utils.ndict import NDict class DatasetDefault(DatasetBase): diff --git a/fuse/data/datasets/dataset_wrap_seq_to_dict.py b/fuse/data/datasets/dataset_wrap_seq_to_dict.py index 672898718..b31578557 100644 --- a/fuse/data/datasets/dataset_wrap_seq_to_dict.py +++ b/fuse/data/datasets/dataset_wrap_seq_to_dict.py @@ -17,15 +17,15 @@ """ -from typing import List, Optional, Union, Sequence -from fuse.data.datasets.caching.samples_cacher import SamplesCacher -from fuse.data.utils.sample import get_sample_id +from typing import List, Optional, Sequence, Union from torch.utils.data import Dataset -from fuse.data.ops.op_base import OpBase +from fuse.data.datasets.caching.samples_cacher import SamplesCacher from fuse.data.datasets.dataset_default import DatasetDefault +from fuse.data.ops.op_base import OpBase from fuse.data.pipelines.pipeline_default import PipelineDefault +from fuse.data.utils.sample import get_sample_id from fuse.utils.ndict import NDict diff --git a/fuse/data/datasets/sample_caching_audit.py b/fuse/data/datasets/sample_caching_audit.py index 33abefeca..1f1698922 100644 --- a/fuse/data/datasets/sample_caching_audit.py +++ b/fuse/data/datasets/sample_caching_audit.py @@ -1,7 +1,9 @@ -from typing import Optional, Any from time import time +from typing import Any, Optional + from deepdiff import DeepDiff -from fuse.data import get_sample_id + +from fuse.data.utils.sample import get_sample_id """ By auditing the samples, "stale" caches can be found, which is very important to detect. diff --git a/fuse/data/datasets/tests/test_dataset_default.py b/fuse/data/datasets/tests/test_dataset_default.py index 5cf490207..6fac23ed7 100644 --- a/fuse/data/datasets/tests/test_dataset_default.py +++ b/fuse/data/datasets/tests/test_dataset_default.py @@ -15,21 +15,22 @@ """ +import os +import tempfile import unittest +from typing import List, Optional, Union -from fuse.utils.rand.seed import Seed - -# from fuse.utils.file_io.file_io import SAFE_save_hdf5, load_hdf5 -from fuse.data.pipelines.pipeline_default import PipelineDefault -from fuse.data import get_sample_id, create_initial_sample import numpy as np -import tempfile -import os -from fuse.data.ops.op_base import OpBase -from typing import List, Union, Optional + from fuse.data.datasets.caching.samples_cacher import SamplesCacher from fuse.data.datasets.dataset_default import DatasetDefault +from fuse.data.ops.op_base import OpBase + +# from fuse.utils.file_io.file_io import SAFE_save_hdf5, load_hdf5 +from fuse.data.pipelines.pipeline_default import PipelineDefault +from fuse.data.utils.sample import create_initial_sample, get_sample_id from fuse.utils.ndict import NDict +from fuse.utils.rand.seed import Seed class OpFakeLoad(OpBase): diff --git a/fuse/data/datasets/tests/test_dataset_default_audit_feature.py b/fuse/data/datasets/tests/test_dataset_default_audit_feature.py index 91382f09e..58b50f942 100644 --- a/fuse/data/datasets/tests/test_dataset_default_audit_feature.py +++ b/fuse/data/datasets/tests/test_dataset_default_audit_feature.py @@ -15,19 +15,20 @@ """ +import os +import tempfile import unittest -from fuse.utils.rand.seed import Seed -from fuse.utils.ndict import NDict +from typing import List, Union -from fuse.data.pipelines.pipeline_default import PipelineDefault -from fuse.data import get_sample_id, create_initial_sample import numpy as np -import tempfile -import os -from fuse.data.ops.op_base import OpBase -from typing import List, Union + from fuse.data.datasets.caching.samples_cacher import SamplesCacher from fuse.data.datasets.dataset_default import DatasetDefault +from fuse.data.ops.op_base import OpBase +from fuse.data.pipelines.pipeline_default import PipelineDefault +from fuse.data.utils.sample import create_initial_sample, get_sample_id +from fuse.utils.ndict import NDict +from fuse.utils.rand.seed import Seed class OpFakeLoad(OpBase): diff --git a/fuse/data/datasets/tests/test_dataset_wrap_seq_to_dict.py b/fuse/data/datasets/tests/test_dataset_wrap_seq_to_dict.py index 11f061ec1..5e8ab1ed3 100644 --- a/fuse/data/datasets/tests/test_dataset_wrap_seq_to_dict.py +++ b/fuse/data/datasets/tests/test_dataset_wrap_seq_to_dict.py @@ -16,17 +16,16 @@ """ import os -import unittest - import random +import tempfile +import unittest import torchvision from torchvision import transforms -from fuse.utils.rand.seed import Seed -from fuse.utils.ndict import NDict -import tempfile from fuse.data.datasets.dataset_wrap_seq_to_dict import DatasetWrapSeqToDict +from fuse.utils.ndict import NDict +from fuse.utils.rand.seed import Seed class TestDatasetWrapSeqToDict(unittest.TestCase): diff --git a/fuse/data/key_types.py b/fuse/data/key_types.py index 918357f9d..68fdcb5d3 100644 --- a/fuse/data/key_types.py +++ b/fuse/data/key_types.py @@ -1,6 +1,7 @@ from abc import ABC, abstractmethod from enum import Enum -from typing import Dict, Sequence, Any +from typing import Any, Dict, Sequence + from fuse.data.patterns import Patterns diff --git a/fuse/data/key_types_for_testing.py b/fuse/data/key_types_for_testing.py index dbc45cb5b..4eb7f4864 100644 --- a/fuse/data/key_types_for_testing.py +++ b/fuse/data/key_types_for_testing.py @@ -1,4 +1,5 @@ from enum import Enum + from fuse.data.key_types import DataTypeBasic, TypeDetectorPatternsBased diff --git a/fuse/data/ops/caching_tools.py b/fuse/data/ops/caching_tools.py index 49fa3b2af..cecaa9c9b 100644 --- a/fuse/data/ops/caching_tools.py +++ b/fuse/data/ops/caching_tools.py @@ -1,11 +1,12 @@ +import hashlib import inspect -from typing import Callable, Any, Type, Optional, Sequence, List -from inspect import stack -import warnings -from fuse.utils.file_io.file_io import load_pickle, save_pickle_safe import os +import warnings +from inspect import stack +from typing import Any, Callable, List, Optional, Sequence, Type + from fuse.utils.cpu_profiling import Timer -import hashlib +from fuse.utils.file_io.file_io import load_pickle, save_pickle_safe def get_function_call_str( diff --git a/fuse/data/ops/hashable_class.py b/fuse/data/ops/hashable_class.py index 0decab3a5..68f45151f 100644 --- a/fuse/data/ops/hashable_class.py +++ b/fuse/data/ops/hashable_class.py @@ -1,4 +1,5 @@ -from typing import Callable, Any +from typing import Any, Callable + from fuse.data.ops import get_function_call_str from fuse.data.ops.caching_tools import get_callers_string_description, value_to_string diff --git a/fuse/data/ops/op_base.py b/fuse/data/ops/op_base.py index ccbdbf08a..c5dc16b8d 100644 --- a/fuse/data/ops/op_base.py +++ b/fuse/data/ops/op_base.py @@ -16,13 +16,14 @@ Created on June 30, 2021 """ -from typing import Union, List, Optional +import inspect +import traceback from abc import abstractmethod +from typing import List, Optional, Union + +from fuse.data.ops.hashable_class import HashableClass from fuse.data.utils.sample import get_sample_id from fuse.utils.ndict import NDict -from fuse.data.ops.hashable_class import HashableClass -import inspect -import traceback class OpBase(HashableClass): diff --git a/fuse/data/ops/ops_aug_common.py b/fuse/data/ops/ops_aug_common.py index 8c53c4967..91e01dc61 100644 --- a/fuse/data/ops/ops_aug_common.py +++ b/fuse/data/ops/ops_aug_common.py @@ -1,13 +1,10 @@ import random from typing import List, Optional, Sequence, Union - -from fuse.utils.rand.param_sampler import RandBool, draw_samples_recursively - from fuse.data.ops.op_base import OpBase, OpReversibleBase, op_call, op_reverse from fuse.data.ops.ops_common import OpRepeat - from fuse.utils.ndict import NDict +from fuse.utils.rand.param_sampler import RandBool, draw_samples_recursively class OpRandApply(OpReversibleBase): diff --git a/fuse/data/ops/ops_aug_tabular.py b/fuse/data/ops/ops_aug_tabular.py index f0a3cbdd2..25b592167 100644 --- a/fuse/data/ops/ops_aug_tabular.py +++ b/fuse/data/ops/ops_aug_tabular.py @@ -1,9 +1,9 @@ import random -import numpy as np from typing import Optional, Sequence -from fuse.data.ops.op_base import OpBase +import numpy as np +from fuse.data.ops.op_base import OpBase from fuse.utils import NDict diff --git a/fuse/data/ops/ops_cast.py b/fuse/data/ops/ops_cast.py index 83e31626c..744cb0456 100644 --- a/fuse/data/ops/ops_cast.py +++ b/fuse/data/ops/ops_cast.py @@ -18,12 +18,13 @@ """ from abc import abstractmethod from typing import Any, List, Optional, Sequence, Union -from fuse.data.ops.op_base import OpReversibleBase -import numpy as np -from fuse.data import OpBase +import numpy as np import torch from torch import Tensor + +from fuse.data import OpBase +from fuse.data.ops.op_base import OpReversibleBase from fuse.utils.ndict import NDict diff --git a/fuse/data/ops/ops_common.py b/fuse/data/ops/ops_common.py index b08eb7855..822821aba 100644 --- a/fuse/data/ops/ops_common.py +++ b/fuse/data/ops/ops_common.py @@ -1,6 +1,8 @@ +import copy import math import numbers import random +from enum import Enum from typing import ( Any, Callable, @@ -12,14 +14,15 @@ Tuple, Union, ) + +import numpy as np +import torch + from fuse.data.key_types import TypeDetectorBase -import copy -from enum import Enum -from .op_base import OpBase, OpReversibleBase, op_call, op_reverse # DataType, from fuse.data.patterns import Patterns from fuse.utils.ndict import NDict -import numpy as np -import torch + +from .op_base import OpBase, OpReversibleBase, op_call, op_reverse # DataType, class OpRepeat(OpReversibleBase): diff --git a/fuse/data/ops/ops_common_for_testing.py b/fuse/data/ops/ops_common_for_testing.py index b2195fa15..c5536c6f9 100644 --- a/fuse/data/ops/ops_common_for_testing.py +++ b/fuse/data/ops/ops_common_for_testing.py @@ -1,7 +1,8 @@ -from fuse.data.ops.ops_common import OpApplyTypes -from fuse.data.key_types_for_testing import type_detector_for_testing from functools import partial +from fuse.data.key_types_for_testing import type_detector_for_testing +from fuse.data.ops.ops_common import OpApplyTypes + OpApplyTypesImaging = partial( OpApplyTypes, type_detector=type_detector_for_testing, diff --git a/fuse/data/ops/ops_debug.py b/fuse/data/ops/ops_debug.py index bf78b380c..702994c04 100644 --- a/fuse/data/ops/ops_debug.py +++ b/fuse/data/ops/ops_debug.py @@ -1,11 +1,13 @@ from abc import abstractmethod -from typing import Hashable, List, Sequence, Optional -from fuse.data.utils.sample import get_sample_id -from fuse.utils import NDict -from fuse.data import OpBase +from typing import Hashable, List, Optional, Sequence + import numpy import torch +from fuse.data import OpBase +from fuse.data.utils.sample import get_sample_id +from fuse.utils import NDict + class OpDebugBase(OpBase): """ diff --git a/fuse/data/ops/ops_read.py b/fuse/data/ops/ops_read.py index 4cc8d0837..6a3045b98 100644 --- a/fuse/data/ops/ops_read.py +++ b/fuse/data/ops/ops_read.py @@ -17,12 +17,13 @@ """ -from typing import Hashable, List, Optional, Dict, Union -from fuse.utils.file_io.file_io import read_dataframe -import pandas as pd +from typing import Dict, Hashable, List, Optional, Union + import h5py +import pandas as pd from fuse.data import OpBase +from fuse.utils.file_io.file_io import read_dataframe from fuse.utils.ndict import NDict @@ -155,7 +156,6 @@ def num_samples(self) -> int: return self._num_samples def __call__(self, sample_dict: NDict) -> Union[None, dict, List[dict]]: - index = sample_dict[self._key_index] for column in self._columns_to_extract: key_to_store = self._rename_columns.get(column, column) diff --git a/fuse/data/ops/tests/test_op_base.py b/fuse/data/ops/tests/test_op_base.py index f9a95841c..2cd83c0d2 100644 --- a/fuse/data/ops/tests/test_op_base.py +++ b/fuse/data/ops/tests/test_op_base.py @@ -1,15 +1,14 @@ import unittest +from typing import List, Union -from typing import Union, List -from fuse.utils.ndict import NDict - -from fuse.data.ops.op_base import OpBase, op_call from fuse.data.key_types import DataTypeBasic -from fuse.data import create_initial_sample from fuse.data.key_types_for_testing import ( DataTypeForTesting, type_detector_for_testing, ) +from fuse.data.ops.op_base import OpBase, op_call +from fuse.data.utils.sample import create_initial_sample +from fuse.utils.ndict import NDict class TestOpBase(unittest.TestCase): diff --git a/fuse/data/ops/tests/test_ops_aug_common.py b/fuse/data/ops/tests/test_ops_aug_common.py index 4b8516c8a..d37e2908f 100644 --- a/fuse/data/ops/tests/test_ops_aug_common.py +++ b/fuse/data/ops/tests/test_ops_aug_common.py @@ -1,19 +1,18 @@ import unittest - -from typing import Optional, Union, List, Any -from fuse.utils.ndict import NDict +from typing import Any, List, Optional, Union from fuse.data.ops.op_base import OpBase, OpReversibleBase, op_call, op_reverse -from fuse.data import create_initial_sample -from fuse.data import OpRepeat from fuse.data.ops.ops_aug_common import ( OpRandApply, + OpRepeatAndSample, OpSample, OpSampleAndRepeat, - OpRepeatAndSample, ) -from fuse.utils.rand.param_sampler import Choice, RandBool, RandInt, Uniform +from fuse.data.ops.ops_common import OpRepeat +from fuse.data.utils.sample import create_initial_sample from fuse.utils import Seed +from fuse.utils.ndict import NDict +from fuse.utils.rand.param_sampler import Choice, RandBool, RandInt, Uniform class OpArgsForTest(OpReversibleBase): diff --git a/fuse/data/ops/tests/test_ops_cast.py b/fuse/data/ops/tests/test_ops_cast.py index 2ea4649b2..59049cb90 100644 --- a/fuse/data/ops/tests/test_ops_cast.py +++ b/fuse/data/ops/tests/test_ops_cast.py @@ -17,14 +17,15 @@ """ import unittest - from typing import List -from fuse.data.ops.op_base import op_call, op_reverse -from fuse.utils.ndict import NDict + +import numpy as np import pandas as pd import torch -import numpy as np + +from fuse.data.ops.op_base import op_call, op_reverse from fuse.data.ops.ops_cast import OpToNumpy, OpToTensor +from fuse.utils.ndict import NDict class TestOpsCast(unittest.TestCase): diff --git a/fuse/data/ops/tests/test_ops_common.py b/fuse/data/ops/tests/test_ops_common.py index 544a04c33..9642a94b3 100644 --- a/fuse/data/ops/tests/test_ops_common.py +++ b/fuse/data/ops/tests/test_ops_common.py @@ -1,14 +1,10 @@ import unittest - -from typing import Optional, OrderedDict, Union, List, Any +from typing import Any, List, Optional, OrderedDict, Union import torch -from fuse.utils.ndict import NDict - -from fuse.data.ops.op_base import OpBase, OpReversibleBase, op_call from fuse.data.key_types_for_testing import DataTypeForTesting - +from fuse.data.ops.op_base import OpBase, OpReversibleBase, op_call from fuse.data.ops.ops_common import ( OpApplyPatterns, OpFunc, @@ -18,6 +14,7 @@ OpReplaceElements, ) from fuse.data.ops.ops_common_for_testing import OpApplyTypesImaging +from fuse.utils.ndict import NDict class OpIncrForTest(OpReversibleBase): diff --git a/fuse/data/ops/tests/test_ops_read.py b/fuse/data/ops/tests/test_ops_read.py index 832792a7c..48d178e65 100644 --- a/fuse/data/ops/tests/test_ops_read.py +++ b/fuse/data/ops/tests/test_ops_read.py @@ -18,11 +18,12 @@ """ import unittest -from fuse.data.ops.op_base import op_call import pandas as pd -from fuse.utils.ndict import NDict + +from fuse.data.ops.op_base import op_call from fuse.data.ops.ops_read import OpReadDataframe +from fuse.utils.ndict import NDict class TestOpsRead(unittest.TestCase): diff --git a/fuse/data/patterns.py b/fuse/data/patterns.py index 28e8e0cd4..3b2796ada 100644 --- a/fuse/data/patterns.py +++ b/fuse/data/patterns.py @@ -1,6 +1,6 @@ +import re from collections import OrderedDict from typing import Any, Sequence -import re class Patterns: diff --git a/fuse/data/pipelines/pipeline_default.py b/fuse/data/pipelines/pipeline_default.py index ac0a320de..7a5302da5 100644 --- a/fuse/data/pipelines/pipeline_default.py +++ b/fuse/data/pipelines/pipeline_default.py @@ -16,14 +16,15 @@ Created on June 30, 2021 """ -from typing import List, Tuple, Union, Optional, Any +import copy +import os +import threading +from typing import Any, List, Optional, Tuple, Union + from fuse.data.ops.op_base import OpBase, OpReversibleBase, op_call, op_reverse +from fuse.utils.cpu_profiling.timer import Timer from fuse.utils.misc.context import DummyContext from fuse.utils.ndict import NDict -from fuse.utils.cpu_profiling.timer import Timer -import os -import copy -import threading class PipelineDefault(OpReversibleBase): diff --git a/fuse/data/pipelines/tests/test_pipeline_default.py b/fuse/data/pipelines/tests/test_pipeline_default.py index 0ee34a35b..41271b1c5 100644 --- a/fuse/data/pipelines/tests/test_pipeline_default.py +++ b/fuse/data/pipelines/tests/test_pipeline_default.py @@ -1,12 +1,11 @@ -import unittest - -from fuse.utils.ndict import NDict -from typing import Any, Union, List import copy +import unittest +from typing import Any, List, Union from unittest.case import expectedFailure from fuse.data.ops.op_base import OpBase, OpReversibleBase from fuse.data.pipelines.pipeline_default import PipelineDefault +from fuse.utils.ndict import NDict class OpSetForTest(OpReversibleBase): diff --git a/fuse/data/tests/test_version.py b/fuse/data/tests/test_version.py index 3ba5f590c..194e60b19 100644 --- a/fuse/data/tests/test_version.py +++ b/fuse/data/tests/test_version.py @@ -19,9 +19,10 @@ import unittest -import fuse.data import pkg_resources # part of setuptools +import fuse.data + class TestVersion(unittest.TestCase): def test_version(self) -> None: diff --git a/fuse/data/tokenizers/modular_tokenizer/inject_utils.py b/fuse/data/tokenizers/modular_tokenizer/inject_utils.py index e9942ef8f..3e9f0a41d 100644 --- a/fuse/data/tokenizers/modular_tokenizer/inject_utils.py +++ b/fuse/data/tokenizers/modular_tokenizer/inject_utils.py @@ -1,13 +1,15 @@ -from typing import Optional, List, Tuple, Dict, Union -from tokenizers import Encoding -import torch import re -from fuse.utils import NDict +from typing import Dict, List, Optional, Tuple, Union +from warnings import warn + +import torch +from tokenizers import Encoding + from fuse.data.tokenizers.modular_tokenizer.modular_tokenizer import ( TypedInput, list_to_tokenizer_string, ) -from warnings import warn +from fuse.utils import NDict class InjectorToModularTokenizerLib: diff --git a/fuse/data/tokenizers/modular_tokenizer/modular_tokenizer.py b/fuse/data/tokenizers/modular_tokenizer/modular_tokenizer.py index 5762250c2..6ab753d1d 100644 --- a/fuse/data/tokenizers/modular_tokenizer/modular_tokenizer.py +++ b/fuse/data/tokenizers/modular_tokenizer/modular_tokenizer.py @@ -1,20 +1,22 @@ +import copy +import json +import os import random -from torch import Tensor +import re +import traceback from collections.abc import Iterable -from tokenizers import Tokenizer, Encoding -import tokenizers +from dataclasses import dataclass +from typing import Any, Dict, Iterator, List, Optional, Set, Tuple, Union from warnings import warn -from typing import Optional, List, Set, Union, Tuple, Any, Iterator, Dict -import json + +import omegaconf +import tokenizers import transformers -import os from omegaconf import OmegaConf -import omegaconf -import copy -import traceback -import re +from tokenizers import Encoding, Tokenizer +from torch import Tensor + from fuse.data.tokenizers.modular_tokenizer.special_tokens import special_wrap_input -from dataclasses import dataclass @dataclass @@ -800,7 +802,6 @@ def add_single_tokenizer( self, tokenizer_info: Dict, ) -> None: - # first we load the new tokenizer # TODO: rename variable diff --git a/fuse/data/tokenizers/modular_tokenizer/op.py b/fuse/data/tokenizers/modular_tokenizer/op.py index 63f983188..fb622a07f 100644 --- a/fuse/data/tokenizers/modular_tokenizer/op.py +++ b/fuse/data/tokenizers/modular_tokenizer/op.py @@ -1,19 +1,19 @@ -from fuse.utils import NDict +import os +import re +from collections import defaultdict +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union +from warnings import warn + +from huggingface_hub import HfApi, snapshot_download +from huggingface_hub.utils import SoftTemporaryDirectory, validate_hf_hub_args + from fuse.data import OpBase, get_sample_id -from fuse.data.tokenizers.modular_tokenizer.modular_tokenizer import ModularTokenizer from fuse.data.tokenizers.modular_tokenizer.inject_utils import ( InjectorToModularTokenizerLib, ) -from huggingface_hub import snapshot_download, HfApi -from huggingface_hub.utils import validate_hf_hub_args, SoftTemporaryDirectory - - -from warnings import warn -from pathlib import Path -from collections import defaultdict -from typing import Any, Tuple, Dict, List, Optional, Union -import os -import re +from fuse.data.tokenizers.modular_tokenizer.modular_tokenizer import ModularTokenizer +from fuse.utils import NDict class ModularTokenizerWithoutInjectOp(OpBase): diff --git a/fuse/data/tokenizers/modular_tokenizer/scripts/add_multi_tokenizer_special_tokens.py b/fuse/data/tokenizers/modular_tokenizer/scripts/add_multi_tokenizer_special_tokens.py index 8b3ebd0d5..8e341b8f0 100644 --- a/fuse/data/tokenizers/modular_tokenizer/scripts/add_multi_tokenizer_special_tokens.py +++ b/fuse/data/tokenizers/modular_tokenizer/scripts/add_multi_tokenizer_special_tokens.py @@ -1,5 +1,7 @@ +from typing import List, Union + import click -from typing import Union, List + from fuse.data.tokenizers.modular_tokenizer.modular_tokenizer import ModularTokenizer diff --git a/fuse/data/tokenizers/modular_tokenizer/scripts/add_tokenizer_to_multi_tokenizer.py b/fuse/data/tokenizers/modular_tokenizer/scripts/add_tokenizer_to_multi_tokenizer.py index 342d8b1ac..e4969c3f1 100644 --- a/fuse/data/tokenizers/modular_tokenizer/scripts/add_tokenizer_to_multi_tokenizer.py +++ b/fuse/data/tokenizers/modular_tokenizer/scripts/add_tokenizer_to_multi_tokenizer.py @@ -1,7 +1,9 @@ +from typing import Any, Dict + import hydra from omegaconf import DictConfig, OmegaConf + from fuse.data.tokenizers.modular_tokenizer.modular_tokenizer import ModularTokenizer -from typing import Dict, Any @hydra.main( diff --git a/fuse/data/tokenizers/modular_tokenizer/scripts/create_multi_tokenizer.py b/fuse/data/tokenizers/modular_tokenizer/scripts/create_multi_tokenizer.py index 665f2e14f..1d1bbc615 100644 --- a/fuse/data/tokenizers/modular_tokenizer/scripts/create_multi_tokenizer.py +++ b/fuse/data/tokenizers/modular_tokenizer/scripts/create_multi_tokenizer.py @@ -1,7 +1,9 @@ +from typing import Any, Dict + import hydra from omegaconf import DictConfig, OmegaConf + from fuse.data.tokenizers.modular_tokenizer.modular_tokenizer import ModularTokenizer -from typing import Dict, Any @hydra.main( diff --git a/fuse/data/utils/collates.py b/fuse/data/utils/collates.py index 371e56248..267b87423 100644 --- a/fuse/data/utils/collates.py +++ b/fuse/data/utils/collates.py @@ -16,16 +16,16 @@ Created on June 30, 2021 """ -from typing import Any, Callable, Dict, List, Sequence, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple import numpy as np import torch -from torch.utils.data._utils.collate import default_collate import torch.nn.functional as F +from torch.utils.data._utils.collate import default_collate -from fuse.utils import NDict +from fuse.data.utils.sample import get_sample_id_key from fuse.utils.data.collate import CollateToBatchList -from fuse.data import get_sample_id_key +from fuse.utils.ndict import NDict class CollateDefault(CollateToBatchList): diff --git a/fuse/data/utils/export.py b/fuse/data/utils/export.py index 122888e34..0fc5bc311 100644 --- a/fuse/data/utils/export.py +++ b/fuse/data/utils/export.py @@ -17,12 +17,12 @@ """ from typing import Optional, Sequence + import pandas as pds +from tqdm import tqdm from fuse.data.datasets.dataset_base import DatasetBase - from fuse.utils.file_io.file_io import save_dataframe -from tqdm import tqdm class ExportDataset: diff --git a/fuse/data/utils/sample.py b/fuse/data/utils/sample.py index 6a2cd6e20..413e2bc11 100644 --- a/fuse/data/utils/sample.py +++ b/fuse/data/utils/sample.py @@ -1,4 +1,5 @@ -from typing import Dict, Hashable, Optional, Union, List +from typing import Dict, Hashable, List, Optional, Union + from fuse.utils.ndict import NDict """ diff --git a/fuse/data/utils/samplers.py b/fuse/data/utils/samplers.py index 4cff330c0..aa77c521e 100644 --- a/fuse/data/utils/samplers.py +++ b/fuse/data/utils/samplers.py @@ -18,9 +18,11 @@ """ import math +from typing import Any, Dict, List, Optional, Sequence, Union + import numpy as np -from typing import Any, List, Optional, Union, Sequence, Dict -from torch.utils.data.sampler import Sampler, BatchSampler +from torch.utils.data.sampler import BatchSampler, Sampler + from fuse.data.datasets.dataset_base import DatasetBase diff --git a/fuse/data/utils/split.py b/fuse/data/utils/split.py index 5beb712ca..922090c31 100644 --- a/fuse/data/utils/split.py +++ b/fuse/data/utils/split.py @@ -14,13 +14,15 @@ import os from typing import Hashable, Sequence -from fuse.data.datasets.dataset_base import DatasetBase -from fuse.utils.file_io.file_io import load_pickle, save_pickle -from sklearn.utils import shuffle -import pandas as pd + import numpy as np +import pandas as pd +from sklearn.utils import shuffle + +from fuse.data.datasets.dataset_base import DatasetBase from fuse.data.utils.export import ExportDataset from fuse.data.utils.sample import get_sample_id_key +from fuse.utils.file_io.file_io import load_pickle, save_pickle def print_folds_stat(db: pd.DataFrame, nfolds: int, key_columns: np.ndarray) -> str: diff --git a/fuse/data/utils/tests/test_collates.py b/fuse/data/utils/tests/test_collates.py index fb6c9d301..dd3227003 100644 --- a/fuse/data/utils/tests/test_collates.py +++ b/fuse/data/utils/tests/test_collates.py @@ -17,20 +17,20 @@ """ -from typing import List, Optional, Union import unittest +from typing import List, Optional, Union -import pandas as pds import numpy as np +import pandas as pds import torch from torch.utils.data.dataloader import DataLoader -from fuse.data.ops.ops_read import OpReadDataframe from fuse.data.datasets.dataset_default import DatasetDefault -from fuse.data.utils.collates import CollateDefault -from fuse.data.pipelines.pipeline_default import PipelineDefault from fuse.data.ops.op_base import OpBase -from fuse.data import get_sample_id +from fuse.data.ops.ops_read import OpReadDataframe +from fuse.data.pipelines.pipeline_default import PipelineDefault +from fuse.data.utils.collates import CollateDefault +from fuse.data.utils.sample import get_sample_id class OpCustomCollateDefTest(OpBase): diff --git a/fuse/data/utils/tests/test_dataset_export.py b/fuse/data/utils/tests/test_dataset_export.py index 0ba583bf1..b03c217e5 100644 --- a/fuse/data/utils/tests/test_dataset_export.py +++ b/fuse/data/utils/tests/test_dataset_export.py @@ -18,16 +18,15 @@ """ import unittest - from tempfile import mkstemp -import pandas as pds +import pandas as pds -from fuse.utils.file_io.file_io import read_dataframe -from fuse.data.ops.ops_read import OpReadDataframe from fuse.data.datasets.dataset_default import DatasetDefault -from fuse.data.utils.export import ExportDataset +from fuse.data.ops.ops_read import OpReadDataframe from fuse.data.pipelines.pipeline_default import PipelineDefault +from fuse.data.utils.export import ExportDataset +from fuse.utils.file_io.file_io import read_dataframe class TestDatasetExport(unittest.TestCase): diff --git a/fuse/data/utils/tests/test_samplers.py b/fuse/data/utils/tests/test_samplers.py index a18a9eed5..2f6ab1845 100644 --- a/fuse/data/utils/tests/test_samplers.py +++ b/fuse/data/utils/tests/test_samplers.py @@ -20,21 +20,21 @@ import os import tempfile import unittest -import pandas as pds + import numpy as np -from tqdm.std import tqdm +import pandas as pds import torchvision -from torchvision import transforms from torch.utils.data.dataloader import DataLoader +from torchvision import transforms +from tqdm.std import tqdm -from fuse.utils import Seed - -from fuse.data.ops.ops_read import OpReadDataframe from fuse.data.datasets.dataset_default import DatasetDefault from fuse.data.datasets.dataset_wrap_seq_to_dict import DatasetWrapSeqToDict -from fuse.data.utils.collates import CollateDefault +from fuse.data.ops.ops_read import OpReadDataframe from fuse.data.pipelines.pipeline_default import PipelineDefault +from fuse.data.utils.collates import CollateDefault from fuse.data.utils.samplers import BatchSamplerDefault +from fuse.utils import Seed class TestSamplers(unittest.TestCase): diff --git a/fuse/dl/cross_validation/pipeline.py b/fuse/dl/cross_validation/pipeline.py index 9c0dfa188..adac38d3d 100644 --- a/fuse/dl/cross_validation/pipeline.py +++ b/fuse/dl/cross_validation/pipeline.py @@ -1,16 +1,16 @@ -from typing import Dict, Callable -from fuse.utils import gpu as FuseUtilsGPU -from sklearn.model_selection import KFold -from functools import partial -from multiprocessing import Process, Queue -from typing import Sequence, Union import os -from fuse.eval.metrics.classification.metrics_ensembling_common import MetricEnsemble from collections import OrderedDict +from functools import partial +from multiprocessing import Process, Queue, set_start_method +from typing import Callable, Dict, Sequence, Union + +from sklearn.model_selection import KFold + from fuse.eval.evaluator import EvaluatorDefault +from fuse.eval.metrics.classification.metrics_ensembling_common import MetricEnsemble +from fuse.utils import gpu as FuseUtilsGPU from fuse.utils.file_io.file_io import create_or_reset_dir from fuse.utils.rand.seed import Seed -from multiprocessing import set_start_method set_start_method("spawn", force=True) diff --git a/fuse/dl/lightning/pl_epoch_summary.py b/fuse/dl/lightning/pl_epoch_summary.py index 02d24ca48..37762f778 100644 --- a/fuse/dl/lightning/pl_epoch_summary.py +++ b/fuse/dl/lightning/pl_epoch_summary.py @@ -16,17 +16,17 @@ Created on June 30, 2021 """ -from copy import deepcopy import os -from typing import Optional, Dict, Union -from fuse.utils.misc.misc import get_pretty_dataframe +from copy import deepcopy +from typing import Dict, Optional, Union +import pandas as pd import pytorch_lightning as pl +import torch from pytorch_lightning.callbacks import Checkpoint - from pytorch_lightning.utilities.rank_zero import rank_zero_only -import torch -import pandas as pd + +from fuse.utils.misc.misc import get_pretty_dataframe class ModelEpochSummary(Checkpoint): diff --git a/fuse/dl/lightning/pl_funcs.py b/fuse/dl/lightning/pl_funcs.py index e6c37a2cc..f9f36111a 100644 --- a/fuse/dl/lightning/pl_funcs.py +++ b/fuse/dl/lightning/pl_funcs.py @@ -21,24 +21,25 @@ """ import os import traceback -from typing import Any, Dict, List, OrderedDict, Sequence, Union, Mapping, TypeVar from statistics import mean -from fuse.data.utils.sample import get_sample_id_key -from fuse.utils.data.collate import uncollate -import pandas as pd +from typing import Any, Dict, List, Mapping, OrderedDict, Sequence, TypeVar, Union -import torch +import pandas as pd import pytorch_lightning as pl -from pytorch_lightning.callbacks import ModelCheckpoint, LearningRateMonitor -from .pl_epoch_summary import ModelEpochSummary +import torch -from fuse.utils import NDict +# for clearml +from clearml import Task +from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint + +from fuse.data.utils.sample import get_sample_id_key from fuse.dl.losses.loss_base import LossBase from fuse.eval import MetricBase from fuse.eval.metrics.utils import PerSampleData +from fuse.utils import NDict +from fuse.utils.data.collate import uncollate -# for clearml -from clearml import Task +from .pl_epoch_summary import ModelEpochSummary TaskInstance = TypeVar("TaskInstance", bound="Task") diff --git a/fuse/dl/lightning/pl_module.py b/fuse/dl/lightning/pl_module.py index ca61d14d7..d0c60ec5f 100644 --- a/fuse/dl/lightning/pl_module.py +++ b/fuse/dl/lightning/pl_module.py @@ -16,10 +16,11 @@ Created on June 30, 2021 """ -import pytorch_lightning as pl -from typing import Optional, Union, Tuple, Callable -from collections import OrderedDict import os +from collections import OrderedDict +from typing import Callable, Optional, Tuple, Union + +import pytorch_lightning as pl from fuse.dl.lightning.pl_funcs import * # noqa from fuse.utils.file_io.file_io import create_dir diff --git a/fuse/dl/losses/contrastive/contrastive_loss.py b/fuse/dl/losses/contrastive/contrastive_loss.py index 76d2e8bf9..dc1f20317 100644 --- a/fuse/dl/losses/contrastive/contrastive_loss.py +++ b/fuse/dl/losses/contrastive/contrastive_loss.py @@ -1,12 +1,12 @@ from typing import Callable, Optional -from fuse.dl.losses.loss_base import LossBase -from fuse.utils.ndict import NDict - import torch import torch.nn.functional as F from torch import Tensor +from fuse.dl.losses.loss_base import LossBase +from fuse.utils.ndict import NDict + def softcrossentropyloss(target: Tensor, logits: Tensor) -> Tensor: """ diff --git a/fuse/dl/losses/loss_base.py b/fuse/dl/losses/loss_base.py index ae4c9d210..ef639cd8f 100644 --- a/fuse/dl/losses/loss_base.py +++ b/fuse/dl/losses/loss_base.py @@ -18,9 +18,11 @@ """ from abc import abstractmethod -from fuse.utils.ndict import NDict + import torch +from fuse.utils.ndict import NDict + class LossBase(torch.nn.Module): """ diff --git a/fuse/dl/losses/loss_warm_up.py b/fuse/dl/losses/loss_warm_up.py index f06e81165..99f6f18bd 100644 --- a/fuse/dl/losses/loss_warm_up.py +++ b/fuse/dl/losses/loss_warm_up.py @@ -1,6 +1,7 @@ +from typing import Any, Dict + import torch from torch import Tensor -from typing import Any, Dict class LossWarmUp(torch.nn.Module): diff --git a/fuse/dl/losses/loss_wrap_to_dict.py b/fuse/dl/losses/loss_wrap_to_dict.py index d3042b3b3..c798999d7 100644 --- a/fuse/dl/losses/loss_wrap_to_dict.py +++ b/fuse/dl/losses/loss_wrap_to_dict.py @@ -1,6 +1,8 @@ from typing import Dict + import torch from torch import nn + from fuse.utils.ndict import NDict diff --git a/fuse/dl/losses/segmentation/loss_dice.py b/fuse/dl/losses/segmentation/loss_dice.py index 15ccfdaad..1a6382081 100644 --- a/fuse/dl/losses/segmentation/loss_dice.py +++ b/fuse/dl/losses/segmentation/loss_dice.py @@ -17,10 +17,11 @@ """ +from typing import List + import torch import torch.nn as nn from torch import Tensor -from typing import List class BinaryDiceLoss(nn.Module): diff --git a/fuse/dl/models/__init__.py b/fuse/dl/models/__init__.py index fed70e8db..a99910cba 100644 --- a/fuse/dl/models/__init__.py +++ b/fuse/dl/models/__init__.py @@ -1,4 +1,4 @@ -from .model_multihead import ModelMultiHead from .model_ensemble import ModelEnsemble +from .model_multihead import ModelMultiHead from .model_multistream import ModelMultistream from .model_siamese import ModelSiamese diff --git a/fuse/dl/models/backbones/__init__.py b/fuse/dl/models/backbones/__init__.py index d4d89e262..1cea6a453 100644 --- a/fuse/dl/models/backbones/__init__.py +++ b/fuse/dl/models/backbones/__init__.py @@ -1,4 +1,4 @@ from .backbone_inception_resnet_v2 import BackboneInceptionResnetV2 from .backbone_mlp import BackboneMultilayerPerceptron -from .backbone_resnet_3d import BackboneResnet3D from .backbone_resnet import BackboneResnet +from .backbone_resnet_3d import BackboneResnet3D diff --git a/fuse/dl/models/backbones/backbone_inception_resnet_v2.py b/fuse/dl/models/backbones/backbone_inception_resnet_v2.py index a8746835a..2a9e97c84 100644 --- a/fuse/dl/models/backbones/backbone_inception_resnet_v2.py +++ b/fuse/dl/models/backbones/backbone_inception_resnet_v2.py @@ -17,14 +17,14 @@ """ -from typing import Callable, Optional, Any, Dict, Tuple, Union +import logging +from typing import Any, Callable, Dict, Optional, Tuple, Union +from urllib.error import URLError import torch -from torch.hub import load_state_dict_from_url import torch.nn as nn from torch import Tensor -from urllib.error import URLError -import logging +from torch.hub import load_state_dict_from_url def make_seq( diff --git a/fuse/dl/models/backbones/backbone_mvit.py b/fuse/dl/models/backbones/backbone_mvit.py index 1949d7bd2..f3c76522f 100644 --- a/fuse/dl/models/backbones/backbone_mvit.py +++ b/fuse/dl/models/backbones/backbone_mvit.py @@ -17,12 +17,12 @@ """ -import torch.nn as nn -from torchvision.models.video.mvit import MViT, MViT_V2_S_Weights from typing import Dict, List -from torch import Tensor + import torch -from torchvision.models.video.mvit import MSBlockConfig +import torch.nn as nn +from torch import Tensor +from torchvision.models.video.mvit import MSBlockConfig, MViT, MViT_V2_S_Weights class BackboneMViT(nn.Module): diff --git a/fuse/dl/models/backbones/backbone_resnet.py b/fuse/dl/models/backbones/backbone_resnet.py index 6258ab177..57a8b2a6b 100644 --- a/fuse/dl/models/backbones/backbone_resnet.py +++ b/fuse/dl/models/backbones/backbone_resnet.py @@ -17,11 +17,11 @@ """ from typing import Optional, Union -import torch.nn as nn -from torchvision.models import ResNet -from torchvision.models.resnet import Bottleneck, BasicBlock, WeightsEnum +import torch.nn as nn from torch import Tensor +from torchvision.models import ResNet +from torchvision.models.resnet import BasicBlock, Bottleneck, WeightsEnum class BackboneResnet(ResNet): diff --git a/fuse/dl/models/backbones/backbone_resnet_3d.py b/fuse/dl/models/backbones/backbone_resnet_3d.py index 9c5e074d3..92a82b9ff 100644 --- a/fuse/dl/models/backbones/backbone_resnet_3d.py +++ b/fuse/dl/models/backbones/backbone_resnet_3d.py @@ -17,9 +17,10 @@ """ +from typing import Callable, List, Optional, Sequence, Tuple, Type + import torch.nn as nn from torch import Tensor -from typing import Tuple, Optional, Callable, List, Sequence, Type class Conv3DSimple(nn.Conv3d): diff --git a/fuse/dl/models/backbones/backbone_transformer.py b/fuse/dl/models/backbones/backbone_transformer.py index 41623a07a..ecfa38974 100644 --- a/fuse/dl/models/backbones/backbone_transformer.py +++ b/fuse/dl/models/backbones/backbone_transformer.py @@ -1,9 +1,10 @@ +from typing import Optional + import torch from torch import nn -from typing import Optional from vit_pytorch.vit import Transformer as _Transformer from vit_pytorch.vit import repeat -from x_transformers import Encoder, CrossAttender, TransformerWrapper +from x_transformers import CrossAttender, Encoder, TransformerWrapper class Transformer(nn.Module): diff --git a/fuse/dl/models/backbones/backbone_unet3d.py b/fuse/dl/models/backbones/backbone_unet3d.py index edb4261f8..2837eb960 100755 --- a/fuse/dl/models/backbones/backbone_unet3d.py +++ b/fuse/dl/models/backbones/backbone_unet3d.py @@ -1,7 +1,8 @@ +from typing import Tuple + import torch import torch.nn as nn import torch.nn.functional as F -from typing import Tuple # Based on the architecture from https://github.com/MrGiovanni/ModelsGenesis torch.use_deterministic_algorithms(False) @@ -9,7 +10,6 @@ class ContBatchNorm3d(nn.modules.batchnorm._BatchNorm): def _check_input_dim(self, input: torch.Tensor) -> None: - if input.dim() != 5: raise ValueError("expected 5D input (got {}D input)".format(input.dim())) # super(ContBatchNorm3d, self)._check_input_dim(input) @@ -111,7 +111,6 @@ def forward(self, x: torch.Tensor, skip_x: torch.Tensor) -> torch.Tensor: class OutputTransition(nn.Module): def __init__(self, inChans: int, n_labels: int): - super(OutputTransition, self).__init__() self.final_conv = nn.Conv3d(inChans, n_labels, kernel_size=1) self.sigmoid = nn.Sigmoid() diff --git a/fuse/dl/models/backbones/backbone_vit.py b/fuse/dl/models/backbones/backbone_vit.py index 23fdebc80..0abe3febd 100644 --- a/fuse/dl/models/backbones/backbone_vit.py +++ b/fuse/dl/models/backbones/backbone_vit.py @@ -1,7 +1,9 @@ from typing import Sequence + import numpy as np import torch -from torch import nn, Tensor +from torch import Tensor, nn + from fuse.dl.models.backbones.backbone_transformer import Transformer diff --git a/fuse/dl/models/heads/__init__.py b/fuse/dl/models/heads/__init__.py index d79d026e2..7863555dd 100644 --- a/fuse/dl/models/heads/__init__.py +++ b/fuse/dl/models/heads/__init__.py @@ -1,5 +1,5 @@ from .common import ClassifierFCN, ClassifierFCN3D, ClassifierMLP -from .heads_1D import Head1D -from .heads_3D import Head3D from .head_dense_segmentation import HeadDenseSegmentation from .head_global_pooling_classifier import HeadGlobalPoolingClassifier +from .heads_1D import Head1D +from .heads_3D import Head3D diff --git a/fuse/dl/models/heads/common.py b/fuse/dl/models/heads/common.py index 3cdaf2fc7..71201989e 100644 --- a/fuse/dl/models/heads/common.py +++ b/fuse/dl/models/heads/common.py @@ -17,10 +17,11 @@ """ -from typing import Optional, Sequence, List +from typing import List, Optional, Sequence + +import torch import torch.nn as nn from torch import Tensor -import torch class ClassifierFCN(nn.Module): diff --git a/fuse/dl/models/heads/head_dense_segmentation.py b/fuse/dl/models/heads/head_dense_segmentation.py index c4ddc4a7d..b72f2e29a 100644 --- a/fuse/dl/models/heads/head_dense_segmentation.py +++ b/fuse/dl/models/heads/head_dense_segmentation.py @@ -17,7 +17,7 @@ """ -from typing import Dict, Tuple, Sequence, Optional, Union +from typing import Dict, Optional, Sequence, Tuple, Union import torch import torch.nn as nn diff --git a/fuse/dl/models/heads/head_global_pooling_classifier.py b/fuse/dl/models/heads/head_global_pooling_classifier.py index 465c5c402..e30ce744a 100644 --- a/fuse/dl/models/heads/head_global_pooling_classifier.py +++ b/fuse/dl/models/heads/head_global_pooling_classifier.py @@ -17,7 +17,7 @@ """ -from typing import Dict, Tuple, Sequence, Optional +from typing import Dict, Optional, Sequence, Tuple import torch import torch.nn as nn diff --git a/fuse/dl/models/heads/heads_1D.py b/fuse/dl/models/heads/heads_1D.py index 17a5029dc..e930989d2 100644 --- a/fuse/dl/models/heads/heads_1D.py +++ b/fuse/dl/models/heads/heads_1D.py @@ -17,11 +17,12 @@ """ +from typing import Dict, Optional, Sequence, Tuple + import torch import torch.nn as nn import torch.nn.functional as F -from typing import Dict, Tuple, Sequence, Optional from fuse.dl.models.heads.common import ClassifierMLP diff --git a/fuse/dl/models/heads/heads_3D.py b/fuse/dl/models/heads/heads_3D.py index f1b0dc5b4..e139e498e 100644 --- a/fuse/dl/models/heads/heads_3D.py +++ b/fuse/dl/models/heads/heads_3D.py @@ -17,14 +17,14 @@ """ -from typing import Dict, Tuple, Sequence, Optional +from typing import Dict, Optional, Sequence, Tuple import torch import torch.nn as nn import torch.nn.functional as F -from fuse.utils.ndict import NDict from fuse.dl.models.heads.common import ClassifierFCN3D, ClassifierMLP +from fuse.utils.ndict import NDict class Head3D(nn.Module): diff --git a/fuse/dl/models/model_ensemble.py b/fuse/dl/models/model_ensemble.py index a92ed7e79..b0c135984 100644 --- a/fuse/dl/models/model_ensemble.py +++ b/fuse/dl/models/model_ensemble.py @@ -19,9 +19,9 @@ import copy import os +from typing import Dict, List, Sequence import torch -from typing import Sequence, Dict, List class ModelEnsemble(torch.nn.Module): diff --git a/fuse/dl/models/model_multihead.py b/fuse/dl/models/model_multihead.py index d2336872f..aaf8777fe 100644 --- a/fuse/dl/models/model_multihead.py +++ b/fuse/dl/models/model_multihead.py @@ -17,7 +17,7 @@ """ -from typing import Sequence, Dict, Tuple, Union +from typing import Dict, Sequence, Tuple, Union import torch diff --git a/fuse/dl/models/model_multistream.py b/fuse/dl/models/model_multistream.py index e3bd72121..518cbcd55 100644 --- a/fuse/dl/models/model_multistream.py +++ b/fuse/dl/models/model_multistream.py @@ -17,8 +17,9 @@ """ +from typing import Callable, Dict, Optional, Sequence, Tuple + import torch -from typing import Sequence, Dict, Tuple, Callable, Optional from fuse.utils.ndict import NDict diff --git a/fuse/dl/models/model_siamese.py b/fuse/dl/models/model_siamese.py index bcb91f438..c965dac77 100644 --- a/fuse/dl/models/model_siamese.py +++ b/fuse/dl/models/model_siamese.py @@ -17,7 +17,7 @@ """ -from typing import Sequence, Dict, Tuple +from typing import Dict, Sequence, Tuple import torch diff --git a/fuse/dl/models/model_wrapper.py b/fuse/dl/models/model_wrapper.py index 3d34a0b9f..f30bad095 100644 --- a/fuse/dl/models/model_wrapper.py +++ b/fuse/dl/models/model_wrapper.py @@ -17,7 +17,7 @@ """ -from typing import Sequence, Dict, Callable, Union, Any +from typing import Any, Callable, Dict, Sequence, Union import torch import torch.nn as nn diff --git a/fuse/dl/templates/walkthrough_template.py b/fuse/dl/templates/walkthrough_template.py index a17ffa476..6c06ca771 100644 --- a/fuse/dl/templates/walkthrough_template.py +++ b/fuse/dl/templates/walkthrough_template.py @@ -17,31 +17,28 @@ """ +import logging import os from typing import OrderedDict -import logging -import torch import pytorch_lightning as pl +import torch import torch.optim as optim from torch.utils.data.dataloader import DataLoader -from fuse.utils.utils_debug import FuseDebug import fuse.utils.gpu as GPU -from fuse.utils.utils_logger import fuse_logger_start -from fuse.utils.file_io.file_io import create_dir, save_dataframe - from fuse.data.datasets.caching.samples_cacher import SamplesCacher from fuse.data.datasets.dataset_default import DatasetDefault from fuse.data.pipelines.pipeline_default import PipelineDefault from fuse.data.utils.collates import CollateDefault from fuse.data.utils.samplers import BatchSamplerDefault - -from fuse.dl.models import ModelMultiHead -from fuse.dl.lightning.pl_module import LightningModuleDefault from fuse.dl.lightning.pl_funcs import convert_predictions_to_dataframe - +from fuse.dl.lightning.pl_module import LightningModuleDefault +from fuse.dl.models import ModelMultiHead from fuse.eval.evaluator import EvaluatorDefault +from fuse.utils.file_io.file_io import create_dir, save_dataframe +from fuse.utils.utils_debug import FuseDebug +from fuse.utils.utils_logger import fuse_logger_start ########################################################################################################### # Fuse diff --git a/fuse/dl/tests/test_cat.py b/fuse/dl/tests/test_cat.py index bf91506b4..2132761f7 100644 --- a/fuse/dl/tests/test_cat.py +++ b/fuse/dl/tests/test_cat.py @@ -17,9 +17,11 @@ """ -import torch -import unittest import random +import unittest + +import torch + from fuse.dl.models.backbones.backbone_transformer import ( CrossAttentionTransformerEncoder, ) diff --git a/fuse/dl/tests/test_vit.py b/fuse/dl/tests/test_vit.py index 2f608d5b6..50fdc8944 100644 --- a/fuse/dl/tests/test_vit.py +++ b/fuse/dl/tests/test_vit.py @@ -18,6 +18,7 @@ """ import unittest + from fuse.dl.models.backbones.backbone_vit import usage_example diff --git a/fuse/eval/__init__.py b/fuse/eval/__init__.py index 536c8ea2e..58c4d2495 100644 --- a/fuse/eval/__init__.py +++ b/fuse/eval/__init__.py @@ -1,9 +1,8 @@ # version -from fuse.version import __version__ - # shortcuts from fuse.eval.metrics.metrics_common import ( MetricBase, MetricCollector, MetricWithCollectorBase, ) +from fuse.version import __version__ diff --git a/fuse/eval/evaluator.py b/fuse/eval/evaluator.py index e96f2c02a..399f005a3 100644 --- a/fuse/eval/evaluator.py +++ b/fuse/eval/evaluator.py @@ -18,6 +18,7 @@ """ import os +import pickle from typing import ( Dict, Hashable, @@ -28,15 +29,12 @@ Sequence, Union, ) -import pickle -import pandas as pd -import numpy as np -from fuse.utils import read_dataframe -from fuse.utils import NDict -from fuse.utils import CollateToBatchList +import numpy as np +import pandas as pd from fuse.eval.metrics.metrics_common import MetricBase +from fuse.utils import CollateToBatchList, NDict, read_dataframe class EvaluatorDefault: diff --git a/fuse/eval/examples/examples.py b/fuse/eval/examples/examples.py index 6417e7f93..848fd42e8 100644 --- a/fuse/eval/examples/examples.py +++ b/fuse/eval/examples/examples.py @@ -19,30 +19,31 @@ import os import pathlib +from collections import OrderedDict from tempfile import mkdtemp from typing import Any, Dict -from collections import OrderedDict - -import pandas as pd import numpy as np -from fuse.eval.metrics.survival.metrics_survival import MetricCIndex - -from fuse.utils import set_seed -from fuse.utils.ndict import NDict +import pandas as pd -from fuse.eval.metrics.metrics_common import GroupAnalysis, CI, Filter -from fuse.eval.metrics.metrics_model_comparison import PairedBootstrap +from fuse.eval.evaluator import EvaluatorDefault +from fuse.eval.metrics.classification.metrics_calibration_common import ( + MetricApplyTemperature, + MetricECE, + MetricFindTemperature, + MetricReliabilityDiagram, +) from fuse.eval.metrics.classification.metrics_classification_common import ( + MetricAccuracy, MetricAUCPR, MetricAUCROC, - MetricAccuracy, + MetricAUCROCMultLabel, + MetricBSS, MetricConfusion, MetricConfusionMatrix, - MetricBSS, MetricROCCurve, - MetricAUCROCMultLabel, ) +from fuse.eval.metrics.classification.metrics_ensembling_common import MetricEnsemble from fuse.eval.metrics.classification.metrics_model_comparison_common import ( MetricDelongsTest, MetricMcnemarsTest, @@ -50,14 +51,11 @@ from fuse.eval.metrics.classification.metrics_thresholding_common import ( MetricApplyThresholds, ) -from fuse.eval.metrics.classification.metrics_calibration_common import ( - MetricReliabilityDiagram, - MetricECE, - MetricFindTemperature, - MetricApplyTemperature, -) -from fuse.eval.metrics.classification.metrics_ensembling_common import MetricEnsemble -from fuse.eval.evaluator import EvaluatorDefault +from fuse.eval.metrics.metrics_common import CI, Filter, GroupAnalysis +from fuse.eval.metrics.metrics_model_comparison import PairedBootstrap +from fuse.eval.metrics.survival.metrics_survival import MetricCIndex +from fuse.utils import set_seed +from fuse.utils.ndict import NDict def example_0() -> Dict[str, Any]: diff --git a/fuse/eval/examples/examples_segmentation.py b/fuse/eval/examples/examples_segmentation.py index 66bd2023f..d2c3667ef 100644 --- a/fuse/eval/examples/examples_segmentation.py +++ b/fuse/eval/examples/examples_segmentation.py @@ -14,28 +14,29 @@ import os import pathlib -from typing import Any, Dict from collections import OrderedDict -from pycocotools.coco import COCO -import numpy as np +from typing import Any, Dict + import nibabel as nib -from fuse.utils import set_seed, NDict +import numpy as np +from pycocotools.coco import COCO +from fuse.eval.evaluator import EvaluatorDefault +from fuse.eval.metrics.detection.metrics_detection_common import ( + MetricDetectionPrecision, + MetricDetectionRecall, +) +from fuse.eval.metrics.segmentation.metrics_instance_segmentation_common import ( + MetricInstanceIouJaccard, +) from fuse.eval.metrics.segmentation.metrics_segmentation_common import ( + Metric2DHausdorff, MetricDice, MetricIouJaccard, MetricOverlap, - Metric2DHausdorff, MetricPixelAccuracy, ) -from fuse.eval.metrics.segmentation.metrics_instance_segmentation_common import ( - MetricInstanceIouJaccard, -) -from fuse.eval.metrics.detection.metrics_detection_common import ( - MetricDetectionPrecision, - MetricDetectionRecall, -) -from fuse.eval.evaluator import EvaluatorDefault +from fuse.utils import NDict, set_seed def example_seg_0() -> Dict[str, Any]: diff --git a/fuse/eval/examples/examples_seq_gen.py b/fuse/eval/examples/examples_seq_gen.py index 32a1b51a3..c0825f555 100644 --- a/fuse/eval/examples/examples_seq_gen.py +++ b/fuse/eval/examples/examples_seq_gen.py @@ -11,22 +11,20 @@ limitations under the License. Created on June 30, 2021 """ -import pandas as pd -from typing import Any, Dict from collections import OrderedDict +from typing import Any, Dict + +import pandas as pd +import torch +from torch.utils.data.dataloader import DataLoader +from fuse.data import CollateDefault, DatasetDefault, OpReadDataframe, PipelineDefault +from fuse.eval.evaluator import EvaluatorDefault from fuse.eval.metrics.sequence_gen.metrics_seq_gen_common import ( - MetricPerplexity, MetricCountSeqAndTokens, + MetricPerplexity, ) -from fuse.eval.evaluator import EvaluatorDefault - - -from fuse.data import PipelineDefault, DatasetDefault, OpReadDataframe, CollateDefault -import torch -from torch.utils.data.dataloader import DataLoader - def example_seq_gen_0(seed: int = 1234) -> Dict[str, Any]: """ diff --git a/fuse/eval/examples/examples_stats.py b/fuse/eval/examples/examples_stats.py index d0fe36388..20bcfe72d 100644 --- a/fuse/eval/examples/examples_stats.py +++ b/fuse/eval/examples/examples_stats.py @@ -17,11 +17,13 @@ """ -from fuse.eval.metrics.regression.metrics import MetricPearsonCorrelation +from collections import OrderedDict + import numpy as np import pandas as pd -from collections import OrderedDict + from fuse.eval.evaluator import EvaluatorDefault +from fuse.eval.metrics.regression.metrics import MetricPearsonCorrelation def example_pearson_correlation() -> float: diff --git a/fuse/eval/metrics/classification/metrics_calibration_common.py b/fuse/eval/metrics/classification/metrics_calibration_common.py index 2fbbef8ca..5dbf921e4 100644 --- a/fuse/eval/metrics/classification/metrics_calibration_common.py +++ b/fuse/eval/metrics/classification/metrics_calibration_common.py @@ -1,9 +1,9 @@ +from functools import partial from typing import Optional, Union from fuse.eval.metrics.libs.calibration import Calibration -from .metrics_classification_common import MetricMultiClassDefault -from functools import partial +from .metrics_classification_common import MetricMultiClassDefault class MetricReliabilityDiagram(MetricMultiClassDefault): diff --git a/fuse/eval/metrics/classification/metrics_classification_common.py b/fuse/eval/metrics/classification/metrics_classification_common.py index 61ea008bc..2742d8514 100644 --- a/fuse/eval/metrics/classification/metrics_classification_common.py +++ b/fuse/eval/metrics/classification/metrics_classification_common.py @@ -17,15 +17,15 @@ """ -from functools import partial -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union, Hashable import traceback +from functools import partial +from typing import Any, Callable, Dict, Hashable, List, Optional, Sequence, Tuple, Union import numpy as np +from sklearn.metrics import balanced_accuracy_score, matthews_corrcoef -from fuse.eval.metrics.metrics_common import MetricDefault, MetricWithCollectorBase from fuse.eval.metrics.libs.classification import MetricsLibClass -from sklearn.metrics import matthews_corrcoef, balanced_accuracy_score +from fuse.eval.metrics.metrics_common import MetricDefault, MetricWithCollectorBase class MetricMultiClassDefault(MetricWithCollectorBase): diff --git a/fuse/eval/metrics/classification/metrics_ensembling_common.py b/fuse/eval/metrics/classification/metrics_ensembling_common.py index 827694085..2966dc05c 100644 --- a/fuse/eval/metrics/classification/metrics_ensembling_common.py +++ b/fuse/eval/metrics/classification/metrics_ensembling_common.py @@ -1,13 +1,12 @@ -from typing import Optional, Sequence, Hashable, List, Dict, Callable, Any -from fuse.eval.metrics.libs.ensembling import Ensembling -from fuse.eval.metrics.utils import PerSampleData - -import pandas as pd +from functools import partial +from typing import Any, Callable, Dict, Hashable, List, Optional, Sequence import numpy as np +import pandas as pd +from fuse.eval.metrics.libs.ensembling import Ensembling from fuse.eval.metrics.metrics_common import MetricDefault -from functools import partial +from fuse.eval.metrics.utils import PerSampleData class MetricEnsemble(MetricDefault): diff --git a/fuse/eval/metrics/classification/metrics_model_comparison_common.py b/fuse/eval/metrics/classification/metrics_model_comparison_common.py index c02ab6626..887ed3cdb 100644 --- a/fuse/eval/metrics/classification/metrics_model_comparison_common.py +++ b/fuse/eval/metrics/classification/metrics_model_comparison_common.py @@ -19,10 +19,11 @@ from typing import Optional, Sequence -from .metrics_classification_common import MetricMultiClassDefault from fuse.eval.metrics.libs.model_comparison import ModelComparison from fuse.eval.metrics.metrics_common import MetricDefault +from .metrics_classification_common import MetricMultiClassDefault + class MetricDelongsTest(MetricMultiClassDefault): def __init__( diff --git a/fuse/eval/metrics/classification/metrics_thresholding_common.py b/fuse/eval/metrics/classification/metrics_thresholding_common.py index 41592329d..55384c59f 100644 --- a/fuse/eval/metrics/classification/metrics_thresholding_common.py +++ b/fuse/eval/metrics/classification/metrics_thresholding_common.py @@ -1,9 +1,10 @@ -from typing import Optional, Sequence, Tuple, Union, Hashable, Dict, Any -from fuse.eval.metrics.libs.thresholding import Thresholding -from fuse.eval.metrics.utils import PerSampleData +from typing import Any, Dict, Hashable, Optional, Sequence, Tuple, Union import numpy as np +from fuse.eval.metrics.libs.thresholding import Thresholding +from fuse.eval.metrics.utils import PerSampleData + from .metrics_classification_common import MetricMultiClassDefault diff --git a/fuse/eval/metrics/detection/metrics_detection_common.py b/fuse/eval/metrics/detection/metrics_detection_common.py index 7bbc1042f..4a9e39474 100644 --- a/fuse/eval/metrics/detection/metrics_detection_common.py +++ b/fuse/eval/metrics/detection/metrics_detection_common.py @@ -1,9 +1,9 @@ from functools import partial from typing import List, Optional -from fuse.eval.metrics.libs.instance_segmentation import MetricsInstanceSegmentaion import numpy as np +from fuse.eval.metrics.libs.instance_segmentation import MetricsInstanceSegmentaion from fuse.eval.metrics.metrics_common import MetricPerSampleDefault diff --git a/fuse/eval/metrics/libs/calibration.py b/fuse/eval/metrics/libs/calibration.py index b9da1a5eb..548e49cf9 100644 --- a/fuse/eval/metrics/libs/calibration.py +++ b/fuse/eval/metrics/libs/calibration.py @@ -1,10 +1,11 @@ -from typing import Sequence, Union, Dict, Optional -import numpy as np -import matplotlib.pyplot as plt import math +from typing import Dict, Optional, Sequence, Union + +import matplotlib.pyplot as plt +import numpy as np import torch -from torch import nn, optim, Tensor from scipy import stats +from torch import Tensor, nn, optim class Calibration: diff --git a/fuse/eval/metrics/libs/classification.py b/fuse/eval/metrics/libs/classification.py index 33431284b..046f295cc 100644 --- a/fuse/eval/metrics/libs/classification.py +++ b/fuse/eval/metrics/libs/classification.py @@ -18,17 +18,14 @@ """ from typing import Dict, Optional, Sequence, Tuple, Union -import pandas as pd +import matplotlib.pyplot as plt import numpy as np - -from sklearn import metrics +import pandas as pd import sklearn +from sklearn import metrics from sklearn.utils.multiclass import type_of_target -import matplotlib.pyplot as plt - - class MetricsLibClass: @staticmethod def auc_roc( diff --git a/fuse/eval/metrics/libs/ensembling.py b/fuse/eval/metrics/libs/ensembling.py index b182d809c..d185fe896 100644 --- a/fuse/eval/metrics/libs/ensembling.py +++ b/fuse/eval/metrics/libs/ensembling.py @@ -1,4 +1,5 @@ from typing import Dict, Optional, Sequence + import numpy as np import scipy diff --git a/fuse/eval/metrics/libs/instance_segmentation.py b/fuse/eval/metrics/libs/instance_segmentation.py index 4b4cbf88e..610a12b91 100644 --- a/fuse/eval/metrics/libs/instance_segmentation.py +++ b/fuse/eval/metrics/libs/instance_segmentation.py @@ -1,7 +1,6 @@ from typing import Dict, Sequence import numpy as np - from pycocotools import mask as maskUtils diff --git a/fuse/eval/metrics/libs/segmentation.py b/fuse/eval/metrics/libs/segmentation.py index 60582d10e..516bcc25f 100644 --- a/fuse/eval/metrics/libs/segmentation.py +++ b/fuse/eval/metrics/libs/segmentation.py @@ -1,7 +1,6 @@ from typing import Dict, Optional, Tuple import numpy as np - from scipy.spatial import distance from scipy.spatial.distance import directed_hausdorff diff --git a/fuse/eval/metrics/libs/stat.py b/fuse/eval/metrics/libs/stat.py index d51f21a8b..9efc82e43 100644 --- a/fuse/eval/metrics/libs/stat.py +++ b/fuse/eval/metrics/libs/stat.py @@ -1,5 +1,6 @@ -import numpy as np from typing import Sequence, Union + +import numpy as np from scipy.stats import pearsonr, spearmanr diff --git a/fuse/eval/metrics/libs/survival.py b/fuse/eval/metrics/libs/survival.py index 9886784d1..cf5f99df2 100644 --- a/fuse/eval/metrics/libs/survival.py +++ b/fuse/eval/metrics/libs/survival.py @@ -1,7 +1,8 @@ -from lifelines.utils import concordance_index -import numpy as np from typing import Sequence +import numpy as np +from lifelines.utils import concordance_index + class MetricsSurvival: @staticmethod diff --git a/fuse/eval/metrics/libs/thresholding.py b/fuse/eval/metrics/libs/thresholding.py index bb819df8f..451f43e15 100644 --- a/fuse/eval/metrics/libs/thresholding.py +++ b/fuse/eval/metrics/libs/thresholding.py @@ -1,7 +1,9 @@ from typing import Dict, Sequence, Union -from fuse.eval.metrics.libs.classification import MetricsLibClass + import numpy as np +from fuse.eval.metrics.libs.classification import MetricsLibClass + class Thresholding: """ diff --git a/fuse/eval/metrics/metrics_common.py b/fuse/eval/metrics/metrics_common.py index e5a4e399e..35525f995 100644 --- a/fuse/eval/metrics/metrics_common.py +++ b/fuse/eval/metrics/metrics_common.py @@ -17,17 +17,17 @@ """ -from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, Hashable, Optional, Sequence, Tuple, Union, List import copy -from fuse.utils import uncollate -import torch.distributed as dist -import pandas as pd -import torch +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, Hashable, List, Optional, Sequence, Tuple, Union + import numpy as np +import pandas as pd import scipy +import torch +import torch.distributed as dist -from fuse.utils import NDict +from fuse.utils import NDict, uncollate class MetricBase(ABC): diff --git a/fuse/eval/metrics/metrics_model_comparison.py b/fuse/eval/metrics/metrics_model_comparison.py index 49e418e93..9925d94c6 100644 --- a/fuse/eval/metrics/metrics_model_comparison.py +++ b/fuse/eval/metrics/metrics_model_comparison.py @@ -16,13 +16,13 @@ Created on June 30, 2021 """ -from typing import Any, Callable, Dict, Optional, Sequence, Union, Hashable +from typing import Any, Callable, Dict, Hashable, Optional, Sequence, Union import numpy as np import pandas as pd -from fuse.eval.metrics.metrics_common import MetricBase, MetricWithCollectorBase from fuse.eval.metrics.libs.model_comparison import ModelComparison +from fuse.eval.metrics.metrics_common import MetricBase, MetricWithCollectorBase class PairedBootstrap(MetricWithCollectorBase): diff --git a/fuse/eval/metrics/regression/metrics.py b/fuse/eval/metrics/regression/metrics.py index f19bc3782..8b3ba4f68 100644 --- a/fuse/eval/metrics/regression/metrics.py +++ b/fuse/eval/metrics/regression/metrics.py @@ -1,6 +1,5 @@ -from typing import List, Optional, Union, Sequence -from fuse.eval.metrics.libs.stat import Stat -from fuse.eval.metrics.metrics_common import MetricDefault +from typing import List, Optional, Sequence, Union + import numpy as np from sklearn.metrics import ( mean_absolute_error, @@ -9,6 +8,9 @@ root_mean_squared_error, ) +from fuse.eval.metrics.libs.stat import Stat +from fuse.eval.metrics.metrics_common import MetricDefault + class MetricPearsonCorrelation(MetricDefault): def __init__( @@ -66,9 +68,7 @@ def mae( mask: Optional[np.ndarray] = None, **kwargs: dict, ) -> float: - if mask is not None: - if isinstance(pred, Sequence): if np.isscalar(pred[0]): pred = np.array(pred) @@ -122,9 +122,7 @@ def mse( mask: Optional[np.ndarray] = None, **kwargs: dict, ) -> float: - if mask is not None: - if isinstance(pred, Sequence): if np.isscalar(pred[0]): pred = np.array(pred) @@ -175,9 +173,7 @@ def rmse( mask: Optional[np.ndarray] = None, **kwargs: dict, ) -> float: - if mask is not None: - if isinstance(pred, Sequence): if np.isscalar(pred[0]): pred = np.array(pred) @@ -232,7 +228,6 @@ def r2( target: Union[List, np.ndarray], **kwargs: dict, ) -> float: - pred = np.array(pred).flatten() target = np.array(target).flatten() diff --git a/fuse/eval/metrics/segmentation/metrics_instance_segmentation_common.py b/fuse/eval/metrics/segmentation/metrics_instance_segmentation_common.py index 2b1997949..3433d6854 100644 --- a/fuse/eval/metrics/segmentation/metrics_instance_segmentation_common.py +++ b/fuse/eval/metrics/segmentation/metrics_instance_segmentation_common.py @@ -1,9 +1,9 @@ from functools import partial from typing import List, Optional -from fuse.eval.metrics.libs.instance_segmentation import MetricsInstanceSegmentaion import numpy as np +from fuse.eval.metrics.libs.instance_segmentation import MetricsInstanceSegmentaion from fuse.eval.metrics.metrics_common import MetricPerSampleDefault diff --git a/fuse/eval/metrics/segmentation/metrics_segmentation_common.py b/fuse/eval/metrics/segmentation/metrics_segmentation_common.py index bea952e04..59efd866c 100644 --- a/fuse/eval/metrics/segmentation/metrics_segmentation_common.py +++ b/fuse/eval/metrics/segmentation/metrics_segmentation_common.py @@ -1,10 +1,10 @@ +from collections import defaultdict from functools import partial from typing import Dict, List, Optional -from collections import defaultdict -from fuse.eval.metrics.libs.segmentation import MetricsSegmentation import numpy as np +from fuse.eval.metrics.libs.segmentation import MetricsSegmentation from fuse.eval.metrics.metrics_common import MetricPerSampleDefault diff --git a/fuse/eval/metrics/sequence_gen/metrics_seq_gen_common.py b/fuse/eval/metrics/sequence_gen/metrics_seq_gen_common.py index 6da0ff1b4..5f48275c2 100644 --- a/fuse/eval/metrics/sequence_gen/metrics_seq_gen_common.py +++ b/fuse/eval/metrics/sequence_gen/metrics_seq_gen_common.py @@ -16,11 +16,12 @@ Created on June 30, 2021 """ -from typing import Optional, Tuple, List, Dict -from functools import partial from copy import copy -import torch +from functools import partial +from typing import Dict, List, Optional, Tuple + import numpy as np +import torch from fuse.eval.metrics.metrics_common import MetricPerBatchDefault @@ -71,7 +72,6 @@ def _count_seq_and_tokens_compute( seq_num: List[np.ndarray], token_num: List[np.ndarray], ) -> dict: - seq_num_total = sum(seq_num) token_num_total = sum(token_num) self._state["seq_num"] += seq_num_total diff --git a/fuse/eval/metrics/stat/metrics_stat_common.py b/fuse/eval/metrics/stat/metrics_stat_common.py index 9164d12d5..9d23132d4 100644 --- a/fuse/eval/metrics/stat/metrics_stat_common.py +++ b/fuse/eval/metrics/stat/metrics_stat_common.py @@ -1,5 +1,6 @@ -from typing import Any, Dict, Hashable, Optional, Sequence from collections import Counter +from typing import Any, Dict, Hashable, Optional, Sequence + from fuse.eval.metrics.metrics_common import MetricWithCollectorBase from fuse.eval.metrics.regression.metrics import MetricPearsonCorrelation # noqa: F401 diff --git a/fuse/eval/metrics/utils.py b/fuse/eval/metrics/utils.py index e837ab4f9..cf745bc3d 100644 --- a/fuse/eval/metrics/utils.py +++ b/fuse/eval/metrics/utils.py @@ -1,6 +1,7 @@ -from typing import Sequence, Optional, Hashable -import numpy as np import copy +from typing import Hashable, Optional, Sequence + +import numpy as np class PerSampleData: diff --git a/fuse/eval/tests/test_eval.py b/fuse/eval/tests/test_eval.py index f2597351d..403e2773f 100644 --- a/fuse/eval/tests/test_eval.py +++ b/fuse/eval/tests/test_eval.py @@ -16,10 +16,10 @@ Created on June 30, 2021 """ -import pandas as pd -from distutils.log import warn import unittest +from distutils.log import warn +import pandas as pd from fuse.eval.examples.examples import ( example_0, @@ -39,7 +39,6 @@ example_14, example_18, ) - from fuse.eval.examples.examples_segmentation import ( example_seg_0, example_seg_1, @@ -47,13 +46,11 @@ example_seg_3, example_seg_4, ) - from fuse.eval.examples.examples_seq_gen import ( example_seq_gen_0, example_seq_gen_1, example_seq_gen_2, ) - from fuse.eval.examples.examples_stats import example_pearson_correlation @@ -121,8 +118,9 @@ def test_eval_example_8(self) -> None: def test_eval_example_9(self) -> None: try: - import fuse.data import torchvision + + import fuse.data except ImportError: warn( " test_eval_example_8: requires fuse-med-ml-data and torchvision packages" diff --git a/fuse/utils/__init__.py b/fuse/utils/__init__.py index da14056b2..571630e03 100644 --- a/fuse/utils/__init__.py +++ b/fuse/utils/__init__.py @@ -1,21 +1,19 @@ # version -from fuse.version import __version__ - # workaround for an issue caused by importing pandas after numpy import pandas as pd -from fuse.utils.ndict import NDict from fuse.utils.data.collate import CollateToBatchList, uncollate - +from fuse.utils.ndict import NDict from fuse.utils.rand.param_sampler import ( - Uniform, - RandInt, - RandBool, Choice, + RandBool, + RandInt, + Uniform, draw_samples_recursively, ) from fuse.utils.rand.seed import Seed +from fuse.version import __version__ set_seed = Seed.set_seed -from fuse.utils.file_io.file_io import read_dataframe, save_dataframe from fuse.utils.data.collate import CollateToBatchList, uncollate +from fuse.utils.file_io.file_io import read_dataframe, save_dataframe diff --git a/fuse/utils/add_preamble.py b/fuse/utils/add_preamble.py index 608fb7cb4..0a0ba6a99 100644 --- a/fuse/utils/add_preamble.py +++ b/fuse/utils/add_preamble.py @@ -17,10 +17,10 @@ """ -import sys +import fileinput import os import os.path as path -import fileinput +import sys from typing import List src_extensions = [".py"] diff --git a/fuse/utils/config_tools/config_tools.py b/fuse/utils/config_tools/config_tools.py index 6aebec6aa..3d10d8b3f 100644 --- a/fuse/utils/config_tools/config_tools.py +++ b/fuse/utils/config_tools/config_tools.py @@ -1,5 +1,5 @@ import runpy -from typing import List, Union, Callable, Dict, Any +from typing import Any, Callable, Dict, List, Union class Config: diff --git a/fuse/utils/config_tools/tests/some_conf_internal_include.py b/fuse/utils/config_tools/tests/some_conf_internal_include.py index 64e2a9db6..8dd07c379 100644 --- a/fuse/utils/config_tools/tests/some_conf_internal_include.py +++ b/fuse/utils/config_tools/tests/some_conf_internal_include.py @@ -1,4 +1,5 @@ import os + from fuse.utils.config_tools import get_config_function diff --git a/fuse/utils/config_tools/tests/test_config.py b/fuse/utils/config_tools/tests/test_config.py index da719b891..273df9157 100644 --- a/fuse/utils/config_tools/tests/test_config.py +++ b/fuse/utils/config_tools/tests/test_config.py @@ -17,10 +17,10 @@ """ +import os import unittest from fuse.utils.config_tools import Config -import os class TestConfig(unittest.TestCase): diff --git a/fuse/utils/cpu_profiling/tests/test_profiler.py b/fuse/utils/cpu_profiling/tests/test_profiler.py index 42cbbfd42..c26c6dc68 100644 --- a/fuse/utils/cpu_profiling/tests/test_profiler.py +++ b/fuse/utils/cpu_profiling/tests/test_profiler.py @@ -1,5 +1,6 @@ # type: ignore import unittest + from fuse.utils.cpu_profiling import Profiler diff --git a/fuse/utils/cpu_profiling/tests/test_timer.py b/fuse/utils/cpu_profiling/tests/test_timer.py index 853c424e8..b3083acea 100644 --- a/fuse/utils/cpu_profiling/tests/test_timer.py +++ b/fuse/utils/cpu_profiling/tests/test_timer.py @@ -1,5 +1,6 @@ import unittest from typing import Union + from fuse.utils.cpu_profiling import Timer diff --git a/fuse/utils/data/collate.py b/fuse/utils/data/collate.py index c1fd8ced1..d73ca191b 100644 --- a/fuse/utils/data/collate.py +++ b/fuse/utils/data/collate.py @@ -17,12 +17,12 @@ """ import logging -from typing import Callable, Dict, List, Sequence, Tuple, Any +from typing import Any, Callable, Dict, List, Sequence, Tuple -from fuse.utils import NDict - -import torch import numpy as np +import torch + +from fuse.utils.ndict import NDict class CollateToBatchList(Callable): diff --git a/fuse/utils/dl/torch_utils.py b/fuse/utils/dl/torch_utils.py index 47a3c6f85..853b0d173 100644 --- a/fuse/utils/dl/torch_utils.py +++ b/fuse/utils/dl/torch_utils.py @@ -1,6 +1,7 @@ +import functools from typing import List, Union + import torch -import functools def select( diff --git a/fuse/utils/file_io/__init__.py b/fuse/utils/file_io/__init__.py index 75717d98b..ef373e43a 100644 --- a/fuse/utils/file_io/__init__.py +++ b/fuse/utils/file_io/__init__.py @@ -1,20 +1,18 @@ +from .compressed import extract_zip_file from .file_io import ( - save_pickle, + create_simple_timestamp_file, + delete_directory_tree, + get_randomized_postfix_name, + load_hdf5, load_pickle, - save_pickle_safe, - save_text_file_safe, - save_text_file, read_simple_float_file, read_simple_int_file, - read_text_file, - get_randomized_postfix_name, read_single_str_line_file, - create_simple_timestamp_file, + read_text_file, save_hdf5_safe, - load_hdf5, - delete_directory_tree, + save_pickle, + save_pickle_safe, + save_text_file, + save_text_file_safe, ) - -from .compressed import extract_zip_file - from .path import change_extension, get_extension, remove_extension diff --git a/fuse/utils/file_io/compressed.py b/fuse/utils/file_io/compressed.py index c568780ae..2218bf298 100644 --- a/fuse/utils/file_io/compressed.py +++ b/fuse/utils/file_io/compressed.py @@ -1,6 +1,7 @@ +import os import zipfile + from tqdm import tqdm -import os def extract_zip_file( diff --git a/fuse/utils/file_io/file_io.py b/fuse/utils/file_io/file_io.py index 8e28c5315..e8a723a4b 100644 --- a/fuse/utils/file_io/file_io.py +++ b/fuse/utils/file_io/file_io.py @@ -1,18 +1,18 @@ -import errno -from typing import Iterable, List, Dict, Optional, Tuple, Union, Any -import pickle import bz2 +import datetime +import errno import gzip -import socket import os -import numpy as np +import pickle +import shutil +import socket import time -import datetime +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union -import pandas as pd -import shutil import h5py import hdf5plugin +import numpy as np +import pandas as pd from scipy.io import arff from fuse.utils.misc.misc import Misc @@ -150,9 +150,10 @@ def get_randomized_postfix_name(filename: str, **additional_rand_state: dict) -> @return: """ - import numpy as np import hashlib + import numpy as np + # out_dir = os.path.dirname(filename) str_for_hash = filename + str(additional_rand_state) str_for_hash += G_host_name + "@" + str(os.getpid()) + str(np.random.random()) diff --git a/fuse/utils/file_io/path.py b/fuse/utils/file_io/path.py index 61647351d..2371effb1 100644 --- a/fuse/utils/file_io/path.py +++ b/fuse/utils/file_io/path.py @@ -1,5 +1,5 @@ -from os.path import join, basename, dirname import re +from os.path import basename, dirname, join def add_base_prefix(filepath: str, prefix: str) -> str: diff --git a/fuse/utils/file_io/tests/test_hdf5.py b/fuse/utils/file_io/tests/test_hdf5.py index e648adc13..9ed58e3cf 100644 --- a/fuse/utils/file_io/tests/test_hdf5.py +++ b/fuse/utils/file_io/tests/test_hdf5.py @@ -15,13 +15,14 @@ """ +import os +import tempfile import unittest -from fuse.utils.rand.seed import Seed -from fuse.utils.file_io.file_io import save_hdf5_safe, load_hdf5 import numpy as np -import tempfile -import os + +from fuse.utils.file_io.file_io import load_hdf5, save_hdf5_safe +from fuse.utils.rand.seed import Seed class TestHDF5IO(unittest.TestCase): diff --git a/fuse/utils/file_io/tests/test_path.py b/fuse/utils/file_io/tests/test_path.py index 6924e7987..f31477a50 100644 --- a/fuse/utils/file_io/tests/test_path.py +++ b/fuse/utils/file_io/tests/test_path.py @@ -15,14 +15,15 @@ """ +import os +import tempfile import unittest -from fuse.utils.rand.seed import Seed -from fuse.utils.file_io.file_io import save_hdf5_safe, load_hdf5 import numpy as np -import tempfile -import os + from fuse.utils.file_io import path +from fuse.utils.file_io.file_io import load_hdf5, save_hdf5_safe +from fuse.utils.rand.seed import Seed class TestPath(unittest.TestCase): @@ -34,7 +35,6 @@ def setUp(self) -> None: pass def test_path_1(self) -> None: - ans = path.add_base_prefix("/a/b/c/de/fg/banana.phone", "hohoho@") self.assertEqual(ans, "/a/b/c/de/fg/hohoho@banana.phone") diff --git a/fuse/utils/interface_validator.py b/fuse/utils/interface_validator.py index 58ae70d10..402342d17 100644 --- a/fuse/utils/interface_validator.py +++ b/fuse/utils/interface_validator.py @@ -1,7 +1,5 @@ -from typing import List, Dict, Callable, Any # , _UnionGenericAlias - - import inspect +from typing import Any, Callable, Dict, List # , _UnionGenericAlias # https://stackoverflow.com/questions/147816/preserving-signatures-of-decorated-functions # https://stackoverflow.com/questions/5929107/decorators-with-parameters diff --git a/fuse/utils/misc/misc.py b/fuse/utils/misc/misc.py index f0fbf614b..15eae8250 100644 --- a/fuse/utils/misc/misc.py +++ b/fuse/utils/misc/misc.py @@ -19,13 +19,12 @@ import logging import sys import time -from IPython import get_ipython from threading import Lock - -from typing import List, Union, Sequence, Optional, Hashable, Any, Iterable +from typing import Any, Hashable, Iterable, List, Optional, Sequence, Union import numpy as np import pandas as pd +from IPython import get_ipython from torch import Tensor from fuse.utils.ndict import NDict diff --git a/fuse/utils/multiprocessing/__init__.py b/fuse/utils/multiprocessing/__init__.py index 54df5d7c7..a6a329e7c 100644 --- a/fuse/utils/multiprocessing/__init__.py +++ b/fuse/utils/multiprocessing/__init__.py @@ -1,2 +1,2 @@ -from .run_multiprocessed import run_multiprocessed, get_from_global_storage from .helpers import get_chunks_ranges +from .run_multiprocessed import get_from_global_storage, run_multiprocessed diff --git a/fuse/utils/multiprocessing/helpers.py b/fuse/utils/multiprocessing/helpers.py index 6c3d6ef1b..a969c18de 100644 --- a/fuse/utils/multiprocessing/helpers.py +++ b/fuse/utils/multiprocessing/helpers.py @@ -1,6 +1,7 @@ -import numpy as np -from typing import List, Tuple import os +from typing import List, Tuple + +import numpy as np def num_available_cores(verbose: bool = True) -> int: diff --git a/fuse/utils/multiprocessing/linux_shared_memory.py b/fuse/utils/multiprocessing/linux_shared_memory.py index 1739d982d..18ed521fe 100644 --- a/fuse/utils/multiprocessing/linux_shared_memory.py +++ b/fuse/utils/multiprocessing/linux_shared_memory.py @@ -1,12 +1,13 @@ import os -from os.path import join, realpath, dirname import shutil +from glob import glob +from os.path import dirname, join, realpath from typing import List +import psutil + # from multiprocessing import Lock from filelock import FileLock -from glob import glob -import psutil SHARED_MEM_FILES_PREFIX = "OUR_SHARED_MEM_@" SHARED_MEM_ACTIVE_ENV_VAR = "ACTIVATE_OUR_SHARED_MEM" diff --git a/fuse/utils/multiprocessing/run_multiprocessed.py b/fuse/utils/multiprocessing/run_multiprocessed.py index ce4cbf13c..5862f4a1c 100644 --- a/fuse/utils/multiprocessing/run_multiprocessed.py +++ b/fuse/utils/multiprocessing/run_multiprocessed.py @@ -1,12 +1,14 @@ import functools -from typing import Any, List, Optional, Union, Tuple, Callable -from fuse.utils.utils_debug import FuseDebug -import torch -from tqdm import tqdm import multiprocessing as mp -from termcolor import cprint import os import traceback +from typing import Any, Callable, List, Optional, Tuple, Union + +import torch +from termcolor import cprint +from tqdm import tqdm + +from fuse.utils.utils_debug import FuseDebug """ global dictionary that stores arguments to a new created process diff --git a/fuse/utils/ndict.py b/fuse/utils/ndict.py index 2f0630a61..32077f8d3 100644 --- a/fuse/utils/ndict.py +++ b/fuse/utils/ndict.py @@ -17,24 +17,25 @@ """ from __future__ import annotations -from _collections_abc import dict_items, dict_keys -import difflib import copy +import difflib import types -from numpy import ndarray -from torch import Tensor from typing import ( Any, Callable, Iterator, + List, + MutableMapping, Optional, Sequence, Union, - List, - MutableMapping, ) +from _collections_abc import dict_items, dict_keys +from numpy import ndarray +from torch import Tensor + class NDict(dict): """ diff --git a/fuse/utils/rand/param_sampler.py b/fuse/utils/rand/param_sampler.py index 2ff914e80..f8271752a 100644 --- a/fuse/utils/rand/param_sampler.py +++ b/fuse/utils/rand/param_sampler.py @@ -16,10 +16,9 @@ Created on June 30, 2021 """ -from abc import ABC, abstractmethod - -from typing import Any, Optional, Sequence, List, Tuple import random +from abc import ABC, abstractmethod +from typing import Any, List, Optional, Sequence, Tuple import numpy as np diff --git a/fuse/utils/rand/seed.py b/fuse/utils/rand/seed.py index e594a7212..ef10bca56 100644 --- a/fuse/utils/rand/seed.py +++ b/fuse/utils/rand/seed.py @@ -1,7 +1,8 @@ -import torch -import numpy as np -import random import os +import random + +import numpy as np +import torch class Seed: diff --git a/fuse/utils/rand/tests/test_param_sampler.py b/fuse/utils/rand/tests/test_param_sampler.py index 5e44d015b..c5fd042d0 100644 --- a/fuse/utils/rand/tests/test_param_sampler.py +++ b/fuse/utils/rand/tests/test_param_sampler.py @@ -17,18 +17,16 @@ """ -import unittest - - import random +import unittest from fuse.utils import ( - Uniform, Choice, - RandInt, RandBool, - draw_samples_recursively, + RandInt, Seed, + Uniform, + draw_samples_recursively, ) diff --git a/fuse/utils/remote_execution/__init__.py b/fuse/utils/remote_execution/__init__.py index c4d6bed46..89b9a44a0 100644 --- a/fuse/utils/remote_execution/__init__.py +++ b/fuse/utils/remote_execution/__init__.py @@ -1,11 +1,11 @@ import os + from fuse.utils.remote_execution.remote_execution import ( - RemoteExecution, RemoteCommand, + RemoteExecution, get_script_runner_path, ) - __all__ = [ "RemoteExecution", "RemoteCommand", diff --git a/fuse/utils/remote_execution/remote_execution.py b/fuse/utils/remote_execution/remote_execution.py index b045e0676..3eea36e10 100644 --- a/fuse/utils/remote_execution/remote_execution.py +++ b/fuse/utils/remote_execution/remote_execution.py @@ -1,10 +1,11 @@ -import os -from typing import Optional, List, Tuple import getpass -from .shell_handler import ShellHandler -from os.path import abspath, join, dirname -from glob import glob +import os from collections import namedtuple +from glob import glob +from os.path import abspath, dirname, join +from typing import List, Optional, Tuple + +from .shell_handler import ShellHandler SCRIPT_RUNNER_PATH = os.path.join( os.path.dirname(os.path.abspath(__file__)), "script_runner" diff --git a/fuse/utils/remote_execution/shell_handler.py b/fuse/utils/remote_execution/shell_handler.py index dbf677687..9fbac2570 100644 --- a/fuse/utils/remote_execution/shell_handler.py +++ b/fuse/utils/remote_execution/shell_handler.py @@ -1,8 +1,9 @@ -import paramiko -from paramiko.channel import ChannelFile import re from typing import List, Tuple +import paramiko +from paramiko.channel import ChannelFile + class ShellHandler: def __init__(self, host: str, user: str, psw: str): diff --git a/fuse/utils/tests/test_interface_validator.py b/fuse/utils/tests/test_interface_validator.py index d6052d43f..87de4cbd0 100644 --- a/fuse/utils/tests/test_interface_validator.py +++ b/fuse/utils/tests/test_interface_validator.py @@ -1,7 +1,8 @@ import unittest -from fuse.utils.interface_validator import InterfaceValidator, validate_signature from typing import Dict +from fuse.utils.interface_validator import InterfaceValidator, validate_signature + class TestInterfaceValidator(unittest.TestCase): def test_1(self) -> None: diff --git a/fuse/utils/tests/test_ndict.py b/fuse/utils/tests/test_ndict.py index 6f63a75b0..cf5aeb6df 100644 --- a/fuse/utils/tests/test_ndict.py +++ b/fuse/utils/tests/test_ndict.py @@ -19,11 +19,11 @@ import unittest - -from fuse.utils.ndict import NDict import numpy import torch +from fuse.utils.ndict import NDict + class TestNDict(unittest.TestCase): """ diff --git a/fuse/utils/utils_hierarchical_dict.py b/fuse/utils/utils_hierarchical_dict.py index f60c2d5f5..e5b23f2e8 100644 --- a/fuse/utils/utils_hierarchical_dict.py +++ b/fuse/utils/utils_hierarchical_dict.py @@ -13,11 +13,11 @@ Created on June 30, 2021 """ -from typing import Any, Callable, Optional, List, Sequence, Union +import warnings +from typing import Any, Callable, List, Optional, Sequence, Union import numpy import torch -import warnings warnings.warn( "FuseUtilsHierarchicalDict is obsolete and soon will be deleted! Please consider transform to NDict." diff --git a/fuse/utils/utils_logger.py b/fuse/utils/utils_logger.py index 6b39d7524..d74e6340c 100644 --- a/fuse/utils/utils_logger.py +++ b/fuse/utils/utils_logger.py @@ -27,12 +27,12 @@ from logging.handlers import RotatingFileHandler from multiprocessing import current_process from shutil import copyfile -from typing import Dict, Optional, Any, List, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union +from pytorch_lightning.utilities.rank_zero import rank_zero_only from termcolor import colored from fuse.utils.file_io.file_io import create_dir, create_or_reset_dir -from pytorch_lightning.utilities.rank_zero import rank_zero_only class ProcessSafeHandler(logging.StreamHandler): diff --git a/fuse_examples/imaging/classification/bright/eval/eval.py b/fuse_examples/imaging/classification/bright/eval/eval.py index eb8d5e264..a6c497df1 100644 --- a/fuse_examples/imaging/classification/bright/eval/eval.py +++ b/fuse_examples/imaging/classification/bright/eval/eval.py @@ -1,26 +1,24 @@ +import csv +import os import pathlib import sys -import os -from typing import List, Tuple, Union from collections import OrderedDict +from typing import List, Tuple, Union -import csv -from fuse.eval.metrics.classification.metrics_thresholding_common import ( - MetricApplyThresholds, -) -from fuse.utils.ndict import NDict -import pandas as pd import numpy as np - -from fuse.utils.file_io.file_io import create_or_reset_dir +import pandas as pd from fuse.eval.evaluator import EvaluatorDefault from fuse.eval.metrics.classification.metrics_classification_common import ( MetricBSS, MetricConfusion, ) +from fuse.eval.metrics.classification.metrics_thresholding_common import ( + MetricApplyThresholds, +) from fuse.eval.metrics.metrics_common import CI - +from fuse.utils.file_io.file_io import create_or_reset_dir +from fuse.utils.ndict import NDict ## Constants # Constants that defines the expected format of the prediction and target files and list the classes for task 1 and task @ diff --git a/fuse_examples/imaging/classification/cmmd/runner.py b/fuse_examples/imaging/classification/cmmd/runner.py index 502c45bcd..889302b84 100644 --- a/fuse_examples/imaging/classification/cmmd/runner.py +++ b/fuse_examples/imaging/classification/cmmd/runner.py @@ -14,50 +14,47 @@ """ -from collections import OrderedDict -import os -import sys import copy -from fuse.eval.metrics.classification.metrics_thresholding_common import ( - MetricApplyThresholds, -) - -from fuse.utils.utils_debug import FuseDebug -from fuse.utils.gpu import choose_and_enable_multiple_gpus - import logging +import os +import sys +from collections import OrderedDict -import torch.optim as optim +import hydra +import torch import torch.nn.functional as F +import torch.optim as optim +from omegaconf import DictConfig, OmegaConf +from pytorch_lightning import Trainer from torch.utils.data.dataloader import DataLoader -from fuse.utils.utils_logger import fuse_logger_start -from fuse.utils import NDict -from fuse.data.utils.samplers import BatchSamplerDefault from fuse.data.utils.collates import CollateDefault +from fuse.data.utils.samplers import BatchSamplerDefault from fuse.data.utils.split import dataset_balanced_division_to_folds +from fuse.dl.lightning.pl_funcs import convert_predictions_to_dataframe +from fuse.dl.lightning.pl_module import LightningModuleDefault +from fuse.dl.losses.loss_default import LossDefault from fuse.dl.models import ModelMultiHead +from fuse.dl.models.backbones.backbone_inception_resnet_v2 import ( + BackboneInceptionResnetV2, +) from fuse.dl.models.heads.head_global_pooling_classifier import ( HeadGlobalPoolingClassifier, ) -from fuse.dl.losses.loss_default import LossDefault - +from fuse.eval.evaluator import EvaluatorDefault from fuse.eval.metrics.classification.metrics_classification_common import ( - MetricAUCROC, MetricAccuracy, + MetricAUCROC, ) -from fuseimg.datasets.cmmd import CMMD -from fuse.dl.models.backbones.backbone_inception_resnet_v2 import ( - BackboneInceptionResnetV2, +from fuse.eval.metrics.classification.metrics_thresholding_common import ( + MetricApplyThresholds, ) -from fuse.dl.lightning.pl_funcs import convert_predictions_to_dataframe -from pytorch_lightning import Trainer -from fuse.dl.lightning.pl_module import LightningModuleDefault +from fuse.utils import NDict from fuse.utils.file_io.file_io import create_dir, load_pickle, save_dataframe -from fuse.eval.evaluator import EvaluatorDefault -import torch -import hydra -from omegaconf import DictConfig, OmegaConf +from fuse.utils.gpu import choose_and_enable_multiple_gpus +from fuse.utils.utils_debug import FuseDebug +from fuse.utils.utils_logger import fuse_logger_start +from fuseimg.datasets.cmmd import CMMD assert ( "CMMD_DATA_PATH" in os.environ diff --git a/fuse_examples/imaging/classification/isic/isic_runner.py b/fuse_examples/imaging/classification/isic/isic_runner.py index 0e69c9c39..b56c4bb09 100644 --- a/fuse_examples/imaging/classification/isic/isic_runner.py +++ b/fuse_examples/imaging/classification/isic/isic_runner.py @@ -17,56 +17,49 @@ """ -import os import copy import logging +import os from typing import OrderedDict, Sequence, Tuple +import pytorch_lightning as pl import torch -import torch.optim as optim +import torch.nn as nn import torch.nn.functional as F +import torch.optim as optim +from pytorch_lightning.utilities.rank_zero import rank_zero_only from torch.utils.data.dataloader import DataLoader from torchvision.models.resnet import ResNet18_Weights +import fuse.utils.gpu as GPU +from fuse.data.utils.collates import CollateDefault +from fuse.dl.lightning.pl_funcs import convert_predictions_to_dataframe +from fuse.dl.lightning.pl_module import LightningModuleDefault +from fuse.dl.losses.loss_default import LossDefault from fuse.dl.models import ModelMultiHead -from fuse.dl.models.backbones.backbone_resnet import BackboneResnet -from fuse.dl.models.heads.head_global_pooling_classifier import ( - HeadGlobalPoolingClassifier, -) from fuse.dl.models.backbones.backbone_inception_resnet_v2 import ( BackboneInceptionResnetV2, ) - -from fuse.eval.metrics.classification.metrics_thresholding_common import ( - MetricApplyThresholds, +from fuse.dl.models.backbones.backbone_resnet import BackboneResnet +from fuse.dl.models.backbones.backbone_vit import ViT +from fuse.dl.models.heads.head_global_pooling_classifier import ( + HeadGlobalPoolingClassifier, ) +from fuse.dl.models.model_wrapper import ModelWrapSeqToDict +from fuse.eval.evaluator import EvaluatorDefault from fuse.eval.metrics.classification.metrics_classification_common import ( MetricAccuracy, MetricAUCROC, MetricROCCurve, ) - +from fuse.eval.metrics.classification.metrics_thresholding_common import ( + MetricApplyThresholds, +) +from fuse.utils.file_io.file_io import create_dir, load_pickle, save_dataframe from fuse.utils.utils_debug import FuseDebug from fuse.utils.utils_logger import fuse_logger_start -from fuse.utils.file_io.file_io import create_dir, save_dataframe, load_pickle -import fuse.utils.gpu as GPU - -from fuse.eval.evaluator import EvaluatorDefault -from fuse.dl.losses.loss_default import LossDefault -from fuse.data.utils.collates import CollateDefault - -import pytorch_lightning as pl -from fuse.dl.lightning.pl_module import LightningModuleDefault -from fuse.dl.lightning.pl_funcs import convert_predictions_to_dataframe -from pytorch_lightning.utilities.rank_zero import rank_zero_only - -from fuseimg.datasets.isic import ISIC, ISICDataModule from fuse_examples.imaging.classification.isic.golden_members import FULL_GOLDEN_MEMBERS - -import torch.nn as nn -from fuse.dl.models.model_wrapper import ModelWrapSeqToDict -from fuse.dl.models.backbones.backbone_vit import ViT - +from fuseimg.datasets.isic import ISIC, ISICDataModule ########################################################################################################### # Fuse diff --git a/fuse_examples/imaging/classification/knight/baseline/fuse_baseline.py b/fuse_examples/imaging/classification/knight/baseline/fuse_baseline.py index 565a5ff63..89dfa5dc9 100644 --- a/fuse_examples/imaging/classification/knight/baseline/fuse_baseline.py +++ b/fuse_examples/imaging/classification/knight/baseline/fuse_baseline.py @@ -1,39 +1,39 @@ -from collections import OrderedDict -import pathlib -from fuse.utils.utils_logger import fuse_logger_start +import copy +import logging import os -import yaml +import pathlib +import shutil +import time +from collections import OrderedDict import pandas as pd -from fuse.dl.models import ModelMultiHead -from fuse.dl.models.backbones.backbone_resnet_3d import BackboneResnet3D -from fuse.dl.models.heads.heads_3D import Head3D -from fuseimg.datasets.knight import KNIGHT -import torch.nn.functional as F +import pytorch_lightning as pl import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +import yaml +from omegaconf import DictConfig from torch.utils.data.dataloader import DataLoader + +import fuse.utils.gpu as GPU from fuse.data.utils.collates import CollateDefault from fuse.data.utils.samplers import BatchSamplerDefault - +from fuse.dl.lightning.pl_module import LightningModuleDefault +from fuse.dl.losses.loss_default import LossDefault +from fuse.dl.models import ModelMultiHead +from fuse.dl.models.backbones.backbone_resnet_3d import BackboneResnet3D +from fuse.dl.models.heads.heads_3D import Head3D from fuse.eval.metrics.classification.metrics_classification_common import ( - MetricAUCROC, MetricAccuracy, + MetricAUCROC, MetricConfusion, ) from fuse.eval.metrics.classification.metrics_thresholding_common import ( MetricApplyThresholds, ) -import torch.optim as optim -import fuse.utils.gpu as GPU from fuse.utils.rand.seed import Seed -import logging -import time -import copy -import shutil -from fuse.dl.losses.loss_default import LossDefault -from fuse.dl.lightning.pl_module import LightningModuleDefault -import pytorch_lightning as pl -from omegaconf import DictConfig +from fuse.utils.utils_logger import fuse_logger_start +from fuseimg.datasets.knight import KNIGHT ## Parameters: ############################################################################## diff --git a/fuse_examples/imaging/classification/knight/baseline/utils.py b/fuse_examples/imaging/classification/knight/baseline/utils.py index ba49c6f1e..292a9f455 100644 --- a/fuse_examples/imaging/classification/knight/baseline/utils.py +++ b/fuse_examples/imaging/classification/knight/baseline/utils.py @@ -1,9 +1,10 @@ -import os import json -import pandas as pd -import numpy as np +import os from typing import Optional +import numpy as np +import pandas as pd + CLINICAL_NAMES = [ "SubjectId", "age", diff --git a/fuse_examples/imaging/classification/knight/eval/eval.py b/fuse_examples/imaging/classification/knight/eval/eval.py index 9d0f4b6f0..4e7a21653 100644 --- a/fuse_examples/imaging/classification/knight/eval/eval.py +++ b/fuse_examples/imaging/classification/knight/eval/eval.py @@ -16,27 +16,25 @@ Created on June 30, 2021 """ +import csv +import os import pathlib import sys -import os -from typing import List, Tuple, Union, Optional from collections import OrderedDict +from functools import partial +from typing import List, Optional, Tuple, Union -import csv -import pandas as pd import numpy as np - -from fuse.utils.ndict import NDict -from fuse.utils.file_io.file_io import create_dir +import pandas as pd from fuse.eval.evaluator import EvaluatorDefault - from fuse.eval.metrics.classification.metrics_classification_common import ( MetricAUCROC, MetricROCCurve, ) from fuse.eval.metrics.metrics_common import CI -from functools import partial +from fuse.utils.file_io.file_io import create_dir +from fuse.utils.ndict import NDict ## Constants # Constants that defines the expected format of the prediction and target files and list the classes for task 1 and task @ diff --git a/fuse_examples/imaging/classification/knight/make_predictions_file.py b/fuse_examples/imaging/classification/knight/make_predictions_file.py index c92014424..6fd2f7f3d 100644 --- a/fuse_examples/imaging/classification/knight/make_predictions_file.py +++ b/fuse_examples/imaging/classification/knight/make_predictions_file.py @@ -19,26 +19,25 @@ import logging import os - from typing import Optional, Union + import pandas as pd +import pytorch_lightning as pl import torch from torch.utils.data.dataloader import DataLoader -from fuse.data.utils.collates import CollateDefault -from fuseimg.datasets.knight import KNIGHT -from fuse.utils.utils_logger import fuse_logger_start +from fuse.data.utils.collates import CollateDefault +from fuse.dl.lightning.pl_module import LightningModuleDefault from fuse.utils.file_io.file_io import save_dataframe - +from fuse.utils.utils_logger import fuse_logger_start +from fuse_examples.imaging.classification.knight.baseline.fuse_baseline import ( + make_model, +) from fuse_examples.imaging.classification.knight.eval.eval import ( TASK1_CLASS_NAMES, TASK2_CLASS_NAMES, ) -from fuse.dl.lightning.pl_module import LightningModuleDefault -import pytorch_lightning as pl -from fuse_examples.imaging.classification.knight.baseline.fuse_baseline import ( - make_model, -) +from fuseimg.datasets.knight import KNIGHT def make_predictions_file( diff --git a/fuse_examples/imaging/classification/knight/make_targets_file.py b/fuse_examples/imaging/classification/knight/make_targets_file.py index ba242af7c..e4c8cc750 100644 --- a/fuse_examples/imaging/classification/knight/make_targets_file.py +++ b/fuse_examples/imaging/classification/knight/make_targets_file.py @@ -18,12 +18,12 @@ """ import logging import os - from typing import Union + import pandas as pd -from fuse.utils.utils_logger import fuse_logger_start from fuse.utils.file_io.file_io import save_dataframe +from fuse.utils.utils_logger import fuse_logger_start def make_targets_file( diff --git a/fuse_examples/imaging/classification/mnist/run_mnist_custom_pl_imp.py b/fuse_examples/imaging/classification/mnist/run_mnist_custom_pl_imp.py index 91809d037..67193864c 100644 --- a/fuse_examples/imaging/classification/mnist/run_mnist_custom_pl_imp.py +++ b/fuse_examples/imaging/classification/mnist/run_mnist_custom_pl_imp.py @@ -24,40 +24,35 @@ import logging import os from typing import Any, List, OrderedDict, Sequence, Tuple -from fuse.utils.ndict import NDict +import pytorch_lightning as pl import torch import torch.nn.functional as F import torch.optim as optim -from torch.utils.data.dataloader import DataLoader -import pytorch_lightning as pl from pytorch_lightning.loggers import CSVLogger, TensorBoardLogger +from torch.utils.data.dataloader import DataLoader +import fuse.dl.lightning.pl_funcs as fuse_pl +import fuse.utils.gpu as GPU +from fuse.data.utils.collates import CollateDefault +from fuse.data.utils.samplers import BatchSamplerDefault +from fuse.dl.losses.loss_default import LossDefault +from fuse.dl.models.model_wrapper import ModelWrapSeqToDict from fuse.eval.evaluator import EvaluatorDefault -from fuse.eval.metrics.classification.metrics_thresholding_common import ( - MetricApplyThresholds, -) from fuse.eval.metrics.classification.metrics_classification_common import ( MetricAccuracy, MetricAUCROC, MetricROCCurve, ) - -from fuse.data.utils.samplers import BatchSamplerDefault -from fuse.data.utils.collates import CollateDefault - -from fuse.dl.losses.loss_default import LossDefault -from fuse.dl.models.model_wrapper import ModelWrapSeqToDict -import fuse.dl.lightning.pl_funcs as fuse_pl - +from fuse.eval.metrics.classification.metrics_thresholding_common import ( + MetricApplyThresholds, +) +from fuse.utils.file_io.file_io import create_dir, save_dataframe +from fuse.utils.ndict import NDict from fuse.utils.utils_debug import FuseDebug from fuse.utils.utils_logger import fuse_logger_start -from fuse.utils.file_io.file_io import create_dir, save_dataframe -import fuse.utils.gpu as GPU - -from fuseimg.datasets.mnist import MNIST - from fuse_examples.imaging.classification.mnist import lenet +from fuseimg.datasets.mnist import MNIST ########################################################################################################### # Fuse diff --git a/fuse_examples/imaging/classification/mnist/simple_mnist_starter.py b/fuse_examples/imaging/classification/mnist/simple_mnist_starter.py index d63c5366b..a48c60d79 100644 --- a/fuse_examples/imaging/classification/mnist/simple_mnist_starter.py +++ b/fuse_examples/imaging/classification/mnist/simple_mnist_starter.py @@ -24,32 +24,29 @@ load datasets from fuse. """ import copy -from typing import OrderedDict, Any, Tuple, Optional -from jsonargparse import CLI +from typing import Any, Optional, OrderedDict, Tuple +import pytorch_lightning as pl +import torch import torch.nn.functional as F import torch.optim as optim -from torch.utils.data.dataloader import DataLoader +from jsonargparse import CLI from torch.utils.data import DistributedSampler -import pytorch_lightning as pl - -from fuse.eval.metrics.classification.metrics_thresholding_common import ( - MetricApplyThresholds, -) -from fuse.eval.metrics.classification.metrics_classification_common import ( - MetricAccuracy, -) +from torch.utils.data.dataloader import DataLoader from fuse.data.utils.collates import CollateDefault from fuse.data.utils.samplers import BatchSamplerDefault +from fuse.dl.lightning.pl_module import LightningModuleDefault from fuse.dl.losses.loss_default import LossDefault from fuse.dl.models.model_wrapper import ModelWrapSeqToDict -from fuse.dl.lightning.pl_module import LightningModuleDefault - -from fuseimg.datasets.mnist import MNIST +from fuse.eval.metrics.classification.metrics_classification_common import ( + MetricAccuracy, +) +from fuse.eval.metrics.classification.metrics_thresholding_common import ( + MetricApplyThresholds, +) from fuse_examples.imaging.classification.mnist import lenet - -import torch +from fuseimg.datasets.mnist import MNIST def perform_softmax(logits: Any) -> Tuple[torch.Tensor, torch.Tensor]: diff --git a/fuse_examples/imaging/classification/stoic21/dataset.py b/fuse_examples/imaging/classification/stoic21/dataset.py index b41c53eaf..1f5636323 100644 --- a/fuse_examples/imaging/classification/stoic21/dataset.py +++ b/fuse_examples/imaging/classification/stoic21/dataset.py @@ -1,5 +1,6 @@ -from typing import Union, Sequence, Optional -from fuse.data import DatasetDefault +from typing import Optional, Sequence, Union + +from fuse.data.datasets.dataset_default import DatasetDefault from fuse.data.utils.split import dataset_balanced_division_to_folds from fuseimg.datasets.stoic21 import STOIC21 diff --git a/fuse_examples/imaging/classification/stoic21/runner_stoic21.py b/fuse_examples/imaging/classification/stoic21/runner_stoic21.py index 48ee57393..0f60135f0 100644 --- a/fuse_examples/imaging/classification/stoic21/runner_stoic21.py +++ b/fuse_examples/imaging/classification/stoic21/runner_stoic21.py @@ -17,46 +17,42 @@ """ +import copy import logging import os from typing import OrderedDict -import copy -from fuse.dl.lightning.pl_funcs import convert_predictions_to_dataframe +import pytorch_lightning as pl import torch import torch.nn.functional as F import torch.optim as optim -from torch.utils.data.dataloader import DataLoader -import pytorch_lightning as pl from pytorch_lightning.loggers import CSVLogger, TensorBoardLogger +from torch.utils.data.dataloader import DataLoader +import fuse.utils.gpu as GPU +import fuse_examples.imaging.classification.stoic21.dataset as dataset +from fuse.data.datasets.dataset_default import DatasetDefault +from fuse.data.utils.collates import CollateDefault +from fuse.data.utils.samplers import BatchSamplerDefault +from fuse.dl.lightning.pl_funcs import convert_predictions_to_dataframe +from fuse.dl.lightning.pl_module import LightningModuleDefault +from fuse.dl.losses.loss_default import LossDefault +from fuse.dl.models import ModelMultiHead +from fuse.dl.models.backbones.backbone_resnet_3d import BackboneResnet3D +from fuse.dl.models.heads.heads_3D import Head3D from fuse.eval.evaluator import EvaluatorDefault -from fuse.eval.metrics.classification.metrics_thresholding_common import ( - MetricApplyThresholds, -) from fuse.eval.metrics.classification.metrics_classification_common import ( MetricAccuracy, MetricAUCROC, MetricROCCurve, ) - -from fuse.data.utils.samplers import BatchSamplerDefault -from fuse.data.utils.collates import CollateDefault -from fuse.data.datasets.dataset_default import DatasetDefault - -from fuse.dl.losses.loss_default import LossDefault -from fuse.dl.models.backbones.backbone_resnet_3d import BackboneResnet3D -from fuse.dl.models import ModelMultiHead -from fuse.dl.models.heads.heads_3D import Head3D -from fuse.dl.lightning.pl_module import LightningModuleDefault - -from fuse.utils.utils_debug import FuseDebug -from fuse.utils.utils_logger import fuse_logger_start +from fuse.eval.metrics.classification.metrics_thresholding_common import ( + MetricApplyThresholds, +) from fuse.utils.file_io.file_io import create_dir, save_dataframe -import fuse.utils.gpu as GPU from fuse.utils.ndict import NDict - -import fuse_examples.imaging.classification.stoic21.dataset as dataset +from fuse.utils.utils_debug import FuseDebug +from fuse.utils.utils_logger import fuse_logger_start ########################################################################################################### # Fuse diff --git a/fuse_examples/imaging/hello_world/hello_world_utils.py b/fuse_examples/imaging/hello_world/hello_world_utils.py index ced52bc56..87a6fe9e8 100644 --- a/fuse_examples/imaging/hello_world/hello_world_utils.py +++ b/fuse_examples/imaging/hello_world/hello_world_utils.py @@ -1,8 +1,9 @@ +from typing import Any, Tuple + import torch import torch.nn as nn import torch.nn.functional as F from torch import Tensor -from typing import Tuple, Any def perform_softmax(output: Any) -> Tuple[Tensor, Tensor]: diff --git a/fuse_examples/imaging/ml_pipelines/mnist/funcs.py b/fuse_examples/imaging/ml_pipelines/mnist/funcs.py index e56ef6e66..394cdc82c 100644 --- a/fuse_examples/imaging/ml_pipelines/mnist/funcs.py +++ b/fuse_examples/imaging/ml_pipelines/mnist/funcs.py @@ -1,7 +1,7 @@ -from typing import Union, Optional -from fuseimg.datasets.mnist import MNIST -from typing import Sequence +from typing import Optional, Sequence, Union + from fuse.data import DatasetDefault +from fuseimg.datasets.mnist import MNIST def create_dataset( diff --git a/fuse_examples/imaging/ml_pipelines/mnist/run_mnist_pipeline.py b/fuse_examples/imaging/ml_pipelines/mnist/run_mnist_pipeline.py index 7e013e4cd..4e79683ce 100644 --- a/fuse_examples/imaging/ml_pipelines/mnist/run_mnist_pipeline.py +++ b/fuse_examples/imaging/ml_pipelines/mnist/run_mnist_pipeline.py @@ -1,14 +1,16 @@ +import os + +from funcs import create_dataset + from fuse.dl.cross_validation.pipeline import run from fuse_examples.imaging.classification.mnist.run_mnist import ( - run_train, - run_infer, - run_eval, - TRAIN_COMMON_PARAMS, - INFER_COMMON_PARAMS, EVAL_COMMON_PARAMS, + INFER_COMMON_PARAMS, + TRAIN_COMMON_PARAMS, + run_eval, + run_infer, + run_train, ) -from funcs import create_dataset -import os ########################################## # Required Parameters diff --git a/fuse_examples/imaging/ml_pipelines/stoic21/run_stoic_pipeline.py b/fuse_examples/imaging/ml_pipelines/stoic21/run_stoic_pipeline.py index d59e507d2..af197e4ee 100644 --- a/fuse_examples/imaging/ml_pipelines/stoic21/run_stoic_pipeline.py +++ b/fuse_examples/imaging/ml_pipelines/stoic21/run_stoic_pipeline.py @@ -1,20 +1,19 @@ +import os + from fuse.dl.cross_validation.pipeline import run -from fuse_examples.imaging.classification.stoic21.runner_stoic21 import ( - run_train, - run_infer, - run_eval, -) from fuse_examples.imaging.classification.stoic21.dataset import ( create_dataset, train_val_test_splits, ) from fuse_examples.imaging.classification.stoic21.runner_stoic21 import ( - TRAIN_COMMON_PARAMS, - INFER_COMMON_PARAMS, - EVAL_COMMON_PARAMS, DATASET_COMMON_PARAMS, + EVAL_COMMON_PARAMS, + INFER_COMMON_PARAMS, + TRAIN_COMMON_PARAMS, + run_eval, + run_infer, + run_train, ) -import os ########################################## # Required Parameters diff --git a/fuse_examples/imaging/oai_example/data/data_ops.py b/fuse_examples/imaging/oai_example/data/data_ops.py index 4c1b543da..9ae72f679 100755 --- a/fuse_examples/imaging/oai_example/data/data_ops.py +++ b/fuse_examples/imaging/oai_example/data/data_ops.py @@ -1,15 +1,17 @@ -from fuse.data.ops.op_base import OpBase -from fuse.utils import NDict -import numpy as np import os -from copy import deepcopy import random +from copy import deepcopy +from typing import List, Tuple, Union + +import numpy as np import pydicom -from scipy.ndimage import zoom import torch.nn.functional as F -from typing import Tuple, Union, List +from scipy.ndimage import zoom from volumentations import Compose +from fuse.data.ops.op_base import OpBase +from fuse.utils import NDict + class OpLoadData(OpBase): """ @@ -55,7 +57,6 @@ def __call__( key: str, to_range: Tuple[float, float], ) -> Union[None, dict, List[dict]]: - img = sample_dict[key] img = np.clip( img, *(np.percentile(img, [0, 95])) diff --git a/fuse_examples/imaging/oai_example/data/oai_ds.py b/fuse_examples/imaging/oai_example/data/oai_ds.py index 6a78f9fd4..3d84c6f34 100755 --- a/fuse_examples/imaging/oai_example/data/oai_ds.py +++ b/fuse_examples/imaging/oai_example/data/oai_ds.py @@ -1,27 +1,28 @@ -from fuse.data.datasets.dataset_default import DatasetDefault -from fuse.data.datasets.caching.samples_cacher import SamplesCacher -from fuse.data import PipelineDefault, OpToTensor -from fuseimg.data.ops.aug.geometry import OpAugAffine2D -from fuse.data.ops.ops_aug_common import OpSample, OpRandApply -from fuse.data.ops.ops_read import OpReadDataframe - -from fuse.data.ops.ops_common import OpLambda, OpRepeat from functools import partial -from typing import Hashable, Optional, Sequence, Union, List -import torch +from typing import Hashable, List, Optional, Sequence, Union + import pandas as pd -from fuse.utils.rand.param_sampler import Uniform, RandInt +import torch +from volumentations import Compose, Flip, GaussianNoise, RandomRotate90, Rotate + +from fuse.data import OpToTensor, PipelineDefault +from fuse.data.datasets.caching.samples_cacher import SamplesCacher +from fuse.data.datasets.dataset_default import DatasetDefault +from fuse.data.ops.ops_aug_common import OpRandApply, OpSample +from fuse.data.ops.ops_common import OpLambda, OpRepeat +from fuse.data.ops.ops_read import OpReadDataframe +from fuse.utils.rand.param_sampler import RandInt, Uniform from fuse_examples.imaging.oai_example.data.data_ops import ( + OpDinoCrops, OpLoadData, - OpNormalizeMRI, - OpResize3D, - OpVolumentation, OpMask3D, - OpDinoCrops, + OpNormalizeMRI, OpRandomCrop, OpRandomFlip, + OpResize3D, + OpVolumentation, ) -from volumentations import Compose, RandomRotate90, GaussianNoise, Flip, Rotate +from fuseimg.data.ops.aug.geometry import OpAugAffine2D class OAI: diff --git a/fuse_examples/imaging/oai_example/data/seg_ds.py b/fuse_examples/imaging/oai_example/data/seg_ds.py index d40e9fefa..127ed3210 100755 --- a/fuse_examples/imaging/oai_example/data/seg_ds.py +++ b/fuse_examples/imaging/oai_example/data/seg_ds.py @@ -1,21 +1,22 @@ -from fuse.data.datasets.dataset_default import DatasetDefault +from functools import partial +from typing import Hashable, Optional, Sequence, Tuple, Union + +import pandas as pd +import torch + +from fuse.data import OpToTensor, PipelineDefault from fuse.data.datasets.caching.samples_cacher import SamplesCacher -from fuse.data import PipelineDefault, OpToTensor +from fuse.data.datasets.dataset_default import DatasetDefault from fuse.data.ops.ops_common import OpLambda -from fuseimg.data.ops.image_loader import OpLoadImage from fuse.data.ops.ops_read import OpReadDataframe - -from functools import partial -from typing import Hashable, Optional, Sequence, Union, Tuple -import torch -import pandas as pd from fuse_examples.imaging.oai_example.data.data_ops import ( OpNormalizeMRI, - OpResize3D, + OpRandomCrop, OpRandomFlip, + OpResize3D, OpSegToOneHot, - OpRandomCrop, ) +from fuseimg.data.ops.image_loader import OpLoadImage class SegOAI: diff --git a/fuse_examples/imaging/oai_example/downstream/classification.py b/fuse_examples/imaging/oai_example/downstream/classification.py index e7d6bc11b..e762e1cd4 100755 --- a/fuse_examples/imaging/oai_example/downstream/classification.py +++ b/fuse_examples/imaging/oai_example/downstream/classification.py @@ -1,36 +1,35 @@ -from fuse.utils.utils_logger import fuse_logger_start +import copy +import logging import os from glob import glob -import pandas as pd -from fuse.dl.models import ModelMultiHead -from fuse.dl.models.heads.heads_3D import Head3D -import torch.nn as nn +import hydra +import pandas as pd +import pytorch_lightning as pl import torch +import torch.nn as nn +import torch.optim as optim +from clearml import Task +from omegaconf import DictConfig +from pytorch_lightning.callbacks import LearningRateMonitor from torch.utils.data.dataloader import DataLoader -from fuse.data.utils.collates import CollateDefault +from fuse.data.utils.collates import CollateDefault +from fuse.dl.lightning.pl_module import LightningModuleDefault +from fuse.dl.losses.loss_default import LossDefault +from fuse.dl.models import ModelMultiHead +from fuse.dl.models.backbones.backbone_unet3d import UNet3D +from fuse.dl.models.heads.heads_3D import Head3D from fuse.eval.metrics.classification.metrics_classification_common import ( - MetricAUCROC, MetricAccuracy, + MetricAUCROC, ) from fuse.eval.metrics.classification.metrics_thresholding_common import ( MetricApplyThresholds, ) -import torch.optim as optim from fuse.utils.rand.seed import Seed -import logging -import copy -from fuse.dl.losses.loss_default import LossDefault -from fuse.dl.lightning.pl_module import LightningModuleDefault -import pytorch_lightning as pl -from pytorch_lightning.callbacks import LearningRateMonitor -import hydra -from omegaconf import DictConfig -from clearml import Task - +from fuse.utils.utils_logger import fuse_logger_start from fuse_examples.imaging.oai_example.data.oai_ds import OAI -from fuse.dl.models.backbones.backbone_unet3d import UNet3D torch.set_float32_matmul_precision("medium") diff --git a/fuse_examples/imaging/oai_example/downstream/segmentation.py b/fuse_examples/imaging/oai_example/downstream/segmentation.py index b68d6c128..4961d4b1e 100755 --- a/fuse_examples/imaging/oai_example/downstream/segmentation.py +++ b/fuse_examples/imaging/oai_example/downstream/segmentation.py @@ -1,32 +1,30 @@ -from fuse.utils.utils_logger import fuse_logger_start +import logging import os from glob import glob -import pandas as pd -from fuse.dl.models import ModelMultiHead -from fuse.utils import NDict -from fuse.dl.models.heads.head_dense_segmentation import HeadDenseSegmentation - -from monai.losses import GeneralizedDiceLoss -import torch.nn as nn +import hydra +import pandas as pd +import pytorch_lightning as pl import torch +import torch.nn as nn +import torch.optim as optim +from clearml import Task +from monai.losses import GeneralizedDiceLoss +from omegaconf import DictConfig +from pytorch_lightning.callbacks import LearningRateMonitor from torch.utils.data.dataloader import DataLoader -from fuse.data.utils.collates import CollateDefault +from fuse.data.utils.collates import CollateDefault +from fuse.dl.lightning.pl_module import LightningModuleDefault +from fuse.dl.losses.loss_default import LossDefault +from fuse.dl.models import ModelMultiHead +from fuse.dl.models.backbones.backbone_unet3d import UNet3D +from fuse.dl.models.heads.head_dense_segmentation import HeadDenseSegmentation from fuse.eval.metrics.segmentation.metrics_segmentation_common import MetricDice -import torch.optim as optim +from fuse.utils import NDict from fuse.utils.rand.seed import Seed -import logging -from fuse.dl.losses.loss_default import LossDefault -from fuse.dl.lightning.pl_module import LightningModuleDefault -import pytorch_lightning as pl -from pytorch_lightning.callbacks import LearningRateMonitor -import hydra -from omegaconf import DictConfig -from clearml import Task +from fuse.utils.utils_logger import fuse_logger_start from fuse_examples.imaging.oai_example.data.seg_ds import SegOAI -from fuse.dl.models.backbones.backbone_unet3d import UNet3D - torch.set_float32_matmul_precision("medium") diff --git a/fuse_examples/imaging/oai_example/self_supervised/dino.py b/fuse_examples/imaging/oai_example/self_supervised/dino.py index d49bbdd22..d4694aece 100755 --- a/fuse_examples/imaging/oai_example/self_supervised/dino.py +++ b/fuse_examples/imaging/oai_example/self_supervised/dino.py @@ -1,30 +1,30 @@ # This file is a modifies version of the Dino example from lightly - https://github.com/lightly-ai/lightly/blob/master/examples/pytorch_lightning/dino.py import copy -import torch -import pytorch_lightning as pl +import os +from glob import glob +from typing import Any + +import hydra import pandas as pd +import pytorch_lightning as pl +import torch +from clearml import Task from lightly.loss import DINOLoss from lightly.models.modules import DINOProjectionHead +from lightly.models.utils import deactivate_requires_grad, update_momentum from lightly.utils.scheduler import cosine_schedule -from lightly.models.utils import deactivate_requires_grad -from lightly.models.utils import update_momentum -from typing import Any -import os -from glob import glob -from fuse.data.utils.collates import CollateDefault -from fuse.dl.models.backbones.backbone_vit import vit_base -from fuse.utils import NDict -from fuse.dl.models.backbones.backbone_resnet_3d import BackboneResnet3D +from omegaconf import DictConfig +from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint from torch.optim.lr_scheduler import CosineAnnealingLR from torch.utils.data.dataloader import DataLoader -from pytorch_lightning.callbacks import ModelCheckpoint, LearningRateMonitor +from fuse.data.utils.collates import CollateDefault +from fuse.dl.models.backbones.backbone_resnet_3d import BackboneResnet3D from fuse.dl.models.backbones.backbone_unet3d import UNet3D +from fuse.dl.models.backbones.backbone_vit import vit_base +from fuse.utils import NDict from fuse_examples.imaging.oai_example.data.oai_ds import OAI -from omegaconf import DictConfig -import hydra -from clearml import Task torch.set_float32_matmul_precision("high") diff --git a/fuse_examples/imaging/segmentation/picai/runner.py b/fuse_examples/imaging/segmentation/picai/runner.py index f2a909d0f..92c5bd85a 100644 --- a/fuse_examples/imaging/segmentation/picai/runner.py +++ b/fuse_examples/imaging/segmentation/picai/runner.py @@ -12,47 +12,47 @@ Created on June 30, 2021 """ -import os import copy -from pathlib import Path +import logging +import os import pickle -import pandas as pd +from collections import OrderedDict +from pathlib import Path + +import hydra import numpy as np -from fuse.utils.utils_debug import FuseDebug -from fuse.utils.gpu import choose_and_enable_multiple_gpus +import pandas as pd -import logging +# from picai_baseline.unet.training_setup.default_hyperparam import \ +# get_default_hyperparams +# from picai_baseline.unet.training_setup.neural_network_selector import \ +# neural_network_for_run +import torch import torch.optim as optim +from omegaconf import DictConfig, OmegaConf +from pytorch_lightning import Trainer from torch.utils.data.dataloader import DataLoader -from fuse.utils.utils_logger import fuse_logger_start -from fuse.utils import NDict -from fuse.data.utils.samplers import BatchSamplerDefault from fuse.data.utils.collates import CollateDefault +from fuse.data.utils.samplers import BatchSamplerDefault from fuse.data.utils.split import dataset_balanced_division_to_folds +from fuse.dl.lightning.pl_funcs import convert_predictions_to_dataframe +from fuse.dl.lightning.pl_module import LightningModuleDefault from fuse.dl.losses.loss_default import LossDefault +from fuse.dl.losses.segmentation.loss_dice import BinaryDiceLoss +from fuse.eval.evaluator import EvaluatorDefault +from fuse.eval.metrics.segmentation.metrics_segmentation_common import MetricDice +from fuse.utils import NDict +from fuse.utils.file_io.file_io import create_dir, load_pickle, save_dataframe +from fuse.utils.gpu import choose_and_enable_multiple_gpus +from fuse.utils.utils_debug import FuseDebug +from fuse.utils.utils_logger import fuse_logger_start # from report_guided_annotation import extract_lesion_candidates from fuse_examples.imaging.segmentation.picai.unet import UNet -from fuse.eval.metrics.segmentation.metrics_segmentation_common import MetricDice -from collections import OrderedDict -from fuse.dl.losses.segmentation.loss_dice import BinaryDiceLoss # from fuse.eval.metrics.detection.metrics_detection_common import MetricDetectionPICAI from fuseimg.datasets.picai import PICAI -from fuse.dl.lightning.pl_funcs import convert_predictions_to_dataframe -from pytorch_lightning import Trainer -from fuse.dl.lightning.pl_module import LightningModuleDefault -from fuse.utils.file_io.file_io import create_dir, load_pickle, save_dataframe -from fuse.eval.evaluator import EvaluatorDefault - -# from picai_baseline.unet.training_setup.default_hyperparam import \ -# get_default_hyperparams -# from picai_baseline.unet.training_setup.neural_network_selector import \ -# neural_network_for_run -import torch -import hydra -from omegaconf import DictConfig, OmegaConf # from monai.losses import DiceFocalLoss diff --git a/fuse_examples/imaging/segmentation/picai/unet.py b/fuse_examples/imaging/segmentation/picai/unet.py index eb169e808..045f11fa7 100644 --- a/fuse_examples/imaging/segmentation/picai/unet.py +++ b/fuse_examples/imaging/segmentation/picai/unet.py @@ -1,7 +1,8 @@ -from torch import nn +import torch.nn.functional as F from monai.networks.nets import UNet as UNetBase +from torch import nn + from fuse.utils.ndict import NDict -import torch.nn.functional as F class UNet(nn.Module): diff --git a/fuse_examples/multimodality/ehr_transformer/dataset.py b/fuse_examples/multimodality/ehr_transformer/dataset.py index 18bf76c4e..165a77ff2 100644 --- a/fuse_examples/multimodality/ehr_transformer/dataset.py +++ b/fuse_examples/multimodality/ehr_transformer/dataset.py @@ -1,30 +1,27 @@ +import glob +import os import pickle from collections import OrderedDict from random import randrange -from typing import Sequence, Any, Tuple, List +from typing import Any, List, Sequence, Tuple -import glob -import os import numpy as np import pandas as pd -from fuse.data import DatasetDefault, PipelineDefault, OpBase -from fuse.data.ops.ops_read import OpReadDataframe +from fuse.data import DatasetDefault, OpBase, PipelineDefault from fuse.data.ops.ops_common import OpLookup, OpSetIfNotExist -from fuse.utils.ndict import NDict - -from fuse.data.utils.split import dataset_balanced_division_to_folds +from fuse.data.ops.ops_read import OpReadDataframe from fuse.data.utils.export import ExportDataset - +from fuse.data.utils.split import dataset_balanced_division_to_folds +from fuse.utils.ndict import NDict from fuse_examples.multimodality.ehr_transformer.utils import ( - seq_translate, + WordVocab, position_idx, - special_tokens, seq_pad, - WordVocab, + seq_translate, + special_tokens, ) - VALID_TESTS_ABOVE_ZERO = [ "pH", "Weight", diff --git a/fuse_examples/multimodality/ehr_transformer/main_train.py b/fuse_examples/multimodality/ehr_transformer/main_train.py index 63410fdef..74a481c33 100644 --- a/fuse_examples/multimodality/ehr_transformer/main_train.py +++ b/fuse_examples/multimodality/ehr_transformer/main_train.py @@ -1,31 +1,28 @@ -from typing import Any, Optional, Tuple -import hydra -from omegaconf import DictConfig from copy import deepcopy +from typing import Any, Optional, Tuple +import hydra +import pytorch_lightning as pl import torch import torch.nn.functional as F - -import pytorch_lightning as pl - +from omegaconf import DictConfig from torch.utils.data.dataloader import DataLoader from fuse.data.utils.collates import CollateDefault from fuse.data.utils.samplers import BatchSamplerDefault -from fuse.dl.models.model_wrapper import ModelWrapSeqToDict from fuse.dl.lightning.pl_module import LightningModuleDefault +from fuse.dl.losses import LossDefault from fuse.dl.models.heads import Head1D +from fuse.dl.models.model_wrapper import ModelWrapSeqToDict from fuse.eval.metrics.classification.metrics_classification_common import MetricAUCROC -from fuse.dl.losses import LossDefault from fuse.utils import NDict - +from fuse_examples.multimodality.ehr_transformer.dataset import PhysioNetCinC from fuse_examples.multimodality.ehr_transformer.model import ( - Embed, - TransformerEncoder, Bert, BertConfig, + Embed, + TransformerEncoder, ) -from fuse_examples.multimodality.ehr_transformer.dataset import PhysioNetCinC def filter_gender_label_unknown(batch_dict: NDict) -> NDict: diff --git a/fuse_examples/multimodality/ehr_transformer/model.py b/fuse_examples/multimodality/ehr_transformer/model.py index 3032caeff..5a023eaf2 100644 --- a/fuse_examples/multimodality/ehr_transformer/model.py +++ b/fuse_examples/multimodality/ehr_transformer/model.py @@ -1,10 +1,11 @@ from typing import Optional + import numpy as np import torch import torch.nn as nn +from transformers.models.bert.modeling_bert import BertConfig, BertEncoder, BertPooler from fuse.dl.models.backbones.backbone_transformer import Transformer -from transformers.models.bert.modeling_bert import BertEncoder, BertPooler, BertConfig from fuse.utils.ndict import NDict diff --git a/fuse_examples/multimodality/ehr_transformer/utils.py b/fuse_examples/multimodality/ehr_transformer/utils.py index 759898a3e..55732972e 100644 --- a/fuse_examples/multimodality/ehr_transformer/utils.py +++ b/fuse_examples/multimodality/ehr_transformer/utils.py @@ -19,9 +19,10 @@ """ import pickle -import tqdm -from typing import Optional, Sequence, List, Union, Tuple from collections import Counter, defaultdict +from typing import List, Optional, Sequence, Tuple, Union + +import tqdm # dictionary of special tokens that will be used in a generation of # patients trajectory sequence diff --git a/fuse_examples/multimodality/image_clinical/dataset.py b/fuse_examples/multimodality/image_clinical/dataset.py index 06859e164..1b19f6fa7 100644 --- a/fuse_examples/multimodality/image_clinical/dataset.py +++ b/fuse_examples/multimodality/image_clinical/dataset.py @@ -1,13 +1,13 @@ from typing import Optional, Sequence, Tuple + +from torch.utils.data.dataloader import DataLoader + from fuse.data import OpBase from fuse.data.utils.collates import CollateDefault from fuse.data.utils.samplers import BatchSamplerDefault from fuse.data.utils.split import dataset_balanced_division_to_folds - from fuseimg.datasets.isic import ISIC -from torch.utils.data.dataloader import DataLoader - def isic_2019_dataloaders( data_path: str, diff --git a/fuse_examples/tests/test_classification_cmmd.py b/fuse_examples/tests/test_classification_cmmd.py index 0e69c05ce..c5a2d9d6e 100644 --- a/fuse_examples/tests/test_classification_cmmd.py +++ b/fuse_examples/tests/test_classification_cmmd.py @@ -18,15 +18,15 @@ """ # FIXME: data_package import multiprocessing -from fuse.utils import NDict -import unittest +import os import shutil import tempfile -import os +import unittest + +from fuse.utils import NDict from fuse.utils.gpu import choose_and_enable_multiple_gpus from fuse.utils.multiprocessing.run_multiprocessed import run_in_subprocess - # os env variable CMMD_DATA_PATH is a path to the stored dataset location # dataset should be download from https://wiki.cancerimagingarchive.net/pages/viewpage.action?pageId=70230508 # download requires NBIA data retriever https://wiki.cancerimagingarchive.net/display/NBIA/Downloading+TCIA+Images @@ -35,9 +35,9 @@ # 2. folder named CMMD which is the downloaded data folder if "CMMD_DATA_PATH" in os.environ: from fuse_examples.imaging.classification.cmmd.runner import ( - run_train, run_eval, run_infer, + run_train, ) diff --git a/fuse_examples/tests/test_classification_isic.py b/fuse_examples/tests/test_classification_isic.py index 7bf661c87..a3a2c00a1 100644 --- a/fuse_examples/tests/test_classification_isic.py +++ b/fuse_examples/tests/test_classification_isic.py @@ -17,24 +17,23 @@ """ -import unittest import os -import tempfile import shutil -from fuse.utils.multiprocessing.run_multiprocessed import run_in_subprocess +import tempfile +import unittest +import fuse.utils.gpu as GPU +from fuse.utils.multiprocessing.run_multiprocessed import run_in_subprocess +from fuse.utils.rand.seed import Seed +from fuse_examples.imaging.classification.isic.golden_members import FULL_GOLDEN_MEMBERS from fuse_examples.imaging.classification.isic.isic_runner import ( - TRAIN_COMMON_PARAMS, - INFER_COMMON_PARAMS, EVAL_COMMON_PARAMS, - run_train, - run_infer, + INFER_COMMON_PARAMS, + TRAIN_COMMON_PARAMS, run_eval, + run_infer, + run_train, ) -from fuse_examples.imaging.classification.isic.golden_members import FULL_GOLDEN_MEMBERS - -import fuse.utils.gpu as GPU -from fuse.utils.rand.seed import Seed def run_isic( diff --git a/fuse_examples/tests/test_classification_knight.py b/fuse_examples/tests/test_classification_knight.py index 9886aede8..9a22927ca 100644 --- a/fuse_examples/tests/test_classification_knight.py +++ b/fuse_examples/tests/test_classification_knight.py @@ -16,21 +16,23 @@ Created on June 30, 2021 """ +import os import pathlib import shutil +import sys import tempfile import unittest -import os + +import wget + from fuse.utils.file_io.file_io import create_dir from fuse.utils.multiprocessing.run_multiprocessed import run_in_subprocess -import sys -import wget # add parent directory to path, so that 'knight' folder is treated as a module sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +import imaging.classification.knight.baseline.fuse_baseline as baseline from imaging.classification.knight.eval.eval import eval from imaging.classification.knight.make_targets_file import make_targets_file -import imaging.classification.knight.baseline.fuse_baseline as baseline class KnightTestTestCase(unittest.TestCase): diff --git a/fuse_examples/tests/test_classification_mnist.py b/fuse_examples/tests/test_classification_mnist.py index 2dfa64f49..8bbcbe33a 100644 --- a/fuse_examples/tests/test_classification_mnist.py +++ b/fuse_examples/tests/test_classification_mnist.py @@ -20,13 +20,11 @@ import shutil import tempfile import unittest -from fuse.utils.multiprocessing.run_multiprocessed import run_in_subprocess - -from fuseimg.datasets.mnist import MNIST +from fuse.utils.multiprocessing.run_multiprocessed import run_in_subprocess from fuse.utils.rand.seed import Seed - from fuse_examples.imaging.classification.mnist.simple_mnist_starter import run_train +from fuseimg.datasets.mnist import MNIST def run_mnist(root: str) -> None: diff --git a/fuse_examples/tests/test_classification_stoic21.py b/fuse_examples/tests/test_classification_stoic21.py index 46a0c182b..a1cd499b2 100644 --- a/fuse_examples/tests/test_classification_stoic21.py +++ b/fuse_examples/tests/test_classification_stoic21.py @@ -16,27 +16,26 @@ Created on June 30, 2021 """ +import os import shutil import tempfile import unittest -import os -from fuse.utils.multiprocessing.run_multiprocessed import run_in_subprocess -from fuse.utils.rand.seed import Seed import fuse.utils.gpu as GPU - import fuse_examples.imaging.classification.stoic21.dataset as dataset +from fuse.utils.multiprocessing.run_multiprocessed import run_in_subprocess +from fuse.utils.rand.seed import Seed if "STOIC21_DATA_PATH" in os.environ: from fuse_examples.imaging.classification.stoic21.runner_stoic21 import ( + DATASET_COMMON_PARAMS, + EVAL_COMMON_PARAMS, + INFER_COMMON_PARAMS, PATHS, TRAIN_COMMON_PARAMS, - run_train, - run_infer, run_eval, - INFER_COMMON_PARAMS, - EVAL_COMMON_PARAMS, - DATASET_COMMON_PARAMS, + run_infer, + run_train, ) diff --git a/fuse_examples/tests/test_ehr_transformer.py b/fuse_examples/tests/test_ehr_transformer.py index a36483881..fffc0b62f 100644 --- a/fuse_examples/tests/test_ehr_transformer.py +++ b/fuse_examples/tests/test_ehr_transformer.py @@ -17,18 +17,17 @@ """ +import os import shutil -import unittest import tempfile -import os +import unittest -from fuse.utils.multiprocessing.run_multiprocessed import run_in_subprocess -from omegaconf import DictConfig import yaml +from omegaconf import DictConfig +from fuse.utils.multiprocessing.run_multiprocessed import run_in_subprocess from fuse_examples.multimodality.ehr_transformer.main_train import main as main_train - if "CINC_TEST_DATA_PATH" in os.environ: TEST_PARAMS = { "CONFIG_PATH": os.path.dirname(__file__) diff --git a/fuse_examples/tests/test_notebook_hello_world.py b/fuse_examples/tests/test_notebook_hello_world.py index 855684801..76557662c 100644 --- a/fuse_examples/tests/test_notebook_hello_world.py +++ b/fuse_examples/tests/test_notebook_hello_world.py @@ -1,5 +1,7 @@ import unittest + from testbook import testbook + from fuse.utils.multiprocessing.run_multiprocessed import run_in_subprocess diff --git a/fuse_examples/tests/test_segmentation_picai.py b/fuse_examples/tests/test_segmentation_picai.py index ae7200ce3..4593f63ad 100644 --- a/fuse_examples/tests/test_segmentation_picai.py +++ b/fuse_examples/tests/test_segmentation_picai.py @@ -18,17 +18,17 @@ """ # FIXME: data_package import multiprocessing -from fuse.utils import NDict -import unittest +import os import shutil import tempfile -import os -from fuse.utils.gpu import choose_and_enable_multiple_gpus -from fuse.utils.multiprocessing.run_multiprocessed import run_in_subprocess +import unittest import hydra from hydra import compose, initialize +from fuse.utils import NDict +from fuse.utils.gpu import choose_and_enable_multiple_gpus +from fuse.utils.multiprocessing.run_multiprocessed import run_in_subprocess # os env variable PICAI_DATA_PATH is a path to the stored dataset location # dataset should be download from https://zenodo.org/record/6517398#.ZGEHVXZBxD8 @@ -36,9 +36,9 @@ # folder named PICAI which is the downloaded data folder with partition for images and labels if "PICAI_DATA_PATH" in os.environ: from fuse_examples.imaging.segmentation.picai.runner import ( - run_train, - run_infer, run_eval, + run_infer, + run_train, ) diff --git a/fuseimg/data/ops/aug/color.py b/fuseimg/data/ops/aug/color.py index af79b2491..0773f8e90 100644 --- a/fuseimg/data/ops/aug/color.py +++ b/fuseimg/data/ops/aug/color.py @@ -1,11 +1,13 @@ from typing import List, Optional + +import torch +from torch import Tensor + from fuse.data.ops.op_base import OpBase +from fuse.data.ops.ops_cast import Cast from fuse.utils.ndict import NDict from fuse.utils.rand.param_sampler import Gaussian -from fuse.data.ops.ops_cast import Cast from fuseimg.data.ops.color import OpClip -from torch import Tensor -import torch class OpAugColor(OpBase): diff --git a/fuseimg/data/ops/aug/geometry.py b/fuseimg/data/ops/aug/geometry.py index 1d96c2904..2ba693223 100644 --- a/fuseimg/data/ops/aug/geometry.py +++ b/fuseimg/data/ops/aug/geometry.py @@ -1,17 +1,15 @@ -from typing import List, Optional, Tuple, Union, Dict, Any - -from PIL import Image +from typing import Any, Dict, List, Optional, Tuple, Union import numpy as np -import torch -import torchvision.transforms.functional as TTF -import torchvision.transforms as transforms import skimage import skimage.transform - -from fuse.utils.ndict import NDict +import torch +import torchvision.transforms as transforms +import torchvision.transforms.functional as TTF +from PIL import Image from fuse.data import OpBase +from fuse.utils.ndict import NDict class OpAugAffine2D(OpBase): diff --git a/fuseimg/data/ops/color.py b/fuseimg/data/ops/color.py index 8d4f68265..a6cabda53 100644 --- a/fuseimg/data/ops/color.py +++ b/fuseimg/data/ops/color.py @@ -1,12 +1,12 @@ -from typing import Tuple, Union, Dict, Any +from typing import Any, Dict, Tuple, Union + import numpy as np import torch -from fuse.utils.ndict import NDict from fuse.data.ops.op_base import OpBase - -from fuseimg.utils.typing.key_types_imaging import DataTypeImaging +from fuse.utils.ndict import NDict from fuseimg.data.ops.ops_common_imaging import OpApplyTypesImaging +from fuseimg.utils.typing.key_types_imaging import DataTypeImaging class OpClip(OpBase): diff --git a/fuseimg/data/ops/debug_ops.py b/fuseimg/data/ops/debug_ops.py index 90c2e4910..c29c63f32 100644 --- a/fuseimg/data/ops/debug_ops.py +++ b/fuseimg/data/ops/debug_ops.py @@ -1,10 +1,11 @@ # type: ignore import cv2 from torch import Tensor + from fuse.data.ops.op_base import OpBase -from fuseimg.utils.typing.key_types_imaging import DataTypeImaging -from fuseimg.data.ops.ops_common_imaging import OpApplyTypesImaging from fuse.utils.ndict import NDict +from fuseimg.data.ops.ops_common_imaging import OpApplyTypesImaging +from fuseimg.utils.typing.key_types_imaging import DataTypeImaging def no_op(input_tensor: Tensor) -> Tensor: diff --git a/fuseimg/data/ops/image_loader.py b/fuseimg/data/ops/image_loader.py index bb777639b..f247a99d8 100644 --- a/fuseimg/data/ops/image_loader.py +++ b/fuseimg/data/ops/image_loader.py @@ -1,13 +1,14 @@ import os -from fuse.data.ops.op_base import OpReversibleBase from typing import Optional -import numpy as np -import nibabel as nib -from fuse.utils.ndict import NDict -from torchvision.io import read_image -from medpy.io import load +import nibabel as nib +import numpy as np import pydicom +from medpy.io import load +from torchvision.io import read_image + +from fuse.data.ops.op_base import OpReversibleBase +from fuse.utils.ndict import NDict class OpLoadImage(OpReversibleBase): diff --git a/fuseimg/data/ops/ops_common_imaging.py b/fuseimg/data/ops/ops_common_imaging.py index 957011b10..28b064f45 100644 --- a/fuseimg/data/ops/ops_common_imaging.py +++ b/fuseimg/data/ops/ops_common_imaging.py @@ -1,6 +1,7 @@ +from functools import partial + from fuse.data.ops.ops_common import OpApplyTypes from fuseimg.utils.typing.key_types_imaging import type_detector_imaging -from functools import partial OpApplyTypesImaging = partial( OpApplyTypes, diff --git a/fuseimg/data/ops/ops_debug.py b/fuseimg/data/ops/ops_debug.py index 657a630e9..f7dc22b5b 100644 --- a/fuseimg/data/ops/ops_debug.py +++ b/fuseimg/data/ops/ops_debug.py @@ -1,16 +1,18 @@ import os -from typing import Callable, Optional, Tuple, Any, Dict +from typing import Any, Callable, Dict, Optional, Tuple + +import matplotlib.pyplot as plt +import numpy +import numpy as np +import plotly.graph_objects as go +import SimpleITK as sitk +import torch + from fuse.data.ops.ops_cast import Cast +from fuse.data.ops.ops_debug import OpDebugBase from fuse.data.utils.sample import get_sample_id -from fuse.utils import NDict from fuse.utils.file_io.file_io import create_dir -from fuse.data.ops.ops_debug import OpDebugBase -import numpy -import torch -import matplotlib.pyplot as plt -import SimpleITK as sitk -import plotly.graph_objects as go -import numpy as np +from fuse.utils.ndict import NDict class OpVis2DImage(OpDebugBase): diff --git a/fuseimg/data/ops/shape_ops.py b/fuseimg/data/ops/shape_ops.py index c277514d4..e6783a3bd 100755 --- a/fuseimg/data/ops/shape_ops.py +++ b/fuseimg/data/ops/shape_ops.py @@ -1,16 +1,16 @@ -from typing import Tuple, List, Dict, Any +from typing import Any, Dict, List, Tuple + import numpy as np -from torch import Tensor import skimage import torch import torchvision.transforms.functional as TTF - -from fuse.utils.ndict import NDict +from skimage import measure +from torch import Tensor from fuse.data.ops.op_base import OpBase -from skimage import measure -from fuseimg.utils.typing.key_types_imaging import DataTypeImaging +from fuse.utils.ndict import NDict from fuseimg.data.ops.ops_common_imaging import OpApplyTypesImaging +from fuseimg.utils.typing.key_types_imaging import DataTypeImaging def sanity_check_HWC(input_tensor: Tensor) -> None: diff --git a/fuseimg/data/ops/tests/test_ops.py b/fuseimg/data/ops/tests/test_ops.py index 15392ebe5..693f167da 100644 --- a/fuseimg/data/ops/tests/test_ops.py +++ b/fuseimg/data/ops/tests/test_ops.py @@ -1,14 +1,13 @@ import unittest +import numpy as np +import torch + from fuse.data.pipelines.pipeline_default import PipelineDefault +from fuse.utils.ndict import NDict from fuseimg.data.ops.color import OpClip, OpToRange from fuseimg.data.ops.shape_ops import OpPad -from fuse.utils.ndict import NDict - -import numpy as np -import torch - class TestOps(unittest.TestCase): def test_basic_1(self) -> None: diff --git a/fuseimg/data/ops/tests/test_pipeline_caching.py b/fuseimg/data/ops/tests/test_pipeline_caching.py index 406a8ebc2..6edd873ad 100644 --- a/fuseimg/data/ops/tests/test_pipeline_caching.py +++ b/fuseimg/data/ops/tests/test_pipeline_caching.py @@ -1,11 +1,9 @@ -import unittest import os import tempfile +import unittest - -from fuse.data.datasets.dataset_default import DatasetDefault from fuse.data.datasets.caching.samples_cacher import SamplesCacher - +from fuse.data.datasets.dataset_default import DatasetDefault from fuseimg.datasets.isic import ISIC diff --git a/fuseimg/datasets/cmmd.py b/fuseimg/datasets/cmmd.py index 0d2fce223..46a5ae795 100644 --- a/fuseimg/datasets/cmmd.py +++ b/fuseimg/datasets/cmmd.py @@ -1,32 +1,34 @@ -from fuse.data.datasets.dataset_default import DatasetDefault +import glob +import os +from functools import partial +from pathlib import Path +from typing import Hashable, Optional, Sequence + +import numpy as np +import pandas as pd +import pydicom +import torch + from fuse.data.datasets.caching.samples_cacher import SamplesCacher -from fuseimg.data.ops.image_loader import OpLoadImage +from fuse.data.datasets.dataset_default import DatasetDefault +from fuse.data.ops.op_base import OpBase +from fuse.data.ops.ops_aug_common import OpRandApply, OpSample +from fuse.data.ops.ops_cast import OpToNumpy, OpToTensor +from fuse.data.ops.ops_common import OpLambda +from fuse.data.ops.ops_read import OpReadDataframe +from fuse.data.pipelines.pipeline_default import PipelineDefault +from fuse.data.utils.sample import get_sample_id +from fuse.utils.ndict import NDict +from fuse.utils.rand.param_sampler import RandBool, RandInt, Uniform +from fuseimg.data.ops.aug.color import OpAugColor +from fuseimg.data.ops.aug.geometry import OpAugAffine2D from fuseimg.data.ops.color import OpNormalizeAgainstSelf +from fuseimg.data.ops.image_loader import OpLoadImage from fuseimg.data.ops.shape_ops import ( - OpFlipBrightSideOnLeft2D, OpFindBiggestNonEmptyBbox2D, + OpFlipBrightSideOnLeft2D, OpResizeAndPad2D, ) -from fuse.data import PipelineDefault, OpToTensor -from fuse.data.ops.ops_common import OpLambda -from fuseimg.data.ops.aug.color import OpAugColor -from fuseimg.data.ops.aug.geometry import OpAugAffine2D -from fuse.data.ops.ops_aug_common import OpSample, OpRandApply -from fuse.data.ops.ops_read import OpReadDataframe -from fuse.data.ops.ops_cast import OpToNumpy -from fuse.data.ops.op_base import OpBase -from fuse.utils import NDict -from functools import partial -from typing import Hashable, Optional, Sequence -import torch -import pandas as pd -import numpy as np -import pydicom -import os -import glob -from pathlib import Path -from fuse.data.utils.sample import get_sample_id -from fuse.utils.rand.param_sampler import Uniform, RandInt, RandBool class OpCMMDSampleIDDecode(OpBase): diff --git a/fuseimg/datasets/isic.py b/fuseimg/datasets/isic.py index bf759e02d..c9462e83e 100644 --- a/fuseimg/datasets/isic.py +++ b/fuseimg/datasets/isic.py @@ -1,32 +1,33 @@ import os +from typing import Hashable, List, Optional, Sequence, Tuple from zipfile import ZipFile -from fuse.utils.file_io.file_io import create_dir -import wget -from typing import Hashable, Optional, Sequence, List, Tuple -import torch + import numpy as np import pytorch_lightning as pl -from fuse.data import DatasetDefault -from fuse.data.ops.ops_cast import OpToTensor -from fuse.data.utils.sample import get_sample_id -from fuse.data.pipelines.pipeline_default import PipelineDefault -from fuse.data.ops.op_base import OpBase +import torch +import wget +from torch.utils.data import DataLoader + from fuse.data.datasets.caching.samples_cacher import SamplesCacher -from fuse.data.ops.ops_aug_common import OpSample, OpRandApply -from fuse.data.ops.ops_read import OpReadDataframe -from fuse.data.ops.ops_common import OpLambda, OpOverrideNaN, OpConcat -from fuseimg.data.ops.color import OpToRange +from fuse.data.datasets.dataset_default import DatasetDefault +from fuse.data.ops.op_base import OpBase +from fuse.data.ops.ops_aug_common import OpRandApply, OpSample from fuse.data.ops.ops_aug_tabular import OpAugOneHot +from fuse.data.ops.ops_cast import OpToTensor +from fuse.data.ops.ops_common import OpConcat, OpLambda, OpOverrideNaN +from fuse.data.ops.ops_read import OpReadDataframe +from fuse.data.pipelines.pipeline_default import PipelineDefault +from fuse.data.utils.collates import CollateDefault +from fuse.data.utils.sample import get_sample_id +from fuse.data.utils.samplers import BatchSamplerDefault from fuse.data.utils.split import dataset_balanced_division_to_folds from fuse.utils import NDict - -from fuseimg.data.ops.image_loader import OpLoadImage +from fuse.utils.file_io.file_io import create_dir +from fuse.utils.rand.param_sampler import RandBool, RandInt, Uniform from fuseimg.data.ops.aug.color import OpAugColor, OpAugGaussian -from fuseimg.data.ops.aug.geometry import OpResizeTo, OpAugAffine2D -from fuse.utils.rand.param_sampler import Uniform, RandInt, RandBool -from fuse.data.utils.samplers import BatchSamplerDefault -from torch.utils.data import DataLoader -from fuse.data.utils.collates import CollateDefault +from fuseimg.data.ops.aug.geometry import OpAugAffine2D, OpResizeTo +from fuseimg.data.ops.color import OpToRange +from fuseimg.data.ops.image_loader import OpLoadImage class OpISICSampleIDDecode(OpBase): diff --git a/fuseimg/datasets/kits21.py b/fuseimg/datasets/kits21.py index c17f140eb..854dd1f62 100644 --- a/fuseimg/datasets/kits21.py +++ b/fuseimg/datasets/kits21.py @@ -16,33 +16,31 @@ Created on June 30, 2021 """ -from functools import partial import os +from functools import partial from typing import Hashable, List, Optional, Sequence, Tuple, Union import numpy as np -import torch -from tqdm import tqdm import skimage import skimage.transform - -from fuse.utils import NDict -from fuse.utils.rand.param_sampler import RandBool, RandInt, Uniform +import torch import wget +from tqdm import tqdm -from fuse.data import DatasetDefault from fuse.data.datasets.caching.samples_cacher import SamplesCacher -from fuse.data import PipelineDefault, OpSampleAndRepeat, OpToTensor, OpRepeat +from fuse.data.datasets.dataset_default import DatasetDefault from fuse.data.ops.op_base import OpReversibleBase -from fuse.data.ops.ops_aug_common import OpSample -from fuse.data.ops.ops_common import OpLambda - +from fuse.data.ops.ops_aug_common import OpSample, OpSampleAndRepeat +from fuse.data.ops.ops_cast import OpToTensor +from fuse.data.ops.ops_common import OpLambda, OpRepeat +from fuse.data.pipelines.pipeline_default import PipelineDefault from fuse.data.utils.sample import get_sample_id - +from fuse.utils.ndict import NDict +from fuse.utils.rand.param_sampler import RandBool, RandInt, Uniform from fuseimg.data.ops.aug.color import OpAugColor from fuseimg.data.ops.aug.geometry import OpAugAffine2D -from fuseimg.data.ops.image_loader import OpLoadImage from fuseimg.data.ops.color import OpClip, OpToRange +from fuseimg.data.ops.image_loader import OpLoadImage class OpKits21SampleIDDecode(OpReversibleBase): diff --git a/fuseimg/datasets/kits21_example.ipynb b/fuseimg/datasets/kits21_example.ipynb index 810dd5338..32bbccdc1 100644 --- a/fuseimg/datasets/kits21_example.ipynb +++ b/fuseimg/datasets/kits21_example.ipynb @@ -47,11 +47,11 @@ "from fuse.data.pipelines.pipeline_default import PipelineDefault\n", "from fuse.data.datasets.dataset_default import DatasetDefault\n", "from fuse.data.ops.op_base import OpBase\n", - "from fuse.data.ops.ops_aug_common import OpSample\n", + "from fuse.data.ops.ops_aug_common import OpSample, OpSampleAndRepeat\n", "from fuse.data.datasets.caching.samples_cacher import SamplesCacher\n", - "from fuse.data.ops.ops_common import OpLambda\n", + "from fuse.data.ops.ops_common import OpLambda, OpRepeat\n", "from fuse.data.utils.samplers import BatchSamplerDefault\n", - "from fuse.data import PipelineDefault, OpSampleAndRepeat, OpToTensor, OpRepeat\n", + "from fuse.data.ops.ops_cast import OpToTensor\n", "from fuse.utils.rand.param_sampler import RandBool, RandInt, Uniform\n", "import torch\n", "import numpy as np\n", diff --git a/fuseimg/datasets/knight.py b/fuseimg/datasets/knight.py index bb72db932..7033a7998 100644 --- a/fuseimg/datasets/knight.py +++ b/fuseimg/datasets/knight.py @@ -1,25 +1,26 @@ import os +from functools import partial from glob import glob from typing import Hashable, Optional, Sequence, Tuple -from fuse.utils.rand.param_sampler import Uniform, RandInt, RandBool -from fuse.utils.ndict import NDict +import numpy as np +import pandas as pd +import torch -from fuse.data import DatasetDefault from fuse.data.datasets.caching.samples_cacher import SamplesCacher -from fuse.data import PipelineDefault, OpToTensor +from fuse.data.datasets.dataset_default import DatasetDefault from fuse.data.ops.op_base import OpBase -from fuse.data.ops.ops_aug_common import OpSample, OpRandApply +from fuse.data.ops.ops_aug_common import OpRandApply, OpSample +from fuse.data.ops.ops_cast import OpToTensor from fuse.data.ops.ops_common import OpLambda, OpZScoreNorm -from fuseimg.data.ops.aug.geometry import OpAugAffine2D, OpRotation3D, OpResizeTo +from fuse.data.pipelines.pipeline_default import PipelineDefault +from fuse.data.utils.sample import get_sample_id +from fuse.utils.ndict import NDict +from fuse.utils.rand.param_sampler import RandBool, RandInt, Uniform from fuseimg.data.ops.aug.color import OpAugGaussian -from fuseimg.data.ops.image_loader import OpLoadImage +from fuseimg.data.ops.aug.geometry import OpAugAffine2D, OpResizeTo, OpRotation3D from fuseimg.data.ops.color import OpClip -import numpy as np -from fuse.data.utils.sample import get_sample_id -from functools import partial -import torch -import pandas as pd +from fuseimg.data.ops.image_loader import OpLoadImage class OpKnightSampleIDDecode(OpBase): diff --git a/fuseimg/datasets/mnist.py b/fuseimg/datasets/mnist.py index 8c64adb90..875e79320 100644 --- a/fuseimg/datasets/mnist.py +++ b/fuseimg/datasets/mnist.py @@ -17,10 +17,11 @@ """ from typing import Optional, Sequence -from fuse.data.datasets.dataset_wrap_seq_to_dict import DatasetWrapSeqToDict -from torchvision import transforms, datasets -from fuse.data import DatasetDefault +from torchvision import datasets, transforms + +from fuse.data.datasets.dataset_default import DatasetDefault +from fuse.data.datasets.dataset_wrap_seq_to_dict import DatasetWrapSeqToDict class MNIST: diff --git a/fuseimg/datasets/picai.py b/fuseimg/datasets/picai.py index 5ad1d140a..06d08511c 100644 --- a/fuseimg/datasets/picai.py +++ b/fuseimg/datasets/picai.py @@ -1,32 +1,31 @@ -from fuse.data.datasets.dataset_default import DatasetDefault -from fuse.data.datasets.caching.samples_cacher import SamplesCacher -from fuseimg.data.ops.color import OpNormalizeAgainstSelf - -from fuseimg.data.ops.aug.geometry import ( - OpAugAffine2D, - OpAugSqueeze3Dto2D, - OpAugUnsqueeze3DFrom2D, -) -from fuse.data import PipelineDefault, OpToTensor, OpRepeat -from fuse.data.ops.ops_common import OpLambda, OpLookup, OpToOneHot - -from fuse.data.ops.ops_aug_common import OpRandApply, OpSampleAndRepeat -from fuse.data.ops.ops_read import OpReadDataframe -from fuse.data.ops.ops_cast import OpToNumpy -from fuse.data.ops.op_base import OpBase -from fuse.utils import NDict +import os from functools import partial -import nibabel as nib from typing import Hashable, Optional, Sequence -import torch -import pandas as pd + +import nibabel as nib import numpy as np +import pandas as pd import skimage -import os -from fuse.data.utils.sample import get_sample_id +import torch from medpy.io import load -from fuse.utils.rand.param_sampler import Uniform, RandInt +from fuse.data.datasets.caching.samples_cacher import SamplesCacher +from fuse.data.datasets.dataset_default import DatasetDefault +from fuse.data.ops.op_base import OpBase +from fuse.data.ops.ops_aug_common import OpRandApply, OpSampleAndRepeat +from fuse.data.ops.ops_cast import OpToNumpy, OpToTensor +from fuse.data.ops.ops_common import OpLambda, OpLookup, OpRepeat, OpToOneHot +from fuse.data.ops.ops_read import OpReadDataframe +from fuse.data.pipelines.pipeline_default import PipelineDefault +from fuse.data.utils.sample import get_sample_id +from fuse.utils.ndict import NDict +from fuse.utils.rand.param_sampler import RandInt, Uniform +from fuseimg.data.ops.aug.geometry import ( + OpAugAffine2D, + OpAugSqueeze3Dto2D, + OpAugUnsqueeze3DFrom2D, +) +from fuseimg.data.ops.color import OpNormalizeAgainstSelf class OpPICAISampleIDDecode(OpBase): diff --git a/fuseimg/datasets/stoic21.py b/fuseimg/datasets/stoic21.py index 348276b1f..50eb39ed3 100644 --- a/fuseimg/datasets/stoic21.py +++ b/fuseimg/datasets/stoic21.py @@ -16,34 +16,30 @@ Created on June 30, 2021 """ +import os from functools import partial from glob import glob -import os -from typing import Hashable, Optional, Sequence, Tuple, List - +from typing import Hashable, List, Optional, Sequence, Tuple import numpy as np -import torch import skimage import skimage.transform +import torch - -from fuse.utils import NDict -from fuse.utils.rand.param_sampler import RandBool, RandInt, Uniform - -from fuse.data import DatasetDefault from fuse.data.datasets.caching.samples_cacher import SamplesCacher -from fuse.data import PipelineDefault, OpToTensor +from fuse.data.datasets.dataset_default import DatasetDefault from fuse.data.ops.op_base import OpBase -from fuse.data.ops.ops_aug_common import OpSample, OpRandApply +from fuse.data.ops.ops_aug_common import OpRandApply, OpSample +from fuse.data.ops.ops_cast import OpToFloat, OpToInt, OpToNumpy, OpToTensor from fuse.data.ops.ops_common import OpConcat, OpLambda, OpLookup, OpToOneHot from fuse.data.ops.ops_read import OpReadDataframe -from fuse.data.ops.ops_cast import OpToFloat, OpToInt, OpToNumpy +from fuse.data.pipelines.pipeline_default import PipelineDefault from fuse.data.utils.sample import get_sample_id - +from fuse.utils.ndict import NDict +from fuse.utils.rand.param_sampler import RandBool, RandInt, Uniform from fuseimg.data.ops.aug.geometry import OpAugAffine2D -from fuseimg.data.ops.image_loader import OpLoadImage from fuseimg.data.ops.color import OpClip, OpToRange +from fuseimg.data.ops.image_loader import OpLoadImage class OpSTOIC21SampleIDDecode(OpBase): diff --git a/fuseimg/datasets/tests/test_datasets.py b/fuseimg/datasets/tests/test_datasets.py index 3c003c791..ebbea7be7 100644 --- a/fuseimg/datasets/tests/test_datasets.py +++ b/fuseimg/datasets/tests/test_datasets.py @@ -1,29 +1,28 @@ import os import pathlib import shutil -from tempfile import mkdtemp import unittest -from fuse.data.utils.sample import get_sample_id -from fuse.utils.file_io.file_io import create_dir +from tempfile import mkdtemp from typing import Any -from fuseimg.datasets.kits21 import KITS21 -from fuseimg.datasets.stoic21 import STOIC21 -from tqdm import trange from testbook import testbook +from tqdm import trange + +from fuse.data.utils.sample import get_sample_id +from fuse.eval.evaluator import EvaluatorDefault from fuse.eval.metrics.stat.metrics_stat_common import MetricUniqueValues +from fuse.utils.file_io.file_io import create_dir from fuse.utils.multiprocessing.run_multiprocessed import ( get_from_global_storage, run_multiprocessed, ) -from fuse.eval.evaluator import EvaluatorDefault - -from fuseimg.datasets.isic import ISIC from fuse_examples.imaging.classification.isic.golden_members import ( FULL_GOLDEN_MEMBERS, TEN_GOLDEN_MEMBERS, ) - +from fuseimg.datasets.isic import ISIC +from fuseimg.datasets.kits21 import KITS21 +from fuseimg.datasets.stoic21 import STOIC21 notebook_path = os.path.join( pathlib.Path(__file__).parent.resolve(), "../kits21_example.ipynb" diff --git a/fuseimg/eval/metrics_vis_2d.py b/fuseimg/eval/metrics_vis_2d.py index 49e4a2d11..671664728 100644 --- a/fuseimg/eval/metrics_vis_2d.py +++ b/fuseimg/eval/metrics_vis_2d.py @@ -1,5 +1,6 @@ from functools import partial from typing import Any, Callable, Dict, Hashable, Optional, Sequence + from fuse.eval.metrics.metrics_common import MetricWithCollectorBase from fuseimg.data.ops.ops_debug import OpVis2DImage diff --git a/fuseimg/utils/align/utils_align_base.py b/fuseimg/utils/align/utils_align_base.py index 03b6306d3..95663c0ec 100644 --- a/fuseimg/utils/align/utils_align_base.py +++ b/fuseimg/utils/align/utils_align_base.py @@ -18,6 +18,7 @@ """ from abc import ABC + import numpy as np diff --git a/fuseimg/utils/align/utils_align_ecc.py b/fuseimg/utils/align/utils_align_ecc.py index 333c7d488..35fda9336 100644 --- a/fuseimg/utils/align/utils_align_ecc.py +++ b/fuseimg/utils/align/utils_align_ecc.py @@ -18,9 +18,10 @@ """ -from fuseimg.utils.align.utils_align_base import AlignMapBase -import numpy as np import cv2 +import numpy as np + +from fuseimg.utils.align.utils_align_base import AlignMapBase class AlignMapECC(AlignMapBase): diff --git a/fuseimg/utils/image_processing.py b/fuseimg/utils/image_processing.py index 8be1865bd..6f5a42b95 100644 --- a/fuseimg/utils/image_processing.py +++ b/fuseimg/utils/image_processing.py @@ -17,12 +17,12 @@ """ -from typing import Union, Tuple, Callable +from typing import Callable, Tuple, Union +import cv2 import numpy as np import skimage import torch -import cv2 def match_img_to_input(im: np.ndarray, input: np.ndarray) -> np.ndarray: @@ -148,6 +148,7 @@ def align_ecc( try: import cv2 + from fuseimg.utils.align.utils_align_ecc import AlignMapECC transformation = transformation or cv2.MOTION_AFFINE diff --git a/fuseimg/utils/typing/key_types_imaging.py b/fuseimg/utils/typing/key_types_imaging.py index 1e8853010..682154537 100644 --- a/fuseimg/utils/typing/key_types_imaging.py +++ b/fuseimg/utils/typing/key_types_imaging.py @@ -1,4 +1,5 @@ from enum import Enum + from fuse.data.key_types import DataTypeBasic, TypeDetectorPatternsBased diff --git a/fuseimg/utils/typing/typed_element.py b/fuseimg/utils/typing/typed_element.py index a9f5498db..9a7a47cbf 100644 --- a/fuseimg/utils/typing/typed_element.py +++ b/fuseimg/utils/typing/typed_element.py @@ -1,5 +1,6 @@ # type: ignore import numpy as np + from fuse.data.key_types import DataTypeBasic from fuse.data.patterns import Patterns diff --git a/run_all_unit_tests.py b/run_all_unit_tests.py index daf3143dc..4fd14be7b 100644 --- a/run_all_unit_tests.py +++ b/run_all_unit_tests.py @@ -4,9 +4,10 @@ """ import logging +import os import sys import unittest -import os + import termcolor import xmlrunner diff --git a/setup.cfg b/setup.cfg index fc9b9cc65..b4c3d4f4e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -101,3 +101,6 @@ disable_error_code = enable_error_code = no-untyped-def + +[isort] +profile = black diff --git a/setup.py b/setup.py index c923f745a..7a3a83524 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ #!/usr/bin/env python import os import pathlib -from setuptools import setup, find_namespace_packages +from setuptools import find_namespace_packages, setup # The directory containing this file HERE = pathlib.Path(__file__).parent