Integrate isort (#385)

.github/workflows/lint.yaml

@@ -42,3 +42,13 @@ jobs:
       - run: pip install --upgrade pip
       - run: pip install nb-clean
       - run: nb-clean check .
+  isort:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.9"
+      - uses: isort/isort-action@master
+        with:
+          isort-version: 5.13.2

.pre-commit-config.yaml

@@ -6,10 +6,6 @@ repos:
       - id: end-of-file-fixer
       - id: mixed-line-ending
       - id: trailing-whitespace
-  - repo: https://github.com/psf/black
-    rev: 22.10.0
-    hooks:
-      - id: black
   - repo: https://github.com/PyCQA/flake8
     rev: 5.0.4
     hooks:
@@ -25,3 +21,11 @@ repos:
         args:
           - --remove-empty-cells
           - --preserve-cell-outputs
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+      - id: black

fuse/data/__init__.py

@@ -1,27 +1,14 @@
 # version
-from fuse.version import __version__
-
-# import shortcuts
-from fuse.data.utils.sample import get_sample_id, set_sample_id, get_sample_id_key
-from fuse.data.utils.sample import (
-    create_initial_sample,
-    get_initial_sample_id,
-    get_initial_sample_id_key,
-    get_specific_sample_from_potentially_morphed,
-)
+from fuse.data.datasets.dataset_default import DatasetBase, DatasetDefault
 from fuse.data.ops.op_base import OpBase  # DataTypeForTesting,
-from fuse.data.ops.ops_common import (
-    OpApplyPatterns,
-    OpLambda,
-    OpFunc,
-    OpRepeat,
-    OpKeepKeypaths,
-    OpSet,
-)
 from fuse.data.ops.ops_aug_common import OpRandApply, OpSample, OpSampleAndRepeat
+from fuse.data.ops.ops_cast import OpToNumpy, OpToTensor
+from fuse.data.ops.ops_common import OpApplyPatterns, OpFunc, OpKeepKeypaths, OpSet
 from fuse.data.ops.ops_read import OpReadDataframe
-from fuse.data.ops.ops_cast import OpToTensor, OpToNumpy
+from fuse.data.pipelines.pipeline_default import PipelineDefault
 from fuse.data.utils.collates import CollateDefault
 from fuse.data.utils.export import ExportDataset
-from fuse.data.pipelines.pipeline_default import PipelineDefault
-from fuse.data.datasets.dataset_default import DatasetBase, DatasetDefault
+from fuse.utils.ndict import NDict  # Fix circular import
+from fuse.version import __version__
+
+# import shortcuts

fuse/data/datasets/__init__.py

@@ -1 +1 @@
-from .dataset_default import DatasetDefault
+from fuse.data.datasets.dataset_default import DatasetDefault

fuse/data/datasets/caching/object_caching_handlers.py

@@ -1,8 +1,10 @@
-from typing import List, Any
+from typing import Any, List
+
 import numpy as np
-from fuse.utils.ndict import NDict
 import torch
 
+from fuse.utils.ndict import NDict
+
 # TODO: support custom _object_requires_hdf5_single
 #      maybe even more flexible (knowing key name etc., patterns, explicit name, regular expr.)
 

fuse/data/datasets/caching/samples_cacher.py

@@ -11,39 +11,41 @@
 limitations under the License.
 Created on June 30, 2021
 """
+import hashlib
+import os
+from collections import OrderedDict
 from functools import partial
-from typing import Hashable, List, Optional, Sequence, Union, Callable, Any, Tuple, Dict
+from glob import glob
+from typing import Any, Callable, Dict, Hashable, List, Optional, Sequence, Tuple, Union
+from warnings import warn
+
+import numpy as np
+import psutil
 
-from fuse.data.pipelines.pipeline_default import PipelineDefault
-from collections import OrderedDict
 from fuse.data.datasets.caching.object_caching_handlers import (
     _object_requires_hdf5_recurse,
 )
-from fuse.utils.ndict import NDict
-import os
-import psutil
+from fuse.data.datasets.sample_caching_audit import SampleCachingAudit
+from fuse.data.pipelines.pipeline_default import PipelineDefault
+from fuse.data.utils.sample import (
+    create_initial_sample,
+    get_initial_sample_id,
+    get_sample_id,
+    get_specific_sample_from_potentially_morphed,
+    set_initial_sample_id,
+)
+from fuse.utils.file_io import delete_directory_tree
 from fuse.utils.file_io.file_io import (
     load_hdf5,
-    save_hdf5_safe,
     load_pickle,
+    save_hdf5_safe,
     save_pickle_safe,
 )
-from fuse.data import (
-    get_sample_id,
-    create_initial_sample,
-    get_specific_sample_from_potentially_morphed,
-)
-import hashlib
-from fuse.utils.file_io import delete_directory_tree
-from glob import glob
 from fuse.utils.multiprocessing.run_multiprocessed import (
-    run_multiprocessed,
     get_from_global_storage,
+    run_multiprocessed,
 )
-from fuse.data.datasets.sample_caching_audit import SampleCachingAudit
-from fuse.data.utils.sample import get_initial_sample_id, set_initial_sample_id
-from warnings import warn
-import numpy as np
+from fuse.utils.ndict import NDict
 
 
 class SamplesCacher:

fuse/data/datasets/caching/tests/test_sample_caching.py

@@ -15,19 +15,19 @@
 
 """
 
+import os
+import tempfile
 import unittest
+from typing import List, Union
 
-from fuse.utils.rand.seed import Seed
-from fuse.data.pipelines.pipeline_default import PipelineDefault
-from fuse.data import get_sample_id, create_initial_sample
 import numpy as np
-import tempfile
-import os
-from fuse.data.ops.op_base import OpBase
-from typing import List, Union
-from fuse.data.datasets.caching.samples_cacher import SamplesCacher
 
+from fuse.data.datasets.caching.samples_cacher import SamplesCacher
+from fuse.data.ops.op_base import OpBase
+from fuse.data.pipelines.pipeline_default import PipelineDefault
+from fuse.data.utils.sample import create_initial_sample, get_sample_id
 from fuse.utils.ndict import NDict
+from fuse.utils.rand.seed import Seed
 
 
 class OpFakeLoad(OpBase):

fuse/data/datasets/dataset_default.py

@@ -16,27 +16,28 @@
 
 """
 
-from typing import Dict, Hashable, List, Optional, Sequence, Union, Any
-
+import copy
+from collections import OrderedDict
+from operator import itemgetter
+from typing import Any, Dict, Hashable, List, Optional, Sequence, Union
 from warnings import warn
+
+import numpy as np
+
+from fuse.data.datasets.caching.samples_cacher import SamplesCacher
 from fuse.data.datasets.dataset_base import DatasetBase
 from fuse.data.ops.ops_common import OpCollectMarker
 from fuse.data.pipelines.pipeline_default import PipelineDefault
-from fuse.data.datasets.caching.samples_cacher import SamplesCacher
-from fuse.utils.ndict import NDict
-from fuse.utils.multiprocessing.run_multiprocessed import (
-    run_multiprocessed,
-    get_from_global_storage,
-)
-from fuse.data import (
-    get_sample_id,
+from fuse.data.utils.sample import (
     create_initial_sample,
+    get_sample_id,
     get_specific_sample_from_potentially_morphed,
 )
-import copy
-from collections import OrderedDict
-import numpy as np
-from operator import itemgetter
+from fuse.utils.multiprocessing.run_multiprocessed import (
+    get_from_global_storage,
+    run_multiprocessed,
+)
+from fuse.utils.ndict import NDict
 
 
 class DatasetDefault(DatasetBase):

fuse/data/datasets/dataset_wrap_seq_to_dict.py

@@ -17,15 +17,15 @@
 
 """
 
-from typing import List, Optional, Union, Sequence
-from fuse.data.datasets.caching.samples_cacher import SamplesCacher
-from fuse.data.utils.sample import get_sample_id
+from typing import List, Optional, Sequence, Union
 
 from torch.utils.data import Dataset
 
-from fuse.data.ops.op_base import OpBase
+from fuse.data.datasets.caching.samples_cacher import SamplesCacher
 from fuse.data.datasets.dataset_default import DatasetDefault
+from fuse.data.ops.op_base import OpBase
 from fuse.data.pipelines.pipeline_default import PipelineDefault
+from fuse.data.utils.sample import get_sample_id
 from fuse.utils.ndict import NDict
 
 

fuse/data/datasets/sample_caching_audit.py

@@ -1,7 +1,9 @@
-from typing import Optional, Any
 from time import time
+from typing import Any, Optional
+
 from deepdiff import DeepDiff
-from fuse.data import get_sample_id
+
+from fuse.data.utils.sample import get_sample_id
 
 """
 By auditing the samples, "stale" caches can be found, which is very important to detect.

fuse/data/datasets/tests/test_dataset_default.py

@@ -15,21 +15,22 @@
 
 """
 
+import os
+import tempfile
 import unittest
+from typing import List, Optional, Union
 
-from fuse.utils.rand.seed import Seed
-
-# from fuse.utils.file_io.file_io import SAFE_save_hdf5, load_hdf5
-from fuse.data.pipelines.pipeline_default import PipelineDefault
-from fuse.data import get_sample_id, create_initial_sample
 import numpy as np
-import tempfile
-import os
-from fuse.data.ops.op_base import OpBase
-from typing import List, Union, Optional
+
 from fuse.data.datasets.caching.samples_cacher import SamplesCacher
 from fuse.data.datasets.dataset_default import DatasetDefault
+from fuse.data.ops.op_base import OpBase
+
+# from fuse.utils.file_io.file_io import SAFE_save_hdf5, load_hdf5
+from fuse.data.pipelines.pipeline_default import PipelineDefault
+from fuse.data.utils.sample import create_initial_sample, get_sample_id
 from fuse.utils.ndict import NDict
+from fuse.utils.rand.seed import Seed
 
 
 class OpFakeLoad(OpBase):

fuse/data/datasets/tests/test_dataset_default_audit_feature.py

@@ -15,19 +15,20 @@
 
 """
 
+import os
+import tempfile
 import unittest
-from fuse.utils.rand.seed import Seed
-from fuse.utils.ndict import NDict
+from typing import List, Union
 
-from fuse.data.pipelines.pipeline_default import PipelineDefault
-from fuse.data import get_sample_id, create_initial_sample
 import numpy as np
-import tempfile
-import os
-from fuse.data.ops.op_base import OpBase
-from typing import List, Union
+
 from fuse.data.datasets.caching.samples_cacher import SamplesCacher
 from fuse.data.datasets.dataset_default import DatasetDefault
+from fuse.data.ops.op_base import OpBase
+from fuse.data.pipelines.pipeline_default import PipelineDefault
+from fuse.data.utils.sample import create_initial_sample, get_sample_id
+from fuse.utils.ndict import NDict
+from fuse.utils.rand.seed import Seed
 
 
 class OpFakeLoad(OpBase):

fuse/data/datasets/tests/test_dataset_wrap_seq_to_dict.py

@@ -16,17 +16,16 @@
 """
 
 import os
-import unittest
-
 import random
+import tempfile
+import unittest
 
 import torchvision
 from torchvision import transforms
-from fuse.utils.rand.seed import Seed
-from fuse.utils.ndict import NDict
 
-import tempfile
 from fuse.data.datasets.dataset_wrap_seq_to_dict import DatasetWrapSeqToDict
+from fuse.utils.ndict import NDict
+from fuse.utils.rand.seed import Seed
 
 
 class TestDatasetWrapSeqToDict(unittest.TestCase):

fuse/data/key_types.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
 from enum import Enum
-from typing import Dict, Sequence, Any
+from typing import Any, Dict, Sequence
+
 from fuse.data.patterns import Patterns
 
 

fuse/data/key_types_for_testing.py

@@ -1,4 +1,5 @@
 from enum import Enum
+
 from fuse.data.key_types import DataTypeBasic, TypeDetectorPatternsBased
 
 

fuse/data/ops/caching_tools.py

@@ -1,11 +1,12 @@
+import hashlib
 import inspect
-from typing import Callable, Any, Type, Optional, Sequence, List
-from inspect import stack
-import warnings
-from fuse.utils.file_io.file_io import load_pickle, save_pickle_safe
 import os
+import warnings
+from inspect import stack
+from typing import Any, Callable, List, Optional, Sequence, Type
+
 from fuse.utils.cpu_profiling import Timer
-import hashlib
+from fuse.utils.file_io.file_io import load_pickle, save_pickle_safe
 
 
 def get_function_call_str(

fuse/data/ops/hashable_class.py

@@ -1,4 +1,5 @@
-from typing import Callable, Any
+from typing import Any, Callable
+
 from fuse.data.ops import get_function_call_str
 from fuse.data.ops.caching_tools import get_callers_string_description, value_to_string
 

fuse/data/ops/op_base.py

@@ -16,13 +16,14 @@
 Created on June 30, 2021
 
 """
-from typing import Union, List, Optional
+import inspect
+import traceback
 from abc import abstractmethod
+from typing import List, Optional, Union
+
+from fuse.data.ops.hashable_class import HashableClass
 from fuse.data.utils.sample import get_sample_id
 from fuse.utils.ndict import NDict
-from fuse.data.ops.hashable_class import HashableClass
-import inspect
-import traceback
 
 
 class OpBase(HashableClass):