Move out dependency of rest modules in extra_requires in forte (#760)

* import error for wikipedia

* import error for transformers

* import error for tqdm

* add more import tests

* remove pytorch installation at .github/workflows/main.yml

* test backbone in an independent job at .github/workflows/main.yml

* install pytest in .github/workflows/main.yml

* remove RemoteProcessor that requires extra module at tests/forte/forte_backbone_test.py

* add requests in remoe extra require at setup.py

* remove init import RemoteProcessor at forte/processors/misc/__init__.py

* try-except ImportError for requests at forte/processors/misc/remote_processor.py

* change relative importing path for RemoteProcessor at tests/forte/remote_processor_test.py

* try-except ImportError for module requests at forte/processors/third_party/machine_translation_processor.py

* try-except ImportError for module requests at forte/processors/data_augment/algorithms/character_flip_op.py

* try-except ImportError for module requests at forte/processors/data_augment/algorithms/distribution_replacement_op.py

* add requests requirement in forte[data_aug] in setup.py

* try-except ImportError for module requests at forte/processors/data_augment/algorithms/typo_replacement_op.py

* pylint: raise errors from errors and disable import-outside-toplevel

* pylint: raise errors from errors and disable import-outside-toplevel

* forte_backbone_test.py  pytest -> regular running python script

* move requests importing inside init for RemoteProcessor

* revert remove init import RemoteProcessor at forte/processors/misc/__init__.py

* remove tensorflow and pytorch version in the test_backbone job

* simplify test_backbone job

.github/workflows/main.yml

@@ -70,13 +70,7 @@ jobs:
           # Simply keep the database file but remove the repo.
           cp stave/simple-backend/db.sqlite3 .
           rm -rf stave
-      - name: Test backbone Forte import test
-        run: |
-          pip install --progress-bar off torch==${{ matrix.torch-version }}
-          # Try to install Forte backbone only and test basic imports.
-          pip install --use-feature=in-tree-build --progress-bar off .
-          pytest tests/forte/forte_backbone_test.py
-          pip uninstall -y torch
+
       - name: Install Forte
         run: |
           pip install --use-feature=in-tree-build --progress-bar off .[data_aug,ir,remote,audio_ext,stave,models,test,wikipedia,nlp,extractor]
@@ -116,7 +110,29 @@ jobs:
             cd ..
             coverage run -m pytest tests/forte/notebooks
           fi
-
+  test_backbone:
+    runs-on: ubuntu-latest
+    env:
+      python-version: 3.7
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ env.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: Test backbone Forte import test
+        run: |
+          # Try to install Forte backbone only and test basic imports.
+          pip install --use-feature=in-tree-build --progress-bar off .
+          python tests/forte/forte_backbone_test.py
 
   test_modules:
     runs-on: ubuntu-latest

forte/datasets/wikipedia/dbpedia/db_utils.py

@@ -21,8 +21,17 @@
 import sys
 from typing import List, Dict, Tuple, Optional
 from urllib.parse import urlparse, parse_qs
+from forte.utils import create_import_error_msg
+
+try:
+    import rdflib
+except ImportError as e:
+    raise ImportError(
+        create_import_error_msg(
+            "rbflib", "wikipedia", "DBpedia dataset supports"
+        )
+    ) from e
 
-import rdflib
 
 dbpedia_prefix = "http://dbpedia.org/resource/"
 state_type = Tuple[rdflib.term.Node, rdflib.term.Node, rdflib.term.Node]

forte/datasets/wikipedia/dbpedia/dbpedia_datasets.py

@@ -31,7 +31,7 @@
     Optional,
 )
 
-import rdflib
+
 from smart_open import open
 
 from forte.common import Resources
@@ -49,6 +49,7 @@
     state_type,
 )
 from forte.processors.base import PackWriter
+from forte.utils import create_import_error_msg
 from ft.onto.wikipedia import (
     WikiPage,
     WikiSection,
@@ -60,6 +61,16 @@
     WikiCategory,
 )
 
+try:
+    import rdflib
+except ImportError as e:
+    raise ImportError(
+        create_import_error_msg(
+            "rbflib", "wikipedia", "DBpedia dataset supports"
+        )
+    ) from e
+
+
 __all__ = [
     "DBpediaWikiReader",
     "WikiPackReader",

forte/processors/data_augment/algorithms/character_flip_op.py

@@ -15,13 +15,14 @@
 import random
 import json
 from typing import Tuple, Any, Dict, Union
-import requests
+
 
 from forte.data.ontology import Annotation
 from forte.common.configuration import Config
 from forte.processors.data_augment.algorithms.single_annotation_op import (
     SingleAnnotationAugmentOp,
 )
+from forte.utils import create_import_error_msg
 
 __all__ = ["CharacterFlipOp"]
 
@@ -45,9 +46,18 @@ class CharacterFlipOp(SingleAnnotationAugmentOp):
     """
 
     def __init__(self, configs: Union[Config, Dict[str, Any]]) -> None:
+        try:
+            import requests  # pylint: disable=import-outside-toplevel
+        except ImportError as e:
+            raise ImportError(
+                create_import_error_msg(
+                    "requests", "data_aug", "data augment support"
+                )
+            ) from e
         super().__init__(configs)
 
         self.dict_path = self.configs["dict_path"]
+
         try:
             r = requests.get(self.dict_path)
             self.data = r.json()

forte/processors/data_augment/algorithms/distribution_replacement_op.py

@@ -14,14 +14,15 @@
 import json
 import random
 from typing import Tuple, Union, Dict, Any
-import requests
+
 from forte.common.configurable import Configurable
 from forte.common.configuration import Config
 from forte.data.ontology import Annotation
 from forte.processors.data_augment.algorithms.single_annotation_op import (
     SingleAnnotationAugmentOp,
 )
 from forte.utils.utils import create_class_with_kwargs
+from forte.utils import create_import_error_msg
 
 __all__ = [
     "DistributionReplacementOp",
@@ -64,6 +65,15 @@ def cofigure_sampler(self) -> None:
         used by the distribution replacement op. The sampler will be set
         according to the configuration values
         """
+        try:
+            import requests  # pylint: disable=import-outside-toplevel
+        except ImportError as e:
+            raise ImportError(
+                create_import_error_msg(
+                    "requests", "data_aug", "data augment support"
+                )
+            ) from e
+
         try:
             if "data_path" in self.configs["sampler_config"]["kwargs"]:
                 distribution_path = self.configs["sampler_config"]["kwargs"][

forte/processors/data_augment/algorithms/typo_replacement_op.py

@@ -17,12 +17,13 @@
 import json
 from typing import Tuple, Union, Dict, Any
 
-import requests
+
 from forte.data.ontology import Annotation
 from forte.processors.data_augment.algorithms.single_annotation_op import (
     SingleAnnotationAugmentOp,
 )
 from forte.common.configuration import Config
+from forte.utils import create_import_error_msg
 
 __all__ = [
     "UniformTypoGenerator",
@@ -50,6 +51,15 @@ class UniformTypoGenerator:
     """
 
     def __init__(self, dict_path: str):
+        try:
+            import requests  # pylint: disable=import-outside-toplevel
+        except ImportError as e:
+            raise ImportError(
+                create_import_error_msg(
+                    "requests", "data_aug", "data augment support"
+                )
+            ) from e
+
         try:
             r = requests.get(dict_path)
             self.data = r.json()

forte/processors/misc/remote_processor.py

@@ -21,14 +21,14 @@
 import json
 import logging
 from typing import Dict, Set, Any, Optional
-import requests
 
 from forte.common import Resources, ProcessorConfigError
 from forte.common.configuration import Config
 from forte.data.data_pack import DataPack
 from forte.processors.base import PackProcessor
 from forte.utils import create_import_error_msg
 
+
 logger = logging.getLogger(__name__)
 
 __all__ = ["RemoteProcessor"]
@@ -53,6 +53,14 @@ class RemoteProcessor(PackProcessor):
 
     def __init__(self):
         super().__init__()
+        try:
+            import requests  # pylint: disable=import-outside-toplevel
+        except ImportError as e:
+            raise ImportError(
+                create_import_error_msg(
+                    "requests", "remote", "Remote Processor"
+                )
+            ) from e
         self._requests: Any = requests
         self._records: Optional[Dict[str, Set[str]]] = None
         self._expectation: Optional[Dict[str, Set[str]]] = None

forte/processors/third_party/machine_translation_processor.py

@@ -17,15 +17,26 @@
 from typing import Dict, Any
 from urllib.parse import urlencode
 import os
-import requests
-from transformers import T5Tokenizer, T5ForConditionalGeneration
+
 from forte.common.configuration import Config
 from forte.common.resources import Resources
 from forte.data.data_pack import DataPack
 from forte.data.multi_pack import MultiPack
 from forte.processors.base import MultiPackProcessor, PackProcessor
+from forte.utils import create_import_error_msg
 from ft.onto.base_ontology import Document, Utterance
 
+try:
+    from transformers import (  # pylint:disable=import-outside-toplevel
+        T5Tokenizer,
+        T5ForConditionalGeneration,
+    )
+except ImportError as err:
+    raise ImportError(
+        create_import_error_msg(
+            "transformers", "data_aug", "Machine Translator"
+        )
+    ) from err
 
 __all__ = ["MicrosoftBingTranslator", "MachineTranslationProcessor"]
 
@@ -80,6 +91,14 @@ def initialize(self, resources: Resources, configs: Config):
             self.out_pack_name = configs.out_pack_name
 
     def _process(self, input_pack: MultiPack):
+        try:
+            import requests  # pylint: disable=import-outside-toplevel
+        except ImportError as e:
+            raise ImportError(
+                create_import_error_msg(
+                    "requests", "data_aug", "data augment support"
+                )
+            ) from e
         query = input_pack.get_pack(self.in_pack_name).text
         params = "?" + urlencode(
             {

forte/trainer/ner_trainer.py

@@ -24,7 +24,7 @@
 from typing import List, Tuple, Iterator, Optional, Dict
 
 import numpy as np
-from tqdm import tqdm
+
 
 from forte.common.configuration import Config
 from forte.common.resources import Resources
@@ -34,6 +34,14 @@
 from forte.utils import create_import_error_msg
 from ft.onto.base_ontology import Token, Sentence
 
+
+try:
+    from tqdm import tqdm
+except ImportError as e:
+    raise ImportError(
+        create_import_error_msg("tqdm", "models", "models")
+    ) from e
+
 try:
     import torch
     from torch.optim import SGD

setup.py

@@ -10,9 +10,9 @@
     sys.exit("Python>=3.6 is required by Forte.")
 
 version = {}
-with open(os.path.join(
-    os.path.dirname(os.path.abspath(__file__)), "forte/version.py"
-)) as fp:
+with open(
+    os.path.join(os.path.dirname(os.path.abspath(__file__)), "forte/version.py")
+) as fp:
     exec(fp.read(), version)
 
 setuptools.setup(
@@ -50,12 +50,10 @@
             "transformers>=4.15.0",
             "nltk",
             "texar-pytorch>=0.1.4",
+            "requests",
         ],
         "ir": ["texar-pytorch>=0.1.4", "tensorflow>=1.15.0"],
-        "remote": [
-            "fastapi>=0.65.2",
-            "uvicorn>=0.14.0",
-        ],
+        "remote": ["fastapi>=0.65.2", "uvicorn>=0.14.0", "requests"],
         "audio_ext": ["soundfile>=0.10.3"],
         "stave": ["stave>=0.0.1.dev12"],
         "models": [