Move TES batch system to a plugin

.gitlab-ci.yml

@@ -166,32 +166,6 @@ batch_systems:
   script:
     - pwd
     - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.3'
-    - wget https://github.com/ohsu-comp-bio/funnel/releases/download/0.10.1/funnel-linux-amd64-0.10.1.tar.gz
-    - tar -xvf funnel-linux-amd64-0.10.1.tar.gz funnel
-    - export FUNNEL_SERVER_USER=toil
-    - export FUNNEL_SERVER_PASSWORD=$(openssl rand -hex 256)
-    - |
-      cat >funnel.conf <<EOF
-      Server:
-        BasicAuth:
-          - User: ${FUNNEL_SERVER_USER}
-            Password: ${FUNNEL_SERVER_PASSWORD}
-      RPCClient:
-        User: ${FUNNEL_SERVER_USER}
-        Password: ${FUNNEL_SERVER_PASSWORD}
-      LocalStorage:
-        AllowedDirs:
-          - $HOME/.aws
-          - ./
-      Compute: manual
-      EOF
-    - ./funnel server run -c funnel.conf &
-    - ./funnel node run -c funnel.conf &
-    - export TOIL_TES_ENDPOINT="http://127.0.0.1:8000"
-    - export TOIL_TES_USER="${FUNNEL_SERVER_USER}"
-    - export TOIL_TES_PASSWORD="${FUNNEL_SERVER_PASSWORD}"
-    - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/batchSystems/batchSystemTest.py"
-    - kill $(jobs -p) || true
 
 slurm_test:
   rules:

docs/appendices/environment_vars.rst

@@ -73,17 +73,6 @@ There are several environment variables that affect the way Toil runs.
 |                                  | instead of polling for running jobs. Default       |
 |                                  | value is set to False.                             |
 +----------------------------------+----------------------------------------------------+
-| TOIL_TES_ENDPOINT                | URL to the TES server to run against when using    |
-|                                  | the ``tes`` batch system.                          |
-+----------------------------------+----------------------------------------------------+
-| TOIL_TES_USER                    | Username to use with HTTP Basic Authentication to  |
-|                                  | log into the TES server.                           |
-+----------------------------------+----------------------------------------------------+
-| TOIL_TES_PASSWORD                | Password to use with HTTP Basic Authentication to  |
-|                                  | log into the TES server.                           |
-+----------------------------------+----------------------------------------------------+
-| TOIL_TES_BEARER_TOKEN            | Token to use to authenticate to the TES server.    |
-+----------------------------------+----------------------------------------------------+
 | TOIL_APPLIANCE_SELF              | The fully qualified reference for the Toil         |
 |                                  | Appliance you wish to use, in the form             |
 |                                  | ``REPO/IMAGE:TAG``.                                |

docs/running/cliOptions.rst

@@ -221,14 +221,6 @@ levels in toil are based on priority from the logging module:
   --kubernetesPodTimeout KUBERNETES_POD_TIMEOUT
                         Seconds to wait for a scheduled Kubernetes pod to
                         start running. (default: 120s)
-  --tesEndpoint TES_ENDPOINT
-                        The http(s) URL of the TES server.
-                        (default: http://<leader IP>:8000)
-  --tesUser TES_USER    User name to use for basic authentication to TES server.
-  --tesPassword TES_PASSWORD
-                        Password to use for basic authentication to TES server.
-  --tesBearerToken TES_BEARER_TOKEN
-                        Bearer token to use for authentication to TES server.
   --awsBatchRegion AWS_BATCH_REGION
                         The AWS region containing the AWS Batch queue to submit
                         to.

requirements.txt

@@ -5,7 +5,6 @@ docker>=3.7.2, <7
 urllib3>=1.26.0,<3
 python-dateutil
 psutil >= 3.0.1, <6
-py-tes>=0.4.2,<1
 PyPubSub >=4.0.3, <5
 addict>=2.2.1, <2.5
 pytz>=2012

setup.cfg

@@ -34,7 +34,6 @@ markers =
     slow
     slurm
     singularity
-    tes
     torque
     wes_server
     cwl_small_log_dir
@@ -68,7 +67,3 @@ no_warn_no_return = True
 
 [mypy-toil.cwl.*]
 strict = True
-
-[mypy-tes]
-ignore_errors = True
-follow_imports=skip

src/toil/batchSystems/contained_executor.py

@@ -14,7 +14,7 @@
 """
 Executor for running inside a container.
 
-Useful for Kubernetes and TES batch systems.
+Useful for Kubernetes batch system and TES batch system plugin.
 """
 import base64
 import logging

src/toil/batchSystems/options.py

@@ -21,8 +21,8 @@
 else:
     from typing_extensions import Protocol
 
-from toil.batchSystems.registry import (BATCH_SYSTEM_FACTORY_REGISTRY,
-                                        BATCH_SYSTEMS,
+from toil.batchSystems.registry import (get_batch_system,
+                                        get_batch_systems,
                                         DEFAULT_BATCH_SYSTEM)
 from toil.lib.threading import cpu_count
 
@@ -55,14 +55,14 @@ def set_batchsystem_options(batch_system: Optional[str], set_option: OptionSette
     """
     if batch_system is not None:
         # Use this batch system
-        batch_system_type = BATCH_SYSTEM_FACTORY_REGISTRY[batch_system]()
+        batch_system_type = get_batch_system(batch_system)
         batch_system_type.setOptions(set_option)
     else:
-        for factory in BATCH_SYSTEM_FACTORY_REGISTRY.values():
+        for name in get_batch_systems():
             # All the batch systems are responsible for setting their own options
             # with their setOptions() class methods.
             try:
-                batch_system_type = factory()
+                batch_system_type = get_batch_system(name)
             except ImportError:
                 # Skip anything we can't import
                 continue
@@ -86,9 +86,9 @@ def add_all_batchsystem_options(parser: Union[ArgumentParser, _ArgumentGroup]) -
         "--batchSystem",
         dest="batchSystem",
         default=DEFAULT_BATCH_SYSTEM,
-        choices=BATCH_SYSTEMS,
+        choices=get_batch_systems(),
         help=f"The type of batch system to run the job(s) with, currently can be one "
-        f"of {', '.join(BATCH_SYSTEMS)}. default={DEFAULT_BATCH_SYSTEM}",
+        f"of {', '.join(get_batch_systems())}. default={DEFAULT_BATCH_SYSTEM}",
     )
     parser.add_argument(
         "--disableHotDeployment",
@@ -175,14 +175,14 @@ def add_all_batchsystem_options(parser: Union[ArgumentParser, _ArgumentGroup]) -
              "systems such as gridengine, htcondor, torque, slurm, and lsf."
     )
 
-    for factory in BATCH_SYSTEM_FACTORY_REGISTRY.values():
+    for name in get_batch_systems():
         # All the batch systems are responsible for adding their own options
         # with the add_options class method.
         try:
-            batch_system_type = factory()
+            batch_system_type = get_batch_system(name)
         except ImportError:
             # Skip anything we can't import
             continue
         # Ask the batch system to create its options in the parser
-        logger.debug('Add options for %s', batch_system_type)
-        batch_system_type.add_options(parser)
+        logger.debug('Add options for %s batch system', name)
+        batch_system_type.add_options(parser)