Merge remote-tracking branch 'upstream/master' into issues/4633-cwl-p…

…ipefish-compatibility
DataBiosphere · Nov 9, 2023 · f93d1c9 · f93d1c9
2 parents 50d6cea + bdb0286
commit f93d1c9
Show file tree

Hide file tree

Showing 15 changed files with 128 additions and 830 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -172,32 +172,6 @@ batch_systems:
   script:
     - pwd
     - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.3'
-    - wget https://github.com/ohsu-comp-bio/funnel/releases/download/0.10.1/funnel-linux-amd64-0.10.1.tar.gz
-    - tar -xvf funnel-linux-amd64-0.10.1.tar.gz funnel
-    - export FUNNEL_SERVER_USER=toil
-    - export FUNNEL_SERVER_PASSWORD=$(openssl rand -hex 256)
-    - |
-      cat >funnel.conf <<EOF
-      Server:
-        BasicAuth:
-          - User: ${FUNNEL_SERVER_USER}
-            Password: ${FUNNEL_SERVER_PASSWORD}
-      RPCClient:
-        User: ${FUNNEL_SERVER_USER}
-        Password: ${FUNNEL_SERVER_PASSWORD}
-      LocalStorage:
-        AllowedDirs:
-          - $HOME/.aws
-          - ./
-      Compute: manual
-      EOF
-    - ./funnel server run -c funnel.conf &
-    - ./funnel node run -c funnel.conf &
-    - export TOIL_TES_ENDPOINT="http://127.0.0.1:8000"
-    - export TOIL_TES_USER="${FUNNEL_SERVER_USER}"
-    - export TOIL_TES_PASSWORD="${FUNNEL_SERVER_PASSWORD}"
-#    - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/batchSystems/batchSystemTest.py src/toil/test/mesos/MesosDataStructuresTest.py"
-    - kill $(jobs -p) || true
 
 slurm_test:
   rules:

diff --git a/contrib/tes/funnel/kubernetes/setup.yml b/contrib/tes/funnel/kubernetes/setup.yml
diff --git a/docs/appendices/environment_vars.rst b/docs/appendices/environment_vars.rst
@@ -73,17 +73,6 @@ There are several environment variables that affect the way Toil runs.
 |                                  | instead of polling for running jobs. Default       |
 |                                  | value is set to False.                             |
 +----------------------------------+----------------------------------------------------+
-| TOIL_TES_ENDPOINT                | URL to the TES server to run against when using    |
-|                                  | the ``tes`` batch system.                          |
-+----------------------------------+----------------------------------------------------+
-| TOIL_TES_USER                    | Username to use with HTTP Basic Authentication to  |
-|                                  | log into the TES server.                           |
-+----------------------------------+----------------------------------------------------+
-| TOIL_TES_PASSWORD                | Password to use with HTTP Basic Authentication to  |
-|                                  | log into the TES server.                           |
-+----------------------------------+----------------------------------------------------+
-| TOIL_TES_BEARER_TOKEN            | Token to use to authenticate to the TES server.    |
-+----------------------------------+----------------------------------------------------+
 | TOIL_APPLIANCE_SELF              | The fully qualified reference for the Toil         |
 |                                  | Appliance you wish to use, in the form             |
 |                                  | ``REPO/IMAGE:TAG``.                                |

diff --git a/docs/running/cliOptions.rst b/docs/running/cliOptions.rst
@@ -221,14 +221,6 @@ levels in toil are based on priority from the logging module:
   --kubernetesPodTimeout KUBERNETES_POD_TIMEOUT
                         Seconds to wait for a scheduled Kubernetes pod to
                         start running. (default: 120s)
-  --tesEndpoint TES_ENDPOINT
-                        The http(s) URL of the TES server.
-                        (default: http://<leader IP>:8000)
-  --tesUser TES_USER    User name to use for basic authentication to TES server.
-  --tesPassword TES_PASSWORD
-                        Password to use for basic authentication to TES server.
-  --tesBearerToken TES_BEARER_TOKEN
-                        Bearer token to use for authentication to TES server.
   --awsBatchRegion AWS_BATCH_REGION
                         The AWS region containing the AWS Batch queue to submit
                         to.

diff --git a/requirements-cwl.txt b/requirements-cwl.txt
@@ -4,5 +4,5 @@ galaxy-tool-util<23
 galaxy-util<23
 ruamel.yaml>=0.15,<=0.18.3
 ruamel.yaml.clib>=0.2.6
-networkx>=2,<2.8.9
+networkx!=2.8.1,<4
 CacheControl[filecache]
diff --git a/requirements-wdl.txt b/requirements-wdl.txt
@@ -1,3 +1,3 @@
-miniwdl==1.10.0
+miniwdl==1.11.1
 wdlparse==0.1.0
 graphlib-backport==1.0 ; python_version < '3.9'
diff --git a/requirements.txt b/requirements.txt
@@ -5,7 +5,6 @@ docker>=3.7.2, <7
 urllib3>=1.26.0,<3
 python-dateutil
 psutil >= 3.0.1, <6
-py-tes>=0.4.2,<1
 PyPubSub >=4.0.3, <5
 addict>=2.2.1, <2.5
 pytz>=2012

diff --git a/setup.cfg b/setup.cfg
@@ -34,7 +34,6 @@ markers =
     slow
     slurm
     singularity
-    tes
     torque
     wes_server
     cwl_small_log_dir
@@ -68,7 +67,3 @@ no_warn_no_return = True
 
 [mypy-toil.cwl.*]
 strict = True
-
-[mypy-tes]
-ignore_errors = True
-follow_imports=skip
diff --git a/src/toil/batchSystems/contained_executor.py b/src/toil/batchSystems/contained_executor.py
@@ -14,7 +14,7 @@
 """
 Executor for running inside a container.
 
-Useful for Kubernetes and TES batch systems.
+Useful for Kubernetes batch system and TES batch system plugin.
 """
 import base64
 import logging