Support Python3.11 and drop Python 3.7

.gitlab-ci.yml

@@ -5,11 +5,11 @@ variables:
   PYTHONIOENCODING: "utf-8"
   DEBIAN_FRONTEND: "noninteractive"
   TOIL_OWNER_TAG: "shared"
-  MAIN_PYTHON_PKG: "python3.9"
+  MAIN_PYTHON_PKG: "python3.11"
   # Used to tell pytest which tests to be run by specifying markers,
   # Allows partitioning of tests to prevent duplicate running of tests in different jobs.
   # Currently specifies special tests that are not run by quick_test_offline.
-  MARKER: "(tes or integrative or encryption or mesos or server_mode or fetchable_appliance or appliance or slow or docker or cwl or singularity or rsync3) and not kubernetes"
+  MARKER: "(tes or integrative or encryption or server_mode or fetchable_appliance or appliance or slow or docker or cwl or singularity or rsync3) and not kubernetes"
   TEST_THREADS: "3"
 before_script:
   # Log where we are running, in case some Kubernetes hosts are busted. IPs are assigned per host.
@@ -68,7 +68,9 @@ lint:
   stage: linting_and_dependencies
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
+    - python3.11 -m pip freeze
+    - python3.11 --version
     - make mypy
     - make docs
     # - make diff_pydocstyle_report
@@ -105,55 +107,57 @@ quick_test_offline:
     - make develop extras=[aws,google,wdl]
     - TOIL_TEST_QUICK=True make test_offline threads="${TEST_THREADS}"
 
-py37_appliance_build:
+py38_appliance_build:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
   stage: basic_tests
   script:
     - pwd
-    - python3.7 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.8 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
 
-py38_appliance_build:
+py39_appliance_build:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
   stage: basic_tests
   script:
     - pwd
-    - python3.8 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
 
-py39_appliance_build:
+py310_appliance_build:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
   stage: basic_tests
   script:
     - pwd
-    - python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
 
-py310_appliance_build:
+py311_appliance_build:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
   stage: basic_tests
   script:
     - pwd
-    - python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
 
-py310_main:
+py311_main:
   rules:
     - if: $CI_PIPELINE_SOURCE != "schedule"
   stage: basic_tests
   script:
     - pwd
-    - python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     - make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
     - TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
 
@@ -163,7 +167,7 @@ batch_systems:
   stage: main_tests
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     - wget https://github.com/ohsu-comp-bio/funnel/releases/download/0.10.1/funnel-linux-amd64-0.10.1.tar.gz
     - tar -xvf funnel-linux-amd64-0.10.1.tar.gz funnel
     - export FUNNEL_SERVER_USER=toil
@@ -188,7 +192,7 @@ batch_systems:
     - export TOIL_TES_ENDPOINT="http://127.0.0.1:8000"
     - export TOIL_TES_USER="${FUNNEL_SERVER_USER}"
     - export TOIL_TES_PASSWORD="${FUNNEL_SERVER_PASSWORD}"
-    - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/batchSystems/batchSystemTest.py src/toil/test/mesos/MesosDataStructuresTest.py"
+#    - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/batchSystems/batchSystemTest.py src/toil/test/mesos/MesosDataStructuresTest.py"
     - kill $(jobs -p) || true
 
 slurm_test:
@@ -262,7 +266,7 @@ wdl:
   script:
     - pwd
     - apt update && apt install -y default-jre
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server]
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests=src/toil/test/wdl/wdltoil_test.py
     - which java &> /dev/null || { echo >&2 "Java is not installed.  Install java to run these tests."; exit 1; }
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/wdl/toilwdlTest.py src/toil/test/wdl/builtinTest.py"  # needs java (default-jre) to run "GATK.jar"
@@ -273,7 +277,7 @@ jobstore:
   stage: main_tests
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1' 
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/jobStores/jobStoreTest.py src/toil/test/sort/sortTest.py"
 
 provisioner:
@@ -282,7 +286,7 @@ provisioner:
   stage: main_tests
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1' 
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/lib/aws/ src/toil/test/provisioners/aws/awsProvisionerTest.py src/toil/test/provisioners/clusterScalerTest.py"
 
 # https://ucsc-ci.com/databiosphere/toil/-/jobs/38672
@@ -294,7 +298,7 @@ jobstore_integration:
   stage: integration
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     - export TOIL_TEST_INTEGRATIVE=True
     - export TOIL_AWS_KEYNAME=id_rsa
     - export TOIL_AWS_ZONE=us-west-2a
@@ -309,7 +313,7 @@ server_integration:
   stage: integration
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     - export TOIL_TEST_INTEGRATIVE=True
     - export TOIL_AWS_KEYNAME=id_rsa
     - export TOIL_AWS_ZONE=us-west-2a
@@ -325,7 +329,7 @@ provisioner_integration:
   stage: integration
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
     - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
     - export LIBPROCESS_IP=127.0.0.1
@@ -342,7 +346,7 @@ google_jobstore:
   stage: integration
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
     - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
     - export LIBPROCESS_IP=127.0.0.1
@@ -352,6 +356,31 @@ google_jobstore:
     - export TOIL_GOOGLE_PROJECTID=toil-dev
     - make test threads="${TEST_THREADS}" tests=src/toil/test/jobStores/jobStoreTest.py::GoogleJobStoreTest
 
+mesos:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG
+  stage: integration
+  script:
+    - pwd
+    - python3.10 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[mesos] packages='htcondor==10.2.3'
+    - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
+    - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
+    - export LIBPROCESS_IP=127.0.0.1
+    - export TOIL_TEST_INTEGRATIVE=True
+    - export TOIL_AWS_KEYNAME=id_rsa
+    - export TOIL_AWS_ZONE=us-west-2a
+    - export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_CREDENTIALS
+    - export TOIL_GOOGLE_KEYNAME=id_rsa
+    - export TOIL_GOOGLE_PROJECTID=toil-dev
+    - make test threads="${TEST_THREADS}" src/toil/test/mesos/MesosDataStructuresTest.py::DataStructuresTest
+    - make test threads="${TEST_THREADS}" src/toil/test/batchSystems/batchSystemTest.py::MesosBatchSystemTest
+    - make test threads="${TEST_THREADS}" src/toil/test/sort/sortTest.py::SortTest::testAwsMesos
+    - make test threads="${TEST_THREADS}" src/toil/test/sort/sortTest.py::SortTest::testFileMesos
+    - make test threads="${TEST_THREADS}" src/toil/test/sort/sortTest.py::SortTest::testGoogleMesos
+    - make test threads="${TEST_THREADS}" src/toil/test/cwl/cwlTest.py::CWLv10Test::test_mesos_cwl_conformance
+    - make test threads="${TEST_THREADS}" src/toil/test/cwl/cwlTest.py::CWLv10Test::test_mesos_cwl_conformance_with_caching
+    - make test threads="${TEST_THREADS}" src/toil/test/src/promisedRequirementTest.py::MesosPromisedRequirementsTest
+
 # Cactus-on-Kubernetes integration (as a script and not a pytest test)
 #cactus_integration:
 #  stage: integration

contrib/toil-ci-prebake/Dockerfile

@@ -32,9 +32,16 @@ RUN apt-get -q -y update && \
 RUN sudo add-apt-repository -y ppa:deadsnakes/ppa && \
     sudo apt-get update && \
     sudo apt-get install -y \
-        python3.7 python3.7-dev python3.7-venv python3.7-distutils \
         python3.8 python3.8-dev python3.8-venv python3.8-distutils \
         python3.9 python3.9-dev python3.9-venv python3.9-distutils \
-        python3.10 python3.10-dev python3.10-venv python3.10-distutils
+        python3.10 python3.10-dev python3.10-venv python3.10-distutils \
+        python3.11 python3.11-dev python3.11-venv python3.11-distutils \
+        python3.12 python3.12-dev python3.12-venv python3.12-distutils
+
+RUN rm -f /usr/bin/python
+RUN ln -s /usr/bin/python3.11 /usr/bin/python
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3.8 get-pip.py && python3.9 get-pip.py && python3.10 get-pip.py && python3.11 get-pip.py && python3.12 get-pip.py # for python 3.12, get-pip seems to remove other pip versions
+RUN python3.12 -m pip install virtualenv -U && python3.8 -m pip install virtualenv && python3.9 -m pip install virtualenv && python3.10 -m pip install virtualenv && python3.11 -m pip install virtualenv # python3.12 needs latest version of virtualenv
+
 
 LABEL maintainers="[email protected] [email protected]"

docker/Dockerfile.py

@@ -28,10 +28,11 @@
 dependencies = ' '.join(['libffi-dev',  # For client side encryption for extras with PyNACL
                          python,
                          f'{python}-dev',
-                         'python3.7-distutils' if python == 'python3.7' else '',
                          'python3.8-distutils' if python == 'python3.8' else '',
                          'python3.9-distutils' if python == 'python3.9' else '',
                          'python3.10-distutils' if python == 'python3.10' else '',
+                         'python3.11-distutils' if python == 'python3.11' else '',
+                         'python3.12-distutils' if python== 'python3.12' else '',
                          'python3-pip',
                          'libssl-dev',
                          'wget',

requirements.txt

@@ -1,8 +1,8 @@
 dill>=0.3.2, <0.4
 requests>=2, <3
-docker>=3.7.2, <6
+docker>=6, <7
 # Work around https://github.com/docker/docker-py/issues/3113
-urllib3>=1.26.0, <2.0.0
+urllib3>=2.0.0,<3
 python-dateutil
 psutil >= 3.0.1, <6
 py-tes>=0.4.2,<1

setup.py

@@ -11,7 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import imp
+# import imp
+import types
+from importlib.machinery import SourceFileLoader
 import os
 from tempfile import NamedTemporaryFile
 
@@ -82,10 +84,10 @@ def run_setup():
           'Operating System :: MacOS :: MacOS X',
           'Operating System :: POSIX',
           'Operating System :: POSIX :: Linux',
-          'Programming Language :: Python :: 3.7',
           'Programming Language :: Python :: 3.8',
           'Programming Language :: Python :: 3.9',
           'Programming Language :: Python :: 3.10',
+          'Programming Language :: Python :: 3.11',
           'Topic :: Scientific/Engineering',
           'Topic :: Scientific/Engineering :: Bio-Informatics',
           'Topic :: Scientific/Engineering :: Astronomy',
@@ -146,7 +148,10 @@ def import_version():
     #     return SourceFileLoader('toil.version', path='src/toil/version.py').load_module()
     #
     # Because SourceFileLoader will error and load "src/toil/__init__.py" .
-    return imp.load_source('toil.version', 'src/toil/version.py')
+    loader = SourceFileLoader('toil.version', 'src/toil/version.py')
+    mod = types.ModuleType(loader.name)
+    loader.exec_module(mod)
+    return mod
 
 
 version = import_version()