From 9391286fc52384dd790ba719412d4648e22d130e Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Mon, 25 Sep 2023 16:27:03 -0700
Subject: [PATCH 01/25] Remove python 3.7 and add python 3.11 and make
 python3.11 the main python package

---
 .gitlab-ci.yml | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a3f4bef60f..d833dc8d77 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,7 +5,7 @@ variables:
   PYTHONIOENCODING: "utf-8"
   DEBIAN_FRONTEND: "noninteractive"
   TOIL_OWNER_TAG: "shared"
-  MAIN_PYTHON_PKG: "python3.9"
+  MAIN_PYTHON_PKG: "python3.11"
   # Used to tell pytest which tests to be run by specifying markers,
   # Allows partitioning of tests to prevent duplicate running of tests in different jobs.
   # Currently specifies special tests that are not run by quick_test_offline.
@@ -97,47 +97,47 @@ quick_test_offline:
     - make develop extras=[aws,google,wdl]
     - TOIL_TEST_QUICK=True make test_offline threads="${TEST_THREADS}"
 
-py37_appliance_build:
+py38_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - python3.7 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.8 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
 
-py38_appliance_build:
+py39_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - python3.8 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
 
-py39_appliance_build:
+py310_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
 
-py310_appliance_build:
+py311_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
 
-py310_main:
+py311_main:
   stage: basic_tests
   script:
     - pwd
-    - python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
     - make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
     - TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
 

From 226b8956b3f969fce20e536924925f4d9b12b7c0 Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Mon, 25 Sep 2023 16:53:46 -0700
Subject: [PATCH 02/25] Move main python package back to 3.9

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d833dc8d77..8c5d7b6026 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,7 +5,7 @@ variables:
   PYTHONIOENCODING: "utf-8"
   DEBIAN_FRONTEND: "noninteractive"
   TOIL_OWNER_TAG: "shared"
-  MAIN_PYTHON_PKG: "python3.11"
+  MAIN_PYTHON_PKG: "python3.9"
   # Used to tell pytest which tests to be run by specifying markers,
   # Allows partitioning of tests to prevent duplicate running of tests in different jobs.
   # Currently specifies special tests that are not run by quick_test_offline.

From 787475f73647fe301ac62bb481c52f7b7dcb29f5 Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Mon, 25 Sep 2023 18:07:33 -0700
Subject: [PATCH 03/25] Incude python3.11 in docker

---
 docker/Dockerfile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile.py b/docker/Dockerfile.py
index 722579ba5d..2117de3f19 100644
--- a/docker/Dockerfile.py
+++ b/docker/Dockerfile.py
@@ -28,10 +28,10 @@
 dependencies = ' '.join(['libffi-dev',  # For client side encryption for extras with PyNACL
                          python,
                          f'{python}-dev',
-                         'python3.7-distutils' if python == 'python3.7' else '',
                          'python3.8-distutils' if python == 'python3.8' else '',
                          'python3.9-distutils' if python == 'python3.9' else '',
                          'python3.10-distutils' if python == 'python3.10' else '',
+                         'python3.11-distutils' if python == 'python3.11' else '',
                          'python3-pip',
                          'libssl-dev',
                          'wget',

From cedbceaea3889872bb77348658ada404229ae425 Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Tue, 26 Sep 2023 12:03:27 -0700
Subject: [PATCH 04/25] Test 3.11 in CI

---
 .gitlab-ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8c5d7b6026..65339014d6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,7 +5,7 @@ variables:
   PYTHONIOENCODING: "utf-8"
   DEBIAN_FRONTEND: "noninteractive"
   TOIL_OWNER_TAG: "shared"
-  MAIN_PYTHON_PKG: "python3.9"
+  MAIN_PYTHON_PKG: "python3.11"
   # Used to tell pytest which tests to be run by specifying markers,
   # Allows partitioning of tests to prevent duplicate running of tests in different jobs.
   # Currently specifies special tests that are not run by quick_test_offline.
@@ -76,6 +76,7 @@ cwl_dependency_is_stand_alone:
   stage: linting_and_dependencies
   script:
     - pwd
+    - python3 --version
     - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[cwl]
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests=src/toil/test/docs/scriptsTest.py::ToilDocumentationTest::testCwlexample
 

From 71b1e0d3857d44b0463fdbb372d09b046ddbfd49 Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Tue, 26 Sep 2023 12:09:27 -0700
Subject: [PATCH 05/25] Add python3.11 to CI dockerfile

---
 contrib/toil-ci-prebake/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contrib/toil-ci-prebake/Dockerfile b/contrib/toil-ci-prebake/Dockerfile
index 4fb4b4f583..30c109a75a 100644
--- a/contrib/toil-ci-prebake/Dockerfile
+++ b/contrib/toil-ci-prebake/Dockerfile
@@ -32,9 +32,9 @@ RUN apt-get -q -y update && \
 RUN sudo add-apt-repository -y ppa:deadsnakes/ppa && \
     sudo apt-get update && \
     sudo apt-get install -y \
-        python3.7 python3.7-dev python3.7-venv python3.7-distutils \
         python3.8 python3.8-dev python3.8-venv python3.8-distutils \
         python3.9 python3.9-dev python3.9-venv python3.9-distutils \
-        python3.10 python3.10-dev python3.10-venv python3.10-distutils
+        python3.10 python3.10-dev python3.10-venv python3.10-distutils \
+        python3.11 python3.11-dev python3.11-venv python3.11-distutils
 
 LABEL maintainers="anovak@soe.ucsc.edu lblauvel@ucsc.edu"

From bc8f03b52ddaee390ad440e30a6a77a895b74c8c Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Tue, 26 Sep 2023 13:13:23 -0700
Subject: [PATCH 06/25] Add 3.11 to setup.py and debugging statements

---
 .gitlab-ci.yml | 4 +++-
 setup.py       | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 65339014d6..54b9fc1c68 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -76,7 +76,9 @@ cwl_dependency_is_stand_alone:
   stage: linting_and_dependencies
   script:
     - pwd
-    - python3 --version
+    - python3.9 --version
+    - python3.10 --version
+    - python3.11 --version
     - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[cwl]
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests=src/toil/test/docs/scriptsTest.py::ToilDocumentationTest::testCwlexample
 
diff --git a/setup.py b/setup.py
index 67c23b0611..e48be9c61f 100755
--- a/setup.py
+++ b/setup.py
@@ -82,10 +82,10 @@ def run_setup():
           'Operating System :: MacOS :: MacOS X',
           'Operating System :: POSIX',
           'Operating System :: POSIX :: Linux',
-          'Programming Language :: Python :: 3.7',
           'Programming Language :: Python :: 3.8',
           'Programming Language :: Python :: 3.9',
           'Programming Language :: Python :: 3.10',
+          'Programming Language :: Python :: 3.11',
           'Topic :: Scientific/Engineering',
           'Topic :: Scientific/Engineering :: Bio-Informatics',
           'Topic :: Scientific/Engineering :: Astronomy',

From 37ff2ae38aef46d15f2b2a1d4c090c7c418ae6bb Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Fri, 13 Oct 2023 11:50:59 -0700
Subject: [PATCH 07/25] Python 3.7 backwards compatibility

---
 contrib/toil-ci-prebake/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contrib/toil-ci-prebake/Dockerfile b/contrib/toil-ci-prebake/Dockerfile
index 30c109a75a..b43f6b1c8f 100644
--- a/contrib/toil-ci-prebake/Dockerfile
+++ b/contrib/toil-ci-prebake/Dockerfile
@@ -32,6 +32,7 @@ RUN apt-get -q -y update && \
 RUN sudo add-apt-repository -y ppa:deadsnakes/ppa && \
     sudo apt-get update && \
     sudo apt-get install -y \
+        python3.7 python3.7-dev python3.7-venv python3.7-distutils \
         python3.8 python3.8-dev python3.8-venv python3.8-distutils \
         python3.9 python3.9-dev python3.9-venv python3.9-distutils \
         python3.10 python3.10-dev python3.10-venv python3.10-distutils \

From 5dddf0bb6559f37f6b416d5b669fcb87092851ed Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Thu, 19 Oct 2023 13:52:14 -0700
Subject: [PATCH 08/25] Update to py 3.12 and run 3.12 on gitlab CI

---
 .gitlab-ci.yml                     | 35 +++++++++++++++---------------
 contrib/toil-ci-prebake/Dockerfile |  3 ++-
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 44a0c60097..3b73aa45fe 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,7 +5,7 @@ variables:
   PYTHONIOENCODING: "utf-8"
   DEBIAN_FRONTEND: "noninteractive"
   TOIL_OWNER_TAG: "shared"
-  MAIN_PYTHON_PKG: "python3.11"
+  MAIN_PYTHON_PKG: "python3.12"
   # Used to tell pytest which tests to be run by specifying markers,
   # Allows partitioning of tests to prevent duplicate running of tests in different jobs.
   # Currently specifies special tests that are not run by quick_test_offline.
@@ -80,9 +80,6 @@ cwl_dependency_is_stand_alone:
   stage: linting_and_dependencies
   script:
     - pwd
-    - python3.9 --version
-    - python3.10 --version
-    - python3.11 --version
     - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[cwl]
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests=src/toil/test/docs/scriptsTest.py::ToilDocumentationTest::testCwlexample
 
@@ -108,64 +105,66 @@ quick_test_offline:
     - make develop extras=[aws,google,wdl]
     - TOIL_TEST_QUICK=True make test_offline threads="${TEST_THREADS}"
 
-py37_appliance_build:
+py38_appliance_build:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
   stage: basic_tests
   script:
     - pwd
-    - python3.7 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.8 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
 
-py38_appliance_build:
+py39_appliance_build:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
   stage: basic_tests
   script:
     - pwd
-    - python3.8 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
 
-py39_appliance_build:
+py310_appliance_build:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
   stage: basic_tests
   script:
     - pwd
-    - python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
 
-py310_appliance_build:
+py311_appliance_build:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
   stage: basic_tests
   script:
     - pwd
-    - python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
 
-py310_main:
-  rules:
-    - if: $CI_PIPELINE_SOURCE != "schedule"
+py312_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.12 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
 
-py311_main:
+py312_main:
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
   stage: basic_tests
   script:
     - pwd
-    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.12 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
     - make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
     - TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
 
diff --git a/contrib/toil-ci-prebake/Dockerfile b/contrib/toil-ci-prebake/Dockerfile
index b43f6b1c8f..619047c6a8 100644
--- a/contrib/toil-ci-prebake/Dockerfile
+++ b/contrib/toil-ci-prebake/Dockerfile
@@ -36,6 +36,7 @@ RUN sudo add-apt-repository -y ppa:deadsnakes/ppa && \
         python3.8 python3.8-dev python3.8-venv python3.8-distutils \
         python3.9 python3.9-dev python3.9-venv python3.9-distutils \
         python3.10 python3.10-dev python3.10-venv python3.10-distutils \
-        python3.11 python3.11-dev python3.11-venv python3.11-distutils
+        python3.11 python3.11-dev python3.11-venv python3.11-distutils \
+        python3.12 python3.12-dev python3.12-venv python3.12-distutils
 
 LABEL maintainers="anovak@soe.ucsc.edu lblauvel@ucsc.edu"

From 681a8ebc8ff3a3124f2fefdf772f1cde2eb931f4 Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Fri, 20 Oct 2023 12:24:05 -0700
Subject: [PATCH 09/25] Comment out fstring and try importlib

---
 docker/Dockerfile.py | 1 +
 requirements.txt     | 2 +-
 setup.py             | 9 +++++++--
 version_template.py  | 2 +-
 4 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/docker/Dockerfile.py b/docker/Dockerfile.py
index 2117de3f19..41ca162f8f 100644
--- a/docker/Dockerfile.py
+++ b/docker/Dockerfile.py
@@ -32,6 +32,7 @@
                          'python3.9-distutils' if python == 'python3.9' else '',
                          'python3.10-distutils' if python == 'python3.10' else '',
                          'python3.11-distutils' if python == 'python3.11' else '',
+                         'python3.12-distutils' if python== 'python3.12' else '',
                          'python3-pip',
                          'libssl-dev',
                          'wget',
diff --git a/requirements.txt b/requirements.txt
index a2c6d9e9ad..8f1cb4203d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@ dill>=0.3.2, <0.4
 requests>=2, <3
 docker>=3.7.2, <6
 # Work around https://github.com/docker/docker-py/issues/3113
-urllib3>=1.26.0, <2.0.0
+urllib3>=2.0.0,<3
 python-dateutil
 psutil >= 3.0.1, <6
 py-tes>=0.4.2,<1
diff --git a/setup.py b/setup.py
index e48be9c61f..0a111fc547 100755
--- a/setup.py
+++ b/setup.py
@@ -11,7 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import imp
+# import imp
+import types
+from importlib.machinery import SourceFileLoader
 import os
 from tempfile import NamedTemporaryFile
 
@@ -146,7 +148,10 @@ def import_version():
     #     return SourceFileLoader('toil.version', path='src/toil/version.py').load_module()
     #
     # Because SourceFileLoader will error and load "src/toil/__init__.py" .
-    return imp.load_source('toil.version', 'src/toil/version.py')
+    loader = SourceFileLoader('toil.version', 'src/toil/version.py')
+    mod = types.ModuleType(loader.name)
+    loader.exec_module(mod)
+    return mod
 
 
 version = import_version()
diff --git a/version_template.py b/version_template.py
index 23f7f1ed7e..2767ba9cca 100644
--- a/version_template.py
+++ b/version_template.py
@@ -77,7 +77,7 @@ def exactPython():
     for. Something like 'python2.7' or 'python3.6'.
     """
     import sys
-    return f'python{sys.version_info[0]}.{sys.version_info[1]}'
+    return 'python{sys.version_info[0]}.{sys.version_info[1]}'
 
 
 def python():

From 5889b976fff3db615b2d1ed0ceca866bd148e5a1 Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Fri, 20 Oct 2023 12:30:51 -0700
Subject: [PATCH 10/25] Debug lint

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3b73aa45fe..b4ad057d63 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -69,6 +69,8 @@ lint:
   script:
     - pwd
     - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - pip freeze
+    - python3.12 --version
     - make mypy
     - make docs
     # - make diff_pydocstyle_report

From feb8d8cfac845b8a66035314794b9ef2bc4c423c Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Fri, 20 Oct 2023 12:41:12 -0700
Subject: [PATCH 11/25] Ensure mypy is using python3.12

---
 .gitlab-ci.yml | 2 +-
 Makefile       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b4ad057d63..72afd0631b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -69,7 +69,7 @@ lint:
   script:
     - pwd
     - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
-    - pip freeze
+    - python3.12 -m pip freeze
     - python3.12 --version
     - make mypy
     - make docs
diff --git a/Makefile b/Makefile
index 2ace5221cd..968365bd2b 100644
--- a/Makefile
+++ b/Makefile
@@ -299,7 +299,7 @@ format: $(wildcard src/toil/cwl/*.py)
 	black $^ contrib/mypy-stubs
 
 mypy:
-	mypy --ignore-missing-imports --no-strict-optional \
+	python3.12 -m mypy --ignore-missing-imports --no-strict-optional \
 		--warn-redundant-casts --warn-unused-ignores \
 		$(CURDIR)/src/toil/cwl/cwltoil.py
 	$(CURDIR)/contrib/admin/mypy-with-ignore.py

From d6940741127488c33f38220fb2dca6681ea3eb86 Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Fri, 20 Oct 2023 12:44:23 -0700
Subject: [PATCH 12/25] Print python version beofre mypy

---
 Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 968365bd2b..02af05b91e 100644
--- a/Makefile
+++ b/Makefile
@@ -299,7 +299,8 @@ format: $(wildcard src/toil/cwl/*.py)
 	black $^ contrib/mypy-stubs
 
 mypy:
-	python3.12 -m mypy --ignore-missing-imports --no-strict-optional \
+	python --version
+	mypy --ignore-missing-imports --no-strict-optional \
 		--warn-redundant-casts --warn-unused-ignores \
 		$(CURDIR)/src/toil/cwl/cwltoil.py
 	$(CURDIR)/contrib/admin/mypy-with-ignore.py

From 74676fb5e36b7ff4132ce1b43695e92ea7393789 Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Mon, 23 Oct 2023 16:47:25 -0700
Subject: [PATCH 13/25] Fix virtualenv, pip for python3.12

---
 .gitlab-ci.yml                     | 19 +++++--------------
 contrib/toil-ci-prebake/Dockerfile |  6 ++++++
 2 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 72afd0631b..f8e2b0d5da 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,7 +5,7 @@ variables:
   PYTHONIOENCODING: "utf-8"
   DEBIAN_FRONTEND: "noninteractive"
   TOIL_OWNER_TAG: "shared"
-  MAIN_PYTHON_PKG: "python3.12"
+  MAIN_PYTHON_PKG: "python3.11"
   # Used to tell pytest which tests to be run by specifying markers,
   # Allows partitioning of tests to prevent duplicate running of tests in different jobs.
   # Currently specifies special tests that are not run by quick_test_offline.
@@ -69,8 +69,8 @@ lint:
   script:
     - pwd
     - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
-    - python3.12 -m pip freeze
-    - python3.12 --version
+    - python3.11 -m pip freeze
+    - python3.11 --version
     - make mypy
     - make docs
     # - make diff_pydocstyle_report
@@ -151,22 +151,13 @@ py311_appliance_build:
     - python setup_gitlab_docker.py
     - make push_docker
 
-py312_appliance_build:
-  stage: basic_tests
-  script:
-    - pwd
-    - python3.12 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
-    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
-    - python setup_gitlab_docker.py
-    - make push_docker
-
-py312_main:
+py311_main:
   rules:
     - if: $CI_PIPELINE_SOURCE != "schedule"
   stage: basic_tests
   script:
     - pwd
-    - python3.12 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
     - make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
     - TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
 
diff --git a/contrib/toil-ci-prebake/Dockerfile b/contrib/toil-ci-prebake/Dockerfile
index 619047c6a8..c5887fa867 100644
--- a/contrib/toil-ci-prebake/Dockerfile
+++ b/contrib/toil-ci-prebake/Dockerfile
@@ -39,4 +39,10 @@ RUN sudo add-apt-repository -y ppa:deadsnakes/ppa && \
         python3.11 python3.11-dev python3.11-venv python3.11-distutils \
         python3.12 python3.12-dev python3.12-venv python3.12-distutils
 
+RUN rm -f /usr/bin/python
+RUN ln -s /usr/bin/python3.12 /usr/bin/python
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py # for python 3.12
+RUN python get-pip.py
+RUN python -m pip install virtualenv -U # default virtualenv is about 2 years old and does not run with python 3.12
+
 LABEL maintainers="anovak@soe.ucsc.edu lblauvel@ucsc.edu"

From 4dcf89e72c304ea85b268cc58f9d964951a42593 Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Mon, 23 Oct 2023 16:58:51 -0700
Subject: [PATCH 14/25] Get rid of mesos tests/builds

---
 .gitlab-ci.yml | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f8e2b0d5da..fbd372c906 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -9,7 +9,7 @@ variables:
   # Used to tell pytest which tests to be run by specifying markers,
   # Allows partitioning of tests to prevent duplicate running of tests in different jobs.
   # Currently specifies special tests that are not run by quick_test_offline.
-  MARKER: "(tes or integrative or encryption or mesos or server_mode or fetchable_appliance or appliance or slow or docker or cwl or singularity or rsync3) and not kubernetes"
+  MARKER: "(tes or integrative or encryption or server_mode or fetchable_appliance or appliance or slow or docker or cwl or singularity or rsync3) and not kubernetes"
   TEST_THREADS: "3"
 before_script:
   # Log where we are running, in case some Kubernetes hosts are busted. IPs are assigned per host.
@@ -68,7 +68,7 @@ lint:
   stage: linting_and_dependencies
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
     - python3.11 -m pip freeze
     - python3.11 --version
     - make mypy
@@ -113,7 +113,7 @@ py38_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - python3.8 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.8 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
@@ -124,7 +124,7 @@ py39_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
@@ -135,7 +135,7 @@ py310_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
@@ -146,7 +146,7 @@ py311_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
@@ -157,7 +157,7 @@ py311_main:
   stage: basic_tests
   script:
     - pwd
-    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
     - make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
     - TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
 
@@ -167,7 +167,7 @@ batch_systems:
   stage: main_tests
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
     - wget https://github.com/ohsu-comp-bio/funnel/releases/download/0.10.1/funnel-linux-amd64-0.10.1.tar.gz
     - tar -xvf funnel-linux-amd64-0.10.1.tar.gz funnel
     - export FUNNEL_SERVER_USER=toil
@@ -192,7 +192,7 @@ batch_systems:
     - export TOIL_TES_ENDPOINT="http://127.0.0.1:8000"
     - export TOIL_TES_USER="${FUNNEL_SERVER_USER}"
     - export TOIL_TES_PASSWORD="${FUNNEL_SERVER_PASSWORD}"
-    - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/batchSystems/batchSystemTest.py src/toil/test/mesos/MesosDataStructuresTest.py"
+#    - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/batchSystems/batchSystemTest.py src/toil/test/mesos/MesosDataStructuresTest.py"
     - kill $(jobs -p) || true
 
 slurm_test:
@@ -266,7 +266,7 @@ wdl:
   script:
     - pwd
     - apt update && apt install -y default-jre
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server]
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests=src/toil/test/wdl/wdltoil_test.py
     - which java &> /dev/null || { echo >&2 "Java is not installed.  Install java to run these tests."; exit 1; }
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/wdl/toilwdlTest.py src/toil/test/wdl/builtinTest.py"  # needs java (default-jre) to run "GATK.jar"
@@ -277,7 +277,7 @@ jobstore:
   stage: main_tests
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1' 
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1' 
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/jobStores/jobStoreTest.py src/toil/test/sort/sortTest.py"
 
 provisioner:
@@ -286,7 +286,7 @@ provisioner:
   stage: main_tests
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1' 
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1' 
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/lib/aws/ src/toil/test/provisioners/aws/awsProvisionerTest.py src/toil/test/provisioners/clusterScalerTest.py"
 
 # https://ucsc-ci.com/databiosphere/toil/-/jobs/38672
@@ -298,7 +298,7 @@ jobstore_integration:
   stage: integration
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
     - export TOIL_TEST_INTEGRATIVE=True
     - export TOIL_AWS_KEYNAME=id_rsa
     - export TOIL_AWS_ZONE=us-west-2a
@@ -313,7 +313,7 @@ server_integration:
   stage: integration
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
     - export TOIL_TEST_INTEGRATIVE=True
     - export TOIL_AWS_KEYNAME=id_rsa
     - export TOIL_AWS_ZONE=us-west-2a
@@ -329,7 +329,7 @@ provisioner_integration:
   stage: integration
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
     - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
     - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
     - export LIBPROCESS_IP=127.0.0.1
@@ -346,7 +346,7 @@ google_jobstore:
   stage: integration
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
     - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
     - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
     - export LIBPROCESS_IP=127.0.0.1

From 767ae8b668f77c9318a0e8281f2c1ea206594d55 Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Mon, 23 Oct 2023 17:05:49 -0700
Subject: [PATCH 15/25] 3.12

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fbd372c906..c7d375db01 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,7 +5,7 @@ variables:
   PYTHONIOENCODING: "utf-8"
   DEBIAN_FRONTEND: "noninteractive"
   TOIL_OWNER_TAG: "shared"
-  MAIN_PYTHON_PKG: "python3.11"
+  MAIN_PYTHON_PKG: "python3.12"
   # Used to tell pytest which tests to be run by specifying markers,
   # Allows partitioning of tests to prevent duplicate running of tests in different jobs.
   # Currently specifies special tests that are not run by quick_test_offline.

From 9c1249d4fc87992698b29f0c10cfcaf2b6ff2bc4 Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Mon, 23 Oct 2023 17:13:05 -0700
Subject: [PATCH 16/25] Revert debug change

---
 version_template.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/version_template.py b/version_template.py
index 2767ba9cca..23f7f1ed7e 100644
--- a/version_template.py
+++ b/version_template.py
@@ -77,7 +77,7 @@ def exactPython():
     for. Something like 'python2.7' or 'python3.6'.
     """
     import sys
-    return 'python{sys.version_info[0]}.{sys.version_info[1]}'
+    return f'python{sys.version_info[0]}.{sys.version_info[1]}'
 
 
 def python():

From 209f4ad849db7fc56027fa58d206afca4c7d9ac2 Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Mon, 23 Oct 2023 18:21:57 -0700
Subject: [PATCH 17/25] Go back to 3.11 and update docker package to make
 requests work again

---
 .gitlab-ci.yml                     | 2 +-
 contrib/toil-ci-prebake/Dockerfile | 6 ++----
 requirements.txt                   | 2 +-
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c7d375db01..fbd372c906 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,7 +5,7 @@ variables:
   PYTHONIOENCODING: "utf-8"
   DEBIAN_FRONTEND: "noninteractive"
   TOIL_OWNER_TAG: "shared"
-  MAIN_PYTHON_PKG: "python3.12"
+  MAIN_PYTHON_PKG: "python3.11"
   # Used to tell pytest which tests to be run by specifying markers,
   # Allows partitioning of tests to prevent duplicate running of tests in different jobs.
   # Currently specifies special tests that are not run by quick_test_offline.
diff --git a/contrib/toil-ci-prebake/Dockerfile b/contrib/toil-ci-prebake/Dockerfile
index c5887fa867..1f474c4898 100644
--- a/contrib/toil-ci-prebake/Dockerfile
+++ b/contrib/toil-ci-prebake/Dockerfile
@@ -40,9 +40,7 @@ RUN sudo add-apt-repository -y ppa:deadsnakes/ppa && \
         python3.12 python3.12-dev python3.12-venv python3.12-distutils
 
 RUN rm -f /usr/bin/python
-RUN ln -s /usr/bin/python3.12 /usr/bin/python
-RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py # for python 3.12
-RUN python get-pip.py
-RUN python -m pip install virtualenv -U # default virtualenv is about 2 years old and does not run with python 3.12
+RUN ln -s /usr/bin/python3.11 /usr/bin/python
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3.12 get-pip.py && python3.12 -m pip install virtualenv -U # for python 3.12, default pip seems to be broken and default virtualenv is about 2 years old and does not run with python 3.12
 
 LABEL maintainers="anovak@soe.ucsc.edu lblauvel@ucsc.edu"
diff --git a/requirements.txt b/requirements.txt
index 8f1cb4203d..6e7cd06ffc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 dill>=0.3.2, <0.4
 requests>=2, <3
-docker>=3.7.2, <6
+docker>=6, <7
 # Work around https://github.com/docker/docker-py/issues/3113
 urllib3>=2.0.0,<3
 python-dateutil

From d5240a9437e3a06db5ea20920f728d54c3be5345 Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Mon, 23 Oct 2023 19:05:29 -0700
Subject: [PATCH 18/25] use an available htcondor package closest to 3.10
 version

---
 .gitlab-ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fbd372c906..1cc153e59e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -68,7 +68,7 @@ lint:
   stage: linting_and_dependencies
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     - python3.11 -m pip freeze
     - python3.11 --version
     - make mypy
@@ -146,7 +146,7 @@ py311_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
+    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
@@ -157,7 +157,7 @@ py311_main:
   stage: basic_tests
   script:
     - pwd
-    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
+    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     - make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
     - TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
 

From 337c67f2190bcfba9f1c4d8259268fcf89e68847 Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Mon, 23 Oct 2023 20:21:57 -0700
Subject: [PATCH 19/25] update htcondor for all

---
 .gitlab-ci.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1cc153e59e..d1d728c5c8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -167,7 +167,7 @@ batch_systems:
   stage: main_tests
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     - wget https://github.com/ohsu-comp-bio/funnel/releases/download/0.10.1/funnel-linux-amd64-0.10.1.tar.gz
     - tar -xvf funnel-linux-amd64-0.10.1.tar.gz funnel
     - export FUNNEL_SERVER_USER=toil
@@ -277,7 +277,7 @@ jobstore:
   stage: main_tests
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1' 
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/jobStores/jobStoreTest.py src/toil/test/sort/sortTest.py"
 
 provisioner:
@@ -286,7 +286,7 @@ provisioner:
   stage: main_tests
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1' 
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/lib/aws/ src/toil/test/provisioners/aws/awsProvisionerTest.py src/toil/test/provisioners/clusterScalerTest.py"
 
 # https://ucsc-ci.com/databiosphere/toil/-/jobs/38672
@@ -298,7 +298,7 @@ jobstore_integration:
   stage: integration
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     - export TOIL_TEST_INTEGRATIVE=True
     - export TOIL_AWS_KEYNAME=id_rsa
     - export TOIL_AWS_ZONE=us-west-2a
@@ -313,7 +313,7 @@ server_integration:
   stage: integration
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     - export TOIL_TEST_INTEGRATIVE=True
     - export TOIL_AWS_KEYNAME=id_rsa
     - export TOIL_AWS_ZONE=us-west-2a
@@ -329,7 +329,7 @@ provisioner_integration:
   stage: integration
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
     - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
     - export LIBPROCESS_IP=127.0.0.1
@@ -346,7 +346,7 @@ google_jobstore:
   stage: integration
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
     - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
     - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
     - export LIBPROCESS_IP=127.0.0.1

From 1f01910c82ab0bb146bcff822eb467b62b744842 Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Tue, 24 Oct 2023 10:32:13 -0700
Subject: [PATCH 20/25] get pip for all python versions

---
 contrib/toil-ci-prebake/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contrib/toil-ci-prebake/Dockerfile b/contrib/toil-ci-prebake/Dockerfile
index 1f474c4898..c78eae9681 100644
--- a/contrib/toil-ci-prebake/Dockerfile
+++ b/contrib/toil-ci-prebake/Dockerfile
@@ -42,5 +42,6 @@ RUN sudo add-apt-repository -y ppa:deadsnakes/ppa && \
 RUN rm -f /usr/bin/python
 RUN ln -s /usr/bin/python3.11 /usr/bin/python
 RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3.12 get-pip.py && python3.12 -m pip install virtualenv -U # for python 3.12, default pip seems to be broken and default virtualenv is about 2 years old and does not run with python 3.12
+RUN python3.8 get-pip.py && python3.9 get-pip.py && python3.10 get-pip.py && python3.11 get-pip.py
 
 LABEL maintainers="anovak@soe.ucsc.edu lblauvel@ucsc.edu"

From cea3ed361a6c57451d5e3fdf3ea63a3065edb3b8 Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Tue, 24 Oct 2023 10:56:14 -0700
Subject: [PATCH 21/25] get virtualenv for all python versions

---
 contrib/toil-ci-prebake/Dockerfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/contrib/toil-ci-prebake/Dockerfile b/contrib/toil-ci-prebake/Dockerfile
index c78eae9681..7338c8acd1 100644
--- a/contrib/toil-ci-prebake/Dockerfile
+++ b/contrib/toil-ci-prebake/Dockerfile
@@ -41,7 +41,8 @@ RUN sudo add-apt-repository -y ppa:deadsnakes/ppa && \
 
 RUN rm -f /usr/bin/python
 RUN ln -s /usr/bin/python3.11 /usr/bin/python
-RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3.12 get-pip.py && python3.12 -m pip install virtualenv -U # for python 3.12, default pip seems to be broken and default virtualenv is about 2 years old and does not run with python 3.12
-RUN python3.8 get-pip.py && python3.9 get-pip.py && python3.10 get-pip.py && python3.11 get-pip.py
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3.8 get-pip.py && python3.9 get-pip.py && python3.10 get-pip.py && python3.11 get-pip.py && python3.12 get-pip.py # for python 3.12, get-pip seems to remove other pip versions
+RUN python3.8 -m pip install virtualenv && python3.9 -m pip install virtualenv && python3.10 -m pip install virtualenv && python3.11 -m pip install virtualenv && python3.12 -m pip install virtualenv -U # python3.12 needs latest version of virtualenv
+
 
 LABEL maintainers="anovak@soe.ucsc.edu lblauvel@ucsc.edu"

From 2f80f79829ad0724bed6c00782947a6f4c3f3353 Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Tue, 24 Oct 2023 10:58:40 -0700
Subject: [PATCH 22/25] needs specific ordering

---
 contrib/toil-ci-prebake/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/toil-ci-prebake/Dockerfile b/contrib/toil-ci-prebake/Dockerfile
index 7338c8acd1..c3f62d3317 100644
--- a/contrib/toil-ci-prebake/Dockerfile
+++ b/contrib/toil-ci-prebake/Dockerfile
@@ -42,7 +42,7 @@ RUN sudo add-apt-repository -y ppa:deadsnakes/ppa && \
 RUN rm -f /usr/bin/python
 RUN ln -s /usr/bin/python3.11 /usr/bin/python
 RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3.8 get-pip.py && python3.9 get-pip.py && python3.10 get-pip.py && python3.11 get-pip.py && python3.12 get-pip.py # for python 3.12, get-pip seems to remove other pip versions
-RUN python3.8 -m pip install virtualenv && python3.9 -m pip install virtualenv && python3.10 -m pip install virtualenv && python3.11 -m pip install virtualenv && python3.12 -m pip install virtualenv -U # python3.12 needs latest version of virtualenv
+RUN python3.12 -m pip install virtualenv -U && python3.8 -m pip install virtualenv && python3.9 -m pip install virtualenv && python3.10 -m pip install virtualenv && python3.11 -m pip install virtualenv # python3.12 needs latest version of virtualenv
 
 
 LABEL maintainers="anovak@soe.ucsc.edu lblauvel@ucsc.edu"

From 0d35cdceea80824f5d783199f5921e4ebf05608c Mon Sep 17 00:00:00 2001
From: stxue1 <stxue@ucsc.edu>
Date: Tue, 24 Oct 2023 13:22:50 -0700
Subject: [PATCH 23/25] Separate mesos tests

---
 .gitlab-ci.yml | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d1d728c5c8..975063a995 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -356,6 +356,31 @@ google_jobstore:
     - export TOIL_GOOGLE_PROJECTID=toil-dev
     - make test threads="${TEST_THREADS}" tests=src/toil/test/jobStores/jobStoreTest.py::GoogleJobStoreTest
 
+mesos:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG
+  stage: integration
+  script:
+    - pwd
+    - python3.10 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[mesos] packages='htcondor==10.2.3'
+    - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
+    - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
+    - export LIBPROCESS_IP=127.0.0.1
+    - export TOIL_TEST_INTEGRATIVE=True
+    - export TOIL_AWS_KEYNAME=id_rsa
+    - export TOIL_AWS_ZONE=us-west-2a
+    - export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_CREDENTIALS
+    - export TOIL_GOOGLE_KEYNAME=id_rsa
+    - export TOIL_GOOGLE_PROJECTID=toil-dev
+    - make test threads="${TEST_THREADS}" src/toil/test/mesos/MesosDataStructuresTest.py::DataStructuresTest
+    - make test threads="${TEST_THREADS}" src/toil/test/batchSystems/batchSystemTest.py::MesosBatchSystemTest
+    - make test threads="${TEST_THREADS}" src/toil/test/sort/sortTest.py::SortTest::testAwsMesos
+    - make test threads="${TEST_THREADS}" src/toil/test/sort/sortTest.py::SortTest::testFileMesos
+    - make test threads="${TEST_THREADS}" src/toil/test/sort/sortTest.py::SortTest::testGoogleMesos
+    - make test threads="${TEST_THREADS}" src/toil/test/cwl/cwlTest.py::CWLv10Test::test_mesos_cwl_conformance
+    - make test threads="${TEST_THREADS}" src/toil/test/cwl/cwlTest.py::CWLv10Test::test_mesos_cwl_conformance_with_caching
+    - make test threads="${TEST_THREADS}" src/toil/test/src/promisedRequirementTest.py::MesosPromisedRequirementsTest
+
 # Cactus-on-Kubernetes integration (as a script and not a pytest test)
 #cactus_integration:
 #  stage: integration

From 908b8ea44ac0d0a8f21bf53e9ec76f798cfb0641 Mon Sep 17 00:00:00 2001
From: stxue1 <122345910+stxue1@users.noreply.github.com>
Date: Wed, 25 Oct 2023 23:12:46 +0000
Subject: [PATCH 24/25] remove 3.7 from CI image

---
 contrib/toil-ci-prebake/Dockerfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/contrib/toil-ci-prebake/Dockerfile b/contrib/toil-ci-prebake/Dockerfile
index c3f62d3317..d8e8d0c9ed 100644
--- a/contrib/toil-ci-prebake/Dockerfile
+++ b/contrib/toil-ci-prebake/Dockerfile
@@ -32,7 +32,6 @@ RUN apt-get -q -y update && \
 RUN sudo add-apt-repository -y ppa:deadsnakes/ppa && \
     sudo apt-get update && \
     sudo apt-get install -y \
-        python3.7 python3.7-dev python3.7-venv python3.7-distutils \
         python3.8 python3.8-dev python3.8-venv python3.8-distutils \
         python3.9 python3.9-dev python3.9-venv python3.9-distutils \
         python3.10 python3.10-dev python3.10-venv python3.10-distutils \

From c67003d99584d040b93af5b6e29b07718bcb1b08 Mon Sep 17 00:00:00 2001
From: stxue1 <122345910+stxue1@users.noreply.github.com>
Date: Wed, 25 Oct 2023 16:17:56 -0700
Subject: [PATCH 25/25] Remove debug statement from makefile

---
 Makefile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Makefile b/Makefile
index 02af05b91e..2ace5221cd 100644
--- a/Makefile
+++ b/Makefile
@@ -299,7 +299,6 @@ format: $(wildcard src/toil/cwl/*.py)
 	black $^ contrib/mypy-stubs
 
 mypy:
-	python --version
 	mypy --ignore-missing-imports --no-strict-optional \
 		--warn-redundant-casts --warn-unused-ignores \
 		$(CURDIR)/src/toil/cwl/cwltoil.py