Fix CI Appliance Builds (#4655)

* Properly build 3.11, fix dependencies and move aws stubs/mock into dev * only keep htcondor installs in appliance builds * Remove unused import * Fix extras_require syntax
DataBiosphere · Oct 30, 2023 · f1d082d · f1d082d
1 parent c4df682
commit f1d082d
Show file tree

Hide file tree

Showing 6 changed files with 23 additions and 24 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -68,7 +68,7 @@ lint:
   stage: linting_and_dependencies
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[all]
     - python3.11 -m pip freeze
     - python3.11 --version
     - make mypy
@@ -113,7 +113,7 @@ py38_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - python3.8 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
+    - python3.8 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
@@ -124,7 +124,7 @@ py39_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
+    - python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
@@ -135,18 +135,16 @@ py310_appliance_build:
   stage: basic_tests
   script:
     - pwd
-    - python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1'
+    - python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
 
 py311_appliance_build:
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
   stage: basic_tests
   script:
     - pwd
-    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
+    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.3'
     # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
     - python setup_gitlab_docker.py
     - make push_docker
@@ -157,7 +155,7 @@ py311_main:
   stage: basic_tests
   script:
     - pwd
-    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
+    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.3'
     - make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
     - TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
 
@@ -167,7 +165,7 @@ batch_systems:
   stage: main_tests
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.3'
     - wget https://github.com/ohsu-comp-bio/funnel/releases/download/0.10.1/funnel-linux-amd64-0.10.1.tar.gz
     - tar -xvf funnel-linux-amd64-0.10.1.tar.gz funnel
     - export FUNNEL_SERVER_USER=toil
@@ -266,7 +264,7 @@ wdl:
   script:
     - pwd
     - apt update && apt install -y default-jre
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server]
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests=src/toil/test/wdl/wdltoil_test.py
     - which java &> /dev/null || { echo >&2 "Java is not installed.  Install java to run these tests."; exit 1; }
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/wdl/toilwdlTest.py src/toil/test/wdl/builtinTest.py"  # needs java (default-jre) to run "GATK.jar"
@@ -277,7 +275,7 @@ jobstore:
   stage: main_tests
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/jobStores/jobStoreTest.py src/toil/test/sort/sortTest.py"
 
 provisioner:
@@ -286,7 +284,7 @@ provisioner:
   stage: main_tests
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/lib/aws/ src/toil/test/provisioners/aws/awsProvisionerTest.py src/toil/test/provisioners/clusterScalerTest.py"
 
 # https://ucsc-ci.com/databiosphere/toil/-/jobs/38672
@@ -298,7 +296,7 @@ jobstore_integration:
   stage: integration
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
     - export TOIL_TEST_INTEGRATIVE=True
     - export TOIL_AWS_KEYNAME=id_rsa
     - export TOIL_AWS_ZONE=us-west-2a
@@ -313,7 +311,7 @@ server_integration:
   stage: integration
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
     - export TOIL_TEST_INTEGRATIVE=True
     - export TOIL_AWS_KEYNAME=id_rsa
     - export TOIL_AWS_ZONE=us-west-2a
@@ -329,7 +327,7 @@ provisioner_integration:
   stage: integration
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
     - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
     - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
     - export LIBPROCESS_IP=127.0.0.1
@@ -346,7 +344,7 @@ google_jobstore:
   stage: integration
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3'
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
     - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
     - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
     - export LIBPROCESS_IP=127.0.0.1
@@ -362,7 +360,7 @@ mesos:
   stage: integration
   script:
     - pwd
-    - python3.10 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[mesos] packages='htcondor==10.2.3'
+    - python3.10 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[mesos]
     - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
     - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
     - export LIBPROCESS_IP=127.0.0.1

diff --git a/requirements-aws.txt b/requirements-aws.txt
@@ -1,4 +1 @@
 boto>=2.48.0, <3
-boto3-stubs[s3,sdb,iam,sts,boto3]>=1.28.3.post2, <2
-mypy-boto3-iam>=1.28.3.post2, <2 # Need to force .post1 to be replaced
-moto>=4.1.11, <5
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -28,3 +28,6 @@ lxml
 docutils>=0.16,<0.21
 pyupgrade
 pytest-xdist
+boto3-stubs[s3,sdb,iam,sts,boto3]>=1.28.3.post2, <2
+mypy-boto3-iam>=1.28.3.post2, <2 # Need to force .post1 to be replaced
+moto>=4.1.11, <5
diff --git a/requirements-mesos.txt b/requirements-mesos.txt
@@ -1 +1 @@
-pymesos>=0.3.15,<0.4
+pymesos>=0.3.15,<0.4 ; python_version < '3.11'
diff --git a/requirements.txt b/requirements.txt
@@ -2,7 +2,7 @@ dill>=0.3.2, <0.4
 requests>=2, <3
 docker>=6, <7
 # Work around https://github.com/docker/docker-py/issues/3113
-urllib3>=2.0.0,<3
+urllib3<3
 python-dateutil
 psutil >= 3.0.1, <6
 py-tes>=0.4.2,<1

diff --git a/setup.py b/setup.py
@@ -49,13 +49,13 @@ def run_setup():
     # to how wheels work, so it is not included in all and
     # must be explicitly installed as an extra
     all_reqs = []
+
     non_htcondor_extras = [
         "aws",
         "cwl",
         "encryption",
         "google",
         "kubernetes",
-        "mesos",
         "wdl",
         "server"
     ]
@@ -64,8 +64,9 @@ def run_setup():
         all_reqs += extras_require[extra]
     # We exclude htcondor from "all" because it can't be on Mac
     extras_require['htcondor:sys_platform!="darwin"'] = get_requirements("htcondor")
+    extras_require['mesos'] = get_requirements("mesos")
+    all_reqs += get_requirements("mesos")
     extras_require["all"] = all_reqs
-
     setup(
         name='toil',
         version=version.distVersion,
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		pymesos>=0.3.15,<0.4
		pymesos>=0.3.15,<0.4 ; python_version < '3.11'