diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 975063a995..4aff012b58 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -68,7 +68,7 @@ lint: stage: linting_and_dependencies script: - pwd - - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3' + - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[all] - python3.11 -m pip freeze - python3.11 --version - make mypy @@ -113,7 +113,7 @@ py38_appliance_build: stage: basic_tests script: - pwd - - python3.8 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1' + - python3.8 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1' # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS - python setup_gitlab_docker.py - make push_docker @@ -124,7 +124,7 @@ py39_appliance_build: stage: basic_tests script: - pwd - - python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1' + - python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1' # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS - python setup_gitlab_docker.py - make push_docker @@ -135,18 +135,16 @@ py310_appliance_build: stage: basic_tests script: - pwd - - python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.0.post1' + - python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1' # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS - python setup_gitlab_docker.py - make push_docker py311_appliance_build: - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH stage: basic_tests script: - pwd - - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3' + - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.3' # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS - python setup_gitlab_docker.py - make push_docker @@ -157,7 +155,7 @@ py311_main: stage: basic_tests script: - pwd - - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3' + - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.3' - make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils" - TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib @@ -167,7 +165,7 @@ batch_systems: stage: main_tests script: - pwd - - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3' + - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.3' - wget https://github.com/ohsu-comp-bio/funnel/releases/download/0.10.1/funnel-linux-amd64-0.10.1.tar.gz - tar -xvf funnel-linux-amd64-0.10.1.tar.gz funnel - export FUNNEL_SERVER_USER=toil @@ -266,7 +264,7 @@ wdl: script: - pwd - apt update && apt install -y default-jre - - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] + - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] - make test threads="${TEST_THREADS}" marker="${MARKER}" tests=src/toil/test/wdl/wdltoil_test.py - which java &> /dev/null || { echo >&2 "Java is not installed. Install java to run these tests."; exit 1; } - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/wdl/toilwdlTest.py src/toil/test/wdl/builtinTest.py" # needs java (default-jre) to run "GATK.jar" @@ -277,7 +275,7 @@ jobstore: stage: main_tests script: - pwd - - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3' + - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/jobStores/jobStoreTest.py src/toil/test/sort/sortTest.py" provisioner: @@ -286,7 +284,7 @@ provisioner: stage: main_tests script: - pwd - - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3' + - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/lib/aws/ src/toil/test/provisioners/aws/awsProvisionerTest.py src/toil/test/provisioners/clusterScalerTest.py" # https://ucsc-ci.com/databiosphere/toil/-/jobs/38672 @@ -298,7 +296,7 @@ jobstore_integration: stage: integration script: - pwd - - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3' + - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] - export TOIL_TEST_INTEGRATIVE=True - export TOIL_AWS_KEYNAME=id_rsa - export TOIL_AWS_ZONE=us-west-2a @@ -313,7 +311,7 @@ server_integration: stage: integration script: - pwd - - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3' + - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] - export TOIL_TEST_INTEGRATIVE=True - export TOIL_AWS_KEYNAME=id_rsa - export TOIL_AWS_ZONE=us-west-2a @@ -329,7 +327,7 @@ provisioner_integration: stage: integration script: - pwd - - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3' + - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa - echo $'Host *\n AddressFamily inet' > /root/.ssh/config - export LIBPROCESS_IP=127.0.0.1 @@ -346,7 +344,7 @@ google_jobstore: stage: integration script: - pwd - - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[aws,google,encryption,cwl,wdl,kubernetes,server] packages='htcondor==10.2.3' + - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa - echo $'Host *\n AddressFamily inet' > /root/.ssh/config - export LIBPROCESS_IP=127.0.0.1 @@ -362,7 +360,7 @@ mesos: stage: integration script: - pwd - - python3.10 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[mesos] packages='htcondor==10.2.3' + - python3.10 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[mesos] - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa - echo $'Host *\n AddressFamily inet' > /root/.ssh/config - export LIBPROCESS_IP=127.0.0.1 diff --git a/requirements-aws.txt b/requirements-aws.txt index 1ae51c64f6..e44f45ccce 100644 --- a/requirements-aws.txt +++ b/requirements-aws.txt @@ -1,4 +1 @@ boto>=2.48.0, <3 -boto3-stubs[s3,sdb,iam,sts,boto3]>=1.28.3.post2, <2 -mypy-boto3-iam>=1.28.3.post2, <2 # Need to force .post1 to be replaced -moto>=4.1.11, <5 diff --git a/requirements-dev.txt b/requirements-dev.txt index 11ab916305..2d28c2da90 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -28,3 +28,6 @@ lxml docutils>=0.16,<0.21 pyupgrade pytest-xdist +boto3-stubs[s3,sdb,iam,sts,boto3]>=1.28.3.post2, <2 +mypy-boto3-iam>=1.28.3.post2, <2 # Need to force .post1 to be replaced +moto>=4.1.11, <5 \ No newline at end of file diff --git a/requirements-mesos.txt b/requirements-mesos.txt index dcc454d8df..3bd0817a21 100644 --- a/requirements-mesos.txt +++ b/requirements-mesos.txt @@ -1 +1 @@ -pymesos>=0.3.15,<0.4 +pymesos>=0.3.15,<0.4 ; python_version < '3.11' diff --git a/requirements.txt b/requirements.txt index 6e7cd06ffc..b46c571093 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ dill>=0.3.2, <0.4 requests>=2, <3 docker>=6, <7 # Work around https://github.com/docker/docker-py/issues/3113 -urllib3>=2.0.0,<3 +urllib3<3 python-dateutil psutil >= 3.0.1, <6 py-tes>=0.4.2,<1 diff --git a/setup.py b/setup.py index 0a111fc547..cde2dd4ee7 100755 --- a/setup.py +++ b/setup.py @@ -49,13 +49,13 @@ def run_setup(): # to how wheels work, so it is not included in all and # must be explicitly installed as an extra all_reqs = [] + non_htcondor_extras = [ "aws", "cwl", "encryption", "google", "kubernetes", - "mesos", "wdl", "server" ] @@ -64,8 +64,9 @@ def run_setup(): all_reqs += extras_require[extra] # We exclude htcondor from "all" because it can't be on Mac extras_require['htcondor:sys_platform!="darwin"'] = get_requirements("htcondor") + extras_require['mesos'] = get_requirements("mesos") + all_reqs += get_requirements("mesos") extras_require["all"] = all_reqs - setup( name='toil', version=version.distVersion,