Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Separate out integration tests to run on a schedule #4612

Merged
merged 2 commits into from
Oct 4, 2023
Merged

Separate out integration tests to run on a schedule #4612

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a0dad79
Reorganize tests and move integration tests to scheduled pipeline runs
adamnovak Oct 3, 2023
24102f4
Also handle tags
adamnovak Oct 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 69 additions & 39 deletions .gitlab-ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ stages:


lint:
rules:
- if: $CI_PIPELINE_SOURCE != "schedule"
stage: linting_and_dependencies
script:
- pwd
Expand All @@ -73,6 +75,8 @@ lint:


cwl_dependency_is_stand_alone:
rules:
- if: $CI_PIPELINE_SOURCE != "schedule"
stage: linting_and_dependencies
script:
- pwd
Expand All @@ -81,13 +85,17 @@ cwl_dependency_is_stand_alone:


wdl_dependency_is_stand_alone:
rules:
- if: $CI_PIPELINE_SOURCE != "schedule"
stage: linting_and_dependencies
script:
- pwd
- ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[wdl]
- make test threads="${TEST_THREADS}" marker="${MARKER}" tests=src/toil/test/wdl/toilwdlTest.py::ToilWdlTest::testMD5sum

quick_test_offline:
rules:
- if: $CI_PIPELINE_SOURCE != "schedule"
stage: basic_tests
script:
- ${MAIN_PYTHON_PKG} -m virtualenv venv
Expand All @@ -98,6 +106,8 @@ quick_test_offline:
- TOIL_TEST_QUICK=True make test_offline threads="${TEST_THREADS}"

py37_appliance_build:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
stage: basic_tests
script:
- pwd
Expand All @@ -107,6 +117,8 @@ py37_appliance_build:
- make push_docker

py38_appliance_build:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
stage: basic_tests
script:
- pwd
Expand All @@ -125,6 +137,8 @@ py39_appliance_build:
- make push_docker

py310_appliance_build:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
stage: basic_tests
script:
- pwd
Expand All @@ -134,6 +148,8 @@ py310_appliance_build:
- make push_docker

py310_main:
rules:
- if: $CI_PIPELINE_SOURCE != "schedule"
stage: basic_tests
script:
- pwd
Expand All @@ -142,6 +158,8 @@ py310_main:
- TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib

batch_systems:
rules:
- if: $CI_PIPELINE_SOURCE != "schedule"
stage: main_tests
script:
- pwd
Expand Down Expand Up @@ -174,23 +192,29 @@ batch_systems:
- kill $(jobs -p) || true

slurm_test:
stage: main_tests
rules:
- if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG
stage: integration
script:
- pwd
- cd contrib/slurm-test/
- docker compose version
- ./slurm_test.sh

cwl_v1.2:
stage: main_tests
rules:
- if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG
stage: integration
script:
- pwd
- ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws]
- python setup_gitlab_docker.py # login to increase the docker.io rate limit
- make test threads="${TEST_THREADS}" tests=src/toil/test/cwl/cwlTest.py::CWLv12Test::test_run_conformance_with_in_place_update

cwl_on_arm:
stage: main_tests
rules:
- if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG
stage: integration
script:
- pwd
- ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws]
Expand All @@ -201,43 +225,15 @@ cwl_on_arm:
- make test threads="${TEST_THREADS}" tests=src/toil/test/cwl/cwlTest.py::CWLOnARMTest

cwl_misc:
rules:
- if: $CI_PIPELINE_SOURCE != "schedule" || $CI_COMMIT_TAG
stage: main_tests
script:
- pwd
- ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws]
- python setup_gitlab_docker.py # login to increase the docker.io rate limit
- make test threads="${TEST_THREADS}" tests='src/toil/test/cwl/cwlTest.py -k "CWLWorkflowTest or cwl_small"'

cwl_v1.0_kubernetes:
stage: main_tests
only: []
script:
- pwd
- ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws,kubernetes]
- export TOIL_KUBERNETES_OWNER=toiltest
- export TOIL_AWS_SECRET_NAME=shared-s3-credentials
- export TOIL_KUBERNETES_HOST_PATH=/data/scratch
- export TOIL_WORKDIR=/var/lib/toil
- export SINGULARITY_CACHEDIR=/var/lib/toil/singularity-cache
- if [[ ! -z "${KUBERNETES_DOCKER_HUB_MIRROR}" ]] ; then export SINGULARITY_DOCKER_HUB_MIRROR="${KUBERNETES_DOCKER_HUB_MIRROR}" ; fi
- mkdir -p ${TOIL_WORKDIR}
- make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py::CWLv10Test::test_kubernetes_cwl_conformance src/toil/test/cwl/cwlTest.py::CWLv10Test::test_kubernetes_cwl_conformance_with_caching"

cwl_v1.1_kubernetes:
stage: main_tests
only: []
script:
- pwd
- ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws,kubernetes]
- export TOIL_KUBERNETES_OWNER=toiltest
- export TOIL_AWS_SECRET_NAME=shared-s3-credentials
- export TOIL_KUBERNETES_HOST_PATH=/data/scratch
- export TOIL_WORKDIR=/var/lib/toil
- export SINGULARITY_CACHEDIR=/var/lib/toil/singularity-cache
- if [[ ! -z "${KUBERNETES_DOCKER_HUB_MIRROR}" ]] ; then export SINGULARITY_DOCKER_HUB_MIRROR="${KUBERNETES_DOCKER_HUB_MIRROR}" ; fi
- mkdir -p ${TOIL_WORKDIR}
- make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py::CWLv11Test::test_kubernetes_cwl_conformance src/toil/test/cwl/cwlTest.py::CWLv11Test::test_kubernetes_cwl_conformance_with_caching"

#cwl_v1.2_kubernetes:
# stage: main_tests
# script:
Expand All @@ -260,6 +256,8 @@ cwl_v1.1_kubernetes:
# expire_in: 14 days

wdl:
rules:
- if: $CI_PIPELINE_SOURCE != "schedule"
stage: main_tests
script:
- pwd
Expand All @@ -269,16 +267,45 @@ wdl:
- which java &> /dev/null || { echo >&2 "Java is not installed. Install java to run these tests."; exit 1; }
- make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/wdl/toilwdlTest.py src/toil/test/wdl/builtinTest.py" # needs java (default-jre) to run "GATK.jar"

jobstore_and_provisioning:
jobstore:
rules:
- if: $CI_PIPELINE_SOURCE != "schedule"
stage: main_tests
script:
- pwd
- ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
- make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/lib/aws/ src/toil/test/jobStores/jobStoreTest.py src/toil/test/sort/sortTest.py src/toil/test/provisioners/aws/awsProvisionerTest.py src/toil/test/provisioners/clusterScalerTest.py"
- ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
- make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/jobStores/jobStoreTest.py src/toil/test/sort/sortTest.py"

provisioner:
rules:
- if: $CI_PIPELINE_SOURCE != "schedule"
stage: main_tests
script:
- pwd
- ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
- make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/lib/aws/ src/toil/test/provisioners/aws/awsProvisionerTest.py src/toil/test/provisioners/clusterScalerTest.py"

# https://ucsc-ci.com/databiosphere/toil/-/jobs/38672
# guessing decorators are masking class as function? ^ also, abstract class is run as normal test? should hide.

integration:
jobstore_integration:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG
stage: integration
script:
- pwd
- ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
- export TOIL_TEST_INTEGRATIVE=True
- export TOIL_AWS_KEYNAME=id_rsa
- export TOIL_AWS_ZONE=us-west-2a
# This reads GITLAB_SECRET_FILE_SSH_KEYS
- python setup_gitlab_ssh.py
- chmod 400 /root/.ssh/id_rsa
- make test threads="${TEST_THREADS}" tests="src/toil/test/jobStores/jobStoreTest.py"

server_integration:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG
stage: integration
script:
- pwd
Expand All @@ -289,11 +316,12 @@ integration:
# This reads GITLAB_SECRET_FILE_SSH_KEYS
- python setup_gitlab_ssh.py
- chmod 400 /root/.ssh/id_rsa
# Test integration with job stores
# Test server and its integration with AWS
- make test threads="${TEST_THREADS}" tests="src/toil/test/jobStores/jobStoreTest.py src/toil/test/server"
- make test threads="${TEST_THREADS}" tests="src/toil/test/server"

provisioner_integration:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG
stage: integration
script:
- pwd
Expand All @@ -309,6 +337,8 @@ provisioner_integration:
# - make test tests=src/toil/test/provisioners/gceProvisionerTest.py # needs env vars set to run

google_jobstore:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG
stage: integration
script:
- pwd
Expand Down