Reenable some Kubernetes CI tests

.gitlab-ci.yml

@@ -398,31 +398,22 @@ mesos:
     - make test threads="${TEST_THREADS}" src/toil/test/src/promisedRequirementTest.py::MesosPromisedRequirementsTest
 
 # Cactus-on-Kubernetes integration (as a script and not a pytest test)
-#cactus_integration:
-#  stage: integration
-#  script:
-#    - set -e
-#    - ${MAIN_PYTHON_PKG} -m virtualenv --system-site-packages venv
-#    - . venv/bin/activate
-#    - pip install -U pip wheel
-#    - pip install .[aws]
-#    - export TOIL_KUBERNETES_OWNER=toiltest
-#    - export TOIL_AWS_SECRET_NAME=shared-s3-credentials
-#    - export TOIL_KUBERNETES_HOST_PATH=/data/scratch
-#    - export TOIL_WORKDIR=/var/lib/toil
-#    - export SINGULARITY_CACHEDIR=/var/lib/toil/singularity-cache
-#    - mkdir -p ${TOIL_WORKDIR}
-#    - BUCKET_NAME=toil-test-$RANDOM-$RANDOM-$RANDOM
-#    - cd
-#    - git clone https://github.com/ComparativeGenomicsToolkit/cactus.git --recursive
-#    - cd cactus
-#    - git fetch origin
-#    - git checkout f5adf4013326322ae58ef1eccb8409b71d761583
-#    - git submodule update --init --recursive
-#    # We can't use setuptools 66 on Ubuntu due to https://github.com/pypa/setuptools/issues/3772
-#    - pip install --upgrade 'setuptools<66' pip
-#    - pip install --upgrade .
-#    - pip install --upgrade numpy psutil # Cactus installs an old psutil that Toil isn't compatible with. TODO: Do we really need Numpy?
-#    - if [[ ! -z "${KUBERNETES_DOCKER_HUB_MIRROR}" ]] ; then export SINGULARITY_DOCKER_HUB_MIRROR="${KUBERNETES_DOCKER_HUB_MIRROR}" ; fi
-#    - toil clean aws:us-west-2:${BUCKET_NAME}
-#    - time cactus --setEnv SINGULARITY_DOCKER_HUB_MIRROR --batchSystem kubernetes --retryCount=3 --consCores 2 --binariesMode singularity --clean always aws:us-west-2:${BUCKET_NAME} examples/evolverMammals.txt examples/evolverMammals.hal --root mr --defaultDisk "8G" --logDebug
+cactus_integration:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH
+      changes:
+        compare_to: 'refs/heads/master'
+        paths:
+          - 'src/toil/test/cactus/test_cactus_integration.py'
+  stage: integration
+  script:
+    - export CACTUS_COMMIT_SHA=f5adf4013326322ae58ef1eccb8409b71d761583
+    - set -e
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[aws]
+    - python setup_gitlab_docker.py  # login to increase the docker.io rate limit
+    # This reads GITLAB_SECRET_FILE_SSH_KEYS
+    - python setup_gitlab_ssh.py
+    - chmod 400 /root/.ssh/id_rsa
+    - make test tests=src/toil/test/cactus/test_cactus_integration.py

src/toil/test/cactus/test_cactus_integration.py

@@ -0,0 +1,58 @@
+import os
+import uuid
+
+from toil.provisioners import cluster_factory
+from toil.test.provisioners.clusterTest import AbstractClusterTest
+
+
+class CactusIntegrationTest(AbstractClusterTest):
+    """
+    Run the Cactus Integration test on a Kubernetes AWS cluster
+    """
+
+    def __init__(self, methodName):
+        super().__init__(methodName=methodName)
+        self.clusterName = "cactus-test-" + str(uuid.uuid4())
+        self.leaderNodeType = "t2.medium"
+        self.clusterType = "kubernetes"
+
+    def setUp(self):
+        super().setUp()
+        self.jobStore = f"aws:{self.awsRegion()}:cluster-{uuid.uuid4()}"
+
+    def test_cactus_integration(self):
+        # Make a cluster with worker nodes
+        self.createClusterUtil(args=["--nodeTypes=t2.xlarge", "-w=1-3"])
+        # get the leader so we know the IP address - we don't need to wait since create cluster
+        # already ensures the leader is running
+        self.cluster = cluster_factory(
+            provisioner="aws", zone=self.zone, clusterName=self.clusterName
+        )
+        self.leader = self.cluster.getLeader()
+
+        CACTUS_COMMIT_SHA = os.environ["CACTUS_COMMIT_SHA"] or "f5adf4013326322ae58ef1eccb8409b71d761583"  # default cactus commit
+
+        # command to install and run cactus on the cluster
+        cactus_command = ("python -m virtualenv --system-site-packages venv && "
+                          ". venv/bin/activate && "
+                          "git clone https://github.com/ComparativeGenomicsToolkit/cactus.git --recursive && "
+                          "cd cactus && "
+                          "git fetch origin && "
+                          f"git checkout {CACTUS_COMMIT_SHA} && "
+                          "git submodule update --init --recursive && "
+                          "pip install --upgrade 'setuptools<66' pip && "
+                          "pip install --upgrade . && "
+                          "pip install --upgrade numpy psutil && "
+                          "time cactus --batchSystem kubernetes --retryCount=3 "
+                          f"--consCores 2 --binariesMode singularity --clean always {self.jobStore} "
+                          "examples/evolverMammals.txt examples/evolverMammals.hal --root mr --defaultDisk 8G --logDebug")
+
+        # run cactus
+        self.sshUtil(
+            [
+                "bash",
+                "-c",
+                cactus_command
+            ]
+        )
+