Merge branch 'master' of github.com:DataBiosphere/toil into issues/45…

…10-ensure-sibling-files
DataBiosphere · Oct 6, 2023 · 4cfb55c · 4cfb55c
2 parents 79ee250 + b73b9ef
commit 4cfb55c
Show file tree

Hide file tree

Showing 46 changed files with 1,151 additions and 650 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -63,6 +63,8 @@ stages:
 
 
 lint:
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
   stage: linting_and_dependencies
   script:
     - pwd
@@ -73,6 +75,8 @@ lint:
 
 
 cwl_dependency_is_stand_alone:
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
   stage: linting_and_dependencies
   script:
     - pwd
@@ -81,13 +85,17 @@ cwl_dependency_is_stand_alone:
 
 
 wdl_dependency_is_stand_alone:
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
   stage: linting_and_dependencies
   script:
     - pwd
     - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[wdl]
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests=src/toil/test/wdl/toilwdlTest.py::ToilWdlTest::testMD5sum
 
 quick_test_offline:
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
   stage: basic_tests
   script:
     - ${MAIN_PYTHON_PKG} -m virtualenv venv
@@ -98,6 +106,8 @@ quick_test_offline:
     - TOIL_TEST_QUICK=True make test_offline threads="${TEST_THREADS}"
 
 py37_appliance_build:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
   stage: basic_tests
   script:
     - pwd
@@ -107,6 +117,8 @@ py37_appliance_build:
     - make push_docker
 
 py38_appliance_build:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
   stage: basic_tests
   script:
     - pwd
@@ -125,6 +137,8 @@ py39_appliance_build:
     - make push_docker
 
 py310_appliance_build:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
   stage: basic_tests
   script:
     - pwd
@@ -134,6 +148,8 @@ py310_appliance_build:
     - make push_docker
 
 py310_main:
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
   stage: basic_tests
   script:
     - pwd
@@ -142,6 +158,8 @@ py310_main:
     - TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
 
 batch_systems:
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
   stage: main_tests
   script:
     - pwd
@@ -174,23 +192,29 @@ batch_systems:
     - kill $(jobs -p) || true
 
 slurm_test:
-  stage: main_tests
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG
+  stage: integration
   script:
     - pwd
     - cd contrib/slurm-test/
     - docker compose version
     - ./slurm_test.sh
 
 cwl_v1.2:
-  stage: main_tests
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG
+  stage: integration
   script:
     - pwd
     - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws]
     - python setup_gitlab_docker.py  # login to increase the docker.io rate limit
     - make test threads="${TEST_THREADS}" tests=src/toil/test/cwl/cwlTest.py::CWLv12Test::test_run_conformance_with_in_place_update
 
 cwl_on_arm:
-  stage: main_tests
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG
+  stage: integration
   script:
     - pwd
     - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws]
@@ -201,43 +225,15 @@ cwl_on_arm:
     - make test threads="${TEST_THREADS}" tests=src/toil/test/cwl/cwlTest.py::CWLOnARMTest
 
 cwl_misc:
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule" || $CI_COMMIT_TAG
   stage: main_tests
   script:
     - pwd
     - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws]
     - python setup_gitlab_docker.py  # login to increase the docker.io rate limit
     - make test threads="${TEST_THREADS}" tests='src/toil/test/cwl/cwlTest.py -k "CWLWorkflowTest or cwl_small"'
 
-cwl_v1.0_kubernetes:
-  stage: main_tests
-  only: []
-  script:
-    - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws,kubernetes]
-    - export TOIL_KUBERNETES_OWNER=toiltest
-    - export TOIL_AWS_SECRET_NAME=shared-s3-credentials
-    - export TOIL_KUBERNETES_HOST_PATH=/data/scratch
-    - export TOIL_WORKDIR=/var/lib/toil
-    - export SINGULARITY_CACHEDIR=/var/lib/toil/singularity-cache
-    - if [[ ! -z "${KUBERNETES_DOCKER_HUB_MIRROR}" ]] ; then export SINGULARITY_DOCKER_HUB_MIRROR="${KUBERNETES_DOCKER_HUB_MIRROR}" ; fi
-    - mkdir -p ${TOIL_WORKDIR}
-    - make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py::CWLv10Test::test_kubernetes_cwl_conformance src/toil/test/cwl/cwlTest.py::CWLv10Test::test_kubernetes_cwl_conformance_with_caching"
-
-cwl_v1.1_kubernetes:
-  stage: main_tests
-  only: []
-  script:
-    - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws,kubernetes]
-    - export TOIL_KUBERNETES_OWNER=toiltest
-    - export TOIL_AWS_SECRET_NAME=shared-s3-credentials
-    - export TOIL_KUBERNETES_HOST_PATH=/data/scratch
-    - export TOIL_WORKDIR=/var/lib/toil
-    - export SINGULARITY_CACHEDIR=/var/lib/toil/singularity-cache
-    - if [[ ! -z "${KUBERNETES_DOCKER_HUB_MIRROR}" ]] ; then export SINGULARITY_DOCKER_HUB_MIRROR="${KUBERNETES_DOCKER_HUB_MIRROR}" ; fi
-    - mkdir -p ${TOIL_WORKDIR}
-    - make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py::CWLv11Test::test_kubernetes_cwl_conformance src/toil/test/cwl/cwlTest.py::CWLv11Test::test_kubernetes_cwl_conformance_with_caching"
-
 #cwl_v1.2_kubernetes:
 #  stage: main_tests
 #  script:
@@ -260,6 +256,8 @@ cwl_v1.1_kubernetes:
 #    expire_in: 14 days
 
 wdl:
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
   stage: main_tests
   script:
     - pwd
@@ -269,16 +267,45 @@ wdl:
     - which java &> /dev/null || { echo >&2 "Java is not installed.  Install java to run these tests."; exit 1; }
     - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/wdl/toilwdlTest.py src/toil/test/wdl/builtinTest.py"  # needs java (default-jre) to run "GATK.jar"
 
-jobstore_and_provisioning:
+jobstore:
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
   stage: main_tests
   script:
     - pwd
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
-    - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/lib/aws/ src/toil/test/jobStores/jobStoreTest.py src/toil/test/sort/sortTest.py src/toil/test/provisioners/aws/awsProvisionerTest.py src/toil/test/provisioners/clusterScalerTest.py"
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1' 
+    - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/jobStores/jobStoreTest.py src/toil/test/sort/sortTest.py"
+
+provisioner:
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
+  stage: main_tests
+  script:
+    - pwd
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1' 
+    - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/lib/aws/ src/toil/test/provisioners/aws/awsProvisionerTest.py src/toil/test/provisioners/clusterScalerTest.py"
+
 # https://ucsc-ci.com/databiosphere/toil/-/jobs/38672
 # guessing decorators are masking class as function?  ^  also, abstract class is run as normal test?  should hide.
 
-integration:
+jobstore_integration:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG
+  stage: integration
+  script:
+    - pwd
+    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
+    - export TOIL_TEST_INTEGRATIVE=True
+    - export TOIL_AWS_KEYNAME=id_rsa
+    - export TOIL_AWS_ZONE=us-west-2a
+    # This reads GITLAB_SECRET_FILE_SSH_KEYS
+    - python setup_gitlab_ssh.py
+    - chmod 400 /root/.ssh/id_rsa
+    - make test threads="${TEST_THREADS}" tests="src/toil/test/jobStores/jobStoreTest.py"
+
+server_integration:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG
   stage: integration
   script:
     - pwd
@@ -289,11 +316,12 @@ integration:
     # This reads GITLAB_SECRET_FILE_SSH_KEYS
     - python setup_gitlab_ssh.py
     - chmod 400 /root/.ssh/id_rsa
-    # Test integration with job stores
     # Test server and its integration with AWS
-    - make test threads="${TEST_THREADS}" tests="src/toil/test/jobStores/jobStoreTest.py src/toil/test/server"
+    - make test threads="${TEST_THREADS}" tests="src/toil/test/server"
 
 provisioner_integration:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG
   stage: integration
   script:
     - pwd
@@ -309,6 +337,8 @@ provisioner_integration:
 #    - make test tests=src/toil/test/provisioners/gceProvisionerTest.py  # needs env vars set to run
 
 google_jobstore:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG
   stage: integration
   script:
     - pwd

diff --git a/attic/toil-sort-example.py b/attic/toil-sort-example.py
@@ -1,6 +1,6 @@
 from __future__ import absolute_import
 from six.moves import xrange
-from argparse import ArgumentParser
+from configargparse import ArgumentParser
 import os
 import logging
 import random

diff --git a/contrib/mypy-stubs/configargparse/__init__.pyi b/contrib/mypy-stubs/configargparse/__init__.pyi
@@ -0,0 +1,3 @@
+from .configargparse import ArgParser as ArgParser
+from .configargparse import YAMLConfigFileParser as YAMLConfigFileParser
+from .configargparse import ArgumentParser as ArgumentParser
diff --git a/contrib/mypy-stubs/configargparse/configargparse.pyi b/contrib/mypy-stubs/configargparse/configargparse.pyi
@@ -0,0 +1,30 @@
+import argparse
+from typing import Sequence, Any, TypeVar, OrderedDict
+
+__all__ = [
+    "ArgumentParser",
+    "YAMLConfigFileParser",
+    "ConfigFileParser"
+]
+_N = TypeVar("_N")
+
+class ConfigFileParser(object):
+    def get_syntax_description(self) -> Any: ...
+    def parse(self, stream: Any) -> Any: ...
+    def serialize(self, items: OrderedDict[Any, Any]) -> Any: ...
+
+class YAMLConfigFileParser(ConfigFileParser):
+    def get_syntax_description(self) -> str: ...
+    def parse(self, stream: Any) -> OrderedDict[Any, Any]: ...
+    def serialize(self, items: OrderedDict[Any, Any], default_flow_style: bool = ...) -> Any: ...
+
+class ArgumentParser(argparse.ArgumentParser):
+    @property
+    def _config_file_parser(self) -> Any: ...
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None: ...
+    # There may be a better way of type hinting this without a type: ignore, but mypy gets unhappy pretty much no matter what as the signatures for parse_args doesn't match with its superclass in argparse
+    def parse_args(self, args: Sequence[str] | None = None, namespace: Namespace | None = None, config_file_contents: str | None = None, env_vars: Any=None) -> Namespace: ... # type: ignore[override]
+
+Namespace = argparse.Namespace
+ArgParser = ArgumentParser
diff --git a/docs/running/cliOptions.rst b/docs/running/cliOptions.rst
@@ -7,10 +7,29 @@ Commandline Options
 
 A quick way to see all of Toil's commandline options is by executing the following on a toil script::
 
-    $ toil example.py --help
+    $ python example.py --help
 
 For a basic toil workflow, Toil has one mandatory argument, the job store.  All other arguments are optional.
 
+The Config File
+-------------
+Instead of changing the arguments on the CLI, Toil offers support for using a configuration file (Note: Support for the
+configuration file and environmental variables require the use of ``configargparse``).
+
+To generate a default configuration file::
+
+    $ toil config [file].yaml
+
+After editing the config file, make Toil take in the new options::
+
+    $ python example.py --config=[file].yaml
+
+If CLI options are used in addition with the configuration file, the CLI options will overwrite the configuration file
+options::
+
+    $ python example.py --config=[file].yaml --maxNodes 20
+    # maxNodes=[20] even though default maxNodes=[10]
+
 The Job Store
 -------------
 
@@ -228,22 +247,19 @@ levels in toil are based on priority from the logging module:
 **Data Storage Options**
 Allows configuring Toil's data storage.
 
-  --linkImports         When using a filesystem based job store, CWL input files
-                        are by default symlinked in. Specifying this option
+  --symlinkImports BOOL When using a filesystem based job store, CWL input files
+                        are by default symlinked in. Setting this option to True
                         instead copies the files into the job store, which may
-                        protect them from being modified externally. When not
-                        specified and as long as caching is enabled, Toil will
+                        protect them from being modified externally. When set
+                        to False and as long as caching is enabled, Toil will
                         protect the file automatically by changing the permissions
-                        to read-only.
-  --moveExports         When using a filesystem based job store, output files
+                        to read-only. (Default=True)
+  --moveOutputs BOOL    When using a filesystem based job store, output files
                         are by default moved to the output directory, and a
                         symlink to the moved exported file is created at the
-                        initial location. Specifying this option instead copies
-                        the files into the output directory. Applies to
-                        filesystem-based job stores only.
-  --disableCaching      Disables caching in the file store. This flag must be
-                        set to use a batch system that does not support
-                        cleanup, such as Parasol.
+                        initial location. Setting this option to True instead
+                        copies the files into the output directory. Applies to
+                        filesystem-based job stores only. (Default=False)
   --caching BOOL        Set caching options. This must be set to "false"
                         to use a batch system that does not support
                         cleanup, such as Parasol. Set to "true" if caching
@@ -280,7 +296,7 @@ autoscaled cluster, as well as parameters to control the level of provisioning.
                         if using auto-scaling. This should be provided as a
                         comma-separated list of the same length as the list of
                         node types. default=0
-  --maxNodes MAXNODES   Maximum number of nodes of each type in the cluster,
+  --maxNodes MAXNODES   Maximum number of nodes of each type in the cluster,                        Maximum number of nodes of each type in the cluster,
                         if using autoscaling, provided as a comma-separated
                         list. The first value is used as a default if the list
                         length is less than the number of nodeTypes.
@@ -363,7 +379,7 @@ from the batch system.
                         Only applicable to jobs that do not specify an
                         explicit value for this requirement. Standard suffixes
                         like K, Ki, M, Mi, G or Gi are supported. Default is
-                        2.0G
+                        2.0Gi
   --defaultCores FLOAT  The default number of CPU cores to dedicate a job.
                         Only applicable to jobs that do not specify an
                         explicit value for this requirement. Fractions of a
@@ -374,7 +390,7 @@ from the batch system.
                         Only applicable to jobs that do not specify an
                         explicit value for this requirement. Standard suffixes
                         like K, Ki, M, Mi, G or Gi are supported. Default is
-                        2.0G
+                        2.0Gi
   --defaultAccelerators ACCELERATOR
                         The default amount of accelerators to request for a
                         job. Only applicable to jobs that do not specify an

diff --git a/requirements.txt b/requirements.txt
@@ -10,4 +10,7 @@ PyPubSub >=4.0.3, <5
 addict>=2.2.1, <2.5
 pytz>=2012
 enlighten>=1.5.2, <2
+configargparse>=1.7,<2
+ruamel.yaml>=0.15
+pyyaml>=6,<7
 typing-extensions>=4.6.2, <5