diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6fdb588440..1d0a7a63b4 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,7 +5,7 @@ To be copied to the [draft changelog](https://github.com/DataBiosphere/toil/wiki ## Reviewer Checklist - + * [ ] Make sure it is coming from `issues/XXXX-fix-the-thing` in the Toil repo, or from an external repo. * [ ] If it is coming from an external repo, make sure to pull it in for CI with: @@ -26,10 +26,10 @@ To be copied to the [draft changelog](https://github.com/DataBiosphere/toil/wiki ## Merger Checklist - + -* [ ] Make sure the PR passes tests. -* [ ] Make sure the PR has been reviewed **since its last modification**. If not, review it. +* [ ] Make sure the PR passed tests, including the Gitlab tests, for the most recent commit in its branch. +* [ ] Make sure the PR has been reviewed. If not, review it. If it has been reviewed and any requested changes seem to have been addressed, proceed. * [ ] Merge with the Github "Squash and merge" feature. * [ ] If there are multiple authors' commits, add [Co-authored-by](https://github.blog/2018-01-29-commit-together-with-co-authors/) to give credit to all contributing authors. * [ ] Copy its recommended changelog entry to the [Draft Changelog](https://github.com/DataBiosphere/toil/wiki/Draft-Changelog). diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 17ad7c2c7d..2e2dc52e1b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,7 +4,18 @@ updates: directory: "/" # Location of package manifests schedule: interval: "daily" + ignore: + # htcondor is not under CI + - dependency-name: "htcondor" + # Google cloud/storage is not under CI + - dependency-name: "apache-libcloud" + - dependency-name: "google-cloud-storage" + - dependency-name: "google-auth" - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" +# Don't constantly rebase all the dependency bump PRs whenever anything else +# merges; this causes too many CI runs. +rebase-strategy: ignore + diff --git a/.github/workflows/autoupdate.yaml b/.github/workflows/autoupdate.yaml deleted file mode 100644 index de09cfc9a4..0000000000 --- a/.github/workflows/autoupdate.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Auto-update PRs when other PRs are merged. -# See https://github.com/marketplace/actions/auto-update - -name: autoupdate -on: - push: - branches: - - master -jobs: - autoupdate: - name: autoupdate - runs-on: ubuntu-22.04 - steps: - - uses: docker://chinthakagodawita/autoupdate-action:v1 - env: - GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' - # Work on PRs against branches that are protected (so they need to be - # up to date) - PR_FILTER: "protected" - # Don't work on draft PRs - PR_READY_STATE: "ready_for_review" - # Don't fail on the triggering commit if it makes a merge conflict - # with a PR and the PR can't update - MERGE_CONFLICT_ACTION: "ignore" diff --git a/docs/contributing/checklists.rst b/docs/contributing/checklists.rst index dedb989482..b3bc2b364f 100644 --- a/docs/contributing/checklists.rst +++ b/docs/contributing/checklists.rst @@ -43,8 +43,8 @@ This checklist is to be kept in sync with the checklist in the pull request temp When merging a PR, do the following: -* |X| Make sure the PR passes tests. -* |X| Make sure the PR has been reviewed **since its last modification**. If not, review it. +* |X| Make sure the PR passed tests, including the Gitlab tests, for the most recent commit in its branch. +* |X| Make sure the PR has been reviewed. If not, review it. If it has been reviewed and any requested changes seem to have been addressed, proceed. * |X| Merge with the Github "Squash and merge" feature. * |X| If there are multiple authors' commits, add `Co-authored-by`_ to give credit to all contributing authors. .. _Co-authored-by: https://github.blog/2018-01-29-commit-together-with-co-authors/ diff --git a/docs/running/cliOptions.rst b/docs/running/cliOptions.rst index e974f7e5cd..bd47282aeb 100644 --- a/docs/running/cliOptions.rst +++ b/docs/running/cliOptions.rst @@ -90,7 +90,9 @@ about the performance of jobs. directory ``toil-`` within workDir. The workflowID is generated by Toil and will be reported in the workflow logs. Default is determined by the - variables (TMPDIR, TEMP, TMP) via mkdtemp. This + variables (TMPDIR, TEMP, TMP) via mkdtemp. For CWL, + the temporary output directory is used instead + (see CWL option ``--tmp-outdir-prefix``). This directory needs to exist on all machines running jobs; if capturing standard output and error from batch system jobs is desired, it will generally need to be on diff --git a/src/toil/cwl/cwltoil.py b/src/toil/cwl/cwltoil.py index daf2b591e9..a9c73b08b7 100644 --- a/src/toil/cwl/cwltoil.py +++ b/src/toil/cwl/cwltoil.py @@ -4122,24 +4122,18 @@ def main(args: Optional[list[str]] = None, stdout: TextIO = sys.stdout) -> int: tmpdir_prefix = options.tmpdir_prefix = ( options.tmpdir_prefix or DEFAULT_TMPDIR_PREFIX ) - - # We need a workdir for the CWL runtime contexts. - if tmpdir_prefix != DEFAULT_TMPDIR_PREFIX: - # if tmpdir_prefix is not the default value, move - # workdir and the default job store under it - workdir = cwltool.utils.create_tmp_dir(tmpdir_prefix) - else: - # Use a directory in the default tmpdir - workdir = mkdtemp() - # Make sure workdir doesn't exist so it can be a job store - os.rmdir(workdir) + tmp_outdir_prefix = options.tmp_outdir_prefix or tmpdir_prefix + workdir = options.workDir or tmp_outdir_prefix if options.jobStore is None: + jobstore = cwltool.utils.create_tmp_dir(tmp_outdir_prefix) + # Make sure directory doesn't exist so it can be a job store + os.rmdir(jobstore) # Pick a default job store specifier appropriate to our choice of batch # system and provisioner and installed modules, given this available # local directory name. Fail if no good default can be used. options.jobStore = generate_default_job_store( - options.batchSystem, options.provisioner, workdir + options.batchSystem, options.provisioner, jobstore ) options.doc_cache = True @@ -4147,17 +4141,6 @@ def main(args: Optional[list[str]] = None, stdout: TextIO = sys.stdout) -> int: options.do_validate = True options.pack = False options.print_subgraph = False - if tmpdir_prefix != DEFAULT_TMPDIR_PREFIX and options.workDir is None: - # We need to override workDir because by default Toil will pick - # somewhere under the system temp directory if unset, ignoring - # --tmpdir-prefix. - # - # If set, workDir needs to exist, so we directly use the prefix - options.workDir = cwltool.utils.create_tmp_dir(tmpdir_prefix) - if tmpdir_prefix != DEFAULT_TMPDIR_PREFIX and options.coordination_dir is None: - # override coordination_dir as default Toil will pick somewhere else - # ignoring --tmpdir_prefix - options.coordination_dir = cwltool.utils.create_tmp_dir(tmpdir_prefix) if options.batchSystem == "kubernetes": # Containers under Kubernetes can only run in Singularity @@ -4175,9 +4158,6 @@ def main(args: Optional[list[str]] = None, stdout: TextIO = sys.stdout) -> int: logger.debug(f"Final job store {options.jobStore} and workDir {options.workDir}") outdir = os.path.abspath(options.outdir or os.getcwd()) - tmp_outdir_prefix = os.path.abspath( - options.tmp_outdir_prefix or DEFAULT_TMPDIR_PREFIX - ) conf_file = getattr(options, "beta_dependency_resolvers_configuration", None) use_conda_dependencies = getattr(options, "beta_conda_dependencies", None) job_script_provider = None