Merge branch 'master' into add-diff

.ci/jenkins.sh

@@ -10,7 +10,7 @@ GALAXY_URL="http://127.0.0.1:${LOCAL_PORT}"
 SSH_MASTER_SOCKET_DIR="${HOME}/.cache/usegalaxy-tools"
 
 # Set to 'centos:7' and set GALAXY_GIT_* below to use a clone
-GALAXY_DOCKER_IMAGE='galaxy/galaxy-min:21.05'
+GALAXY_DOCKER_IMAGE='galaxy/galaxy-min:23.0'
 # Disable if using a locally built image e.g. for debugging
 GALAXY_DOCKER_IMAGE_PULL=true
 
@@ -22,8 +22,6 @@ GALAXY_TEMPLATE_DB='galaxy.sqlite'
 
 # Need to run dev until 0.10.4
 #EPHEMERIS="git+https://github.com/galaxyproject/ephemeris.git"
-# Fix for not installing Conda deps: https://github.com/galaxyproject/ephemeris/pull/181
-EPHEMERIS="git+https://github.com/mvdbeek/ephemeris.git@fix_option_parsing_and_tool_id_handling"
 
 # Should be set by Jenkins, so the default here is for development
 : ${GIT_COMMIT:=$(git rev-parse HEAD)}
@@ -474,12 +472,14 @@ function run_mounted_galaxy() {
         -e "GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION=sqlite:////galaxy/server/database/${GALAXY_TEMPLATE_DB}" \
         -e "GALAXY_CONFIG_OVERRIDE_INTEGRATED_TOOL_PANEL_CONFIG=/tmp/integrated_tool_panel.xml" \
         -e "GALAXY_CONFIG_OVERRIDE_SHED_TOOL_CONFIG_FILE=${SHED_TOOL_CONFIG}" \
+        -e "GALAXY_CONFIG_OVERRIDE_MIGRATED_TOOLS_CONFIG=/abcdef" \
         -e "GALAXY_CONFIG_OVERRIDE_TOOL_SHEDS_CONFIG_FILE=/tool_sheds_conf.xml" \
         -e "GALAXY_CONFIG_OVERRIDE_SHED_TOOL_DATA_TABLE_CONFIG=${SHED_TOOL_DATA_TABLE_CONFIG}" \
         -e "GALAXY_CONFIG_OVERRIDE_SHED_DATA_MANAGER_CONFIG_FILE=${SHED_DATA_MANAGER_CONFIG}" \
         -e "GALAXY_CONFIG_TOOL_DATA_PATH=/tmp/tool-data" \
         -e "GALAXY_CONFIG_INSTALL_DATABASE_CONNECTION=sqlite:///${INSTALL_DATABASE}" \
         -e "GALAXY_CONFIG_MASTER_API_KEY=${API_KEY:=deadbeef}" \
+        -e "GALAXY_CONFIG_FILE=config/galaxy.yml.sample" \
         -e "${CONDA_ENV_OPTION}" \
         ${CONDA_EXEC_OPTION} \
         -v "${OVERLAYFS_MOUNT}:/cvmfs/${REPO}" \
@@ -488,7 +488,7 @@ function run_mounted_galaxy() {
         -v "${GALAXY_SOURCE_TMPDIR}:/galaxy/server" \
         -v "${GALAXY_DATABASE_TMPDIR}:/galaxy/server/database" \
         --workdir /galaxy/server \
-        "$GALAXY_DOCKER_IMAGE" ./.venv/bin/uwsgi --yaml config/galaxy.yml.sample
+        "$GALAXY_DOCKER_IMAGE" ./.venv/bin/gunicorn 'galaxy.webapps.galaxy.fast_factory:factory\(\)' --timeout 300 --pythonpath lib -k galaxy.webapps.galaxy.workers.Worker -b 0.0.0.0:8080
     GALAXY_CONTAINER_UP=true
 }
 
@@ -511,29 +511,22 @@ function run_cloudve_galaxy() {
         -e "GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION=sqlite:////galaxy/server/database/${GALAXY_TEMPLATE_DB}" \
         -e "GALAXY_CONFIG_OVERRIDE_INTEGRATED_TOOL_PANEL_CONFIG=/tmp/integrated_tool_panel.xml" \
         -e "GALAXY_CONFIG_OVERRIDE_SHED_TOOL_CONFIG_FILE=${SHED_TOOL_CONFIG}" \
+        -e "GALAXY_CONFIG_OVERRIDE_MIGRATED_TOOLS_CONFIG=/abcdef" \
         -e "GALAXY_CONFIG_OVERRIDE_TOOL_SHEDS_CONFIG_FILE=/tool_sheds_conf.xml" \
         -e "GALAXY_CONFIG_OVERRIDE_SHED_TOOL_DATA_TABLE_CONFIG=${SHED_TOOL_DATA_TABLE_CONFIG}" \
         -e "GALAXY_CONFIG_OVERRIDE_SHED_DATA_MANAGER_CONFIG_FILE=${SHED_DATA_MANAGER_CONFIG}" \
         -e "GALAXY_CONFIG_TOOL_DATA_PATH=/tmp/tool-data" \
         -e "GALAXY_CONFIG_INSTALL_DATABASE_CONNECTION=sqlite:///${INSTALL_DATABASE}" \
         -e "GALAXY_CONFIG_MASTER_API_KEY=${API_KEY:=deadbeef}" \
+        -e "GALAXY_CONFIG_FILE=/galaxy/server/lib/galaxy/config/sample/galaxy.yml.sample" \
         -e "${CONDA_ENV_OPTION}" \
         ${CONDA_EXEC_OPTION} \
         -v "${OVERLAYFS_MOUNT}:/cvmfs/${REPO}" \
         -v "${WORKDIR}/tool_sheds_conf.xml:/tool_sheds_conf.xml" \
         -v "${WORKDIR}/condarc:${CONDARC_MOUNT_PATH}" \
         -v "${GALAXY_DATABASE_TMPDIR}:/galaxy/server/database" \
-        "$GALAXY_DOCKER_IMAGE" ./.venv/bin/uwsgi --http :8080 \
-            --virtualenv /galaxy/server/.venv --pythonpath /galaxy/server/lib \
-            --master --offload-threads 2 --processes 1 --threads 4 --enable-threads \
-            --buffer-size 16384 --thunder-lock --die-on-term --py-call-osafterfork \
-            --module 'galaxy.webapps.galaxy.buildapp:uwsgi_app\(\)' \
-            --hook-master-start '"unix_signal:2 gracefully_kill_them_all"' \
-            --hook-master-start '"unix_signal:15 gracefully_kill_them_all"' \
-            --static-map '/static/style=/galaxy/server/static/style/blue' \
-            --static-map '/static=/galaxy/server/static' \
-            --set 'galaxy_config_file=/galaxy/server/config/galaxy.yml' \
-            --set 'galaxy_root=/galaxy/server'
+        --workdir /galaxy/server \
+        "$GALAXY_DOCKER_IMAGE" ./.venv/bin/gunicorn 'galaxy.webapps.galaxy.fast_factory:factory\(\)' --timeout 300 --pythonpath lib -k galaxy.webapps.galaxy.workers.Worker -b 0.0.0.0:8080
         #"$GALAXY_DOCKER_IMAGE" ./.venv/bin/uwsgi --yaml config/galaxy.yml
         # TODO: double quoting above probably breaks non-local mode
     GALAXY_CONTAINER_UP=true
@@ -642,7 +635,7 @@ function install_tools() {
     for tool_yaml in "${TOOL_YAMLS[@]}"; do
         log "Installing tools in ${tool_yaml}"
         # FIXME: after https://github.com/galaxyproject/ephemeris/pull/181 is merged you would need to remove
-        # --skip_install_resolver_dependencies for install_resolver_dependencies in tools.yaml to work
+        # --skip_install_resolver_dependencies for install_resolver_dependencies: true in tools.yaml to work
         log_exec shed-tools install --skip_install_resolver_dependencies -v -g "$GALAXY_URL" -a "$API_KEY" -t "$tool_yaml" || {
             log_error "Tool installation failed"
             show_logs

.github/pull_request_template.md

@@ -0,0 +1,7 @@
+TODO: Please replace this header with a description of your pull request.
+
+## Installation sequence for `tool-installers`
+- [ ] Test using `@galaxybot test this`
+- [ ] Inspect CI output for expected changes
+- [ ] Deploy using `@galaxybot deploy this` if test install was successful
+- [ ] Merge this PR

.github/workflows/static.yml

@@ -0,0 +1,52 @@
+# Simple workflow for deploying static content to GitHub Pages
+name: Deploy static content to Pages
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["master"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  # Single deploy job since we're just deploying
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.11'
+          architecture: 'x64'
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Install pyyaml
+        run: pip install PyYAML
+      - name: Build API
+        run:
+          python scripts/api.py
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+        with:
+          # Upload entire repository
+          path: '.'
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2

Makefile

@@ -1,22 +1,23 @@
 GALAXY_SERVER := https://usegalaxy.*
+TOOLSET := usegalaxy.org
 
 
 help:
 	@egrep '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[94m%-16s\033[0m %s\n", $$1, $$2}'
 
 lint: ## Lint all yaml files
-	find ./$(TOOLSET) -name '*.yml' | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix-lockfile.py
-	find ./$(TOOLSET) -name '*.yml' | grep '^\./[^/]*/' | xargs -n 1 -P 8 -I{} pykwalify -d '{}' -s .schema.yml
+	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix-lockfile.py
+	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 -I{} pykwalify -d '{}' -s .schema.yml
 
 fix: ## Fix all lockfiles and add any missing revisions
 	@# Generates the lockfile or updates it if it is missing tools
-	find ./$(TOOLSET) -name '*.yml' | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix-lockfile.py
+	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix-lockfile.py
 	@# --without says only add those hashes for those missing hashes (zB new tools)
-	find ./$(TOOLSET) -name '*.yml' | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update-tool.py --without
+	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update-tool.py --without
 
 fix-no-deps:
-	find ./$(TOOLSET) -name '*.yml' | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix-lockfile.py --no-install-repository-dependencies --no-install-resolver-dependencies
-	find ./$(TOOLSET) -name '*.yml' | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update-tool.py --without
+	find ./$(TOOLSET) -name '*.yml'! -path .//.schema.yml  | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix-lockfile.py --no-install-repository-dependencies --no-install-resolver-dependencies
+	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update-tool.py --without
 
 #install:
 	#@echo "Installing any updated versions of $<"
@@ -25,9 +26,9 @@ fix-no-deps:
 
 update-trusted: ## Run the update script for a subset of repos
 	@# Missing --without, so this updates all tools in the file.
-	find ./$(TOOLSET) -name '*.yml' | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update-tool.py --owner $(OWNER)
+	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update-tool.py --owner $(OWNER)
 
 update-all: ## Run the update script for all repos
-	find ./$(TOOLSET) -name '*.yml' | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update-tool.py
+	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update-tool.py
 
 .PHONY: lint update-trusted update-all help fix

README.md

@@ -32,18 +32,40 @@ In your destination you should set:
 - Only IUC tools are automatically updated with the latest version each week
 - Use the provided `requirements.txt` to install dependences needed for the make targets
 
-### Updating an Existing Tool
-
-1. Edit the .yaml.lock file to add the latest/specific changeset revision for the tool. You can use `python scripts/update-tool.py --owner <repo-owner> --name <repo-name> <file.yaml.lock>` in order to do this if you just want to add the latest revision.
-2. Open a pull request
+## Requesting a new tool or updating a tool
+
+*Anyone* can request tool installations or updates on [usegalaxy.org](https://usegalaxy.org/) or [test.galaxyproject.org](https://test.galaxyproject.org).
+In the commands below fill the `{server_name}` as appropriate (usegalaxy.org, test.galaxyproject.org)
+
+1. Fork and clone [usegalaxy-tools](https://github.com/galaxyproject/usegalaxy-tools)
+1. Create/activate a virtualenv and `pip install -r requirements.txt`
+1. You are either installing a new repo or updating existing repo
+    - **NEW REPO**
+        1. If this is a new a section without an existing yml file create a new one like this:
+            1. Determine the desired section label
+            1. Normalize the section label to an ID/filename with [this process](https://github.com/galaxyproject/usegalaxy-tools/issues/9#issuecomment-500847395)
+            1. Create `{server_name}/<section_id>.yml` setting `tool_panel_section_label` from the section label obtained in previous step (see existing yml files for exact syntax)
+            1. Continue with the steps below
+        1. Add the entry for the new tool to the section yml (only the yml, not the yml.lock) [example](https://github.com/galaxyproject/usegalaxy-tools/pull/86/files#diff-7de70f8620e8ba71104b398d57087611R25-R26)
+        1. Run `$ make TOOLSET={server_name} fix` (this will fill the yml.lock )
+        1. Then `$ git add <file>` only the updates that you care about.
+    - **UPDATE REPO**
+        1. Find the yml and yml.lock files with the repository entries. Add the changeset hash of repo's desired *installable revision* to the yml.lock file [example](https://github.com/galaxyproject/usegalaxy-tools/pull/80/files#diff-2e7bd27ec27fa6be24b5689cebc77defR62-R64)
+        - Alternatively, run `make TOOLSET={server_name} OWNER={repo_owner} update-trusted` for every owner in your yml file, then run `$ make TOOLSET={server_name} fix`, and then `$ git add <file>` only the updates that you care about.
+1. Run `make TOOLSET={server_name} lint`
+1. Commit `{server_name}/<repo>.yaml{.lock}`
+1. Create a PR against the `master` branch of [usegalaxy-tools](https://github.com/galaxyproject/usegalaxy-tools)
+    - Use PR labels as appropriate
+    - To aid PR mergers, you can include information on tools in the repo's use of `$GALAXY_SLOTS`, or even PR any needed update(s) to [Main's job_conf.xml](https://github.com/galaxyproject/usegalaxy-playbook/blob/master/env/main/templates/galaxy/config/job_conf.xml.j2) as explained in the "[Determine tool requirements](#determine-tool-requirements)" section once the test installation (via Travis) succeeds (see details below)
+1. Once the PR is merged and the tool appears on [usegalaxy.org](https://usegalaxy.org/) or [test.galaxyproject.org](https://test.galaxyproject.org), test to ensure the tool works
 
 ### Requesting a New Tool
 
 - If you just want the latest version:
 	- Edit the .yaml file to add name/owner/section
 - If you want a specific version:
 	- Edit the .yaml file to add name/owner/section
-	- Run `make fix` (or `make fix-no-deps` for non-Conda toolsets like `cloud`)
+	- Run `make fix-no-deps`
 	- Edit the .yaml.lock to correct the version number.
 - Open a pull request
 

cloud/bed.yml.lock

@@ -7,5 +7,6 @@ tools:
   owner: iuc
   revisions:
   - ce3c7f062223
+  - 07e8b80f278c
   tool_panel_section_id: bed
   tool_panel_section_label: BED

cloud/collection_operations.yml.lock

@@ -25,5 +25,6 @@ tools:
   owner: iuc
   revisions:
   - 071084070619
+  - 58228a4d58fe
   tool_panel_section_id: collection_operations
   tool_panel_section_label: Collection Operations

cloud/convert_formats.yml

@@ -0,0 +1,4 @@
+tool_panel_section_label: Convert Formats
+tools:
+- name: gtftobed12
+  owner: iuc

cloud/convert_formats.yml.lock

@@ -0,0 +1,10 @@
+install_repository_dependencies: false
+install_resolver_dependencies: false
+install_tool_dependencies: false
+tool_panel_section_label: Convert Formats
+tools:
+- name: gtftobed12
+  owner: iuc
+  revisions:
+  - b026dae67fba
+  tool_panel_section_label: Convert Formats

cloud/datamash.yml.lock

@@ -6,18 +6,20 @@ tools:
 - name: datamash_reverse
   owner: iuc
   revisions:
-  - 0f1724dd59d2
+  - f198773344e4
   tool_panel_section_id: datamash
   tool_panel_section_label: Datamash
 - name: datamash_transpose
   owner: iuc
   revisions:
+  - 374cb875d38a
+  - ac092723240d
   - 22c2a1ac7ae3
   tool_panel_section_id: datamash
   tool_panel_section_label: Datamash
 - name: datamash_ops
   owner: iuc
   revisions:
-  - 562f3c677828
+  - 746e8e4bf929
   tool_panel_section_id: datamash
   tool_panel_section_label: Datamash

cloud/expressiontools.yml

@@ -0,0 +1,6 @@
+tool_panel_section_label: Expression Tools
+tools:
+- name: map_param_value
+  owner: iuc
+- name: compose_text_param
+  owner: iuc

cloud/expressiontools.yml.lock

@@ -0,0 +1,15 @@
+install_repository_dependencies: false
+install_resolver_dependencies: false
+install_tool_dependencies: false
+tool_panel_section_label: Expression Tools
+tools:
+- name: map_param_value
+  owner: iuc
+  revisions:
+  - a01f088d0e5e
+  tool_panel_section_label: Expression Tools
+- name: compose_text_param
+  owner: iuc
+  revisions:
+  - e188c9826e0f
+  tool_panel_section_label: Expression Tools

cloud/fasta_fastq.yml

@@ -18,3 +18,11 @@ tools:
   owner: iuc
 - name: trim_galore
   owner: bgruening
+- name: umi_tools_count
+  owner: iuc
+- name: umi_tools_extract
+  owner: iuc
+- name: seq_filter_by_id
+  owner: peterjc
+- name: length_and_gc_content
+  owner: iuc

cloud/fasta_fastq.yml.lock

@@ -7,6 +7,9 @@ tools:
   owner: lparsons
   revisions:
   - 093678460093
+  - 135b80fb1ac2
+  - c5e2e5902201
+  - 660cffd8d92a
   tool_panel_section_id: fasta_fastq
   tool_panel_section_label: FASTA/FASTQ
 - name: fastq_filter
@@ -57,3 +60,32 @@ tools:
   - 084bbd8ba7b8
   tool_panel_section_id: fasta_fastq
   tool_panel_section_label: FASTA/FASTQ
+- name: umi_tools_count
+  owner: iuc
+  revisions:
+  - 8250ea3a1501
+  - b557acca0b56
+  - a535cf1e5da4
+  tool_panel_section_id: fasta_fastq
+  tool_panel_section_label: FASTA/FASTQ
+- name: umi_tools_extract
+  owner: iuc
+  revisions:
+  - 6a675c3aa610
+  - 99aea37a7ff7
+  - f3759eec3018
+  tool_panel_section_id: fasta_fastq
+  tool_panel_section_label: FASTA/FASTQ
+- name: seq_filter_by_id
+  owner: peterjc
+  revisions:
+  - 141612f8c3e3
+  tool_panel_section_id: fasta_fastq
+  tool_panel_section_label: FASTA/FASTQ
+- name: length_and_gc_content
+  owner: iuc
+  revisions:
+  - e3ba567abdf5
+  - f088370d2a3c
+  tool_panel_section_id: fasta_fastq
+  tool_panel_section_label: FASTA/FASTQ

cloud/fastq_quality_control.yml.lock

@@ -8,19 +8,25 @@ tools:
   revisions:
   - 75c93c70d094
   - 9a913cdee30e
+  - df99138d2776
+  - abfd8a6544d7
+  - 5e33b465d8d5
+  - 1c2db0054039
   tool_panel_section_id: fastq_quality_control
   tool_panel_section_label: FASTQ Quality Control
 - name: fastp
   owner: iuc
   revisions:
+  - 65b93b623c77
   - dbf9c561ef29
   tool_panel_section_id: fastq_quality_control
   tool_panel_section_label: FASTQ Quality Control
 - name: fastqc
   owner: devteam
   revisions:
-  - e7b2202befea
   - 3d0c7bdf12f5
+  - e7b2202befea
+  - 9da02be9c6cc
   tool_panel_section_id: fastq_quality_control
   tool_panel_section_label: FASTQ Quality Control
 - name: trimmomatic