Merge pull request galaxyproject#15639 from jmchilton/tool_shed_2

Tool Shed 2.0

.ci/flake8_ignorelist.txt

@@ -11,6 +11,7 @@ database
 doc/build
 eggs
 lib/galaxy/web/proxy/js/node_modules
+lib/tool_shed/test/test_data/repos
 static/maps
 static/scripts
 test/functional/tools/cwl_tools/v1.?/

.flake8

@@ -7,3 +7,4 @@
 # W503 is line breaks before binary operators, which has been reversed in PEP 8.
 # D** are docstring linting - which we mostly ignore except D302. (Hopefully we will solve more over time).
 ignore = B008,E203,E402,E501,W503,D100,D101,D102,D103,D104,D105,D106,D107,D200,D201,D202,D204,D205,D206,D207,D208,D209,D210,D211,D300,D301,D400,D401,D402,D403,D412,D413
+exclude = lib/tool_shed/test/test_data/repos

.github/workflows/lint_openapi_schema.yml

@@ -51,6 +51,9 @@ jobs:
       - name: Build typescript schema
         run: make update-client-api-schema
         working-directory: 'galaxy root'
+      - name: Diff...
+        run: git diff
+        working-directory: 'galaxy root'
       - name: Check for changes
         run: |
           if [[ `git status --porcelain` ]]; then

.github/workflows/toolshed.yaml

@@ -20,7 +20,23 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.7']
+        include:
+        - test-install-client: 'galaxy_api'
+          python-version: '3.7'
+          shed-api: 'v1'
+          shed-browser: 'twill'
+        - test-install-client: 'standalone'
+          python-version: '3.8'
+          shed-api: 'v1'
+          shed-browser: 'twill'
+        - test-install-client: 'galaxy_api'
+          python-version: '3.9'
+          shed-api: 'v2'
+          shed-browser: 'playwright'
+        - test-install-client: 'standalone'
+          python-version: '3.10'
+          shed-api: 'v2'
+          shed-browser: 'playwright'
     services:
       postgres:
         image: postgres:13
@@ -52,11 +68,25 @@ jobs:
         with:
           path: 'galaxy root/.venv'
           key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-toolshed
+          key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy_root/requirements.txt') }}-toolshed
+      - name: Install dependencies
+        run: ./scripts/common_startup.sh --skip-client-build
+        working-directory: 'galaxy root'
+      - name: Build Frontend
+        run: '. .venv/bin/activate && cd lib/tool_shed/webapp/frontend && yarn && make client'
+        working-directory: 'galaxy root'
+      - name: Install playwright
+        run: '. .venv/bin/activate && playwright install'
+        working-directory: 'galaxy root'
       - name: Run tests
         run: './run_tests.sh -toolshed'
+        env:
+          TOOL_SHED_TEST_INSTALL_CLIENT: ${{ matrix.test-install-client }}
+          TOOL_SHED_API_VERSION: ${{ matrix.shed-api }}
+          TOOL_SHED_TEST_BROWSER: ${{ matrix.shed-browser }}
         working-directory: 'galaxy root'
       - uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: Toolshed test results (${{ matrix.python-version }})
+          name: Toolshed test results (${{ matrix.python-version }}, ${{ matrix.test-install-client }})
           path: 'galaxy root/run_toolshed_tests.html'

.isort.cfg

@@ -10,5 +10,5 @@ profile=black
 reverse_relative=true
 skip_gitignore=true
 # Make isort run faster by skipping database
-skip_glob=database/*
+skip_glob=database/*,lib/tool_shed/test/test_data/repos/*
 src_paths=lib

.redocly.lint-ignore.yaml

@@ -14,3 +14,6 @@ _schema.yaml:
       #/paths/~1api~1histories~1{history_id}~1contents~1{history_content_id}~1metadata_file
     - '#/paths/~1api~1histories~1{history_id}~1contents~1{id}~1validate'
     - '#/paths/~1api~1histories~1{history_id}~1contents~1{type}s~1{id}'
+_shed_schema.yaml:
+  no-empty-servers:
+    - '#/openapi'

.vscode/shed.code-snippets

@@ -0,0 +1,41 @@
+{
+    "shedcomp": {
+        "prefix": "shed_component",
+        "body": [
+            "<script setup lang=\"ts\">",
+            "\t$0",
+            "</script>",
+            "<template>",
+            "</template>"
+        ],
+       "description": "outline of a tool shed component"
+    },
+    "shedpage": {
+        "prefix": "shed_page",
+        "body": [
+            "<script setup lang=\"ts\">",
+            "import PageContainer from \"@/components/PageContainer.vue\"",
+            "</script>",
+            "<template>",
+            "    <page-container>",
+            "        $0",
+            "    </page-container>",
+            "</template>"
+        ],
+       "description": "outline of a tool shed page"
+    },
+    "shedfetcher": {
+        "prefix": "shed_fetcher",
+        "body": [
+            "import { fetcher } from \"@/schema\"",
+            "const fetcher = fetcher.path(\"$1\").method(\"get\").create()"
+        ],
+        "description": "Import shed fetcher and instantiate with a path"
+    },
+    "shedrouter": {
+        "prefix": "shed_router",
+        "body": [
+            "import router from \"@/router\""
+        ]
+    }
+}

Makefile

@@ -182,17 +182,22 @@ endif
 
 build-api-schema:
 	$(IN_VENV) python scripts/dump_openapi_schema.py _schema.yaml
+	$(IN_VENV) python scripts/dump_openapi_schema.py --app shed _shed_schema.yaml
 
 remove-api-schema:
 	rm _schema.yaml
+	rm _shed_schema.yaml
 
 update-client-api-schema: client-node-deps build-api-schema
 	$(IN_VENV) cd client && node openapi_to_schema.mjs ../_schema.yaml > src/schema/schema.ts && npx prettier --write src/schema/schema.ts
+	$(IN_VENV) cd client && node openapi_to_schema.mjs ../_shed_schema.yaml > ../lib/tool_shed/webapp/frontend/src/schema/schema.ts && npx prettier --write ../lib/tool_shed/webapp/frontend/src/schema/schema.ts
 	$(MAKE) remove-api-schema
 
 lint-api-schema: build-api-schema
 	$(IN_VENV) npx --yes @redocly/cli lint _schema.yaml
+	$(IN_VENV) npx --yes @redocly/cli lint _shed_schema.yaml
 	$(IN_VENV) codespell -I .ci/ignore-spelling.txt _schema.yaml
+	$(IN_VENV) codespell -I .ci/ignore-spelling.txt _shed_schema.yaml
 	$(MAKE) remove-api-schema
 
 update-navigation-schema: client-node-deps

lib/galaxy/app.py

@@ -84,7 +84,10 @@
     install_model_scoped_session,
 )
 from galaxy.model.tags import GalaxyTagHandler
-from galaxy.model.tool_shed_install import mapping as install_mapping
+from galaxy.model.tool_shed_install import (
+    HasToolBox,
+    mapping as install_mapping,
+)
 from galaxy.objectstore import (
     BaseObjectStore,
     build_object_store_from_config,
@@ -105,8 +108,10 @@
     VaultFactory,
 )
 from galaxy.tool_shed.cache import ToolShedRepositoryCache
+from galaxy.tool_shed.galaxy_install.client import InstallationTarget
 from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager
 from galaxy.tool_shed.galaxy_install.update_repository_manager import UpdateRepositoryManager
+from galaxy.tool_util.data import ToolDataTableManager as BaseToolDataTableManager
 from galaxy.tool_util.deps import containers
 from galaxy.tool_util.deps.dependencies import AppInfo
 from galaxy.tool_util.deps.views import DependencyResolversView
@@ -214,14 +219,13 @@ def configure_sentry_client(self):
             )
 
 
-class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMixin):
+class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMixin, HasToolBox):
     """Encapsulates the state of a minimal Galaxy application"""
 
     model: GalaxyModelMapping
     config: config.GalaxyAppConfiguration
     tool_cache: ToolCache
     job_config: jobs.JobConfiguration
-    toolbox: tools.ToolBox
     toolbox_search: ToolBoxSearch
     container_finder: containers.ContainerFinder
     install_model: ModelMapping
@@ -297,15 +301,12 @@ def _configure_tool_config_files(self):
             self.config.tool_configs.append(self.config.migrated_tools_config)
 
     def _configure_toolbox(self):
-        if not isinstance(self, BasicSharedApp):
-            raise Exception("Must inherit from BasicSharedApp")
-
         self.citations_manager = CitationsManager(self)
         self.biotools_metadata_source = get_galaxy_biotools_metadata_source(self.config)
 
         self.dynamic_tools_manager = DynamicToolManager(self)
         self._toolbox_lock = threading.RLock()
-        self.toolbox = tools.ToolBox(self.config.tool_configs, self.config.tool_path, self)
+        self._toolbox = tools.ToolBox(self.config.tool_configs, self.config.tool_path, self)
         galaxy_root_dir = os.path.abspath(self.config.root)
         file_path = os.path.abspath(self.config.file_path)
         app_info = AppInfo(
@@ -345,6 +346,10 @@ def _configure_toolbox(self):
             ToolBoxSearch(self.toolbox, index_dir=self.config.tool_search_index_dir, index_help=index_help),
         )
 
+    @property
+    def toolbox(self) -> tools.ToolBox:
+        return self._toolbox
+
     def reindex_tool_search(self) -> None:
         # Call this when tools are added or removed.
         self.toolbox_search.build_index(tool_cache=self.tool_cache, toolbox=self.toolbox)
@@ -366,7 +371,7 @@ def _set_enabled_container_types(self):
 
     def _configure_tool_data_tables(self, from_shed_config):
         # Initialize tool data tables using the config defined by self.config.tool_data_table_config_path.
-        self.tool_data_tables = ToolDataTableManager(
+        self.tool_data_tables: BaseToolDataTableManager = ToolDataTableManager(
             tool_data_path=self.config.tool_data_path,
             config_filename=self.config.tool_data_table_config_path,
             other_config_dict=self.config,
@@ -488,7 +493,7 @@ def _wait_for_database(self, url):
                 time.sleep(pause)
 
     @property
-    def tool_dependency_dir(self):
+    def tool_dependency_dir(self) -> Optional[str]:
         return self.toolbox.dependency_manager.default_base_path
 
     def _shutdown_object_store(self):
@@ -498,7 +503,7 @@ def _shutdown_model(self):
         self.model.engine.dispose()
 
 
-class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
+class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication, InstallationTarget[tools.ToolBox]):
     """Extends the MinimalGalaxyApplication with most managers that are not tied to a web or job handling context."""
 
     model: GalaxyModelMapping
@@ -685,7 +690,7 @@ def __init__(self, **kwargs) -> None:
         self.watchers = self._register_singleton(ConfigWatchers)
         self._configure_toolbox()
         # Load Data Manager
-        self.data_managers = self._register_singleton(DataManagers)
+        self.data_managers = self._register_singleton(DataManagers)  # type: ignore[type-abstract]
         # Load the update repository manager.
         self.update_repository_manager = self._register_singleton(
             UpdateRepositoryManager, UpdateRepositoryManager(self)

lib/galaxy/app_unittest_utils/galaxy_mock.py

@@ -94,7 +94,7 @@ class MockApp(di.Container, GalaxyDataTestApp):
     config: "MockAppConfig"
     amqp_type: str
     job_search: Optional[JobSearch] = None
-    toolbox: ToolBox
+    _toolbox: ToolBox
     tool_cache: ToolCache
     install_model: ModelMapping
     watchers: ConfigWatchers
@@ -110,6 +110,7 @@ def __init__(self, config=None, **kwargs) -> None:
         super().__init__()
         config = config or MockAppConfig(**kwargs)
         GalaxyDataTestApp.__init__(self, config=config, **kwargs)
+        self.install_model = self.model
         self[BasicSharedApp] = cast(BasicSharedApp, self)
         self[MinimalManagerApp] = cast(MinimalManagerApp, self)  # type: ignore[type-abstract]
         self[StructuredApp] = cast(StructuredApp, self)  # type: ignore[type-abstract]
@@ -153,6 +154,10 @@ def url_for(*args, **kwds):
 
         self.url_for = url_for
 
+    @property
+    def toolbox(self) -> ToolBox:
+        return self._toolbox
+
     def wait_for_toolbox_reload(self, toolbox):
         # TODO: If the tpm test case passes, does the operation really
         # need to wait.