From 7cda336f1fe3ee60fbdb86a660f33db6b557d637 Mon Sep 17 00:00:00 2001
From: Martin Cech <marten@bx.psu.edu>
Date: Tue, 15 Oct 2024 11:40:39 +0200
Subject: [PATCH 01/84] add a hint to tool schema

about inability to change conditionals during runtime
---
 lib/galaxy/tool_util/xsd/galaxy.xsd | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index c8090c919568..0d5dc9e51079 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -312,8 +312,6 @@ A generalisation for macro tokens, templates and xml macros, i.e.
 `<macro name="an_xml_macro" type="xml">` is identical to `<xml name="an_xml_macro">`,
 `<macro name="a_template" type="template">` is identical to `<template name="a_template">`, and
 `<macro name="a_token" type="xml">` is identical to `<token name="a_token">`.
-
-Note that 
        ]]></xs:documentation>
         </xs:annotation>
       </xs:element>
@@ -3499,9 +3497,13 @@ Data source HTTP action (e.g. ``get`` or ``put``) to use.</xs:documentation>
     <xs:annotation>
       <xs:documentation xml:lang="en"><![CDATA[
 
-This is a container for conditional parameters in the tool (must contain 'when'
-tag sets) - the command line (or portions thereof) are then wrapped in an if-else
-statement. A good example tool that demonstrates many conditional parameters is
+This is a container for conditional parameters in the tool (must contain ``when``
+tag sets) - the command line is then wrapped in an if-else statement.
+
+Note that value of a conditional cannot be changed on the workflow run form.
+If you expect users to interact with the element during runtime consider using ``sections`` instead.
+
+An example tool that demonstrates conditional parameters is
 [biom_convert.xml](https://github.com/galaxyproject/tools-iuc/blob/main/tools/biom_format/biom_convert.xml).
 
 ```xml
@@ -3608,7 +3610,7 @@ Referenced parameter to pass method.]]></xs:documentation>
           <xs:annotation>
             <xs:documentation xml:lang="en"><![CDATA[Infrequently used option to dynamically access Galaxy internals, this should be avoided.
 
-Is referenced parameter is the same group.]]></xs:documentation>
+Is referenced parameter in the same group.]]></xs:documentation>
           </xs:annotation>
         </xs:attribute>
         <xs:attribute name="label" type="xs:string">
@@ -6286,7 +6288,7 @@ Therefore a filter for such a variable looks like the following example.
         <xs:documentation xml:lang="en"><![CDATA[
         A string `[reverse_][SORT_COMP_]SORTBY` describing the desired sort order of the collection elements.
         `SORTBY` can be `filename`, `name`, `designation`, `dbkey` and the optional `SORT_COMP` can be either
-        `lexical` or `numeric`. Default is lexical sorting by filename. 
+        `lexical` or `numeric`. Default is lexical sorting by filename.
         Note that lexical sorting is case sensitive, i.e. upper case characters come before lower case characters (e.g. "Apple" < "Banana" < "apple" < "banana").
         ]]></xs:documentation>
       </xs:annotation>

From c6ead0028791fdc7427a4421d45cd8277c8d3a48 Mon Sep 17 00:00:00 2001
From: Martin Cech <marten@bx.psu.edu>
Date: Tue, 15 Oct 2024 12:03:25 +0200
Subject: [PATCH 02/84] run make config-revuild

---
 doc/source/admin/galaxy_options.rst           | 5 +----
 lib/galaxy/config/sample/galaxy.yml.sample    | 5 ++---
 lib/galaxy/config/sample/tool_shed.yml.sample | 4 +++-
 3 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst
index 45386d72d861..c906f982b362 100644
--- a/doc/source/admin/galaxy_options.rst
+++ b/doc/source/admin/galaxy_options.rst
@@ -5124,7 +5124,7 @@
     'deferred' are "materialized" (or undeferred) by the workflow
     scheduler. This might be a lengthy process. Setting this to 'True'
     will place the invocation back in the queue after materialization
-    before scheduling the workflow so it is less  likely to starve
+    before scheduling the workflow so it is less likely to starve
     other workflow scheduling. Ideally, Galaxy would allow more fine
     grain control of handlers but until then, this provides a way to
     tip the balance between "doing more work" and "being more fair".
@@ -5658,6 +5658,3 @@
     to the user. Currently only affects s3fs file sources.
 :Default: ``60``
 :Type: int
-
-
-
diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample
index d6718cf176e7..d2e71539c7df 100644
--- a/lib/galaxy/config/sample/galaxy.yml.sample
+++ b/lib/galaxy/config/sample/galaxy.yml.sample
@@ -191,7 +191,7 @@ gravity:
 
     # Routes file to monitor.
     # Should be set to the same path as ``interactivetools_map`` in the ``galaxy:`` section. This is ignored if
-    # ``interactivetools_map is set.
+    # ``interactivetools_map is set``.
     # sessions: database/interactivetools_map.sqlite
 
     # Include verbose messages in gx-it-proxy
@@ -2752,7 +2752,7 @@ galaxy:
   # 'deferred' are "materialized" (or undeferred) by the workflow
   # scheduler. This might be a lengthy process. Setting this to 'True'
   # will place the invocation back in the queue after materialization
-  # before scheduling the workflow so it is less  likely to starve other
+  # before scheduling the workflow so it is less likely to starve other
   # workflow scheduling. Ideally, Galaxy would allow more fine grain
   # control of handlers but until then, this provides a way to tip the
   # balance between "doing more work" and "being more fair". The default
@@ -3006,4 +3006,3 @@ galaxy:
   # but outdated contents might be displayed to the user. Currently only
   # affects s3fs file sources.
   #file_source_listings_expiry_time: 60
-
diff --git a/lib/galaxy/config/sample/tool_shed.yml.sample b/lib/galaxy/config/sample/tool_shed.yml.sample
index bf16f6bfabab..bc995ed597d0 100644
--- a/lib/galaxy/config/sample/tool_shed.yml.sample
+++ b/lib/galaxy/config/sample/tool_shed.yml.sample
@@ -54,6 +54,9 @@ tool_shed:
   # options.
   #whoosh_index_dir: database/toolshed_whoosh_indexes
 
+  # Cache directory for Pydantic model objects.
+  #model_cache_dir: database/model_cache
+
   # For searching repositories at /api/repositories:
   #repo_name_boost: 0.9
 
@@ -426,4 +429,3 @@ tool_shed:
   # The value of this option will be resolved with respect to
   # <config_dir>.
   #datatypes_config_file: datatypes_conf.xml
-

From d3e0e28ce358dc3c220986050bd559f844da4bd0 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Mon, 14 Oct 2024 09:53:49 -0400
Subject: [PATCH 03/84] Do not skip CI tests on missing tools.

---
 .github/workflows/api.yaml        |  1 +
 lib/galaxy_test/api/test_tools.py | 13 -------------
 2 files changed, 1 insertion(+), 13 deletions(-)

diff --git a/.github/workflows/api.yaml b/.github/workflows/api.yaml
index 8eb5a2dd564a..0ca4399f9314 100644
--- a/.github/workflows/api.yaml
+++ b/.github/workflows/api.yaml
@@ -18,6 +18,7 @@ env:
   GALAXY_TEST_DBURI: 'postgresql://postgres:postgres@localhost:5432/galaxy?client_encoding=utf8'
   GALAXY_TEST_RAISE_EXCEPTION_ON_HISTORYLESS_HDA: '1'
   GALAXY_CONFIG_SQLALCHEMY_WARN_20: '1'
+  GALAXY_TEST_REQUIRE_ALL_NEEDED_TOOLS: '1'
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py
index 46c6cdad3473..b99a059a65a7 100644
--- a/lib/galaxy_test/api/test_tools.py
+++ b/lib/galaxy_test/api/test_tools.py
@@ -29,7 +29,6 @@
     BaseDatasetCollectionPopulator,
     DatasetCollectionPopulator,
     DatasetPopulator,
-    LibraryPopulator,
     skip_without_tool,
     stage_rules_example,
 )
@@ -1027,18 +1026,6 @@ def test_data_column_defaults(self):
                 content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output[0])
                 assert "parameter: 1,2" in content
 
-    @skip_without_tool("library_data")
-    def test_library_data_param(self):
-        with self.dataset_populator.test_history(require_new=False) as history_id:
-            ld = LibraryPopulator(self.galaxy_interactor).new_library_dataset("lda_test_library")
-            inputs = {"library_dataset": ld["ldda_id"], "library_dataset_multiple": [ld["ldda_id"], ld["ldda_id"]]}
-            response = self._run("library_data", history_id, inputs, assert_ok=True)
-            output = response["outputs"]
-            output_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output[0])
-            assert output_content == "TestData\n", output_content
-            output_multiple_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output[1])
-            assert output_multiple_content == "TestData\nTestData\n", output_multiple_content
-
     @skip_without_tool("cat1")
     def test_run_cat1(self):
         with self.dataset_populator.test_history(require_new=False) as history_id:

From 251410b20ef5856d49c6f1f6fe9a73be3879ee35 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 23 Oct 2024 12:34:45 -0400
Subject: [PATCH 04/84] Better search stuff in Tool Shed 2.0.

---
 lib/tool_shed/util/shed_index.py              | 10 ++++---
 .../src/components/RepositoriesGrid.vue       | 22 ++++++++++++---
 .../pages/RepositoriesByCategory.vue          | 11 +++++---
 .../components/pages/RepositoriesBySearch.vue | 27 ++++++++++++++++---
 4 files changed, 58 insertions(+), 12 deletions(-)

diff --git a/lib/tool_shed/util/shed_index.py b/lib/tool_shed/util/shed_index.py
index ee94d4c1a785..11faf5d9c8c2 100644
--- a/lib/tool_shed/util/shed_index.py
+++ b/lib/tool_shed/util/shed_index.py
@@ -56,6 +56,8 @@ def build_index(whoosh_index_dir, file_path, hgweb_config_dir, hgweb_repo_prefix
     execution_timer = ExecutionTimer()
     with repo_index.searcher() as searcher:
         for repo in get_repos(sa_session, file_path, hgweb_config_dir, hgweb_repo_prefix, **kwargs):
+            if repo is None:
+                continue
             tools_list = repo.pop("tools_list")
             repo_id = repo["id"]
             indexed_document = searcher.document(id=repo_id)
@@ -119,9 +121,11 @@ def get_repos(sa_session, file_path, hgweb_config_dir, hgweb_repo_prefix, **kwar
         full_last_updated = repo.update_time.strftime("%Y-%m-%d %I:%M %p")
 
         # Load all changesets of the repo for lineage.
-        repo_path = os.path.join(
-            hgweb_config_dir, hgwcm.get_entry(os.path.join(hgweb_repo_prefix, repo.user.username, repo.name))
-        )
+        try:
+            entry = hgwcm.get_entry(os.path.join(hgweb_repo_prefix, repo.user.username, repo.name))
+        except Exception:
+            return None
+        repo_path = os.path.join(hgweb_config_dir, entry)
         hg_repo = hg.repository(ui.ui(), repo_path.encode("utf-8"))
         lineage = []
         for changeset in hg_repo.changelog:
diff --git a/lib/tool_shed/webapp/frontend/src/components/RepositoriesGrid.vue b/lib/tool_shed/webapp/frontend/src/components/RepositoriesGrid.vue
index f68dab181613..972677babae9 100644
--- a/lib/tool_shed/webapp/frontend/src/components/RepositoriesGrid.vue
+++ b/lib/tool_shed/webapp/frontend/src/components/RepositoriesGrid.vue
@@ -14,6 +14,7 @@ interface RepositoriesGridProps {
     rows: Array<RepositoryGridItem>
     noDataLabel?: string
     debug?: boolean
+    allowSearch?: boolean
 }
 
 interface ScrollDetails {
@@ -29,6 +30,7 @@ const compProps = withDefaults(defineProps<RepositoriesGridProps>(), {
     debug: false,
     onScroll: null as OnScroll | null,
     noDataLabel: "No repositories found",
+    allowSearch: false,
 })
 
 const pagination = { rowsPerPage: 0 }
@@ -44,7 +46,7 @@ const NAME_COLUMN: QTableColumn = {
     name: "name",
     label: "Name",
     align: "left",
-    field: "name",
+    field: "effectiveName",
 }
 
 const columns = computed(() => {
@@ -71,10 +73,14 @@ async function onVirtualScroll(details: ScrollDetails) {
     }
 }
 
+const search = ref("")
+
 // Adapt the rows with doubleIndex so
 const adaptedRows = computed(() =>
     compProps.rows.map((r) => {
-        return { doubleIndex: r.index * 2, ...r }
+        // create this effective name so we are filtering on this when searching...
+        const effectiveName = r.owner + " " + r.name
+        return { doubleIndex: r.index * 2, effectiveName: effectiveName, ...r }
     })
 )
 </script>
@@ -83,10 +89,10 @@ const adaptedRows = computed(() =>
         <q-table
             v-if="loading || rows.length > 0"
             style="height: 85vh"
-            :title="title"
             :rows="adaptedRows"
             :columns="columns"
             :loading="tableLoading"
+            :filter="search"
             row-key="newIndex"
             virtual-scroll
             :virtual-scroll-item-size="48"
@@ -98,6 +104,16 @@ const adaptedRows = computed(() =>
             hide-header
             hide-bottom
         >
+            <template #top>
+                <div class="row col-12">
+                    <div class="q-table__control col-8">
+                        <div class="q-table__title">{{ title }}</div>
+                    </div>
+                    <div class="col-4 justify-end q-pa-md" v-if="allowSearch">
+                        <q-input v-model="search" autogrow label="Filter"></q-input>
+                    </div>
+                </div>
+            </template>
             <template #body="props">
                 <q-tr :props="props" :key="`m_${props.row.index}`">
                     <q-td v-for="col in props.cols" :key="col.name" :props="props">
diff --git a/lib/tool_shed/webapp/frontend/src/components/pages/RepositoriesByCategory.vue b/lib/tool_shed/webapp/frontend/src/components/pages/RepositoriesByCategory.vue
index 899341171af9..cb3046247e07 100644
--- a/lib/tool_shed/webapp/frontend/src/components/pages/RepositoriesByCategory.vue
+++ b/lib/tool_shed/webapp/frontend/src/components/pages/RepositoriesByCategory.vue
@@ -1,8 +1,8 @@
 <script setup lang="ts">
-import { computed } from "vue"
+import { computed, ref } from "vue"
 import { useCategoriesStore } from "@/stores"
 import RepositoriesGrid from "@/components/RepositoriesGrid.vue"
-import { nodeToRow } from "@/components/RepositoriesGridInterface"
+import { nodeToRow, type RepositoryGridItem } from "@/components/RepositoriesGridInterface"
 import ErrorBanner from "@/components/ErrorBanner.vue"
 import LoadingDiv from "@/components/LoadingDiv.vue"
 import { graphql } from "@/gql"
@@ -67,6 +67,11 @@ const rows = computed(() => {
     <loading-div v-if="loading" />
     <error-banner error="Failed to load repository" v-else-if="error"> </error-banner>
     <q-page class="q-pa-md" v-if="categoryName">
-        <repositories-grid :title="`Repositories for ${categoryName}`" :rows="rows" :on-scroll="onScroll" />
+        <repositories-grid
+            :title="`Repositories for ${categoryName}`"
+            :rows="rows"
+            :on-scroll="onScroll"
+            :allow-search="true"
+        />
     </q-page>
 </template>
diff --git a/lib/tool_shed/webapp/frontend/src/components/pages/RepositoriesBySearch.vue b/lib/tool_shed/webapp/frontend/src/components/pages/RepositoriesBySearch.vue
index f98aec5f0007..bf8ca25559f3 100644
--- a/lib/tool_shed/webapp/frontend/src/components/pages/RepositoriesBySearch.vue
+++ b/lib/tool_shed/webapp/frontend/src/components/pages/RepositoriesBySearch.vue
@@ -8,6 +8,7 @@ import { notifyOnCatch } from "@/util"
 
 const query = ref("")
 const page = ref(1)
+const searching = ref(false)
 const fetchedLastPage = ref(false)
 const hits = ref([] as Array<RepositorySearchHit>)
 
@@ -15,7 +16,11 @@ type RepositorySearchHit = components["schemas"]["RepositorySearchHit"]
 
 async function doQuery() {
     const queryValue = query.value
-
+    if (!queryValue) {
+        hits.value = []
+        return
+    }
+    searching.value = true
     try {
         const { data } = await ToolShedApi().GET("/api/repositories", {
             params: {
@@ -39,10 +44,12 @@ async function doQuery() {
             }
             page.value = page.value + 1
         } else {
-            throw Error("Server response structure error.")
+            throw Error("Server response structure error for [" + queryValue + "]")
         }
     } catch (e) {
         notifyOnCatch(e)
+    } finally {
+        searching.value = false
     }
 }
 
@@ -88,12 +95,26 @@ function rowsFunc() {
     return rows
 }*/
 
+const noDataLabel = computed<string>(() => {
+    if (searching.value) {
+        return "Searching..."
+    } else {
+        return "No repositories matching query"
+    }
+})
+
 const rows = realRows
 </script>
 <template>
     <page-container>
         <q-input debounce="20" filled v-model="query" label="Search Repositories" />
-        <repository-grid v-if="query && query.length > 1" :rows="rows" title="Search Results" :on-scroll="OnScrollImpl">
+        <repository-grid
+            v-if="query && query.length > 1"
+            :rows="rows"
+            title="Search Results"
+            :on-scroll="OnScrollImpl"
+            :no-data-label="noDataLabel"
+        >
         </repository-grid>
     </page-container>
 </template>

From 3d6579b948a8348a6002bf71762b8b3bafacfa55 Mon Sep 17 00:00:00 2001
From: Ahmed Awan <qe66653@umbc.edu>
Date: Thu, 24 Oct 2024 10:08:13 -0400
Subject: [PATCH 05/84] [WIP] Add `galaxy-upload` link to `UploadModal`

---
 client/src/components/Upload/UploadModal.vue | 22 +++++++++++++-------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/client/src/components/Upload/UploadModal.vue b/client/src/components/Upload/UploadModal.vue
index a78b0ce0adea..258e7f78d868 100644
--- a/client/src/components/Upload/UploadModal.vue
+++ b/client/src/components/Upload/UploadModal.vue
@@ -1,4 +1,5 @@
 <script setup>
+import { BModal } from "bootstrap-vue";
 import { storeToRefs } from "pinia";
 import { ref, watch } from "vue";
 
@@ -71,7 +72,7 @@ defineExpose({
 </script>
 
 <template>
-    <b-modal
+    <BModal
         v-model="showModal"
         :static="options.modalStatic"
         header-class="no-separator"
@@ -81,12 +82,17 @@ defineExpose({
         no-enforce-focus
         hide-footer>
         <template v-slot:modal-header>
-            <h2 class="title h-sm" tabindex="0">
-                {{ options.title }}
-                <span v-if="currentHistory">
-                    to <b>{{ currentHistory.name }}</b>
-                </span>
-            </h2>
+            <div class="d-flex justify-content-between w-100">
+                <h2 class="title h-sm" tabindex="0">
+                    {{ options.title }}
+                    <span v-if="currentHistory">
+                        to <b>{{ currentHistory.name }}</b>
+                    </span>
+                </h2>
+                <a href="https://galaxy-upload.readthedocs.io/en/latest/" target="_blank">
+                    Check out the <code>galaxy-upload</code> util here!
+                </a>
+            </div>
         </template>
         <UploadContainer
             v-if="currentHistoryId"
@@ -95,7 +101,7 @@ defineExpose({
             :current-history-id="currentHistoryId"
             v-bind="options"
             @dismiss="dismiss" />
-    </b-modal>
+    </BModal>
 </template>
 
 <style>

From f518ef292b5fd3b70c187baecca7d1ebf8ae99ca Mon Sep 17 00:00:00 2001
From: Ahmed Awan <qe66653@umbc.edu>
Date: Thu, 24 Oct 2024 10:28:30 -0400
Subject: [PATCH 06/84] add `<HelpText>` to the link

---
 client/src/components/Help/terms.yml         |  5 +++++
 client/src/components/Upload/UploadModal.vue | 11 ++++++++---
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/client/src/components/Help/terms.yml b/client/src/components/Help/terms.yml
index 8d1e9ca05d92..0ca9bd36bc72 100644
--- a/client/src/components/Help/terms.yml
+++ b/client/src/components/Help/terms.yml
@@ -106,3 +106,8 @@ galaxy:
       failed: |
         This state indicates there was a problem scheduling the workflow invocation. No additional datasets
         or jobs will be created for this workflow invocation.
+  upload:
+    galaxyUploadUtil: |
+      A utility for uploading files to a Galaxy server from the command line. Use ``galaxy-upload`` to upload
+      file(s) to a Galaxy server, and ``galaxy-history-search``, a helper utility to find Galaxy histories
+      to pass to the ``galaxy-upload`` command.
diff --git a/client/src/components/Upload/UploadModal.vue b/client/src/components/Upload/UploadModal.vue
index 258e7f78d868..f3b8a15907d8 100644
--- a/client/src/components/Upload/UploadModal.vue
+++ b/client/src/components/Upload/UploadModal.vue
@@ -9,6 +9,8 @@ import { useUserHistories } from "@/composables/userHistories";
 import { useUserStore } from "@/stores/userStore";
 import { wait } from "@/utils/utils";
 
+import ExternalLink from "../ExternalLink.vue";
+import HelpText from "../Help/HelpText.vue";
 import UploadContainer from "./UploadContainer.vue";
 
 const { currentUser } = storeToRefs(useUserStore());
@@ -89,9 +91,12 @@ defineExpose({
                         to <b>{{ currentHistory.name }}</b>
                     </span>
                 </h2>
-                <a href="https://galaxy-upload.readthedocs.io/en/latest/" target="_blank">
-                    Check out the <code>galaxy-upload</code> util here!
-                </a>
+                <span>
+                    <ExternalLink href="https://galaxy-upload.readthedocs.io/en/latest/"> Click here </ExternalLink>
+                    to check out the
+                    <HelpText uri="galaxy.upload.galaxyUploadUtil" text="galaxy-upload" />
+                    util!
+                </span>
             </div>
         </template>
         <UploadContainer

From da82ed1214caaec365af4e40ff1400d481f3e84a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 25 Oct 2024 10:59:48 +0000
Subject: [PATCH 07/84] Bump http-proxy-middleware from 2.0.6 to 2.0.7 in
 /client

Bumps [http-proxy-middleware](https://github.com/chimurai/http-proxy-middleware) from 2.0.6 to 2.0.7.
- [Release notes](https://github.com/chimurai/http-proxy-middleware/releases)
- [Changelog](https://github.com/chimurai/http-proxy-middleware/blob/v2.0.7/CHANGELOG.md)
- [Commits](https://github.com/chimurai/http-proxy-middleware/compare/v2.0.6...v2.0.7)

---
updated-dependencies:
- dependency-name: http-proxy-middleware
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 client/yarn.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/client/yarn.lock b/client/yarn.lock
index bbefc80fa80b..46d7ba982257 100644
--- a/client/yarn.lock
+++ b/client/yarn.lock
@@ -6890,9 +6890,9 @@ http-proxy-agent@^5.0.0:
     debug "4"
 
 http-proxy-middleware@^2.0.3:
-  version "2.0.6"
-  resolved "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz"
-  integrity sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw==
+  version "2.0.7"
+  resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz#915f236d92ae98ef48278a95dedf17e991936ec6"
+  integrity sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==
   dependencies:
     "@types/http-proxy" "^1.17.8"
     http-proxy "^1.18.1"

From 84b05313c1ca854adabbff1b6813354b204c7030 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 28 Oct 2024 15:46:55 +0100
Subject: [PATCH 08/84] Fix very slow workflow editor loading

Loading the editor for my usegalaxy.org account takes 20 seconds. I've
narrowed this down to the unnecessary join against the workflow table
and loading all steps. I'm also returning the needed columns directly.
---
 .../webapps/galaxy/controllers/workflow.py    | 54 +++++++++++++------
 1 file changed, 39 insertions(+), 15 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py
index 085834f67ec0..5928e925df58 100644
--- a/lib/galaxy/webapps/galaxy/controllers/workflow.py
+++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py
@@ -1,8 +1,13 @@
 import logging
 
 from markupsafe import escape
-from sqlalchemy import desc
-from sqlalchemy.orm import joinedload
+from sqlalchemy import (
+    and_,
+    desc,
+    false,
+    func,
+    select,
+)
 
 from galaxy import (
     model,
@@ -166,13 +171,32 @@ def editor(self, trans, id=None, workflow_id=None, version=None, **kwargs):
 
         # The following query loads all user-owned workflows,
         # So that they can be copied or inserted in the workflow editor.
-        workflows = (
-            trans.sa_session.query(model.StoredWorkflow)
-            .filter_by(user=trans.user, deleted=False, hidden=False)
-            .order_by(desc(model.StoredWorkflow.table.c.update_time))
-            .options(joinedload(model.StoredWorkflow.latest_workflow).joinedload(model.Workflow.steps))
-            .all()
+        assert trans.user  # help out type checker, require_login means we will have a user
+        workflow_stmnt = (
+            select(
+                model.StoredWorkflow.id,
+                model.StoredWorkflow.latest_workflow_id,
+                model.StoredWorkflow.name,
+                func.coalesce(func.count(model.WorkflowStep.id), 0).label("step_count"),
+            )
+            .join(
+                model.WorkflowStep,
+                model.StoredWorkflow.latest_workflow_id == model.WorkflowStep.workflow_id,
+                isouter=True,
+            )
+            .where(
+                and_(
+                    model.StoredWorkflow.user_id == trans.user.id,
+                    model.StoredWorkflow.deleted == false(),
+                    model.StoredWorkflow.hidden == false(),
+                )
+            )
+            .group_by(
+                model.StoredWorkflow.id,
+            )
+            .order_by(desc(model.StoredWorkflow.update_time))
         )
+        workflow_results = trans.sa_session.execute(workflow_stmnt).all()
 
         # create workflow module models
         module_sections = []
@@ -219,13 +243,13 @@ def editor(self, trans, id=None, workflow_id=None, version=None, **kwargs):
         # create workflow models
         workflows = [
             {
-                "id": trans.security.encode_id(workflow.id),
-                "latest_id": trans.security.encode_id(workflow.latest_workflow.id),
-                "step_count": len(workflow.latest_workflow.steps),
-                "name": workflow.name,
+                "id": trans.security.encode_id(stored_workflow_id),
+                "latest_id": trans.security.encode_id(latest_workflow_id),
+                "step_count": step_count,
+                "name": workflow_name,
             }
-            for workflow in workflows
-            if new_workflow or workflow.id != stored.id
+            for stored_workflow_id, latest_workflow_id, workflow_name, step_count in workflow_results
+            if not stored or stored_workflow_id != stored.id
         ]
 
         # build workflow editor model
@@ -236,7 +260,7 @@ def editor(self, trans, id=None, workflow_id=None, version=None, **kwargs):
         }
 
         # for existing workflow add its data to the model
-        if new_workflow is False:
+        if stored:
             editor_config.update(
                 {
                     "id": trans.security.encode_id(stored.id),

From 28596d30668be79b92b344856e6b307a735da488 Mon Sep 17 00:00:00 2001
From: Ahmed Awan <qe66653@umbc.edu>
Date: Tue, 29 Oct 2024 11:39:43 -0500
Subject: [PATCH 09/84] Add a filter for `labels` (tags) in Tool Search

Fixes https://github.com/galaxyproject/galaxy/issues/19069
---
 client/src/components/Panels/utilities.ts | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/client/src/components/Panels/utilities.ts b/client/src/components/Panels/utilities.ts
index 95da1acbbf69..27dde0d1cf86 100644
--- a/client/src/components/Panels/utilities.ts
+++ b/client/src/components/Panels/utilities.ts
@@ -15,6 +15,7 @@ import levenshteinDistance from "@/utils/levenshtein";
 const FILTER_KEYS = {
     id: ["id", "tool_id"],
     panel_section_name: ["section", "panel_section_name"],
+    labels: ["label", "labels", "tag"],
 };
 const STRING_REPLACEMENTS: string[] = [" ", "-", "\\(", "\\)", "'", ":", `"`];
 const MINIMUM_DL_LENGTH = 5; // for Demerau-Levenshtein distance
@@ -286,7 +287,14 @@ export function searchToolsByKeys(
                 if (key === "combined") {
                     actualValue = `${tool.name.trim()} ${tool.description.trim()}`.toLowerCase();
                 } else {
-                    actualValue = (tool[key as keyof Tool] as string)?.trim().toLowerCase();
+                    const toolVal = tool[key as keyof Tool];
+                    if (typeof toolVal === "string") {
+                        actualValue = toolVal.trim().toLowerCase();
+                    } else if (Array.isArray(toolVal)) {
+                        actualValue = toolVal.join(" ").trim().toLowerCase();
+                    } else if (typeof toolVal === "number") {
+                        actualValue = toolVal.toString().trim().toLowerCase();
+                    }
                 }
 
                 // get all (space separated) words in actualValue for tool (for DL)

From eb65bb8764e56b517b2a49fb396ba45ef57be222 Mon Sep 17 00:00:00 2001
From: Ahmed Awan <qe66653@umbc.edu>
Date: Tue, 29 Oct 2024 12:47:17 -0500
Subject: [PATCH 10/84] [24.0] Fetch invocation id only once on the job info
 page

... instead of polling while the job is non terminal; we only need the id once.

Fixes https://github.com/galaxyproject/galaxy/issues/18921
---
 client/src/components/JobInformation/JobInformation.vue | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/client/src/components/JobInformation/JobInformation.vue b/client/src/components/JobInformation/JobInformation.vue
index 759e59b97418..7d1cade4942a 100644
--- a/client/src/components/JobInformation/JobInformation.vue
+++ b/client/src/components/JobInformation/JobInformation.vue
@@ -41,7 +41,7 @@ const metadataDetail = ref({
 
 function updateJob(newJob) {
     job.value = newJob;
-    if (newJob) {
+    if (newJob && !invocationId.value) {
         fetchInvocation(newJob.id);
     }
 }
@@ -57,6 +57,7 @@ function filterMetadata(jobMessages) {
     });
 }
 
+/** Fetches the invocation for the given job id to get the associated invocation id */
 async function fetchInvocation(jobId) {
     if (jobId) {
         const invocation = await invocationForJob({ jobId: jobId });

From d022254709adb464cf521ed5b61c790530711fff Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 16 Oct 2024 15:46:29 -0400
Subject: [PATCH 11/84] Refactor complex method for clarity.

---
 lib/galaxy/tools/parameters/meta.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/tools/parameters/meta.py b/lib/galaxy/tools/parameters/meta.py
index f2d8ba1a68d1..f6e8b70ca950 100644
--- a/lib/galaxy/tools/parameters/meta.py
+++ b/lib/galaxy/tools/parameters/meta.py
@@ -161,6 +161,14 @@ def is_batch(value):
 ExpandedT = Tuple[List[ToolStateJobInstanceT], Optional[matching.MatchingCollections]]
 
 
+def expand_flat_parameters_to_nested(incoming_copy: ToolRequestT) -> Dict[str, Any]:
+    nested_dict: Dict[str, Any] = {}
+    for incoming_key, incoming_value in incoming_copy.items():
+        if not incoming_key.startswith("__"):
+            process_key(incoming_key, incoming_value=incoming_value, d=nested_dict)
+    return nested_dict
+
+
 def expand_meta_parameters(trans, tool, incoming: ToolRequestT) -> ExpandedT:
     """
     Take in a dictionary of raw incoming parameters and expand to a list
@@ -176,11 +184,7 @@ def expand_meta_parameters(trans, tool, incoming: ToolRequestT) -> ExpandedT:
     # order matters, so the following reorders incoming
     # according to tool.inputs (which is ordered).
     incoming_copy = incoming.copy()
-    nested_dict: Dict[str, Any] = {}
-    for incoming_key, incoming_value in incoming_copy.items():
-        if not incoming_key.startswith("__"):
-            process_key(incoming_key, incoming_value=incoming_value, d=nested_dict)
-
+    nested_dict = expand_flat_parameters_to_nested(incoming_copy)
     reordered_incoming = {}
 
     def visitor(input, value, prefix, prefixed_name, prefixed_label, error, **kwargs):

From 6d7842e66fd7be15ddc9a9dbe08246749a4d7e85 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Sun, 13 Oct 2024 21:39:47 -0400
Subject: [PATCH 12/84] Numerous fixes for tool input format "21.01"
 parameters.

---
 lib/galaxy/tools/__init__.py                  |   4 +-
 lib/galaxy/tools/parameters/__init__.py       |  39 ++--
 lib/galaxy/tools/parameters/meta.py           | 138 ++++++++++---
 lib/galaxy/tools/parameters/wrapped.py        |   9 +-
 lib/galaxy/tools/wrappers.py                  |   2 +
 lib/galaxy/util/permutations.py               | 132 +++++++++----
 lib/galaxy_test/api/conftest.py               |  15 +-
 lib/galaxy_test/api/test_tool_execute.py      | 187 +++++++++++++++++-
 lib/galaxy_test/api/test_tools.py             | 106 ----------
 .../test_extended_metadata_mapping.py         |  15 ++
 10 files changed, 451 insertions(+), 196 deletions(-)

diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 24ddffd5875a..68bf154b0f7d 100644
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -1836,7 +1836,9 @@ def expand_incoming(
         # Expand these out to individual parameters for given jobs (tool executions).
         expanded_incomings: List[ToolStateJobInstanceT]
         collection_info: Optional[MatchingCollections]
-        expanded_incomings, collection_info = expand_meta_parameters(request_context, self, incoming)
+        expanded_incomings, collection_info = expand_meta_parameters(
+            request_context, self, incoming, input_format=input_format
+        )
 
         self._ensure_expansion_is_valid(expanded_incomings, rerun_remap_job_id)
 
diff --git a/lib/galaxy/tools/parameters/__init__.py b/lib/galaxy/tools/parameters/__init__.py
index 9de78f2fbff5..ee3a8709817d 100644
--- a/lib/galaxy/tools/parameters/__init__.py
+++ b/lib/galaxy/tools/parameters/__init__.py
@@ -62,6 +62,8 @@ def visit_input_values(
     context=None,
     no_replacement_value=REPLACE_ON_TRUTHY,
     replace_optional_connections=False,
+    allow_case_inference=False,
+    unset_value=None,
 ):
     """
     Given a tools parameter definition (`inputs`) and a specific set of
@@ -158,7 +160,7 @@ def visit_input_values(
     """
 
     def callback_helper(input, input_values, name_prefix, label_prefix, parent_prefix, context=None, error=None):
-        value = input_values.get(input.name)
+        value = input_values.get(input.name, unset_value)
         args = {
             "input": input,
             "parent": input_values,
@@ -182,13 +184,23 @@ def callback_helper(input, input_values, name_prefix, label_prefix, parent_prefi
             input_values[input.name] = input.value
 
     def get_current_case(input, input_values):
+        test_parameter = input.test_param
+        test_parameter_name = test_parameter.name
         try:
-            return input.get_current_case(input_values[input.test_param.name])
+            if test_parameter_name not in input_values and allow_case_inference:
+                return input.get_current_case(test_parameter.get_initial_value(None, input_values))
+            else:
+                return input.get_current_case(input_values[test_parameter_name])
         except (KeyError, ValueError):
             return -1
 
     context = ExpressionContext(input_values, context)
-    payload = {"context": context, "no_replacement_value": no_replacement_value}
+    payload = {
+        "context": context,
+        "no_replacement_value": no_replacement_value,
+        "allow_case_inference": allow_case_inference,
+        "unset_value": unset_value,
+    }
     for input in inputs.values():
         if isinstance(input, Repeat) or isinstance(input, UploadDataset):
             values = input_values[input.name] = input_values.get(input.name, [])
@@ -411,16 +423,15 @@ def populate_state(
             group_state = state[input.name]
             if input.type == "repeat":
                 repeat_input = cast(Repeat, input)
-                if (
-                    len(incoming[repeat_input.name]) > repeat_input.max
-                    or len(incoming[repeat_input.name]) < repeat_input.min
+                repeat_name = repeat_input.name
+                repeat_incoming = incoming.get(repeat_name) or []
+                if repeat_incoming and (
+                    len(repeat_incoming) > repeat_input.max or len(repeat_incoming) < repeat_input.min
                 ):
-                    errors[repeat_input.name] = (
-                        "The number of repeat elements is outside the range specified by the tool."
-                    )
+                    errors[repeat_name] = "The number of repeat elements is outside the range specified by the tool."
                 else:
                     del group_state[:]
-                    for rep in incoming[repeat_input.name]:
+                    for rep in repeat_incoming:
                         new_state: ToolStateJobInstancePopulatedT = {}
                         group_state.append(new_state)
                         repeat_errors: ParameterValidationErrorsT = {}
@@ -454,10 +465,13 @@ def populate_state(
                         current_case = conditional_input.get_current_case(value)
                         group_state = state[conditional_input.name] = {}
                         cast_errors: ParameterValidationErrorsT = {}
+                        incoming_for_conditional = cast(
+                            ToolStateJobInstanceT, incoming.get(conditional_input.name) or {}
+                        )
                         populate_state(
                             request_context,
                             conditional_input.cases[current_case].inputs,
-                            cast(ToolStateJobInstanceT, incoming.get(conditional_input.name)),
+                            incoming_for_conditional,
                             group_state,
                             cast_errors,
                             context=context,
@@ -475,10 +489,11 @@ def populate_state(
             elif input.type == "section":
                 section_input = cast(Section, input)
                 section_errors: ParameterValidationErrorsT = {}
+                incoming_for_state = cast(ToolStateJobInstanceT, incoming.get(section_input.name) or {})
                 populate_state(
                     request_context,
                     section_input.inputs,
-                    cast(ToolStateJobInstanceT, incoming.get(section_input.name)),
+                    incoming_for_state,
                     group_state,
                     section_errors,
                     context=context,
diff --git a/lib/galaxy/tools/parameters/meta.py b/lib/galaxy/tools/parameters/meta.py
index f6e8b70ca950..b74df54fa269 100644
--- a/lib/galaxy/tools/parameters/meta.py
+++ b/lib/galaxy/tools/parameters/meta.py
@@ -19,10 +19,19 @@
     matching,
     subcollections,
 )
-from galaxy.util import permutations
+from galaxy.util.permutations import (
+    build_combos,
+    input_classification,
+    is_in_state,
+    state_copy,
+    state_get_value,
+    state_remove_value,
+    state_set_value,
+)
 from . import visit_input_values
 from .wrapped import process_key
 from .._types import (
+    InputFormatT,
     ToolRequestT,
     ToolStateJobInstanceT,
 )
@@ -169,7 +178,7 @@ def expand_flat_parameters_to_nested(incoming_copy: ToolRequestT) -> Dict[str, A
     return nested_dict
 
 
-def expand_meta_parameters(trans, tool, incoming: ToolRequestT) -> ExpandedT:
+def expand_meta_parameters(trans, tool, incoming: ToolRequestT, input_format: InputFormatT) -> ExpandedT:
     """
     Take in a dictionary of raw incoming parameters and expand to a list
     of expanded incoming parameters (one set of parameters per tool
@@ -184,29 +193,24 @@ def expand_meta_parameters(trans, tool, incoming: ToolRequestT) -> ExpandedT:
     # order matters, so the following reorders incoming
     # according to tool.inputs (which is ordered).
     incoming_copy = incoming.copy()
-    nested_dict = expand_flat_parameters_to_nested(incoming_copy)
-    reordered_incoming = {}
-
-    def visitor(input, value, prefix, prefixed_name, prefixed_label, error, **kwargs):
-        if prefixed_name in incoming_copy:
-            reordered_incoming[prefixed_name] = incoming_copy[prefixed_name]
-            del incoming_copy[prefixed_name]
+    if input_format == "legacy":
+        nested_dict = expand_flat_parameters_to_nested(incoming_copy)
+    else:
+        nested_dict = incoming_copy
 
-    visit_input_values(inputs=tool.inputs, input_values=nested_dict, callback=visitor)
-    reordered_incoming.update(incoming_copy)
+    collections_to_match = matching.CollectionsToMatch()
 
-    def classifier(input_key):
-        value = incoming[input_key]
+    def classifier_from_value(value, input_key):
         if isinstance(value, dict) and "values" in value:
             # Explicit meta wrapper for inputs...
             is_batch = value.get("batch", False)
             is_linked = value.get("linked", True)
             if is_batch and is_linked:
-                classification = permutations.input_classification.MATCHED
+                classification = input_classification.MATCHED
             elif is_batch:
-                classification = permutations.input_classification.MULTIPLIED
+                classification = input_classification.MULTIPLIED
             else:
-                classification = permutations.input_classification.SINGLE
+                classification = input_classification.SINGLE
             if __collection_multirun_parameter(value):
                 collection_value = value["values"][0]
                 values = __expand_collection_parameter(
@@ -215,17 +219,29 @@ def classifier(input_key):
             else:
                 values = value["values"]
         else:
-            classification = permutations.input_classification.SINGLE
+            classification = input_classification.SINGLE
             values = value
         return classification, values
 
-    collections_to_match = matching.CollectionsToMatch()
+    nested = input_format != "legacy"
+    if not nested:
+        reordered_incoming = reorder_parameters(tool, incoming_copy, nested_dict, nested)
+        incoming_template = reordered_incoming
+
+        def classifier_flat(input_key):
+            return classifier_from_value(incoming[input_key], input_key)
 
-    # Stick an unexpanded version of multirun keys so they can be replaced,
-    # by expand_mult_inputs.
-    incoming_template = reordered_incoming
+        single_inputs, matched_multi_inputs, multiplied_multi_inputs = split_inputs_flat(
+            incoming_template, classifier_flat
+        )
+    else:
+        reordered_incoming = reorder_parameters(tool, incoming_copy, nested_dict, nested)
+        incoming_template = reordered_incoming
+        single_inputs, matched_multi_inputs, multiplied_multi_inputs = split_inputs_nested(
+            tool.inputs, incoming_template, classifier_from_value
+        )
 
-    expanded_incomings = permutations.expand_multi_inputs(incoming_template, classifier)
+    expanded_incomings = build_combos(single_inputs, matched_multi_inputs, multiplied_multi_inputs, nested=nested)
     if collections_to_match.has_collections():
         collection_info = trans.app.dataset_collection_manager.match_collections(collections_to_match)
     else:
@@ -233,6 +249,84 @@ def classifier(input_key):
     return expanded_incomings, collection_info
 
 
+def reorder_parameters(tool, incoming, nested_dict, nested):
+    # If we're going to multiply input dataset combinations
+    # order matters, so the following reorders incoming
+    # according to tool.inputs (which is ordered).
+    incoming_copy = state_copy(incoming, nested)
+
+    reordered_incoming = {}
+
+    def visitor(input, value, prefix, prefixed_name, prefixed_label, error, **kwargs):
+        if is_in_state(incoming_copy, prefixed_name, nested):
+            value_to_copy_over = state_get_value(incoming_copy, prefixed_name, nested)
+            state_set_value(reordered_incoming, prefixed_name, value_to_copy_over, nested)
+            state_remove_value(incoming_copy, prefixed_name, nested)
+
+    visit_input_values(inputs=tool.inputs, input_values=nested_dict, callback=visitor)
+
+    def merge_into(from_object, into_object):
+        if isinstance(from_object, dict):
+            for key, value in from_object.items():
+                if key not in into_object:
+                    into_object[key] = value
+                else:
+                    into_target = into_object[key]
+                    merge_into(value, into_target)
+        elif isinstance(from_object, list):
+            for index in from_object:
+                if len(into_object) <= index:
+                    into_object.append(from_object[index])
+                else:
+                    merge_into(from_object[index], into_object[index])
+
+    merge_into(incoming_copy, reordered_incoming)
+    return reordered_incoming
+
+
+def split_inputs_flat(inputs: Dict[str, Any], classifier):
+    single_inputs: Dict[str, Any] = {}
+    matched_multi_inputs: Dict[str, Any] = {}
+    multiplied_multi_inputs: Dict[str, Any] = {}
+
+    for input_key in inputs:
+        input_type, expanded_val = classifier(input_key)
+        if input_type == input_classification.SINGLE:
+            single_inputs[input_key] = expanded_val
+        elif input_type == input_classification.MATCHED:
+            matched_multi_inputs[input_key] = expanded_val
+        elif input_type == input_classification.MULTIPLIED:
+            multiplied_multi_inputs[input_key] = expanded_val
+
+    return (single_inputs, matched_multi_inputs, multiplied_multi_inputs)
+
+
+def split_inputs_nested(inputs, nested_dict, classifier):
+    single_inputs: Dict[str, Any] = {}
+    matched_multi_inputs: Dict[str, Any] = {}
+    multiplied_multi_inputs: Dict[str, Any] = {}
+    unset_value = object()
+
+    def visitor(input, value, prefix, prefixed_name, prefixed_label, error, **kwargs):
+        if value is unset_value:
+            # don't want to inject extra nulls into state
+            return
+
+        input_type, expanded_val = classifier(value, prefixed_name)
+        if input_type == input_classification.SINGLE:
+            single_inputs[prefixed_name] = expanded_val
+        elif input_type == input_classification.MATCHED:
+            matched_multi_inputs[prefixed_name] = expanded_val
+        elif input_type == input_classification.MULTIPLIED:
+            multiplied_multi_inputs[prefixed_name] = expanded_val
+
+    visit_input_values(
+        inputs=inputs, input_values=nested_dict, callback=visitor, allow_case_inference=True, unset_value=unset_value
+    )
+    single_inputs_nested = expand_flat_parameters_to_nested(single_inputs)
+    return (single_inputs_nested, matched_multi_inputs, multiplied_multi_inputs)
+
+
 def __expand_collection_parameter(trans, input_key, incoming_val, collections_to_match, linked=False):
     # If subcollectin multirun of data_collection param - value will
     # be "hdca_id|subcollection_type" else it will just be hdca_id
diff --git a/lib/galaxy/tools/parameters/wrapped.py b/lib/galaxy/tools/parameters/wrapped.py
index 11fa98c0e644..d23e9be5edf9 100644
--- a/lib/galaxy/tools/parameters/wrapped.py
+++ b/lib/galaxy/tools/parameters/wrapped.py
@@ -25,6 +25,10 @@
     InputValueWrapper,
     SelectToolParameterWrapper,
 )
+from galaxy.util.permutations import (
+    looks_like_flattened_repeat_key,
+    split_flattened_repeat_key,
+)
 
 PARAMS_UNWRAPPED = object()
 
@@ -172,10 +176,9 @@ def process_key(incoming_key: str, incoming_value: Any, d: Dict[str, Any]):
             # In case we get an empty repeat after we already filled in a repeat element
             return
         d[incoming_key] = incoming_value
-    elif key_parts[0].rsplit("_", 1)[-1].isdigit():
+    elif looks_like_flattened_repeat_key(key_parts[0]):
         # Repeat
-        input_name, _index = key_parts[0].rsplit("_", 1)
-        index = int(_index)
+        input_name, index = split_flattened_repeat_key(key_parts[0])
         d.setdefault(input_name, [])
         newlist: List[Dict[Any, Any]] = [{} for _ in range(index + 1)]
         d[input_name].extend(newlist[len(d[input_name]) :])
diff --git a/lib/galaxy/tools/wrappers.py b/lib/galaxy/tools/wrappers.py
index 6dd47e7f9bb4..f5d1d6a5a0bb 100644
--- a/lib/galaxy/tools/wrappers.py
+++ b/lib/galaxy/tools/wrappers.py
@@ -802,6 +802,8 @@ def __init__(self, input_datasets: Optional[Dict[str, Any]] = None) -> None:
             self.identifier_key_dict = {}
 
     def identifier(self, dataset_value: str, input_values: Dict[str, str]) -> Optional[str]:
+        if isinstance(dataset_value, list):
+            raise TypeError(f"Expected {dataset_value} to be hashable")
         element_identifier = None
         if identifier_key := self.identifier_key_dict.get(dataset_value, None):
             element_identifier = input_values.get(identifier_key, None)
diff --git a/lib/galaxy/util/permutations.py b/lib/galaxy/util/permutations.py
index 5dd1b11ee8fa..92573d8c6dbc 100644
--- a/lib/galaxy/util/permutations.py
+++ b/lib/galaxy/util/permutations.py
@@ -7,10 +7,8 @@
 with itertools product and permutations. These are open questions.
 """
 
-from typing import (
-    Dict,
-    TypeVar,
-)
+import copy
+from typing import Tuple
 
 from galaxy.exceptions import MessageException
 from galaxy.util.bunch import Bunch
@@ -21,47 +19,20 @@
     MULTIPLIED="multiplied",
 )
 
-# generic type of splitting input dictionary
-T = TypeVar("T")
-
 
 class InputMatchedException(MessageException):
     """Indicates problem matching inputs while building up inputs
     permutations."""
 
 
-def expand_multi_inputs(inputs: Dict[str, T], classifier, key_filter=None):
-    key_filter = key_filter or (lambda x: True)
-
-    single_inputs, matched_multi_inputs, multiplied_multi_inputs = __split_inputs(inputs, classifier, key_filter)
-
+def build_combos(single_inputs, matched_multi_inputs, multiplied_multi_inputs, nested):
     # Build up every combination of inputs to be run together.
-    input_combos = __extend_with_matched_combos(single_inputs, matched_multi_inputs)
-    input_combos = __extend_with_multiplied_combos(input_combos, multiplied_multi_inputs)
-
+    input_combos = __extend_with_matched_combos(single_inputs, matched_multi_inputs, nested)
+    input_combos = __extend_with_multiplied_combos(input_combos, multiplied_multi_inputs, nested)
     return input_combos
 
 
-def __split_inputs(inputs: Dict[str, T], classifier, key_filter):
-    key_filter = key_filter or (lambda x: True)
-
-    single_inputs: Dict[str, T] = {}
-    matched_multi_inputs: Dict[str, T] = {}
-    multiplied_multi_inputs: Dict[str, T] = {}
-
-    for input_key in filter(key_filter, inputs):
-        input_type, expanded_val = classifier(input_key)
-        if input_type == input_classification.SINGLE:
-            single_inputs[input_key] = expanded_val
-        elif input_type == input_classification.MATCHED:
-            matched_multi_inputs[input_key] = expanded_val
-        elif input_type == input_classification.MULTIPLIED:
-            multiplied_multi_inputs[input_key] = expanded_val
-
-    return (single_inputs, matched_multi_inputs, multiplied_multi_inputs)
-
-
-def __extend_with_matched_combos(single_inputs, multi_inputs):
+def __extend_with_matched_combos(single_inputs, multi_inputs, nested):
     """
 
     {a => 1, b => 2} and {c => {3, 4}, d => {5, 6}}
@@ -81,7 +52,7 @@ def __extend_with_matched_combos(single_inputs, multi_inputs):
     first_multi_value = multi_inputs.get(first_multi_input_key)
 
     for value in first_multi_value:
-        new_inputs = __copy_and_extend_inputs(single_inputs, first_multi_input_key, value)
+        new_inputs = __copy_and_extend_inputs(single_inputs, first_multi_input_key, value, nested=nested)
         matched_multi_inputs.append(new_inputs)
 
     for multi_input_key, multi_input_values in multi_inputs.items():
@@ -94,12 +65,12 @@ def __extend_with_matched_combos(single_inputs, multi_inputs):
             )
 
         for index, value in enumerate(multi_input_values):
-            matched_multi_inputs[index][multi_input_key] = value
+            state_set_value(matched_multi_inputs[index], multi_input_key, value, nested)
 
     return matched_multi_inputs
 
 
-def __extend_with_multiplied_combos(input_combos, multi_inputs):
+def __extend_with_multiplied_combos(input_combos, multi_inputs, nested):
     combos = input_combos
 
     for multi_input_key, multi_input_value in multi_inputs.items():
@@ -107,7 +78,7 @@ def __extend_with_multiplied_combos(input_combos, multi_inputs):
 
         for combo in combos:
             for input_value in multi_input_value:
-                iter_combo = __copy_and_extend_inputs(combo, multi_input_key, input_value)
+                iter_combo = __copy_and_extend_inputs(combo, multi_input_key, input_value, nested)
                 iter_combos.append(iter_combo)
 
         combos = iter_combos
@@ -115,7 +86,84 @@ def __extend_with_multiplied_combos(input_combos, multi_inputs):
     return combos
 
 
-def __copy_and_extend_inputs(inputs, key, value):
-    new_inputs = dict(inputs)
-    new_inputs[key] = value
+def __copy_and_extend_inputs(inputs, key, value, nested):
+    # can't deepcopy dicts with our models for reason I don't understand,
+    # test_map_over_two_collections_unlinked breaks if I try to combine these two branches of the if
+    new_inputs = state_copy(inputs, nested)
+    state_set_value(new_inputs, key, value, nested)
     return new_inputs
+
+
+def state_copy(inputs, nested):
+    # can't deepcopy dicts with our models for reason I don't understand,
+    # test_map_over_two_collections_unlinked breaks if I try to combine these two branches of the if
+    if nested:
+        state_dict_copy = copy.deepcopy(inputs)
+    else:
+        state_dict_copy = dict(inputs)
+    return state_dict_copy
+
+
+def state_set_value(state_dict, key, value, nested):
+    if "|" not in key or not nested:
+        state_dict[key] = value
+    else:
+        first, rest = key.split("|", 1)
+        if first not in state_dict and looks_like_flattened_repeat_key(first):
+            repeat_name, index = split_flattened_repeat_key(first)
+            if repeat_name not in state_dict:
+                state_dict[repeat_name] = []
+            repeat_state = state_dict[repeat_name]
+            while len(repeat_state) <= index:
+                repeat_state.append({})
+            state_set_value(repeat_state[index], rest, value, nested)
+        else:
+            state_set_value(state_dict[first], rest, value, nested)
+
+
+def state_remove_value(state_dict, key, nested):
+    if "|" not in key or not nested:
+        del state_dict[key]
+    else:
+        first, rest = key.split("|", 1)
+        child_dict = state_dict[first]
+        # repeats?
+        if "|" in rest:
+            state_remove_value(child_dict, rest, nested)
+        else:
+            del child_dict[rest]
+            if len(child_dict) == 0:
+                del state_dict[first]
+
+
+def state_get_value(state_dict, key, nested):
+    if "|" not in key or not nested:
+        return state_dict[key]
+    else:
+        first, rest = key.split("|", 1)
+        if first not in state_dict and looks_like_flattened_repeat_key(first):
+            repeat_name, index = split_flattened_repeat_key(first)
+            return state_get_value(state_dict[repeat_name][index], rest, nested)
+        else:
+            return state_get_value(state_dict[first], rest, nested)
+
+
+def is_in_state(state_dict, key, nested):
+    if not state_dict:
+        return False
+    if "|" not in key or not nested:
+        return key in state_dict
+    else:
+        first, rest = key.split("|", 1)
+        # repeats?
+        is_in_state(state_dict.get(first), rest, nested)
+
+
+def looks_like_flattened_repeat_key(key: str) -> bool:
+    return key.rsplit("_", 1)[-1].isdigit()
+
+
+def split_flattened_repeat_key(key: str) -> Tuple[str, int]:
+    input_name, _index = key.rsplit("_", 1)
+    index = int(_index)
+    return input_name, index
diff --git a/lib/galaxy_test/api/conftest.py b/lib/galaxy_test/api/conftest.py
index a011532c7af5..74d8958b9158 100644
--- a/lib/galaxy_test/api/conftest.py
+++ b/lib/galaxy_test/api/conftest.py
@@ -157,15 +157,24 @@ def tool_input_format(request) -> Iterator[DescribeToolInputs]:
 def check_required_tools(anonymous_galaxy_interactor, request):
     for marker in request.node.iter_markers():
         if marker.name == "requires_tool_id":
-            tool_id = marker.args[0]
+            tool_id = _requires_marker_to_effective_tool_id(anonymous_galaxy_interactor, marker)
             check_missing_tool(tool_id not in get_tool_ids(anonymous_galaxy_interactor))
 
 
 @pytest.fixture
-def required_tool_ids(request) -> List[str]:
+def required_tool_ids(anonymous_galaxy_interactor, request) -> List[str]:
     tool_ids = []
     for marker in request.node.iter_markers():
         if marker.name == "requires_tool_id":
-            tool_id = marker.args[0]
+            tool_id = _requires_marker_to_effective_tool_id(anonymous_galaxy_interactor, marker)
             tool_ids.append(tool_id)
     return tool_ids
+
+
+def _requires_marker_to_effective_tool_id(anonymous_galaxy_interactor, marker):
+    tool_id = marker.args[0]
+    if "|" in tool_id:
+        any_of_tool_ids = tool_id.split("|")
+        all_tool_ids = get_tool_ids(anonymous_galaxy_interactor)
+        tool_id = [t for t in any_of_tool_ids if t in all_tool_ids][0]
+    return tool_id
diff --git a/lib/galaxy_test/api/test_tool_execute.py b/lib/galaxy_test/api/test_tool_execute.py
index 3d9c60f0ef79..95bf43e27921 100644
--- a/lib/galaxy_test/api/test_tool_execute.py
+++ b/lib/galaxy_test/api/test_tool_execute.py
@@ -7,12 +7,17 @@
 files, etc..).
 """
 
+from dataclasses import dataclass
 from typing import List
 
+import pytest
+
 from galaxy_test.base.decorators import requires_tool_id
 from galaxy_test.base.populators import (
+    DescribeToolExecution,
     DescribeToolInputs,
     RequiredTool,
+    SrcDict,
     TargetHistory,
 )
 
@@ -104,28 +109,43 @@ def test_identifier_map_over_multiple_input_in_conditional(
 
 
 @requires_tool_id("identifier_multiple_in_repeat")
-def test_identifier_multiple_reduce_in_repeat_new_payload_form(
-    target_history: TargetHistory, required_tool: RequiredTool
+def test_identifier_multiple_reduce_in_repeat(
+    target_history: TargetHistory, required_tool: RequiredTool, tool_input_format: DescribeToolInputs
 ):
     hdca = target_history.with_pair()
-    execute = required_tool.execute.with_nested_inputs(
+    inputs = tool_input_format.when.nested(
         {
             "the_repeat": [{"the_data": {"input1": hdca.src_dict}}],
         }
+    ).when.flat(
+        {
+            "the_repeat_0|the_data|input1": hdca.src_dict,
+        }
     )
+    execute = required_tool.execute.with_inputs(inputs)
     execute.assert_has_single_job.assert_has_single_output.with_contents_stripped("forward\nreverse")
 
 
 @requires_tool_id("output_action_change_format")
-def test_map_over_with_output_format_actions(target_history: TargetHistory, required_tool: RequiredTool):
+def test_map_over_with_output_format_actions(
+    target_history: TargetHistory, required_tool: RequiredTool, tool_input_format: DescribeToolInputs
+):
     hdca = target_history.with_pair()
     for use_action in ["do", "dont"]:
-        execute = required_tool.execute.with_inputs(
+        inputs = tool_input_format.when.flat(
             {
                 "input_cond|dispatch": use_action,
                 "input_cond|input": {"batch": True, "values": [hdca.src_dict]},
             }
+        ).when.nested(
+            {
+                "input_cond": {
+                    "dispatch": use_action,
+                    "input": {"batch": True, "values": [hdca.src_dict]},
+                }
+            }
         )
+        execute = required_tool.execute.with_inputs(inputs)
         execute.assert_has_n_jobs(2).assert_creates_n_implicit_collections(1)
         expected_extension = "txt" if (use_action == "do") else "data"
         execute.assert_has_job(0).with_single_output.with_file_ext(expected_extension)
@@ -196,7 +216,7 @@ def test_identifier_with_multiple_normal_datasets(target_history: TargetHistory,
     execute.assert_has_single_job.assert_has_single_output.with_contents_stripped("Normal HDA1\nNormal HDA2")
 
 
-@requires_tool_id("cat1")
+@requires_tool_id("cat|cat1")
 def test_map_over_empty_collection(target_history: TargetHistory, required_tool: RequiredTool):
     hdca = target_history.with_list([])
     inputs = {
@@ -204,7 +224,160 @@ def test_map_over_empty_collection(target_history: TargetHistory, required_tool:
     }
     execute = required_tool.execute.with_inputs(inputs)
     execute.assert_has_n_jobs(0)
-    execute.assert_creates_implicit_collection(0).named("Concatenate datasets on collection 1")
+    name = execute.assert_creates_implicit_collection(0).details["name"]
+    assert "Concatenate datasets" in name
+    assert "on collection 1" in name
+
+
+@dataclass
+class MultiRunInRepeatFixtures:
+    repeat_datasets: List[SrcDict]
+    common_dataset: SrcDict
+
+
+@pytest.fixture
+def multi_run_in_repeat_datasets(target_history: TargetHistory) -> MultiRunInRepeatFixtures:
+    dataset1 = target_history.with_dataset("123").src_dict
+    dataset2 = target_history.with_dataset("456").src_dict
+    common_dataset = target_history.with_dataset("Common").src_dict
+    return MultiRunInRepeatFixtures([dataset1, dataset2], common_dataset)
+
+
+@requires_tool_id("cat|cat1")
+def test_multi_run_in_repeat(
+    required_tool: RequiredTool,
+    multi_run_in_repeat_datasets: MultiRunInRepeatFixtures,
+    tool_input_format: DescribeToolInputs,
+):
+    inputs = tool_input_format.when.flat(
+        {
+            "input1": {"batch": False, "values": [multi_run_in_repeat_datasets.common_dataset]},
+            "queries_0|input2": {"batch": True, "values": multi_run_in_repeat_datasets.repeat_datasets},
+        }
+    ).when.nested(
+        {
+            "input1": {"batch": False, "values": [multi_run_in_repeat_datasets.common_dataset]},
+            "queries": [
+                {
+                    "input2": {"batch": True, "values": multi_run_in_repeat_datasets.repeat_datasets},
+                }
+            ],
+        }
+    )
+    execute = required_tool.execute.with_inputs(inputs)
+    _check_multi_run_in_repeat(execute)
+
+
+@requires_tool_id("cat|cat1")
+def test_multi_run_in_repeat_mismatch(
+    required_tool: RequiredTool,
+    multi_run_in_repeat_datasets: MultiRunInRepeatFixtures,
+    tool_input_format: DescribeToolInputs,
+):
+    """Same test as above but without the batch wrapper around the common dataset shared between multirun."""
+    inputs = tool_input_format.when.flat(
+        {
+            "input1": multi_run_in_repeat_datasets.common_dataset,
+            "queries_0|input2": {"batch": True, "values": multi_run_in_repeat_datasets.repeat_datasets},
+        }
+    ).when.nested(
+        {
+            "input1": multi_run_in_repeat_datasets.common_dataset,
+            "queries": [
+                {
+                    "input2": {"batch": True, "values": multi_run_in_repeat_datasets.repeat_datasets},
+                }
+            ],
+        }
+    )
+    execute = required_tool.execute.with_inputs(inputs)
+    _check_multi_run_in_repeat(execute)
+
+
+def _check_multi_run_in_repeat(execute: DescribeToolExecution):
+    execute.assert_has_n_jobs(2)
+    execute.assert_has_job(0).with_single_output.with_contents_stripped("Common\n123")
+    execute.assert_has_job(1).with_single_output.with_contents_stripped("Common\n456")
+
+
+@dataclass
+class TwoMultiRunsFixture:
+    first_two_datasets: List[SrcDict]
+    second_two_datasets: List[SrcDict]
+
+
+@pytest.fixture
+def two_multi_run_datasets(target_history: TargetHistory) -> TwoMultiRunsFixture:
+    dataset1 = target_history.with_dataset("123").src_dict
+    dataset2 = target_history.with_dataset("456").src_dict
+    dataset3 = target_history.with_dataset("789").src_dict
+    dataset4 = target_history.with_dataset("0ab").src_dict
+    return TwoMultiRunsFixture([dataset1, dataset2], [dataset3, dataset4])
+
+
+@requires_tool_id("cat|cat1")
+def test_multirun_on_multiple_inputs(
+    required_tool: RequiredTool,
+    two_multi_run_datasets: TwoMultiRunsFixture,
+    tool_input_format: DescribeToolInputs,
+):
+    inputs = tool_input_format.when.flat(
+        {
+            "input1": {"batch": True, "values": two_multi_run_datasets.first_two_datasets},
+            "queries_0|input2": {"batch": True, "values": two_multi_run_datasets.second_two_datasets},
+        }
+    ).when.nested(
+        {
+            "input1": {"batch": True, "values": two_multi_run_datasets.first_two_datasets},
+            "queries": [
+                {"input2": {"batch": True, "values": two_multi_run_datasets.second_two_datasets}},
+            ],
+        }
+    )
+    execute = required_tool.execute.with_inputs(inputs)
+    execute.assert_has_n_jobs(2)
+    execute.assert_has_job(0).with_single_output.with_contents_stripped("123\n789")
+    execute.assert_has_job(1).with_single_output.with_contents_stripped("456\n0ab")
+
+
+@requires_tool_id("cat|cat1")
+def test_multirun_on_multiple_inputs_unlinked(
+    required_tool: RequiredTool,
+    two_multi_run_datasets: TwoMultiRunsFixture,
+    tool_input_format: DescribeToolInputs,
+):
+    inputs = tool_input_format.when.flat(
+        {
+            "input1": {"batch": True, "linked": False, "values": two_multi_run_datasets.first_two_datasets},
+            "queries_0|input2": {"batch": True, "linked": False, "values": two_multi_run_datasets.second_two_datasets},
+        }
+    ).when.nested(
+        {
+            "input1": {"batch": True, "linked": False, "values": two_multi_run_datasets.first_two_datasets},
+            "queries": [
+                {"input2": {"batch": True, "linked": False, "values": two_multi_run_datasets.second_two_datasets}},
+            ],
+        }
+    )
+    execute = required_tool.execute.with_inputs(inputs)
+    execute.assert_has_n_jobs(4)
+    execute.assert_has_job(0).with_single_output.with_contents_stripped("123\n789")
+    execute.assert_has_job(1).with_single_output.with_contents_stripped("123\n0ab")
+    execute.assert_has_job(2).with_single_output.with_contents_stripped("456\n789")
+    execute.assert_has_job(3).with_single_output.with_contents_stripped("456\n0ab")
+
+
+@requires_tool_id("cat|cat1")
+def test_map_over_collection(
+    target_history: TargetHistory, required_tool: RequiredTool, tool_input_format: DescribeToolInputs
+):
+    hdca = target_history.with_pair(["123", "456"])
+    inputs = tool_input_format.when.any({"input1": {"batch": True, "values": [hdca.src_dict]}})
+    execute = required_tool.execute.with_inputs(inputs)
+    execute.assert_has_n_jobs(2).assert_creates_n_implicit_collections(1)
+    output_collection = execute.assert_creates_implicit_collection(0)
+    output_collection.assert_has_dataset_element("forward").with_contents_stripped("123")
+    output_collection.assert_has_dataset_element("reverse").with_contents_stripped("456")
 
 
 @requires_tool_id("gx_repeat_boolean_min")
diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py
index 002b9d873389..0f261de94016 100644
--- a/lib/galaxy_test/api/test_tools.py
+++ b/lib/galaxy_test/api/test_tools.py
@@ -71,21 +71,6 @@ def _build_pair(self, history_id, contents):
         hdca_id = create_response.json()["outputs"][0]["id"]
         return hdca_id
 
-    def _run_and_check_simple_collection_mapping(self, history_id, inputs):
-        create = self._run_cat(history_id, inputs=inputs, assert_ok=True)
-        outputs = create["outputs"]
-        jobs = create["jobs"]
-        implicit_collections = create["implicit_collections"]
-        assert len(jobs) == 2
-        assert len(outputs) == 2
-        assert len(implicit_collections) == 1
-        output1 = outputs[0]
-        output2 = outputs[1]
-        output1_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output1)
-        output2_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output2)
-        assert output1_content.strip() == "123"
-        assert output2_content.strip() == "456"
-
     def _run_cat(self, history_id, inputs, assert_ok=False, **kwargs):
         return self._run("cat", history_id, inputs, assert_ok=assert_ok, **kwargs)
 
@@ -1503,56 +1488,6 @@ def test_multirun_non_data_parameter(self, history_id):
         ]
         assert sorted(len(c.split("\n")) for c in outputs_contents) == [1, 2, 3]
 
-    @skip_without_tool("cat1")
-    def test_multirun_in_repeat(self):
-        history_id, common_dataset, repeat_datasets = self._setup_repeat_multirun()
-        inputs = {
-            "input1": common_dataset,
-            "queries_0|input2": {"batch": True, "values": repeat_datasets},
-        }
-        self._check_repeat_multirun(history_id, inputs)
-
-    @skip_without_tool("cat1")
-    def test_multirun_in_repeat_mismatch(self):
-        history_id, common_dataset, repeat_datasets = self._setup_repeat_multirun()
-        inputs = {
-            "input1": {"batch": False, "values": [common_dataset]},
-            "queries_0|input2": {"batch": True, "values": repeat_datasets},
-        }
-        self._check_repeat_multirun(history_id, inputs)
-
-    @skip_without_tool("cat1")
-    def test_multirun_on_multiple_inputs(self):
-        history_id, first_two, second_two = self._setup_two_multiruns()
-        inputs = {
-            "input1": {"batch": True, "values": first_two},
-            "queries_0|input2": {"batch": True, "values": second_two},
-        }
-        outputs = self._cat1_outputs(history_id, inputs=inputs)
-        assert len(outputs) == 2
-        outputs_contents = [
-            self.dataset_populator.get_history_dataset_content(history_id, dataset=o).strip() for o in outputs
-        ]
-        assert "123\n789" in outputs_contents
-        assert "456\n0ab" in outputs_contents
-
-    @skip_without_tool("cat1")
-    def test_multirun_on_multiple_inputs_unlinked(self):
-        history_id, first_two, second_two = self._setup_two_multiruns()
-        inputs = {
-            "input1": {"batch": True, "linked": False, "values": first_two},
-            "queries_0|input2": {"batch": True, "linked": False, "values": second_two},
-        }
-        outputs = self._cat1_outputs(history_id, inputs=inputs)
-        outputs_contents = [
-            self.dataset_populator.get_history_dataset_content(history_id, dataset=o).strip() for o in outputs
-        ]
-        assert len(outputs) == 4
-        assert "123\n789" in outputs_contents
-        assert "456\n0ab" in outputs_contents
-        assert "123\n0ab" in outputs_contents
-        assert "456\n789" in outputs_contents
-
     @skip_without_tool("dbkey_output_action")
     def test_dynamic_parameter_error_handling(self):
         # Run test with valid index once, then supply invalid dbkey and invalid table
@@ -1676,47 +1611,6 @@ def _verify_element(self, history_id, element, **props):
             for key, value in props.items():
                 assert details[key] == value
 
-    def _setup_repeat_multirun(self):
-        history_id = self.dataset_populator.new_history()
-        new_dataset1 = self.dataset_populator.new_dataset(history_id, content="123")
-        new_dataset2 = self.dataset_populator.new_dataset(history_id, content="456")
-        common_dataset = self.dataset_populator.new_dataset(history_id, content="Common")
-        return (
-            history_id,
-            dataset_to_param(common_dataset),
-            [dataset_to_param(new_dataset1), dataset_to_param(new_dataset2)],
-        )
-
-    def _check_repeat_multirun(self, history_id, inputs):
-        outputs = self._cat1_outputs(history_id, inputs=inputs)
-        assert len(outputs) == 2
-        output1 = outputs[0]
-        output2 = outputs[1]
-        output1_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output1)
-        output2_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output2)
-        assert output1_content.strip() == "Common\n123"
-        assert output2_content.strip() == "Common\n456"
-
-    def _setup_two_multiruns(self):
-        history_id = self.dataset_populator.new_history()
-        new_dataset1 = self.dataset_populator.new_dataset(history_id, content="123")
-        new_dataset2 = self.dataset_populator.new_dataset(history_id, content="456")
-        new_dataset3 = self.dataset_populator.new_dataset(history_id, content="789")
-        new_dataset4 = self.dataset_populator.new_dataset(history_id, content="0ab")
-        return (
-            history_id,
-            [dataset_to_param(new_dataset1), dataset_to_param(new_dataset2)],
-            [dataset_to_param(new_dataset3), dataset_to_param(new_dataset4)],
-        )
-
-    @skip_without_tool("cat")
-    def test_map_over_collection(self, history_id):
-        hdca_id = self._build_pair(history_id, ["123", "456"])
-        inputs = {
-            "input1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]},
-        }
-        self._run_and_check_simple_collection_mapping(history_id, inputs)
-
     @skip_without_tool("output_filter_with_input")
     def test_map_over_with_output_filter_no_filtering(self, history_id):
         hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, wait=True).json()["outputs"][0][
diff --git a/test/integration/test_extended_metadata_mapping.py b/test/integration/test_extended_metadata_mapping.py
index b4e088146492..6dd1cbf6c747 100644
--- a/test/integration/test_extended_metadata_mapping.py
+++ b/test/integration/test_extended_metadata_mapping.py
@@ -29,3 +29,18 @@ def test_map_over_collection(self, history_id):
             "input1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]},
         }
         self._run_and_check_simple_collection_mapping(history_id, inputs)
+
+    def _run_and_check_simple_collection_mapping(self, history_id, inputs):
+        create = self._run_cat(history_id, inputs=inputs, assert_ok=True)
+        outputs = create["outputs"]
+        jobs = create["jobs"]
+        implicit_collections = create["implicit_collections"]
+        assert len(jobs) == 2
+        assert len(outputs) == 2
+        assert len(implicit_collections) == 1
+        output1 = outputs[0]
+        output2 = outputs[1]
+        output1_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output1)
+        output2_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output2)
+        assert output1_content.strip() == "123"
+        assert output2_content.strip() == "456"

From 5b08e9b9bb626f9a38b0bc7d40056a7e528302e7 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Mon, 28 Oct 2024 16:29:16 -0400
Subject: [PATCH 13/84] Integrate some new Pydantic models into Planemo lint.

---
 lib/galaxy/tool_util/linters/tests.py       |  62 ++++++++++
 lib/galaxy/tool_util/parameters/__init__.py |   6 +-
 lib/galaxy/tool_util/parameters/case.py     |   9 +-
 lib/galaxy/tool_util/parameters/models.py   |  13 +--
 lib/galaxy/tool_util/parameters/state.py    |  47 ++++----
 test/unit/tool_util/test_tool_linters.py    | 120 ++++++++++++++++----
 6 files changed, 199 insertions(+), 58 deletions(-)

diff --git a/lib/galaxy/tool_util/linters/tests.py b/lib/galaxy/tool_util/linters/tests.py
index 90cec700b06b..f6f483279003 100644
--- a/lib/galaxy/tool_util/linters/tests.py
+++ b/lib/galaxy/tool_util/linters/tests.py
@@ -1,5 +1,6 @@
 """This module contains a linting functions for tool tests."""
 
+from io import StringIO
 from typing import (
     Iterator,
     List,
@@ -8,6 +9,8 @@
 )
 
 from galaxy.tool_util.lint import Linter
+from galaxy.tool_util.parameters import validate_test_cases_for_tool_source
+from galaxy.tool_util.verify.assertion_models import assertion_list
 from galaxy.util import asbool
 from ._util import is_datasource
 
@@ -134,6 +137,65 @@ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
                     )
 
 
+class TestsAssertionValidation(Linter):
+    @classmethod
+    def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
+        try:
+            raw_tests_dict = tool_source.parse_tests_to_dict()
+        except Exception:
+            lint_ctx.warn("Failed to parse test dictionaries from tool - cannot lint assertions")
+            return
+        assert "tests" in raw_tests_dict
+        for test_idx, test in enumerate(raw_tests_dict["tests"], start=1):
+            # TODO: validate command, command_version, element tests. What about children?
+            for output in test["outputs"]:
+                asserts_raw = output.get("attributes", {}).get("assert_list") or []
+                to_yaml_assertions = []
+                for raw_assert in asserts_raw:
+                    to_yaml_assertions.append({"that": raw_assert["tag"], **raw_assert.get("attributes", {})})
+                try:
+                    assertion_list.model_validate(to_yaml_assertions)
+                except Exception as e:
+                    error_str = _cleanup_pydantic_error(e)
+                    lint_ctx.warn(
+                        f"Test {test_idx}: failed to validate assertions. Validation errors are [{error_str}]"
+                    )
+
+
+class TestsCaseValidation(Linter):
+    @classmethod
+    def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
+        try:
+            validation_results = validate_test_cases_for_tool_source(tool_source, use_latest_profile=True)
+        except Exception as e:
+            lint_ctx.warn(
+                f"Serious problem parsing tool source or tests - cannot validate test cases. The exception is [{e}]",
+                linter=cls.name(),
+            )
+            return
+        for test_idx, validation_result in enumerate(validation_results, start=1):
+            error = validation_result.validation_error
+            if error:
+                error_str = _cleanup_pydantic_error(error)
+                lint_ctx.warn(
+                    f"Test {test_idx}: failed to validate test parameters against inputs - tests won't run on a modern Galaxy tool profile version. Validation errors are [{error_str}]",
+                    linter=cls.name(),
+                )
+
+
+def _cleanup_pydantic_error(error) -> str:
+    full_validation_error = f"{error}"
+    new_error = StringIO("")
+    for line in full_validation_error.splitlines():
+        # this repeated over and over isn't useful in the context of how we're building the dynamic models,
+        # tool authors should not be looking up pydantic docs on models they cannot even really inspect
+        if line.strip().startswith("For further information visit https://errors.pydantic"):
+            continue
+        else:
+            new_error.write(f"{line}\n")
+    return new_error.getvalue().strip()
+
+
 class TestsExpectNumOutputs(Linter):
     @classmethod
     def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
diff --git a/lib/galaxy/tool_util/parameters/__init__.py b/lib/galaxy/tool_util/parameters/__init__.py
index 45cd770a5e3f..22dd7e6053aa 100644
--- a/lib/galaxy/tool_util/parameters/__init__.py
+++ b/lib/galaxy/tool_util/parameters/__init__.py
@@ -1,4 +1,7 @@
-from .case import test_case_state
+from .case import (
+    test_case_state,
+    validate_test_cases_for_tool_source,
+)
 from .convert import (
     decode,
     dereference,
@@ -139,6 +142,7 @@
     "ToolParameterT",
     "to_json_schema_string",
     "test_case_state",
+    "validate_test_cases_for_tool_source",
     "RequestToolState",
     "RequestInternalToolState",
     "RequestInternalDereferencedToolState",
diff --git a/lib/galaxy/tool_util/parameters/case.py b/lib/galaxy/tool_util/parameters/case.py
index d1ff72b67d24..c371d5e3b12e 100644
--- a/lib/galaxy/tool_util/parameters/case.py
+++ b/lib/galaxy/tool_util/parameters/case.py
@@ -164,12 +164,12 @@ def test_case_state(
 
 
 def test_case_validation(
-    test_dict: ToolSourceTest, tool_parameter_bundle: List[ToolParameterT], profile: str
+    test_dict: ToolSourceTest, tool_parameter_bundle: List[ToolParameterT], profile: str, name: Optional[str] = None
 ) -> TestCaseStateValidationResult:
     test_case_state_and_warnings = test_case_state(test_dict, tool_parameter_bundle, profile, validate=False)
     exception: Optional[Exception] = None
     try:
-        test_case_state_and_warnings.tool_state.validate(tool_parameter_bundle)
+        test_case_state_and_warnings.tool_state.validate(tool_parameter_bundle, name=name)
         for input_name in test_case_state_and_warnings.unhandled_inputs:
             raise Exception(f"Invalid parameter name found {input_name}")
     except Exception as e:
@@ -323,8 +323,9 @@ def _input_for(flat_state_path: str, inputs: ToolSourceTestInputs) -> Optional[T
 
 
 def validate_test_cases_for_tool_source(
-    tool_source: ToolSource, use_latest_profile: bool = False
+    tool_source: ToolSource, use_latest_profile: bool = False, name: Optional[str] = None
 ) -> List[TestCaseStateValidationResult]:
+    name = name or f"PydanticModelFor[{tool_source.parse_id()}]"
     tool_parameter_bundle = input_models_for_tool_source(tool_source)
     if use_latest_profile:
         # this might get old but it is fine, just needs to be updated when test case changes are made
@@ -334,6 +335,6 @@ def validate_test_cases_for_tool_source(
     test_cases: List[ToolSourceTest] = tool_source.parse_tests_to_dict()["tests"]
     results_by_test: List[TestCaseStateValidationResult] = []
     for test_case in test_cases:
-        validation_result = test_case_validation(test_case, tool_parameter_bundle.parameters, profile)
+        validation_result = test_case_validation(test_case, tool_parameter_bundle.parameters, profile, name=name)
         results_by_test.append(validation_result)
     return results_by_test
diff --git a/lib/galaxy/tool_util/parameters/models.py b/lib/galaxy/tool_util/parameters/models.py
index 5019a92f5f70..d0e7e6bbcb8f 100644
--- a/lib/galaxy/tool_util/parameters/models.py
+++ b/lib/galaxy/tool_util/parameters/models.py
@@ -1489,8 +1489,8 @@ def create_model_strict(*args, **kwd) -> Type[BaseModel]:
 
 def create_model_factory(state_representation: StateRepresentationT):
 
-    def create_method(tool: ToolParameterBundle, name: str = DEFAULT_MODEL_NAME) -> Type[BaseModel]:
-        return create_field_model(tool.parameters, name, state_representation)
+    def create_method(tool: ToolParameterBundle, name: Optional[str] = None) -> Type[BaseModel]:
+        return create_field_model(tool.parameters, name or DEFAULT_MODEL_NAME, state_representation)
 
     return create_method
 
@@ -1546,15 +1546,14 @@ def validate_against_model(pydantic_model: Type[BaseModel], parameter_state: Dic
 
 class ValidationFunctionT(Protocol):
 
-    def __call__(self, tool: ToolParameterBundle, request: RawStateDict, name: str = DEFAULT_MODEL_NAME) -> None: ...
+    def __call__(self, tool: ToolParameterBundle, request: RawStateDict, name: Optional[str] = None) -> None: ...
 
 
 def validate_model_type_factory(state_representation: StateRepresentationT) -> ValidationFunctionT:
 
-    def validate_request(tool: ToolParameterBundle, request: Dict[str, Any], name: str = DEFAULT_MODEL_NAME) -> None:
-        pydantic_model = create_field_model(
-            tool.parameters, name=DEFAULT_MODEL_NAME, state_representation=state_representation
-        )
+    def validate_request(tool: ToolParameterBundle, request: Dict[str, Any], name: Optional[str] = None) -> None:
+        name = name or DEFAULT_MODEL_NAME
+        pydantic_model = create_field_model(tool.parameters, name=name, state_representation=state_representation)
         validate_against_model(pydantic_model, request)
 
     return validate_request
diff --git a/lib/galaxy/tool_util/parameters/state.py b/lib/galaxy/tool_util/parameters/state.py
index 3edc96f69c6e..df040e1803ca 100644
--- a/lib/galaxy/tool_util/parameters/state.py
+++ b/lib/galaxy/tool_util/parameters/state.py
@@ -6,6 +6,7 @@
     Any,
     Dict,
     List,
+    Optional,
     Type,
     Union,
 )
@@ -42,8 +43,8 @@ def __init__(self, input_state: Dict[str, Any]):
     def _validate(self, pydantic_model: Type[BaseModel]) -> None:
         validate_against_model(pydantic_model, self.input_state)
 
-    def validate(self, parameters: HasToolParameters) -> None:
-        base_model = self.parameter_model_for(parameters)
+    def validate(self, parameters: HasToolParameters, name: Optional[str] = None) -> None:
+        base_model = self.parameter_model_for(parameters, name=name)
         if base_model is None:
             raise NotImplementedError(
                 f"Validating tool state against state representation {self.state_representation} is not implemented."
@@ -56,17 +57,17 @@ def state_representation(self) -> StateRepresentationT:
         """Get state representation of the inputs."""
 
     @classmethod
-    def parameter_model_for(cls, parameters: HasToolParameters) -> Type[BaseModel]:
+    def parameter_model_for(cls, parameters: HasToolParameters, name: Optional[str] = None) -> Type[BaseModel]:
         bundle: ToolParameterBundle
         if isinstance(parameters, list):
             bundle = ToolParameterBundleModel(parameters=parameters)
         else:
             bundle = parameters
-        return cls._parameter_model_for(bundle)
+        return cls._parameter_model_for(bundle, name=name)
 
     @classmethod
     @abstractmethod
-    def _parameter_model_for(cls, parameters: ToolParameterBundle) -> Type[BaseModel]:
+    def _parameter_model_for(cls, parameters: ToolParameterBundle, name: Optional[str] = None) -> Type[BaseModel]:
         """Return a model type for this tool state kind."""
 
 
@@ -74,70 +75,70 @@ class RequestToolState(ToolState):
     state_representation: Literal["request"] = "request"
 
     @classmethod
-    def _parameter_model_for(cls, parameters: ToolParameterBundle) -> Type[BaseModel]:
-        return create_request_model(parameters)
+    def _parameter_model_for(cls, parameters: ToolParameterBundle, name: Optional[str] = None) -> Type[BaseModel]:
+        return create_request_model(parameters, name)
 
 
 class RequestInternalToolState(ToolState):
     state_representation: Literal["request_internal"] = "request_internal"
 
     @classmethod
-    def _parameter_model_for(cls, parameters: ToolParameterBundle) -> Type[BaseModel]:
-        return create_request_internal_model(parameters)
+    def _parameter_model_for(cls, parameters: ToolParameterBundle, name: Optional[str] = None) -> Type[BaseModel]:
+        return create_request_internal_model(parameters, name)
 
 
 class LandingRequestToolState(ToolState):
     state_representation: Literal["landing_request"] = "landing_request"
 
     @classmethod
-    def _parameter_model_for(cls, parameters: ToolParameterBundle) -> Type[BaseModel]:
-        return create_landing_request_model(parameters)
+    def _parameter_model_for(cls, parameters: ToolParameterBundle, name: Optional[str] = None) -> Type[BaseModel]:
+        return create_landing_request_model(parameters, name)
 
 
 class LandingRequestInternalToolState(ToolState):
     state_representation: Literal["landing_request_internal"] = "landing_request_internal"
 
     @classmethod
-    def _parameter_model_for(cls, parameters: ToolParameterBundle) -> Type[BaseModel]:
-        return create_landing_request_internal_model(parameters)
+    def _parameter_model_for(cls, parameters: ToolParameterBundle, name: Optional[str] = None) -> Type[BaseModel]:
+        return create_landing_request_internal_model(parameters, name)
 
 
 class RequestInternalDereferencedToolState(ToolState):
     state_representation: Literal["request_internal_dereferenced"] = "request_internal_dereferenced"
 
     @classmethod
-    def _parameter_model_for(cls, parameters: ToolParameterBundle) -> Type[BaseModel]:
-        return create_request_internal_dereferenced_model(parameters)
+    def _parameter_model_for(cls, parameters: ToolParameterBundle, name: Optional[str] = None) -> Type[BaseModel]:
+        return create_request_internal_dereferenced_model(parameters, name)
 
 
 class JobInternalToolState(ToolState):
     state_representation: Literal["job_internal"] = "job_internal"
 
     @classmethod
-    def _parameter_model_for(cls, parameters: ToolParameterBundle) -> Type[BaseModel]:
-        return create_job_internal_model(parameters)
+    def _parameter_model_for(cls, parameters: ToolParameterBundle, name: Optional[str] = None) -> Type[BaseModel]:
+        return create_job_internal_model(parameters, name)
 
 
 class TestCaseToolState(ToolState):
     state_representation: Literal["test_case_xml"] = "test_case_xml"
 
     @classmethod
-    def _parameter_model_for(cls, parameters: ToolParameterBundle) -> Type[BaseModel]:
+    def _parameter_model_for(cls, parameters: ToolParameterBundle, name: Optional[str] = None) -> Type[BaseModel]:
         # implement a test case model...
-        return create_test_case_model(parameters)
+        return create_test_case_model(parameters, name)
 
 
 class WorkflowStepToolState(ToolState):
     state_representation: Literal["workflow_step"] = "workflow_step"
 
     @classmethod
-    def _parameter_model_for(cls, parameters: ToolParameterBundle) -> Type[BaseModel]:
-        return create_workflow_step_model(parameters)
+    def _parameter_model_for(cls, parameters: ToolParameterBundle, name: Optional[str] = None) -> Type[BaseModel]:
+        return create_workflow_step_model(parameters, name)
 
 
 class WorkflowStepLinkedToolState(ToolState):
     state_representation: Literal["workflow_step_linked"] = "workflow_step_linked"
 
     @classmethod
-    def _parameter_model_for(cls, parameters: ToolParameterBundle) -> Type[BaseModel]:
-        return create_workflow_step_linked_model(parameters)
+    def _parameter_model_for(cls, parameters: ToolParameterBundle, name: Optional[str] = None) -> Type[BaseModel]:
+        return create_workflow_step_linked_model(parameters, name)
diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py
index 7e6d3b62ec93..d693bae26390 100644
--- a/test/unit/tool_util/test_tool_linters.py
+++ b/test/unit/tool_util/test_tool_linters.py
@@ -26,7 +26,9 @@
     xsd,
 )
 from galaxy.tool_util.loader_directory import load_tool_sources_from_path
+from galaxy.tool_util.parser.interface import ToolSource
 from galaxy.tool_util.parser.xml import XmlToolSource
+from galaxy.tool_util.unittest_utils import functional_test_tool_path
 from galaxy.util import (
     ElementTree,
     submodules,
@@ -698,6 +700,11 @@
 TESTS_ABSENT = """
 <tool id="id" name="name"/>
 """
+TESTS_ABSENT_YAML = """
+class: GalaxyTool
+name: name
+id: id
+"""
 TESTS_ABSENT_DATA_SOURCE = """
 <tool id="id" name="name" tool_type="data_source"/>
 """
@@ -794,6 +801,38 @@
     </tests>
 </tool>
 """
+INVALID_CENTER_OF_MASS = """
+<tool id="id" name="name">
+    <outputs>
+        <data name="out_image"/>
+    </outputs>
+    <tests>
+        <test>
+            <output name="out_image">
+                <assert_contents>
+                    <has_image_center_of_mass center_of_mass="511.07, 223.34, 2.3" />
+                </assert_contents>
+            </output>
+        </test>
+    </tests>
+</tool>
+"""
+VALID_CENTER_OF_MASS = """
+<tool id="id" name="name">
+    <outputs>
+        <data name="out_image"/>
+    </outputs>
+    <tests>
+        <test>
+            <output name="out_image">
+                <assert_contents>
+                    <has_image_center_of_mass center_of_mass="511.07, 223.34" />
+                </assert_contents>
+            </output>
+        </test>
+    </tests>
+</tool>
+"""
 TESTS_VALID = """
 <tool id="id" name="name">
     <outputs>
@@ -801,7 +840,7 @@
     </outputs>
     <tests>
         <test>
-            <output name="test"/>
+            <output name="test" value="empty.txt" />
         </test>
     </tests>
 </tool>
@@ -950,6 +989,19 @@ def get_xml_tool_source(xml_string: str) -> XmlToolSource:
     return XmlToolSource(get_xml_tree(xml_string))
 
 
+def get_tool_source(source_contents: str) -> ToolSource:
+    if "GalaxyTool" in source_contents:
+        with tempfile.NamedTemporaryFile(mode="w", suffix="tool.yml") as tmp:
+            tmp.write(source_contents)
+            tmp.flush()
+            tool_sources = load_tool_sources_from_path(tmp.name)
+            assert len(tool_sources) == 1, "Expected 1 tool source"
+            tool_source = tool_sources[0][1]
+            return tool_source
+    else:
+        return get_xml_tool_source(source_contents)
+
+
 def run_lint_module(lint_ctx, lint_module, lint_target):
     lint_tool_source_with_modules(lint_ctx, lint_target, list({lint_module, xsd}))
 
@@ -1751,13 +1803,14 @@ def test_stdio_invalid_match(lint_ctx):
 
 
 def test_tests_absent(lint_ctx):
-    tool_source = get_xml_tool_source(TESTS_ABSENT)
-    run_lint_module(lint_ctx, tests, tool_source)
-    assert "No tests found, most tools should define test cases." in lint_ctx.warn_messages
-    assert not lint_ctx.info_messages
-    assert not lint_ctx.valid_messages
-    assert len(lint_ctx.warn_messages) == 1
-    assert not lint_ctx.error_messages
+    for test_contents in [TESTS_ABSENT, TESTS_ABSENT_YAML]:
+        tool_source = get_tool_source(test_contents)
+        run_lint_module(lint_ctx, tests, tool_source)
+        assert "No tests found, most tools should define test cases." in lint_ctx.warn_messages
+        assert not lint_ctx.info_messages
+        assert not lint_ctx.valid_messages
+        assert len(lint_ctx.warn_messages) == 1
+        assert not lint_ctx.error_messages
 
 
 def test_tests_data_source(lint_ctx):
@@ -1791,7 +1844,6 @@ def test_tests_param_output_names(lint_ctx):
     )
     assert not lint_ctx.info_messages
     assert len(lint_ctx.valid_messages) == 1
-    assert not lint_ctx.warn_messages
     assert len(lint_ctx.error_messages) == 6
 
 
@@ -1806,7 +1858,7 @@ def test_tests_expect_failure_output(lint_ctx):
     )
     assert not lint_ctx.info_messages
     assert not lint_ctx.valid_messages
-    assert len(lint_ctx.warn_messages) == 1
+    assert len(lint_ctx.warn_messages) == 3
     assert len(lint_ctx.error_messages) == 2
 
 
@@ -1854,10 +1906,24 @@ def test_tests_asserts(lint_ctx):
     assert "Test 1: 'has_size' must not specify 'value' and 'size'" in lint_ctx.error_messages
     assert "Test 1: 'has_n_columns' needs to specify 'n', 'min', or 'max'" in lint_ctx.error_messages
     assert "Test 1: 'has_n_lines' needs to specify 'n', 'min', or 'max'" in lint_ctx.error_messages
-    assert not lint_ctx.warn_messages
     assert len(lint_ctx.error_messages) == 9
 
 
+def test_tests_assertion_models_valid(lint_ctx):
+    tool_source = get_xml_tool_source(VALID_CENTER_OF_MASS)
+    run_lint_module(lint_ctx, tests, tool_source)
+    assert len(lint_ctx.error_messages) == 0
+    assert len(lint_ctx.warn_messages) == 0
+
+
+def test_tests_assertion_models_invalid(lint_ctx):
+    tool_source = get_xml_tool_source(INVALID_CENTER_OF_MASS)
+    run_lint_module(lint_ctx, tests, tool_source)
+    assert len(lint_ctx.error_messages) == 0
+    assert len(lint_ctx.warn_messages) == 1
+    assert "Test 1: failed to validate assertions. Validation errors are " in lint_ctx.warn_messages
+
+
 def test_tests_output_type_mismatch(lint_ctx):
     tool_source = get_xml_tool_source(TESTS_OUTPUT_TYPE_MISMATCH)
     run_lint_module(lint_ctx, tests, tool_source)
@@ -1869,7 +1935,6 @@ def test_tests_output_type_mismatch(lint_ctx):
         "Test 1: test collection output 'data_name' does not correspond to a 'output_collection' output, but a 'data'"
         in lint_ctx.error_messages
     )
-    assert not lint_ctx.warn_messages
     assert len(lint_ctx.error_messages) == 2
 
 
@@ -1892,7 +1957,6 @@ def test_tests_discover_outputs(lint_ctx):
         "Test 5: test collection 'collection_name' must contain nested 'element' tags and/or element children with a 'count' attribute"
         in lint_ctx.error_messages
     )
-    assert not lint_ctx.warn_messages
     assert len(lint_ctx.error_messages) == 4
 
 
@@ -1911,7 +1975,6 @@ def test_tests_compare_attrib_incompatibility(lint_ctx):
     assert 'Test 1: Attribute sort is incompatible with compare="contains".' in lint_ctx.error_messages
     assert not lint_ctx.info_messages
     assert len(lint_ctx.valid_messages) == 1
-    assert not lint_ctx.warn_messages
     assert len(lint_ctx.error_messages) == 2
 
 
@@ -2077,14 +2140,8 @@ def test_tool_and_macro_xml(lint_ctx_xpath, lint_ctx):
 
 
 def test_linting_yml_tool(lint_ctx):
-    with tempfile.TemporaryDirectory() as tmp:
-        tool_path = os.path.join(tmp, "tool.yml")
-        with open(tool_path, "w") as tmpf:
-            tmpf.write(YAML_TOOL)
-        tool_sources = load_tool_sources_from_path(tmp)
-        assert len(tool_sources) == 1, "Expected 1 tool source"
-        tool_source = tool_sources[0][1]
-        lint_tool_source_with(lint_ctx, tool_source)
+    tool_source = get_tool_source(YAML_TOOL)
+    lint_tool_source_with(lint_ctx, tool_source)
     assert "Tool defines a version [1.0]." in lint_ctx.valid_messages
     assert "Tool defines a name [simple_constructs_y]." in lint_ctx.valid_messages
     assert "Tool defines an id [simple_constructs_y]." in lint_ctx.valid_messages
@@ -2145,7 +2202,7 @@ def test_skip_by_module(lint_ctx):
 def test_list_linters():
     linter_names = Linter.list_listers()
     # make sure to add/remove a test for new/removed linters if this number changes
-    assert len(linter_names) == 132
+    assert len(linter_names) == 134
     assert "Linter" not in linter_names
     # make sure that linters from all modules are available
     for prefix in [
@@ -2164,6 +2221,23 @@ def test_list_linters():
         assert len([x for x in linter_names if x.startswith(prefix)])
 
 
+def test_linting_functional_tool_multi_select(lint_ctx):
+    tool_source = functional_test_tool_source("multi_select.xml")
+    run_lint_module(lint_ctx, tests, tool_source)
+    warn_message = lint_ctx.warn_messages[0]
+    assert (
+        "Test 2: failed to validate test parameters against inputs - tests won't run on a modern Galaxy tool profile version. Validation errors are [5 validation errors for"
+        in str(warn_message)
+    )
+
+
+def functional_test_tool_source(name: str) -> ToolSource:
+    tool_sources = load_tool_sources_from_path(functional_test_tool_path(name))
+    assert len(tool_sources) == 1, "Expected 1 tool source"
+    tool_source = tool_sources[0][1]
+    return tool_source
+
+
 def test_linter_module_list():
     linter_modules = submodules.import_submodules(galaxy.tool_util.linters)
     linter_module_names = [m.__name__.split(".")[-1] for m in linter_modules]

From 7858f2656976e19fc5e724742c6a02a23a990fc0 Mon Sep 17 00:00:00 2001
From: Wolfgang Maier <maierw@informatik.uni-freiburg.de>
Date: Wed, 6 Mar 2024 10:47:15 +0100
Subject: [PATCH 14/84] Improve usability of Directory datatype

---
 .../config/sample/datatypes_conf.xml.sample   |   9 +-
 .../converters/archive_to_directory.xml       |  33 +++++
 .../datatypes/converters/tar_to_directory.xml |   2 +-
 lib/galaxy/datatypes/data.py                  | 137 ++++++++++++++++++
 4 files changed, 176 insertions(+), 5 deletions(-)
 create mode 100644 lib/galaxy/datatypes/converters/archive_to_directory.xml

diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample
index 0e026b5a7b0a..b62d930bb515 100644
--- a/lib/galaxy/config/sample/datatypes_conf.xml.sample
+++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample
@@ -288,13 +288,14 @@
       <display file="qiime/qiime2/q2view.xml"/>
     </datatype>
     <datatype extension='qiime2.tabular' type="galaxy.datatypes.qiime2:QIIME2Metadata" display_in_upload="true"/>
-    <datatype extension="zip" type="galaxy.datatypes.binary:CompressedZipArchive" display_in_upload="true"/>
+    <datatype extension="zip" type="galaxy.datatypes.binary:CompressedZipArchive" display_in_upload="true">
+        <converter file="archive_to_directory.xml" target_datatype="directory" />
+    </datatype>
     <datatype extension="ncbi_genome_dataset.zip" type="galaxy.datatypes.binary:CompressedZipArchive" subclass="true" display_in_upload="true"/>
     <datatype extension="tar" type="galaxy.datatypes.binary:CompressedArchive" subclass="true" display_in_upload="true">
-      <converter file="tar_to_directory.xml" target_datatype="directory"/>
-    </datatype>
-    <datatype extension="directory" type="galaxy.datatypes.data:Directory">
+      <converter file="archive_to_directory.xml" target_datatype="directory"/>
     </datatype>
+    <datatype extension="directory" type="galaxy.datatypes.data:Directory"/>
     <datatype extension="yaml" type="galaxy.datatypes.text:Yaml" display_in_upload="true" />
     <!-- Proteomics Datatypes -->
     <datatype extension="mrm" type="galaxy.datatypes.tabular:Tabular" display_in_upload="true" subclass="true"/>
diff --git a/lib/galaxy/datatypes/converters/archive_to_directory.xml b/lib/galaxy/datatypes/converters/archive_to_directory.xml
new file mode 100644
index 000000000000..9ccdba28a30e
--- /dev/null
+++ b/lib/galaxy/datatypes/converters/archive_to_directory.xml
@@ -0,0 +1,33 @@
+<tool id="CONVERTER_archive_to_directory" name="Unpack archive to directory" version="1.0.0" profile="21.09">
+    <!-- Use compression_utils instead of shell commands (tar/unzip) so we can verify safety of results -->
+    <requirements>
+        <requirement type="package" version="23.2.1">galaxy-util</requirement>
+    </requirements>
+    <command><![CDATA[
+        mkdir '$output1.files_path' &&
+        cd '$output1.files_path' &&
+        python -c "from galaxy.util.compression_utils import CompressedFile; CompressedFile('$input1').extract('.')"
+    ]]></command>
+    <configfiles>
+        <configfile filename="metadata_json"><![CDATA[{
+    "output1": {
+        "name": "$input1.name unpacked to $__target_datatype__",
+        "ext": "$__target_datatype__"
+    }
+}]]></configfile>
+    </configfiles>
+    <inputs>
+        <param format="tar,zip" name="input1" type="data"/>
+    </inputs>
+    <outputs provided_metadata_file="metadata_json">
+        <data format="auto" name="output1"/>
+    </outputs>
+    <tests>
+        <test>
+            <param name="input1" ftype="tar" value="testdir1.tar"/>
+            <output name="output1" ftype="directory" value="testdir1.tar.directory"/>
+        </test>
+    </tests>
+    <help>
+    </help>
+</tool>
diff --git a/lib/galaxy/datatypes/converters/tar_to_directory.xml b/lib/galaxy/datatypes/converters/tar_to_directory.xml
index 59354b39b5fd..0160746283da 100644
--- a/lib/galaxy/datatypes/converters/tar_to_directory.xml
+++ b/lib/galaxy/datatypes/converters/tar_to_directory.xml
@@ -1,7 +1,7 @@
 <tool id="CONVERTER_tar_to_directory" name="Convert tar to directory" version="1.0.1" profile="17.05">
     <!-- Don't use tar directly so we can verify safety of results - tar -xzf '$input1'; -->
     <requirements>
-        <requirement type="package" version="23.1">galaxy-util</requirement>
+        <requirement type="package" version="23.2.1">galaxy-util</requirement>
     </requirements>
     <command>
         mkdir '$output1.files_path';
diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py
index 0024adea6155..eb4d23c9926b 100644
--- a/lib/galaxy/datatypes/data.py
+++ b/lib/galaxy/datatypes/data.py
@@ -1212,6 +1212,143 @@ def regex_line_dataprovider(
 class Directory(Data):
     """Class representing a directory of files."""
 
+    # The behavior of this class is intended to be similar to that of
+    # composite types, but with arbitrary structure of extra_files.
+
+    # A prioritized list of files in the directory that, if present, can serve
+    # as a replacement for a composite type's primary dataset.
+    recognized_index_files = []
+
+    # Directories converted from archives, and possibly others, have a single
+    # root folder inside the directory. The root_folder attribute lets tools
+    # access that folder without first exploring the directory tree themselves.
+    # Will be set to the empty string for flat directories.
+    MetadataElement(
+        name="root_folder",
+        default=None,
+        desc="Name of the root folder of the directory",
+        readonly=True,
+        optional=False,
+        visible=False,
+    )
+
+    # Capture the actual index file obtained from the recognized_index_files
+    # list for a concrete directory.
+    MetadataElement(
+        name="index_file",
+        default=None,
+        desc="Name of the index file relative to the root folder",
+        readonly=True,
+        optional=False,
+        visible=False,
+    )
+
+    MetadataElement(
+        name="folder_size",
+        default=0,
+        desc="Total size of the folder in bytes",
+        readonly=True,
+        optional=False,
+        visible=False,
+    )
+
+    @classmethod
+    def _get_size(cls, root_path):
+        total_size = 0
+        for dirpath, dirnames, filenames in os.walk(root_path):
+            for f in filenames:
+                fp = os.path.join(dirpath, f)
+                # skip if it is symbolic link
+                if not os.path.islink(fp):
+                    total_size += os.path.getsize(fp)
+        return total_size
+
+    def set_meta(self, dataset: DatasetProtocol, **kwd):
+        efp = dataset.extra_files_path
+        efp_items = os.listdir(efp)
+        root_folder_name = efp_items[0]
+        if len(efp_items) == 1 and os.path.isdir(os.path.join(efp, root_folder_name)):
+            dataset.metadata.root_folder = root_folder_name
+        else:
+            dataset.metadata.root_folder = ""
+        index_file = None
+        for f in self.recognized_index_files:
+            if os.path.isfile(os.path.join(efp, root_folder_name, f)):
+                index_file = f
+                break
+        dataset.metadata.index_file = index_file
+        dataset.metadata.folder_size = self._get_size(os.path.join(efp, dataset.metadata.root_folder))
+
+    def set_peek(self, dataset: DatasetProtocol, **kwd) -> None:
+        if not dataset.dataset.purged:
+            dataset.peek = f"{dataset.metadata.root_folder}"
+            dataset.blurb = nice_size(dataset.metadata.folder_size)
+        else:
+            dataset.peek = "file does not exist"
+            dataset.blurb = "file purged from disk"
+
+    def _archive_main_file(
+        self, archive: ZipstreamWrapper, display_name: str, data_filename: str
+    ) -> Tuple[bool, str, str]:
+        """Overwrites the method to not do anything.
+
+        No main file gets added to a directory archive.
+        """
+        error, msg, messagetype = False, "", ""
+        return error, msg, messagetype
+
+    def display_data(
+        self,
+        trans,
+        dataset: DatasetHasHidProtocol,
+        preview: bool = False,
+        filename: Optional[str] = None,
+        to_ext: Optional[str] = None,
+        **kwd,
+    ):
+        headers = kwd.get("headers", {})
+        # Prevent IE8 from sniffing content type since we're explicit about it.  This prevents intentionally text/plain
+        # content from being rendered in the browser
+        headers["X-Content-Type-Options"] = "nosniff"
+        if to_ext:
+            # Download the directory structure as an archive
+            trans.log_event(f"Download directory for dataset id: {str(dataset.id)}")
+            return self._archive_composite_dataset(trans, dataset, headers, do_action=kwd.get("do_action", "zip"))
+        if preview:
+            root_folder = dataset.metadata.root_folder
+            index_file = dataset.metadata.index_file
+            if not filename or filename == 'index':
+                if root_folder is not None and index_file:
+                    # display the index file in lieu of the empty primary dataset
+                    file_path = os.path.join(dataset.extra_files_path, root_folder, index_file)
+                    self._clean_and_set_mime_type(trans, "text/plain", headers)  # type: ignore[arg-type]
+                    return self._yield_user_file_content(
+                        trans,
+                        dataset,
+                        file_path,
+                        headers
+                    ), headers
+                elif root_folder:
+                    # delegate to the parent method, which knows how to display
+                    # a directory
+                    filename = root_folder
+                else:
+                    # No meaningful display available, show an info message instead
+                    self._clean_and_set_mime_type(trans, "text/plain", headers) # type: ignore[arg-type]
+                    if root_folder is None:
+                        return util.smart_str("Cannot generate preview with incomplete metadata. Resetting metadata may help.")
+                    elif self.recognized_index_files:
+                        return util.smart_str("None of the known key files for the datatype present. Is the datatype format set correctly?")
+                    else:
+                        return util.smart_str("No preview available for this dataset."), headers
+
+        return super().display_data(
+            trans,
+            dataset=dataset,
+            filename=filename,
+            **kwd,
+        )
+
 
 class GenericAsn1(Text):
     """Class for generic ASN.1 text format"""

From 7833391dc747a8032619a30cbef28a66ffae0b9b Mon Sep 17 00:00:00 2001
From: Wolfgang Maier <maierw@informatik.uni-freiburg.de>
Date: Fri, 8 Mar 2024 22:51:30 +0100
Subject: [PATCH 15/84] Use dataset's total_size property instead of
 calculating it again

---
 lib/galaxy/datatypes/data.py | 23 +----------------------
 1 file changed, 1 insertion(+), 22 deletions(-)

diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py
index eb4d23c9926b..11c676f141ca 100644
--- a/lib/galaxy/datatypes/data.py
+++ b/lib/galaxy/datatypes/data.py
@@ -1243,26 +1243,6 @@ class Directory(Data):
         visible=False,
     )
 
-    MetadataElement(
-        name="folder_size",
-        default=0,
-        desc="Total size of the folder in bytes",
-        readonly=True,
-        optional=False,
-        visible=False,
-    )
-
-    @classmethod
-    def _get_size(cls, root_path):
-        total_size = 0
-        for dirpath, dirnames, filenames in os.walk(root_path):
-            for f in filenames:
-                fp = os.path.join(dirpath, f)
-                # skip if it is symbolic link
-                if not os.path.islink(fp):
-                    total_size += os.path.getsize(fp)
-        return total_size
-
     def set_meta(self, dataset: DatasetProtocol, **kwd):
         efp = dataset.extra_files_path
         efp_items = os.listdir(efp)
@@ -1277,12 +1257,11 @@ def set_meta(self, dataset: DatasetProtocol, **kwd):
                 index_file = f
                 break
         dataset.metadata.index_file = index_file
-        dataset.metadata.folder_size = self._get_size(os.path.join(efp, dataset.metadata.root_folder))
 
     def set_peek(self, dataset: DatasetProtocol, **kwd) -> None:
         if not dataset.dataset.purged:
             dataset.peek = f"{dataset.metadata.root_folder}"
-            dataset.blurb = nice_size(dataset.metadata.folder_size)
+            dataset.blurb = nice_size(dataset.dataset.total_size)
         else:
             dataset.peek = "file does not exist"
             dataset.blurb = "file purged from disk"

From fcdc27d92ca1abdfc452598b69e374aaab83b88c Mon Sep 17 00:00:00 2001
From: Wolfgang Maier <maierw@informatik.uni-freiburg.de>
Date: Fri, 8 Mar 2024 23:21:14 +0100
Subject: [PATCH 16/84] Add auto-converters for tar.gz and tar.bz2

as suggested by @bernt-matthias
---
 lib/galaxy/config/sample/datatypes_conf.xml.sample | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample
index b62d930bb515..a8e798143f8b 100644
--- a/lib/galaxy/config/sample/datatypes_conf.xml.sample
+++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample
@@ -292,7 +292,7 @@
         <converter file="archive_to_directory.xml" target_datatype="directory" />
     </datatype>
     <datatype extension="ncbi_genome_dataset.zip" type="galaxy.datatypes.binary:CompressedZipArchive" subclass="true" display_in_upload="true"/>
-    <datatype extension="tar" type="galaxy.datatypes.binary:CompressedArchive" subclass="true" display_in_upload="true">
+    <datatype extension="tar" auto_compressed_types="gz,bz2" type="galaxy.datatypes.binary:CompressedArchive" subclass="true" display_in_upload="true">
       <converter file="archive_to_directory.xml" target_datatype="directory"/>
     </datatype>
     <datatype extension="directory" type="galaxy.datatypes.data:Directory"/>

From 2ff6d33d6922a92ea406fd2d547ccc49686ec6af Mon Sep 17 00:00:00 2001
From: Wolfgang Maier <maierw@informatik.uni-freiburg.de>
Date: Fri, 8 Mar 2024 23:39:40 +0100
Subject: [PATCH 17/84] Make new converter testable

---
 lib/galaxy/datatypes/converters/archive_to_directory.xml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/galaxy/datatypes/converters/archive_to_directory.xml b/lib/galaxy/datatypes/converters/archive_to_directory.xml
index 9ccdba28a30e..d0c56c154c09 100644
--- a/lib/galaxy/datatypes/converters/archive_to_directory.xml
+++ b/lib/galaxy/datatypes/converters/archive_to_directory.xml
@@ -18,6 +18,9 @@
     </configfiles>
     <inputs>
         <param format="tar,zip" name="input1" type="data"/>
+        <param name="__target_datatype__" type="select" label="Target data type">
+            <option value="directory">directory</option>
+        </param>
     </inputs>
     <outputs provided_metadata_file="metadata_json">
         <data format="auto" name="output1"/>
@@ -25,6 +28,7 @@
     <tests>
         <test>
             <param name="input1" ftype="tar" value="testdir1.tar"/>
+            <param name="__target_datatype__" value="directory"/>
             <output name="output1" ftype="directory" value="testdir1.tar.directory"/>
         </test>
     </tests>

From 8e538f2fc774b6b008230b383538f7ffde1e49cc Mon Sep 17 00:00:00 2001
From: Wolfgang Maier <maierw@informatik.uni-freiburg.de>
Date: Fri, 8 Mar 2024 23:46:57 +0100
Subject: [PATCH 18/84] Add file_ext and missing type annotation

---
 lib/galaxy/datatypes/data.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py
index 11c676f141ca..7c9fa7f80194 100644
--- a/lib/galaxy/datatypes/data.py
+++ b/lib/galaxy/datatypes/data.py
@@ -1212,12 +1212,14 @@ def regex_line_dataprovider(
 class Directory(Data):
     """Class representing a directory of files."""
 
+    file_ext = "directory"
+
     # The behavior of this class is intended to be similar to that of
     # composite types, but with arbitrary structure of extra_files.
 
     # A prioritized list of files in the directory that, if present, can serve
     # as a replacement for a composite type's primary dataset.
-    recognized_index_files = []
+    recognized_index_files: List[str] = []
 
     # Directories converted from archives, and possibly others, have a single
     # root folder inside the directory. The root_folder attribute lets tools

From 7484a6c82fb5b8414fc5698afe5f6c3f15108b11 Mon Sep 17 00:00:00 2001
From: Wolfgang Maier <maierw@informatik.uni-freiburg.de>
Date: Fri, 8 Mar 2024 23:54:19 +0100
Subject: [PATCH 19/84] Fix linting

---
 lib/galaxy/datatypes/data.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py
index 7c9fa7f80194..eb282fcfaa46 100644
--- a/lib/galaxy/datatypes/data.py
+++ b/lib/galaxy/datatypes/data.py
@@ -1315,7 +1315,7 @@ def display_data(
                     filename = root_folder
                 else:
                     # No meaningful display available, show an info message instead
-                    self._clean_and_set_mime_type(trans, "text/plain", headers) # type: ignore[arg-type]
+                    self._clean_and_set_mime_type(trans, "text/plain", headers)  # type: ignore[arg-type]
                     if root_folder is None:
                         return util.smart_str("Cannot generate preview with incomplete metadata. Resetting metadata may help.")
                     elif self.recognized_index_files:

From 566d526692e254bd010d8b8e688c5cf44c9d0902 Mon Sep 17 00:00:00 2001
From: Wolfgang Maier <maierw@informatik.uni-freiburg.de>
Date: Sat, 9 Mar 2024 00:05:09 +0100
Subject: [PATCH 20/84] Fix code formatting

---
 lib/galaxy/datatypes/data.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py
index eb282fcfaa46..3212ddea409d 100644
--- a/lib/galaxy/datatypes/data.py
+++ b/lib/galaxy/datatypes/data.py
@@ -1298,17 +1298,12 @@ def display_data(
         if preview:
             root_folder = dataset.metadata.root_folder
             index_file = dataset.metadata.index_file
-            if not filename or filename == 'index':
+            if not filename or filename == "index":
                 if root_folder is not None and index_file:
                     # display the index file in lieu of the empty primary dataset
                     file_path = os.path.join(dataset.extra_files_path, root_folder, index_file)
                     self._clean_and_set_mime_type(trans, "text/plain", headers)  # type: ignore[arg-type]
-                    return self._yield_user_file_content(
-                        trans,
-                        dataset,
-                        file_path,
-                        headers
-                    ), headers
+                    return self._yield_user_file_content(trans, dataset, file_path, headers), headers
                 elif root_folder:
                     # delegate to the parent method, which knows how to display
                     # a directory
@@ -1317,9 +1312,13 @@ def display_data(
                     # No meaningful display available, show an info message instead
                     self._clean_and_set_mime_type(trans, "text/plain", headers)  # type: ignore[arg-type]
                     if root_folder is None:
-                        return util.smart_str("Cannot generate preview with incomplete metadata. Resetting metadata may help.")
+                        return util.smart_str(
+                            "Cannot generate preview with incomplete metadata. Resetting metadata may help."
+                        )
                     elif self.recognized_index_files:
-                        return util.smart_str("None of the known key files for the datatype present. Is the datatype format set correctly?")
+                        return util.smart_str(
+                            "None of the known key files for the datatype present. Is the datatype format set correctly?"
+                        )
                     else:
                         return util.smart_str("No preview available for this dataset."), headers
 

From 4f648855591b89a86b03266aa5fd65b1fa601f79 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 9 Oct 2024 10:32:42 +0200
Subject: [PATCH 21/84] Drop index-related metadata from Directory datatype

This will go into a sub-datatype when needed.
---
 lib/galaxy/datatypes/data.py | 72 ------------------------------------
 1 file changed, 72 deletions(-)

diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py
index 3212ddea409d..c074bbdc4b1a 100644
--- a/lib/galaxy/datatypes/data.py
+++ b/lib/galaxy/datatypes/data.py
@@ -1217,10 +1217,6 @@ class Directory(Data):
     # The behavior of this class is intended to be similar to that of
     # composite types, but with arbitrary structure of extra_files.
 
-    # A prioritized list of files in the directory that, if present, can serve
-    # as a replacement for a composite type's primary dataset.
-    recognized_index_files: List[str] = []
-
     # Directories converted from archives, and possibly others, have a single
     # root folder inside the directory. The root_folder attribute lets tools
     # access that folder without first exploring the directory tree themselves.
@@ -1234,17 +1230,6 @@ class Directory(Data):
         visible=False,
     )
 
-    # Capture the actual index file obtained from the recognized_index_files
-    # list for a concrete directory.
-    MetadataElement(
-        name="index_file",
-        default=None,
-        desc="Name of the index file relative to the root folder",
-        readonly=True,
-        optional=False,
-        visible=False,
-    )
-
     def set_meta(self, dataset: DatasetProtocol, **kwd):
         efp = dataset.extra_files_path
         efp_items = os.listdir(efp)
@@ -1253,12 +1238,6 @@ def set_meta(self, dataset: DatasetProtocol, **kwd):
             dataset.metadata.root_folder = root_folder_name
         else:
             dataset.metadata.root_folder = ""
-        index_file = None
-        for f in self.recognized_index_files:
-            if os.path.isfile(os.path.join(efp, root_folder_name, f)):
-                index_file = f
-                break
-        dataset.metadata.index_file = index_file
 
     def set_peek(self, dataset: DatasetProtocol, **kwd) -> None:
         if not dataset.dataset.purged:
@@ -1278,57 +1257,6 @@ def _archive_main_file(
         error, msg, messagetype = False, "", ""
         return error, msg, messagetype
 
-    def display_data(
-        self,
-        trans,
-        dataset: DatasetHasHidProtocol,
-        preview: bool = False,
-        filename: Optional[str] = None,
-        to_ext: Optional[str] = None,
-        **kwd,
-    ):
-        headers = kwd.get("headers", {})
-        # Prevent IE8 from sniffing content type since we're explicit about it.  This prevents intentionally text/plain
-        # content from being rendered in the browser
-        headers["X-Content-Type-Options"] = "nosniff"
-        if to_ext:
-            # Download the directory structure as an archive
-            trans.log_event(f"Download directory for dataset id: {str(dataset.id)}")
-            return self._archive_composite_dataset(trans, dataset, headers, do_action=kwd.get("do_action", "zip"))
-        if preview:
-            root_folder = dataset.metadata.root_folder
-            index_file = dataset.metadata.index_file
-            if not filename or filename == "index":
-                if root_folder is not None and index_file:
-                    # display the index file in lieu of the empty primary dataset
-                    file_path = os.path.join(dataset.extra_files_path, root_folder, index_file)
-                    self._clean_and_set_mime_type(trans, "text/plain", headers)  # type: ignore[arg-type]
-                    return self._yield_user_file_content(trans, dataset, file_path, headers), headers
-                elif root_folder:
-                    # delegate to the parent method, which knows how to display
-                    # a directory
-                    filename = root_folder
-                else:
-                    # No meaningful display available, show an info message instead
-                    self._clean_and_set_mime_type(trans, "text/plain", headers)  # type: ignore[arg-type]
-                    if root_folder is None:
-                        return util.smart_str(
-                            "Cannot generate preview with incomplete metadata. Resetting metadata may help."
-                        )
-                    elif self.recognized_index_files:
-                        return util.smart_str(
-                            "None of the known key files for the datatype present. Is the datatype format set correctly?"
-                        )
-                    else:
-                        return util.smart_str("No preview available for this dataset."), headers
-
-        return super().display_data(
-            trans,
-            dataset=dataset,
-            filename=filename,
-            **kwd,
-        )
-
 
 class GenericAsn1(Text):
     """Class for generic ASN.1 text format"""

From 5530e154c031bf1217f00f26d6a14cdf840b173a Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Thu, 10 Oct 2024 13:10:22 +0200
Subject: [PATCH 22/84] Drop root_folder metadata from Directory datatype

This should probably go in sub-classes that expect specific directory structures.
---
 lib/galaxy/datatypes/data.py | 24 +-----------------------
 1 file changed, 1 insertion(+), 23 deletions(-)

diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py
index c074bbdc4b1a..255f145144ce 100644
--- a/lib/galaxy/datatypes/data.py
+++ b/lib/galaxy/datatypes/data.py
@@ -1217,31 +1217,9 @@ class Directory(Data):
     # The behavior of this class is intended to be similar to that of
     # composite types, but with arbitrary structure of extra_files.
 
-    # Directories converted from archives, and possibly others, have a single
-    # root folder inside the directory. The root_folder attribute lets tools
-    # access that folder without first exploring the directory tree themselves.
-    # Will be set to the empty string for flat directories.
-    MetadataElement(
-        name="root_folder",
-        default=None,
-        desc="Name of the root folder of the directory",
-        readonly=True,
-        optional=False,
-        visible=False,
-    )
-
-    def set_meta(self, dataset: DatasetProtocol, **kwd):
-        efp = dataset.extra_files_path
-        efp_items = os.listdir(efp)
-        root_folder_name = efp_items[0]
-        if len(efp_items) == 1 and os.path.isdir(os.path.join(efp, root_folder_name)):
-            dataset.metadata.root_folder = root_folder_name
-        else:
-            dataset.metadata.root_folder = ""
-
     def set_peek(self, dataset: DatasetProtocol, **kwd) -> None:
         if not dataset.dataset.purged:
-            dataset.peek = f"{dataset.metadata.root_folder}"
+            dataset.peek = f"{dataset.extra_files_path}"
             dataset.blurb = nice_size(dataset.dataset.total_size)
         else:
             dataset.peek = "file does not exist"

From 4a506fe4fc8425db53186cb181a91b0646256670 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Thu, 10 Oct 2024 13:19:02 +0200
Subject: [PATCH 23/84] Do not set peek for directories

---
 lib/galaxy/datatypes/data.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py
index 255f145144ce..27ed78e3aa61 100644
--- a/lib/galaxy/datatypes/data.py
+++ b/lib/galaxy/datatypes/data.py
@@ -1219,7 +1219,6 @@ class Directory(Data):
 
     def set_peek(self, dataset: DatasetProtocol, **kwd) -> None:
         if not dataset.dataset.purged:
-            dataset.peek = f"{dataset.extra_files_path}"
             dataset.blurb = nice_size(dataset.dataset.total_size)
         else:
             dataset.peek = "file does not exist"

From 5ce45817f8051f462a745e46f8df05b251491060 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Fri, 11 Oct 2024 10:42:07 +0200
Subject: [PATCH 24/84] Set output format to directory

---
 lib/galaxy/datatypes/converters/archive_to_directory.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/datatypes/converters/archive_to_directory.xml b/lib/galaxy/datatypes/converters/archive_to_directory.xml
index d0c56c154c09..45d4a18f4562 100644
--- a/lib/galaxy/datatypes/converters/archive_to_directory.xml
+++ b/lib/galaxy/datatypes/converters/archive_to_directory.xml
@@ -23,7 +23,7 @@
         </param>
     </inputs>
     <outputs provided_metadata_file="metadata_json">
-        <data format="auto" name="output1"/>
+        <data format="directory" name="output1"/>
     </outputs>
     <tests>
         <test>

From 673dd37a036ad9dd6ef0469afe8f269828915647 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Fri, 11 Oct 2024 12:24:45 +0200
Subject: [PATCH 25/84] Copy metadata to galaxy.json in archive_to_directory
 converter

---
 lib/galaxy/datatypes/converters/archive_to_directory.xml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/datatypes/converters/archive_to_directory.xml b/lib/galaxy/datatypes/converters/archive_to_directory.xml
index 45d4a18f4562..b346b4a49b62 100644
--- a/lib/galaxy/datatypes/converters/archive_to_directory.xml
+++ b/lib/galaxy/datatypes/converters/archive_to_directory.xml
@@ -4,6 +4,7 @@
         <requirement type="package" version="23.2.1">galaxy-util</requirement>
     </requirements>
     <command><![CDATA[
+        cp '$metadata_json' galaxy.json &&
         mkdir '$output1.files_path' &&
         cd '$output1.files_path' &&
         python -c "from galaxy.util.compression_utils import CompressedFile; CompressedFile('$input1').extract('.')"
@@ -22,8 +23,8 @@
             <option value="directory">directory</option>
         </param>
     </inputs>
-    <outputs provided_metadata_file="metadata_json">
-        <data format="directory" name="output1"/>
+    <outputs>
+        <data format="auto" name="output1"/>
     </outputs>
     <tests>
         <test>

From 75f9de3f1e759561557078635c8120148eccb9fe Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Fri, 11 Oct 2024 12:59:30 +0200
Subject: [PATCH 26/84] Revert back to use provided_metadata_file

---
 lib/galaxy/datatypes/converters/archive_to_directory.xml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/galaxy/datatypes/converters/archive_to_directory.xml b/lib/galaxy/datatypes/converters/archive_to_directory.xml
index b346b4a49b62..d0c56c154c09 100644
--- a/lib/galaxy/datatypes/converters/archive_to_directory.xml
+++ b/lib/galaxy/datatypes/converters/archive_to_directory.xml
@@ -4,7 +4,6 @@
         <requirement type="package" version="23.2.1">galaxy-util</requirement>
     </requirements>
     <command><![CDATA[
-        cp '$metadata_json' galaxy.json &&
         mkdir '$output1.files_path' &&
         cd '$output1.files_path' &&
         python -c "from galaxy.util.compression_utils import CompressedFile; CompressedFile('$input1').extract('.')"
@@ -23,7 +22,7 @@
             <option value="directory">directory</option>
         </param>
     </inputs>
-    <outputs>
+    <outputs provided_metadata_file="metadata_json">
         <data format="auto" name="output1"/>
     </outputs>
     <tests>

From dfd2184b10482f79b153733d6d11598bb925b8dd Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Fri, 11 Oct 2024 16:33:25 +0200
Subject: [PATCH 27/84] Remove set_peek method from Directory datatype

---
 lib/galaxy/datatypes/data.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py
index 27ed78e3aa61..66a6d9caa18e 100644
--- a/lib/galaxy/datatypes/data.py
+++ b/lib/galaxy/datatypes/data.py
@@ -1214,16 +1214,6 @@ class Directory(Data):
 
     file_ext = "directory"
 
-    # The behavior of this class is intended to be similar to that of
-    # composite types, but with arbitrary structure of extra_files.
-
-    def set_peek(self, dataset: DatasetProtocol, **kwd) -> None:
-        if not dataset.dataset.purged:
-            dataset.blurb = nice_size(dataset.dataset.total_size)
-        else:
-            dataset.peek = "file does not exist"
-            dataset.blurb = "file purged from disk"
-
     def _archive_main_file(
         self, archive: ZipstreamWrapper, display_name: str, data_filename: str
     ) -> Tuple[bool, str, str]:

From ccd7b5254e58b27b9b2654147aae83d3f1cb5454 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Tue, 15 Oct 2024 11:10:18 +0200
Subject: [PATCH 28/84] Add roundtrip test for downloading directories

Compressed (Upload) -> Directory (Unpack) -> Compressed (Download)
---
 lib/galaxy/datatypes/data.py             |  1 +
 lib/galaxy_test/api/test_tools_upload.py | 40 ++++++++++++++++++++++++
 2 files changed, 41 insertions(+)

diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py
index 66a6d9caa18e..e5fea2612ace 100644
--- a/lib/galaxy/datatypes/data.py
+++ b/lib/galaxy/datatypes/data.py
@@ -465,6 +465,7 @@ def _serve_file_download(self, headers, data, trans, to_ext, file_size, **kwd):
         composite_extensions = trans.app.datatypes_registry.get_composite_extensions()
         composite_extensions.append("html")  # for archiving composite datatypes
         composite_extensions.append("data_manager_json")  # for downloading bundles if bundled.
+        composite_extensions.append("directory")  # for downloading directories.
 
         if data.extension in composite_extensions:
             return self._archive_composite_dataset(trans, data, headers, do_action=kwd.get("do_action", "zip"))
diff --git a/lib/galaxy_test/api/test_tools_upload.py b/lib/galaxy_test/api/test_tools_upload.py
index ed32bc92caf0..57ffece984dc 100644
--- a/lib/galaxy_test/api/test_tools_upload.py
+++ b/lib/galaxy_test/api/test_tools_upload.py
@@ -2,12 +2,14 @@
 import os
 import urllib.parse
 from base64 import b64encode
+from typing import cast
 
 import pytest
 from tusclient import client
 
 from galaxy.tool_util.verify.test_data import TestDataResolver
 from galaxy.util import UNKNOWN
+from galaxy.util.compression_utils import decompress_bytes_to_directory
 from galaxy.util.unittest_utils import (
     skip_if_github_down,
     skip_if_site_down,
@@ -639,6 +641,44 @@ def test_upload_composite_from_bad_tar(self, history_id):
             details = self.dataset_populator.get_history_dataset_details(history_id, dataset=dataset, assert_ok=False)
             assert details["state"] == "error"
 
+    def test_upload_zip_directory_roundtrip(self, history_id):
+        testdir = TestDataResolver().get_filename("testdir1.zip")
+        with open(testdir, "rb") as fh:
+            details = self._upload_and_get_details(fh, api="fetch", history_id=history_id, ext="zip", assert_ok=True)
+        assert details["file_ext"] == "zip"
+
+        # Convert/unpack zip to directory.
+        payload = {
+            "src": "hda",
+            "id": details["id"],
+            "source_type": "zip",
+            "target_type": "directory",
+            "history_id": history_id,
+        }
+        create_response = self._post("tools/CONVERTER_archive_to_directory/convert", data=payload)
+        self.dataset_populator.wait_for_job(create_response.json()["jobs"][0]["id"], assert_ok=True)
+        create_response.raise_for_status()
+        directory_dataset = create_response.json()["outputs"][0]
+
+        # Download (compressed) directory and verify structure.
+        content = self.dataset_populator.get_history_dataset_content(
+            history_id, dataset=directory_dataset, to_ext="directory", type="bytes"
+        )
+        dir_path = decompress_bytes_to_directory(cast(bytes, content))
+        expected_directory_structure = {
+            "file1": "File",
+            "file2": "File",
+            "dir1": "Directory",
+            "dir1/file3": "File",
+        }
+        assert dir_path.endswith("testdir1")
+        for path, entry_class in expected_directory_structure.items():
+            path = os.path.join(dir_path, path)
+            if entry_class == "Directory":
+                assert os.path.isdir(path)
+            else:
+                assert os.path.isfile(path)
+
     def test_upload_dbkey(self):
         with self.dataset_populator.test_history() as history_id:
             payload = self.dataset_populator.upload_payload(history_id, "Test123", dbkey="hg19")

From 9ef9a435ef8475196ad83eac39efcbe4601e4612 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 30 Oct 2024 16:57:44 +0100
Subject: [PATCH 29/84] Add TAR and ZIP roundtrip tests

---
 lib/galaxy_test/api/test_tools_upload.py | 99 ++++++++++++++++--------
 1 file changed, 66 insertions(+), 33 deletions(-)

diff --git a/lib/galaxy_test/api/test_tools_upload.py b/lib/galaxy_test/api/test_tools_upload.py
index 57ffece984dc..952f2066bde7 100644
--- a/lib/galaxy_test/api/test_tools_upload.py
+++ b/lib/galaxy_test/api/test_tools_upload.py
@@ -1,5 +1,6 @@
 import json
 import os
+import tempfile
 import urllib.parse
 from base64 import b64encode
 from typing import cast
@@ -10,6 +11,7 @@
 from galaxy.tool_util.verify.test_data import TestDataResolver
 from galaxy.util import UNKNOWN
 from galaxy.util.compression_utils import decompress_bytes_to_directory
+from galaxy.util.hash_util import md5_hash_file
 from galaxy.util.unittest_utils import (
     skip_if_github_down,
     skip_if_site_down,
@@ -31,6 +33,14 @@
 B64_FOR_1_2_3 = b64encode(b"1 2 3").decode("utf-8")
 URI_FOR_1_2_3 = f"base64://{B64_FOR_1_2_3}"
 
+EXPECTED_TAR_CONTENTS = {
+    "testdir": "Directory",
+    "testdir/c": "Directory",
+    "testdir/a": "File",
+    "testdir/b": "File",
+    "testdir/c/d": "File",
+}
+
 
 class TestToolsUpload(ApiTestCase):
     dataset_populator: DatasetPopulator
@@ -606,18 +616,11 @@ def _check_testdir_composite(self, dataset, history_id):
         assert content.strip() == "Test123"
         extra_files = self.dataset_populator.get_history_dataset_extra_files(history_id, dataset_id=dataset["id"])
         assert len(extra_files) == 5, extra_files
-        expected_contents = {
-            "testdir": "Directory",
-            "testdir/c": "Directory",
-            "testdir/a": "File",
-            "testdir/b": "File",
-            "testdir/c/d": "File",
-        }
         found_files = set()
         for extra_file in extra_files:
             path = extra_file["path"]
-            assert path in expected_contents
-            assert extra_file["class"] == expected_contents[path]
+            assert path in EXPECTED_TAR_CONTENTS
+            assert extra_file["class"] == EXPECTED_TAR_CONTENTS[path]
             found_files.add(path)
 
         assert len(found_files) == 5, found_files
@@ -641,44 +644,74 @@ def test_upload_composite_from_bad_tar(self, history_id):
             details = self.dataset_populator.get_history_dataset_details(history_id, dataset=dataset, assert_ok=False)
             assert details["state"] == "error"
 
-    def test_upload_zip_directory_roundtrip(self, history_id):
+    def test_upload_tar_roundtrip(self, history_id):
+        testdir = TestDataResolver().get_filename("testdir.tar")
+        expected_size = os.path.getsize(testdir)
+        with open(testdir, "rb") as fh:
+            details = self._upload_and_get_details(fh, api="fetch", history_id=history_id, assert_ok=True)
+        assert details["file_ext"] == "tar"
+        assert details["file_size"] == expected_size
+        content = cast(
+            bytes, self.dataset_populator.get_history_dataset_content(history_id, dataset=details, type="bytes")
+        )
+        # Make sure we got the expected content size.
+        assert len(content) == expected_size
+
+        # Make sure we get the expected contents.
+        dir_path = decompress_bytes_to_directory(content)
+        assert dir_path.endswith("testdir")
+        for path, entry_class in EXPECTED_TAR_CONTENTS.items():
+            path = os.path.join(dir_path, os.path.pardir, path)
+            if entry_class == "Directory":
+                assert os.path.isdir(path)
+            else:
+                assert os.path.isfile(path)
+
+        # Make sure the hash of the content matches the hash of the original file.
+        expected_hash = md5_hash_file(testdir)
+        assert expected_hash is not None
+        self._assert_content_matches_hash(content, expected_hash)
+
+    def _assert_content_matches_hash(self, content: bytes, expected_hash: str):
+        with tempfile.NamedTemporaryFile("wb") as temp:
+            temp.write(content)
+            actual_hash = md5_hash_file(temp.name)
+            assert actual_hash == expected_hash
+
+    def test_upload_zip_roundtrip(self, history_id):
         testdir = TestDataResolver().get_filename("testdir1.zip")
+        expected_size = os.path.getsize(testdir)
         with open(testdir, "rb") as fh:
-            details = self._upload_and_get_details(fh, api="fetch", history_id=history_id, ext="zip", assert_ok=True)
+            details = self._upload_and_get_details(fh, api="fetch", history_id=history_id, assert_ok=True)
         assert details["file_ext"] == "zip"
-
-        # Convert/unpack zip to directory.
-        payload = {
-            "src": "hda",
-            "id": details["id"],
-            "source_type": "zip",
-            "target_type": "directory",
-            "history_id": history_id,
-        }
-        create_response = self._post("tools/CONVERTER_archive_to_directory/convert", data=payload)
-        self.dataset_populator.wait_for_job(create_response.json()["jobs"][0]["id"], assert_ok=True)
-        create_response.raise_for_status()
-        directory_dataset = create_response.json()["outputs"][0]
-
-        # Download (compressed) directory and verify structure.
-        content = self.dataset_populator.get_history_dataset_content(
-            history_id, dataset=directory_dataset, to_ext="directory", type="bytes"
+        assert details["file_size"] == expected_size
+        content = cast(
+            bytes, self.dataset_populator.get_history_dataset_content(history_id, dataset=details, type="bytes")
         )
-        dir_path = decompress_bytes_to_directory(cast(bytes, content))
-        expected_directory_structure = {
+        # Make sure we got the expected content size.
+        assert len(content) == expected_size
+
+        # Make sure we get the expected contents.
+        dir_path = decompress_bytes_to_directory(content)
+        assert dir_path.endswith("testdir1")
+        EXPECTED_ZIP_CONTENTS = {
             "file1": "File",
             "file2": "File",
-            "dir1": "Directory",
+            "dir1/": "Directory",
             "dir1/file3": "File",
         }
-        assert dir_path.endswith("testdir1")
-        for path, entry_class in expected_directory_structure.items():
+        for path, entry_class in EXPECTED_ZIP_CONTENTS.items():
             path = os.path.join(dir_path, path)
             if entry_class == "Directory":
                 assert os.path.isdir(path)
             else:
                 assert os.path.isfile(path)
 
+        # Make sure the hash of the content matches the hash of the original file.
+        expected_hash = md5_hash_file(testdir)
+        assert expected_hash is not None
+        self._assert_content_matches_hash(content, expected_hash)
+
     def test_upload_dbkey(self):
         with self.dataset_populator.test_history() as history_id:
             payload = self.dataset_populator.upload_payload(history_id, "Test123", dbkey="hg19")

From 48ed0682dc0451430634e28d149812c7ce5671af Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 22 Oct 2024 13:16:22 -0400
Subject: [PATCH 30/84] Move TRS import into WorkflowContentManager

which fixes missing source metadata when importing a workflow through a
landing page.
---
 lib/galaxy/app.py                            |  2 +-
 lib/galaxy/app_unittest_utils/galaxy_mock.py |  1 +
 lib/galaxy/managers/landing.py               | 24 +----
 lib/galaxy/managers/workflows.py             | 96 +++++++++++++++-----
 lib/galaxy/model/__init__.py                 |  2 +-
 lib/galaxy/webapps/base/controller.py        |  2 +-
 lib/galaxy/webapps/galaxy/api/workflows.py   | 39 +++-----
 lib/galaxy/workflow/modules.py               |  2 +-
 lib/galaxy/workflow/trs_proxy.py             |  3 +
 lib/galaxy_test/api/test_workflows.py        |  4 +-
 test/unit/app/managers/test_landing.py       |  2 +-
 test/unit/workflows/test_modules.py          |  4 +-
 test/unit/workflows/workflow_support.py      |  2 +
 13 files changed, 104 insertions(+), 79 deletions(-)

diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py
index fcbd1a1b7cb0..285e2eed8a1c 100644
--- a/lib/galaxy/app.py
+++ b/lib/galaxy/app.py
@@ -666,6 +666,7 @@ def __init__(self, configure_logging=True, use_converters=True, use_display_appl
             InstalledRepositoryManager, InstalledRepositoryManager(self)
         )
         self.dynamic_tool_manager = self._register_singleton(DynamicToolManager)
+        self.trs_proxy = self._register_singleton(TrsProxy, TrsProxy(self.config))
         self._configure_datatypes_registry(
             use_converters=use_converters,
             use_display_applications=use_display_applications,
@@ -843,7 +844,6 @@ def __init__(self, **kwargs) -> None:
         # Must be initialized after job_config.
         self.workflow_scheduling_manager = scheduling_manager.WorkflowSchedulingManager(self)
 
-        self.trs_proxy = self._register_singleton(TrsProxy, TrsProxy(self.config))
         # We need InteractiveToolManager before the job handler starts
         self.interactivetool_manager = InteractiveToolManager(self)
         # Start the job manager
diff --git a/lib/galaxy/app_unittest_utils/galaxy_mock.py b/lib/galaxy/app_unittest_utils/galaxy_mock.py
index f709f213ead4..ba89f2453b3e 100644
--- a/lib/galaxy/app_unittest_utils/galaxy_mock.py
+++ b/lib/galaxy/app_unittest_utils/galaxy_mock.py
@@ -156,6 +156,7 @@ def __init__(self, config=None, **kwargs) -> None:
         self.interactivetool_manager = Bunch(create_interactivetool=lambda *args, **kwargs: None)
         self.is_job_handler = False
         self.biotools_metadata_source = None
+        self.trs_proxy = Bunch()
         set_thread_app(self)
 
         def url_for(*args, **kwds):
diff --git a/lib/galaxy/managers/landing.py b/lib/galaxy/managers/landing.py
index 9facbac703c6..5012d2280134 100644
--- a/lib/galaxy/managers/landing.py
+++ b/lib/galaxy/managers/landing.py
@@ -4,7 +4,6 @@
 )
 from uuid import uuid4
 
-import yaml
 from pydantic import UUID4
 from sqlalchemy import select
 
@@ -15,10 +14,7 @@
     ObjectNotFound,
     RequestParameterMissingException,
 )
-from galaxy.managers.workflows import (
-    WorkflowContentsManager,
-    WorkflowCreateOptions,
-)
+from galaxy.managers.workflows import WorkflowContentsManager
 from galaxy.model import (
     ToolLandingRequest as ToolLandingRequestModel,
     WorkflowLandingRequest as WorkflowLandingRequestModel,
@@ -106,22 +102,10 @@ def _ensure_workflow(self, trans: ProvidesUserContext, request: WorkflowLandingR
         if request.workflow_source_type == "trs_url" and isinstance(trans.app, StructuredApp):
             # trans is always structured app except for unit test
             assert request.workflow_source
-            trs_id, trs_version = request.workflow_source.rsplit("/", 1)
-            _, trs_id, trs_version = trans.app.trs_proxy.get_trs_id_and_version_from_trs_url(request.workflow_source)
-            workflow = self.workflow_contents_manager.get_workflow_by_trs_id_and_version(
-                self.sa_session, trs_id=trs_id, trs_version=trs_version, user_id=trans.user and trans.user.id
+            workflow = self.workflow_contents_manager.get_or_create_workflow_from_trs(
+                trans, trs_url=request.workflow_source
             )
-            if not workflow:
-                data = trans.app.trs_proxy.get_version_from_trs_url(request.workflow_source)
-                as_dict = yaml.safe_load(data)
-                raw_workflow_description = self.workflow_contents_manager.normalize_workflow_format(trans, as_dict)
-                created_workflow = self.workflow_contents_manager.build_workflow_from_raw_description(
-                    trans,
-                    raw_workflow_description,
-                    WorkflowCreateOptions(),
-                )
-                workflow = created_workflow.workflow
-            request.workflow_id = workflow.id
+            request.workflow_id = workflow.latest_workflow_id
 
     def get_tool_landing_request(self, trans: ProvidesUserContext, uuid: UUID4) -> ToolLandingRequest:
         request = self._get_claimed_tool_landing_request(trans, uuid)
diff --git a/lib/galaxy/managers/workflows.py b/lib/galaxy/managers/workflows.py
index d69e8afcab70..48be30372d67 100644
--- a/lib/galaxy/managers/workflows.py
+++ b/lib/galaxy/managers/workflows.py
@@ -89,7 +89,6 @@
     text_column_filter,
 )
 from galaxy.model.item_attrs import UsesAnnotations
-from galaxy.model.scoped_session import galaxy_scoped_session
 from galaxy.schema.invocation import InvocationCancellationUserRequest
 from galaxy.schema.schema import WorkflowIndexQueryPayload
 from galaxy.structured_app import MinimalManagerApp
@@ -139,6 +138,7 @@
     attach_ordered_steps,
     has_cycles,
 )
+from galaxy.workflow.trs_proxy import TrsProxy
 
 log = logging.getLogger(__name__)
 
@@ -598,8 +598,10 @@ def add_serializers(self):
 
 
 class WorkflowContentsManager(UsesAnnotations):
-    def __init__(self, app: MinimalManagerApp):
+
+    def __init__(self, app: MinimalManagerApp, trs_proxy: TrsProxy):
         self.app = app
+        self.trs_proxy = trs_proxy
         self._resource_mapper_function = get_resource_mapper_function(app)
 
     def ensure_raw_description(self, dict_or_raw_description):
@@ -814,18 +816,17 @@ def _workflow_from_raw_description(
         workflow.license = data.get("license")
         workflow.creator_metadata = data.get("creator")
 
-        if hasattr(workflow_state_resolution_options, "archive_source"):
-            if workflow_state_resolution_options.archive_source:
-                source_metadata = {}
-                if workflow_state_resolution_options.archive_source == "trs_tool":
-                    source_metadata["trs_tool_id"] = workflow_state_resolution_options.trs_tool_id
-                    source_metadata["trs_version_id"] = workflow_state_resolution_options.trs_version_id
-                    source_metadata["trs_server"] = workflow_state_resolution_options.trs_server
-                    source_metadata["trs_url"] = workflow_state_resolution_options.trs_url
-                elif not workflow_state_resolution_options.archive_source.startswith("file://"):  # URL import
-                    source_metadata["url"] = workflow_state_resolution_options.archive_source
-                workflow_state_resolution_options.archive_source = None  # so trs_id is not set for subworkflows
-                workflow.source_metadata = source_metadata  # type:ignore[assignment]
+        if getattr(workflow_state_resolution_options, "archive_source", None):
+            source_metadata = {}
+            if workflow_state_resolution_options.archive_source in ("trs_tool", "trs_url"):
+                source_metadata["trs_tool_id"] = workflow_state_resolution_options.trs_tool_id
+                source_metadata["trs_version_id"] = workflow_state_resolution_options.trs_version_id
+                source_metadata["trs_server"] = workflow_state_resolution_options.trs_server
+                source_metadata["trs_url"] = workflow_state_resolution_options.trs_url
+            elif not workflow_state_resolution_options.archive_source.startswith("file://"):  # URL import
+                source_metadata["url"] = workflow_state_resolution_options.archive_source
+            workflow_state_resolution_options.archive_source = None  # so trs_id is not set for subworkflows
+            workflow.source_metadata = source_metadata
 
         # Assume no errors until we find a step that has some
         workflow.has_errors = False
@@ -2016,9 +2017,54 @@ def get_all_tools(self, workflow):
                 tools.extend(self.get_all_tools(step.subworkflow))
         return tools
 
+    def get_or_create_workflow_from_trs(
+        self,
+        trans: ProvidesUserContext,
+        trs_url: Optional[str],
+        trs_id: Optional[str] = None,
+        trs_version: Optional[str] = None,
+        trs_server: Optional[str] = None,
+    ):
+        user_id = trans.user and trans.user.id
+        assert user_id, "Cannot create workflow for anonymous user"
+        if not trs_url:
+            assert trs_server and trs_id and trs_version, "trs_url or trs_server, trs_version and trs_id must be passed"
+            server = self.trs_proxy.get_server(trs_server)
+            trs_url = server.get_trs_url(trs_id, trs_version)
+        else:
+            _, trs_id, trs_version = self.trs_proxy.get_trs_id_and_version_from_trs_url(trs_url=trs_url)
+        assert trs_id and trs_version and trs_url
+
+        workflow = self.get_workflow_by_trs_id_and_version(trs_id=trs_id, trs_version=trs_version, user_id=user_id)
+        if not workflow:
+            workflow = self.create_workflow_from_trs_url(trans, trs_url, trs_server)
+        return workflow
+
+    def create_workflow_from_trs_url(
+        self, trans: ProvidesUserContext, trs_url: str, trs_server: Optional[str] = None
+    ) -> StoredWorkflow:
+        _, trs_tool_id, trs_version_id = self.trs_proxy.get_trs_id_and_version_from_trs_url(trs_url=trs_url)
+        data = self.trs_proxy.get_version_from_trs_url(trs_url)
+        as_dict = yaml.safe_load(data)
+        raw_workflow_description = self.normalize_workflow_format(trans, as_dict)
+        created_workflow = self.build_workflow_from_raw_description(
+            trans,
+            raw_workflow_description,
+            WorkflowCreateOptions(
+                trs_tool_id=trs_tool_id,
+                trs_version_id=trs_version_id,
+                trs_url=trs_url,
+                trs_server=trs_server,
+                archive_source="trs_url",
+            ),
+        )
+        return created_workflow.stored_workflow
+
     def get_workflow_by_trs_id_and_version(
-        self, sa_session: galaxy_scoped_session, trs_id: str, trs_version: str, user_id: Optional[int] = None
-    ) -> Optional[model.Workflow]:
+        self, trs_id: str, trs_version: str, user_id: Optional[int] = None
+    ) -> Optional[model.StoredWorkflow]:
+        sa_session = self.app.model.session
+
         def to_json(column, keys: List[str]):
             assert sa_session.bind
             if sa_session.bind.dialect.name == "postgresql":
@@ -2028,12 +2074,12 @@ def to_json(column, keys: List[str]):
                 return cast.astext
             else:
                 for key in keys:
-                    column = column.__getitem__(key)
+                    column = func.json_extract(column, f"$.{key}")
                 return column
 
         stmnt = (
-            select(model.Workflow)
-            .join(model.StoredWorkflow, model.Workflow.stored_workflow_id == model.StoredWorkflow.id)
+            select(model.StoredWorkflow)
+            .join(model.Workflow, model.Workflow.id == model.StoredWorkflow.latest_workflow_id)
             .filter(
                 and_(
                     to_json(model.Workflow.source_metadata, ["trs_tool_id"]) == trs_id,
@@ -2043,7 +2089,7 @@ def to_json(column, keys: List[str]):
         )
         if user_id:
             stmnt = stmnt.filter(
-                model.StoredWorkflow.user_id == user_id, model.StoredWorkflow.latest_workflow_id == model.Workflow.id
+                model.StoredWorkflow.user_id == user_id,
             )
         else:
             stmnt = stmnt.filter(model.StoredWorkflow.importable == true())
@@ -2104,11 +2150,11 @@ class WorkflowCreateOptions(WorkflowStateResolutionOptions):
     shed_tool_conf: Optional[str] = None
 
     # for workflows imported by archive source
-    archive_source: Optional[str] = ""
-    trs_tool_id: str = ""
-    trs_version_id: str = ""
-    trs_server: str = ""
-    trs_url: str = ""
+    archive_source: Optional[str] = None
+    trs_tool_id: Optional[str] = None
+    trs_version_id: Optional[str] = None
+    trs_server: Optional[str] = None
+    trs_url: Optional[str] = None
 
     @property
     def is_importable(self):
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index e97cf0438c2a..ffa93ecc7225 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -7817,7 +7817,7 @@ class Workflow(Base, Dictifiable, RepresentById):
     reports_config: Mapped[Optional[bytes]] = mapped_column(JSONType)
     creator_metadata: Mapped[Optional[bytes]] = mapped_column(JSONType)
     license: Mapped[Optional[str]] = mapped_column(TEXT)
-    source_metadata: Mapped[Optional[bytes]] = mapped_column(JSONType)
+    source_metadata: Mapped[Optional[Dict[str, str]]] = mapped_column(JSONType)
     uuid: Mapped[Optional[Union[UUID, str]]] = mapped_column(UUIDType)
 
     steps = relationship(
diff --git a/lib/galaxy/webapps/base/controller.py b/lib/galaxy/webapps/base/controller.py
index 59bb189d1952..9fdfa1079f6e 100644
--- a/lib/galaxy/webapps/base/controller.py
+++ b/lib/galaxy/webapps/base/controller.py
@@ -1003,7 +1003,7 @@ def _workflow_to_dict(self, trans, stored):
         """
         Converts a workflow to a dict of attributes suitable for exporting.
         """
-        workflow_contents_manager = workflows.WorkflowContentsManager(self.app)
+        workflow_contents_manager = workflows.WorkflowContentsManager(self.app, self.app.trs_proxy)
         return workflow_contents_manager.workflow_to_dict(
             trans,
             stored,
diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py
index 6f94550e2134..1a8bafd38f90 100644
--- a/lib/galaxy/webapps/galaxy/api/workflows.py
+++ b/lib/galaxy/webapps/galaxy/api/workflows.py
@@ -245,29 +245,14 @@ def create(self, trans: GalaxyWebTransaction, payload=None, **kwd):
                     payload["workflow"] = workflow_src
                     return self.__api_import_new_workflow(trans, payload, **kwd)
                 elif archive_source == "trs_tool":
-                    server = None
-                    trs_tool_id = None
-                    trs_version_id = None
-                    import_source = None
-                    if "trs_url" in payload:
-                        parts = self.app.trs_proxy.match_url(
-                            payload["trs_url"], trans.app.config.fetch_url_allowlist_ips
-                        )
-                        if parts:
-                            server = self.app.trs_proxy.server_from_url(parts["trs_base_url"])
-                            trs_tool_id = parts["tool_id"]
-                            trs_version_id = parts["version_id"]
-                            payload["trs_tool_id"] = trs_tool_id
-                            payload["trs_version_id"] = trs_version_id
-                        else:
-                            raise exceptions.RequestParameterInvalidException(f"Invalid TRS URL {payload['trs_url']}.")
-                    else:
-                        trs_server = payload.get("trs_server")
-                        server = self.app.trs_proxy.get_server(trs_server)
-                        trs_tool_id = payload.get("trs_tool_id")
-                        trs_version_id = payload.get("trs_version_id")
-
-                    archive_data = server.get_version_descriptor(trs_tool_id, trs_version_id)
+                    workflow = self.workflow_contents_manager.get_or_create_workflow_from_trs(
+                        trans,
+                        trs_url=payload.get("trs_url"),
+                        trs_id=payload.get("trs_tool_id"),
+                        trs_version=payload.get("trs_version_id"),
+                        trs_server=payload.get("trs_server"),
+                    )
+                    return self.__api_import_response(workflow)
                 else:
                     try:
                         archive_data = stream_url_to_str(
@@ -602,13 +587,15 @@ def __api_import_from_archive(self, trans: GalaxyWebTransaction, archive_data, s
         workflow, missing_tool_tups = self._workflow_from_dict(
             trans, raw_workflow_description, workflow_create_options, source=source
         )
-        workflow_id = workflow.id
-        workflow = workflow.latest_workflow
+        return self.__api_import_response(workflow)
 
+    def __api_import_response(self, stored_workflow: model.StoredWorkflow):
+        workflow = stored_workflow.latest_workflow
+        assert workflow
         response = {
             "message": f"Workflow '{workflow.name}' imported successfully.",
             "status": "success",
-            "id": trans.security.encode_id(workflow_id),
+            "id": self.app.security.encode_id(stored_workflow.id),
         }
         if workflow.has_errors:
             response["message"] = "Imported, but some steps in this workflow have validation errors."
diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py
index 448d07ecbe90..f210755d0482 100644
--- a/lib/galaxy/workflow/modules.py
+++ b/lib/galaxy/workflow/modules.py
@@ -715,7 +715,7 @@ def get_all_outputs(self, data_only=False):
         if hasattr(self.subworkflow, "workflow_outputs"):
             from galaxy.managers.workflows import WorkflowContentsManager
 
-            workflow_contents_manager = WorkflowContentsManager(self.trans.app)
+            workflow_contents_manager = WorkflowContentsManager(self.trans.app, self.trans.app.trs_proxy)
             subworkflow_dict = workflow_contents_manager._workflow_to_dict_editor(
                 trans=self.trans,
                 stored=self.subworkflow.stored_workflow,
diff --git a/lib/galaxy/workflow/trs_proxy.py b/lib/galaxy/workflow/trs_proxy.py
index 550238ffa229..bc4875b95f4b 100644
--- a/lib/galaxy/workflow/trs_proxy.py
+++ b/lib/galaxy/workflow/trs_proxy.py
@@ -140,6 +140,9 @@ def get_version_descriptor(self, tool_id, version_id, **kwd):
         )
         return self._get(trs_api_url)["content"]
 
+    def get_trs_url(self, tool_id: str, version_id: str):
+        return f"{self._get_tool_api_endpoint(tool_id)}/versions/{version_id}"
+
     def _quote(self, tool_id, **kwd):
         if asbool(kwd.get("tool_id_b64_encoded", False)):
             import base64
diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py
index 8408837df347..ec44b1e5a5ff 100644
--- a/lib/galaxy_test/api/test_workflows.py
+++ b/lib/galaxy_test/api/test_workflows.py
@@ -1234,7 +1234,7 @@ def test_trs_import_from_dockstore_trs_url(self):
             == "#workflow/github.com/jmchilton/galaxy-workflow-dockstore-example-1/mycoolworkflow"
         )
         assert original_workflow.get("source_metadata").get("trs_version_id") == "master"
-        assert original_workflow.get("source_metadata").get("trs_server") == ""
+        assert not original_workflow.get("source_metadata").get("trs_server")
         assert original_workflow.get("source_metadata").get("trs_url") == (
             "https://dockstore.org/api/ga4gh/trs/v2/tools/"
             "%23workflow%2Fgithub.com%2Fjmchilton%2Fgalaxy-workflow-dockstore-example-1%2Fmycoolworkflow/"
@@ -1264,7 +1264,7 @@ def test_trs_import_from_workflowhub_trs_url(self):
         assert "COVID-19: variation analysis reporting" in original_workflow["name"]
         assert original_workflow.get("source_metadata").get("trs_tool_id") == "109"
         assert original_workflow.get("source_metadata").get("trs_version_id") == "5"
-        assert original_workflow.get("source_metadata").get("trs_server") == ""
+        assert not original_workflow.get("source_metadata").get("trs_server")
         assert (
             original_workflow.get("source_metadata").get("trs_url")
             == "https://workflowhub.eu/ga4gh/trs/v2/tools/109/versions/5"
diff --git a/test/unit/app/managers/test_landing.py b/test/unit/app/managers/test_landing.py
index 46762c110a6d..f2ccb059b4bf 100644
--- a/test/unit/app/managers/test_landing.py
+++ b/test/unit/app/managers/test_landing.py
@@ -41,7 +41,7 @@ class TestLanding(BaseTestCase):
 
     def setUp(self):
         super().setUp()
-        self.workflow_contents_manager = WorkflowContentsManager(self.app)
+        self.workflow_contents_manager = WorkflowContentsManager(self.app, self.app.trs_proxy)
         self.landing_manager = LandingRequestManager(
             self.trans.sa_session, self.app.security, self.workflow_contents_manager
         )
diff --git a/test/unit/workflows/test_modules.py b/test/unit/workflows/test_modules.py
index 6a1b5b653ce7..8052c6b11950 100644
--- a/test/unit/workflows/test_modules.py
+++ b/test/unit/workflows/test_modules.py
@@ -428,7 +428,9 @@ def _output_step(step_input_def, step_output_def) -> Dict[str, Any]:
 @pytest.mark.parametrize("test_case", _construct_steps_for_map_over())
 def test_subworkflow_map_over_type(test_case):
     trans = MockTrans()
-    new_steps = WorkflowContentsManager(app=trans.app)._resolve_collection_type(test_case.steps)
+    new_steps = WorkflowContentsManager(app=trans.app, trs_proxy=trans.app.trs_proxy)._resolve_collection_type(
+        test_case.steps
+    )
     assert (
         new_steps[1]["outputs"][0].get("collection_type") == test_case.expected_collection_type
     ), "Expected collection_type '{}' for a '{}' input module, a '{}' input and a '{}' output, got collection_type '{}' instead".format(
diff --git a/test/unit/workflows/workflow_support.py b/test/unit/workflows/workflow_support.py
index 05064e722fac..a3b8374637df 100644
--- a/test/unit/workflows/workflow_support.py
+++ b/test/unit/workflows/workflow_support.py
@@ -6,12 +6,14 @@
 from galaxy.app_unittest_utils import galaxy_mock
 from galaxy.managers.workflows import WorkflowsManager
 from galaxy.model.base import transaction
+from galaxy.util.bunch import Bunch
 from galaxy.workflow.modules import module_factory
 
 
 class MockTrans:
     def __init__(self):
         self.app = MockApp()
+        self.app.trs_proxy = Bunch()
         self.sa_session = self.app.model.context
         self._user = None
 

From 14eb0c3b6fe2eef0a3ad6b952d265e01582daedd Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 30 Oct 2024 18:12:28 +0100
Subject: [PATCH 31/84] Flush temporary file before testing hashes

---
 lib/galaxy_test/api/test_tools_upload.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/galaxy_test/api/test_tools_upload.py b/lib/galaxy_test/api/test_tools_upload.py
index 952f2066bde7..15d0198f9c31 100644
--- a/lib/galaxy_test/api/test_tools_upload.py
+++ b/lib/galaxy_test/api/test_tools_upload.py
@@ -675,6 +675,7 @@ def test_upload_tar_roundtrip(self, history_id):
     def _assert_content_matches_hash(self, content: bytes, expected_hash: str):
         with tempfile.NamedTemporaryFile("wb") as temp:
             temp.write(content)
+            temp.flush()
             actual_hash = md5_hash_file(temp.name)
             assert actual_hash == expected_hash
 

From 2cae82cfa9fd7a7f6fcd0dd1a436579490214837 Mon Sep 17 00:00:00 2001
From: Ahmed Awan <qe66653@umbc.edu>
Date: Wed, 30 Oct 2024 13:27:01 -0500
Subject: [PATCH 32/84] use a watcher to fetch invocation id just once

---
 .../JobInformation/JobInformation.vue         | 29 ++++++++++---------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/client/src/components/JobInformation/JobInformation.vue b/client/src/components/JobInformation/JobInformation.vue
index 7d1cade4942a..3c0c8d407695 100644
--- a/client/src/components/JobInformation/JobInformation.vue
+++ b/client/src/components/JobInformation/JobInformation.vue
@@ -5,7 +5,7 @@ import { JobDetailsProvider } from "components/providers/JobProvider";
 import UtcDate from "components/UtcDate";
 import { formatDuration, intervalToDuration } from "date-fns";
 import JOB_STATES_MODEL from "utils/job-states-model";
-import { computed, ref } from "vue";
+import { computed, ref, watch } from "vue";
 
 import { invocationForJob } from "@/api/invocations";
 
@@ -13,7 +13,7 @@ import DecodedId from "../DecodedId.vue";
 import CodeRow from "./CodeRow.vue";
 
 const job = ref(null);
-const invocationId = ref(null);
+const invocationId = ref(undefined);
 
 const props = defineProps({
     job_id: {
@@ -41,9 +41,6 @@ const metadataDetail = ref({
 
 function updateJob(newJob) {
     job.value = newJob;
-    if (newJob && !invocationId.value) {
-        fetchInvocation(newJob.id);
-    }
 }
 
 function filterMetadata(jobMessages) {
@@ -57,15 +54,21 @@ function filterMetadata(jobMessages) {
     });
 }
 
-/** Fetches the invocation for the given job id to get the associated invocation id */
-async function fetchInvocation(jobId) {
-    if (jobId) {
-        const invocation = await invocationForJob({ jobId: jobId });
-        if (invocation) {
-            invocationId.value = invocation.id;
+// Fetches the invocation for the given job id to get the associated invocation id
+watch(
+    () => props.job_id,
+    async (newId, oldId) => {
+        if (newId && (invocationId.value === undefined || newId !== oldId)) {
+            const invocation = await invocationForJob({ jobId: newId });
+            if (invocation) {
+                invocationId.value = invocation.id;
+            } else {
+                invocationId.value = null;
+            }
         }
-    }
-}
+    },
+    { immediate: true }
+);
 </script>
 
 <template>

From 3e31b44af3d64b2a97f3ca6f7674bc82ce147399 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Tue, 29 Oct 2024 18:33:53 +0100
Subject: [PATCH 33/84] Allow deferred datasets to behave as URI

---
 lib/galaxy/model/__init__.py         |  7 +++++++
 lib/galaxy/tool_util/xsd/galaxy.xsd  | 19 +++++++++++++++++++
 lib/galaxy/tools/evaluation.py       | 16 +++++++++++++++-
 lib/galaxy/tools/parameters/basic.py |  7 +++++++
 lib/galaxy/tools/wrappers.py         |  5 +++++
 5 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index bcf956d2aae4..cc3252aadd76 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -4685,6 +4685,13 @@ def ext(self):
     def has_deferred_data(self):
         return self.dataset.state == Dataset.states.DEFERRED
 
+    @property
+    def deferred_source_uri(self):
+        if self.has_deferred_data:
+            # Assuming the first source is the deferred source
+            return self.sources[0].source_uri
+        return None
+
     @property
     def state(self):
         # self._state holds state that should only affect this particular dataset association, not the dataset state itself
diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index c8090c919568..c4dfe86e0da0 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -4314,6 +4314,25 @@ template if the parameter is ``false`` or not checked by the user. Used only
 when the ``type`` attribute value is ``boolean``.</xs:documentation>
           </xs:annotation>
         </xs:attribute>
+        <xs:attribute name="allow_uri_if_prefixed" type="xs:string">
+          <xs:annotation>
+            <xs:documentation xml:lang="en"><![CDATA[
+When using `deferred` datasets, Galaxy will try to materialize then into files when
+running the tool. In case we don't want to download them. The `allow_uri_if_prefixed` attribute
+can be used to avoid materialization and pass the URI as is to the tool.
+This is useful when the tool can handle the URI directly. Example:
+
+```xml
+<param name="input" type="data" format="txt" allow_uri_if_prefixed="http|s3" />
+```
+
+You can specify multiple prefixes separated by `|`. The URI will be passed to the tool
+as is if it starts with any of the specified prefixes.
+
+This attribute is only valid for `data` parameters.
+         ]]></xs:documentation>
+          </xs:annotation>
+        </xs:attribute>
         <xs:attribute name="size" type="xs:string" gxdocs:deprecated="true">
           <!-- TODO: can be integer or integerxinteger -->
           <xs:annotation>
diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py
index 28a4b27c4b0d..67d486627622 100644
--- a/lib/galaxy/tools/evaluation.py
+++ b/lib/galaxy/tools/evaluation.py
@@ -314,7 +314,8 @@ def _deferred_objects(
         deferred_objects: Dict[str, DeferrableObjectsT] = {}
         for key, value in input_datasets.items():
             if value is not None and value.state == model.Dataset.states.DEFERRED:
-                deferred_objects[key] = value
+                if self._should_materialize_deferred_input(key, value):
+                    deferred_objects[key] = value
 
         def find_deferred_collections(input, value, context, prefixed_name=None, **kwargs):
             if (
@@ -327,6 +328,19 @@ def find_deferred_collections(input, value, context, prefixed_name=None, **kwarg
 
         return deferred_objects
 
+    def _should_materialize_deferred_input(self, input_name: str, input_value: DeferrableObjectsT) -> bool:
+        """
+        We can skip materializing some deferred datasets if the input can work with URIs that are prefixed
+        with a known prefix set in `allow_uri_if_prefixed`.
+        """
+        deferred_input = self.tool.inputs[input_name]
+        if isinstance(deferred_input, DataToolParameter) and isinstance(input_value, model.DatasetInstance):
+            source_uri = input_value.sources[0].source_uri or ""
+            for prefix in deferred_input.allow_uri_if_prefixed:
+                if source_uri.startswith(prefix):
+                    return False
+        return True
+
     def __walk_inputs(self, inputs, input_values, func):
         def do_walk(inputs, input_values):
             """
diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py
index 7ed9c657691a..bc41cdf0bd75 100644
--- a/lib/galaxy/tools/parameters/basic.py
+++ b/lib/galaxy/tools/parameters/basic.py
@@ -2068,6 +2068,7 @@ def __init__(self, tool, input_source, trans=None):
         self.tag = tag
         self.is_dynamic = True
         self._parse_options(input_source)
+        self._parse_allow_uri_if_prefixed(input_source)
         # Load conversions required for the dataset input
         self.conversions = []
         self.default_object = input_source.parse_default()
@@ -2089,6 +2090,12 @@ def __init__(self, tool, input_source, trans=None):
                     )
                 self.conversions.append((name, conv_extension, [conv_type]))
 
+    def _parse_allow_uri_if_prefixed(self, input_source):
+        # In case of deferred datasets, if the source URI is prefixed with one of the values in this list,
+        # the dataset will behave as an URI and will not be materialized into a file path.
+        allow_uri_if_prefixed = input_source.get("allow_uri_if_prefixed", None)
+        self.allow_uri_if_prefixed = allow_uri_if_prefixed.split("|") if allow_uri_if_prefixed else []
+
     def from_json(self, value, trans, other_values=None):
         session = trans.sa_session
 
diff --git a/lib/galaxy/tools/wrappers.py b/lib/galaxy/tools/wrappers.py
index 6dd47e7f9bb4..6ad72645d7bf 100644
--- a/lib/galaxy/tools/wrappers.py
+++ b/lib/galaxy/tools/wrappers.py
@@ -471,6 +471,11 @@ def is_of_type(self, *exts: str) -> bool:
         return self.dataset.datatype.matches_any(datatypes)
 
     def __str__(self) -> str:
+        return self._path_or_uri()
+
+    def _path_or_uri(self) -> str:
+        if self.unsanitized.has_deferred_data:
+            return self.unsanitized.deferred_source_uri or ""
         if self.false_path is not None:
             return self.false_path
         else:

From 154a39047836dcaec449b00c4f552253cc0e0b76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?David=20L=C3=B3pez?=
 <46503462+davelopez@users.noreply.github.com>
Date: Tue, 29 Oct 2024 19:01:16 +0100
Subject: [PATCH 34/84] Fix typo

Co-authored-by: Nicola Soranzo <nicola.soranzo@gmail.com>
---
 lib/galaxy/tool_util/xsd/galaxy.xsd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index c4dfe86e0da0..8b9a185a4c17 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -4317,7 +4317,7 @@ when the ``type`` attribute value is ``boolean``.</xs:documentation>
         <xs:attribute name="allow_uri_if_prefixed" type="xs:string">
           <xs:annotation>
             <xs:documentation xml:lang="en"><![CDATA[
-When using `deferred` datasets, Galaxy will try to materialize then into files when
+When using `deferred` datasets, Galaxy will try to materialize them into files when
 running the tool. In case we don't want to download them. The `allow_uri_if_prefixed` attribute
 can be used to avoid materialization and pass the URI as is to the tool.
 This is useful when the tool can handle the URI directly. Example:

From c96ced84faf6c283836cd356529b53a5aca03539 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 30 Oct 2024 10:10:17 +0100
Subject: [PATCH 35/84] Rename allow_uri_if_prefixed to allow_uri_if_protocol

Co-authored-by: Nicola Soranzo <nicola.soranzo@earlham.ac.uk>
---
 lib/galaxy/tool_util/xsd/galaxy.xsd  | 6 +++---
 lib/galaxy/tools/evaluation.py       | 4 ++--
 lib/galaxy/tools/parameters/basic.py | 8 ++++----
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index 8b9a185a4c17..3cfb3823b5da 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -4314,16 +4314,16 @@ template if the parameter is ``false`` or not checked by the user. Used only
 when the ``type`` attribute value is ``boolean``.</xs:documentation>
           </xs:annotation>
         </xs:attribute>
-        <xs:attribute name="allow_uri_if_prefixed" type="xs:string">
+        <xs:attribute name="allow_uri_if_protocol" type="xs:string">
           <xs:annotation>
             <xs:documentation xml:lang="en"><![CDATA[
 When using `deferred` datasets, Galaxy will try to materialize them into files when
-running the tool. In case we don't want to download them. The `allow_uri_if_prefixed` attribute
+running the tool. In case we don't want to download them, the `allow_uri_if_protocol` attribute
 can be used to avoid materialization and pass the URI as is to the tool.
 This is useful when the tool can handle the URI directly. Example:
 
 ```xml
-<param name="input" type="data" format="txt" allow_uri_if_prefixed="http|s3" />
+<param name="input" type="data" format="txt" allow_uri_if_protocol="http|https|s3" />
 ```
 
 You can specify multiple prefixes separated by `|`. The URI will be passed to the tool
diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py
index 67d486627622..144a2d1cfaa9 100644
--- a/lib/galaxy/tools/evaluation.py
+++ b/lib/galaxy/tools/evaluation.py
@@ -331,12 +331,12 @@ def find_deferred_collections(input, value, context, prefixed_name=None, **kwarg
     def _should_materialize_deferred_input(self, input_name: str, input_value: DeferrableObjectsT) -> bool:
         """
         We can skip materializing some deferred datasets if the input can work with URIs that are prefixed
-        with a known prefix set in `allow_uri_if_prefixed`.
+        with a known prefix set in `allow_uri_if_protocol`.
         """
         deferred_input = self.tool.inputs[input_name]
         if isinstance(deferred_input, DataToolParameter) and isinstance(input_value, model.DatasetInstance):
             source_uri = input_value.sources[0].source_uri or ""
-            for prefix in deferred_input.allow_uri_if_prefixed:
+            for prefix in deferred_input.allow_uri_if_protocol:
                 if source_uri.startswith(prefix):
                     return False
         return True
diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py
index bc41cdf0bd75..ce9c6d086837 100644
--- a/lib/galaxy/tools/parameters/basic.py
+++ b/lib/galaxy/tools/parameters/basic.py
@@ -2068,7 +2068,7 @@ def __init__(self, tool, input_source, trans=None):
         self.tag = tag
         self.is_dynamic = True
         self._parse_options(input_source)
-        self._parse_allow_uri_if_prefixed(input_source)
+        self._parse_allow_uri_if_protocol(input_source)
         # Load conversions required for the dataset input
         self.conversions = []
         self.default_object = input_source.parse_default()
@@ -2090,11 +2090,11 @@ def __init__(self, tool, input_source, trans=None):
                     )
                 self.conversions.append((name, conv_extension, [conv_type]))
 
-    def _parse_allow_uri_if_prefixed(self, input_source):
+    def _parse_allow_uri_if_protocol(self, input_source):
         # In case of deferred datasets, if the source URI is prefixed with one of the values in this list,
         # the dataset will behave as an URI and will not be materialized into a file path.
-        allow_uri_if_prefixed = input_source.get("allow_uri_if_prefixed", None)
-        self.allow_uri_if_prefixed = allow_uri_if_prefixed.split("|") if allow_uri_if_prefixed else []
+        allow_uri_if_protocol = input_source.get("allow_uri_if_protocol", None)
+        self.allow_uri_if_protocol = allow_uri_if_protocol.split("|") if allow_uri_if_protocol else []
 
     def from_json(self, value, trans, other_values=None):
         session = trans.sa_session

From f1daa44d473b430311d2aeb670cf9a132b5d8d6c Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 30 Oct 2024 10:16:50 +0100
Subject: [PATCH 36/84] Use commas to list protocols in allow_uri_if_protocol

Co-authored-by: Bjoern Gruening <bjoern.gruening@gmail.com>
---
 lib/galaxy/tool_util/xsd/galaxy.xsd  | 4 ++--
 lib/galaxy/tools/parameters/basic.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index 3cfb3823b5da..9de63fe28d73 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -4323,10 +4323,10 @@ can be used to avoid materialization and pass the URI as is to the tool.
 This is useful when the tool can handle the URI directly. Example:
 
 ```xml
-<param name="input" type="data" format="txt" allow_uri_if_protocol="http|https|s3" />
+<param name="input" type="data" format="txt" allow_uri_if_protocol="http,https,s3" />
 ```
 
-You can specify multiple prefixes separated by `|`. The URI will be passed to the tool
+You can specify multiple prefixes separated by comma. The URI will be passed to the tool
 as is if it starts with any of the specified prefixes.
 
 This attribute is only valid for `data` parameters.
diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py
index ce9c6d086837..d3bb7f119c58 100644
--- a/lib/galaxy/tools/parameters/basic.py
+++ b/lib/galaxy/tools/parameters/basic.py
@@ -2094,7 +2094,7 @@ def _parse_allow_uri_if_protocol(self, input_source):
         # In case of deferred datasets, if the source URI is prefixed with one of the values in this list,
         # the dataset will behave as an URI and will not be materialized into a file path.
         allow_uri_if_protocol = input_source.get("allow_uri_if_protocol", None)
-        self.allow_uri_if_protocol = allow_uri_if_protocol.split("|") if allow_uri_if_protocol else []
+        self.allow_uri_if_protocol = allow_uri_if_protocol.split(",") if allow_uri_if_protocol else []
 
     def from_json(self, value, trans, other_values=None):
         session = trans.sa_session

From b3a70863fcb3850511757d2026d6d9fcb400410e Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 30 Oct 2024 11:41:01 +0100
Subject: [PATCH 37/84] Add test for parsing allow_uri_if_protocol

---
 test/unit/app/tools/test_parameter_parsing.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/test/unit/app/tools/test_parameter_parsing.py b/test/unit/app/tools/test_parameter_parsing.py
index 842e7df91844..e33e183fc962 100644
--- a/test/unit/app/tools/test_parameter_parsing.py
+++ b/test/unit/app/tools/test_parameter_parsing.py
@@ -452,3 +452,12 @@ def test_tool_collection(self):
         )
         assert param.type == "data_collection"
         assert param.collection_types == ["list", "list:paired"]
+
+    def test_data_allow_uri_if_protocol(self):
+        param = self._parameter_for(
+            xml="""
+            <param name="deferred" type="data" allow_uri_if_protocol="https,s3">
+            </param>
+        """
+        )
+        assert param.allow_uri_if_protocol == ["https", "s3"]

From 305dd2b181562b9a94dc0a514ce88f0cef53db5d Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 30 Oct 2024 11:45:53 +0100
Subject: [PATCH 38/84] Add `has_deferred_data` flag to MockDataset and
 FakeDatasetAssociation

---
 lib/galaxy/model/__init__.py         | 1 +
 test/unit/app/tools/test_wrappers.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index cc3252aadd76..7656a98fbed1 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -2737,6 +2737,7 @@ class FakeDatasetAssociation:
     def __init__(self, dataset: Optional["Dataset"] = None) -> None:
         self.dataset = dataset
         self.metadata: Dict = {}
+        self.has_deferred_data = False
 
     def get_file_name(self, sync_cache: bool = True) -> str:
         assert self.dataset
diff --git a/test/unit/app/tools/test_wrappers.py b/test/unit/app/tools/test_wrappers.py
index 746fef91f480..7fc20e8fa061 100644
--- a/test/unit/app/tools/test_wrappers.py
+++ b/test/unit/app/tools/test_wrappers.py
@@ -310,6 +310,7 @@ def __init__(self):
         self.extra_files_path = MOCK_DATASET_EXTRA_FILES_PATH
         self.ext = MOCK_DATASET_EXT
         self.tags = []
+        self.has_deferred_data = False
 
     def get_file_name(self, sync_cache=True):
         return MOCK_DATASET_PATH

From fd14e606e84c7e9cc0a6eecd91840369d6212dd0 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 30 Oct 2024 12:11:16 +0100
Subject: [PATCH 39/84] Fix handling for missing deferred input in
 ToolEvaluator

---
 lib/galaxy/tools/evaluation.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py
index 144a2d1cfaa9..5b278b91d9e6 100644
--- a/lib/galaxy/tools/evaluation.py
+++ b/lib/galaxy/tools/evaluation.py
@@ -333,7 +333,11 @@ def _should_materialize_deferred_input(self, input_name: str, input_value: Defer
         We can skip materializing some deferred datasets if the input can work with URIs that are prefixed
         with a known prefix set in `allow_uri_if_protocol`.
         """
-        deferred_input = self.tool.inputs[input_name]
+        deferred_input = self.tool.inputs.get(input_name)
+        if not deferred_input:
+            # TODO: Can this ever happen? It seems like it happens in the test suite.
+            # For example, in test_metadata_validator_on_deferred_input
+            return True
         if isinstance(deferred_input, DataToolParameter) and isinstance(input_value, model.DatasetInstance):
             source_uri = input_value.sources[0].source_uri or ""
             for prefix in deferred_input.allow_uri_if_protocol:

From 678caa415f3167c8c6d6a2c3909058ba2a246479 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 30 Oct 2024 16:27:55 +0100
Subject: [PATCH 40/84] Add test for allow_uri_if_protocol with deferred input

---
 lib/galaxy_test/api/test_tools.py              | 18 ++++++++++++++++++
 .../parameters/gx_allow_uri_if_protocol.xml    | 13 +++++++++++++
 2 files changed, 31 insertions(+)
 create mode 100644 test/functional/tools/parameters/gx_allow_uri_if_protocol.xml

diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py
index 002b9d873389..41d610f7b29e 100644
--- a/lib/galaxy_test/api/test_tools.py
+++ b/lib/galaxy_test/api/test_tools.py
@@ -2573,6 +2573,24 @@ def test_metadata_validator_can_fail_on_deferred_input(self, history_id):
         job_details = self.dataset_populator.get_job_details(job_id=job_id).json()
         assert job_details["state"] == "failed"
 
+    @skip_without_tool("gx_allow_uri_if_protocol")
+    def test_allow_uri_if_protocol_on_deferred_input(self, history_id):
+        source_uri = "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/simple_line.txt"
+        deferred_hda = self.dataset_populator.create_deferred_hda(history_id, source_uri, ext="txt")
+
+        inputs = {"input1": dataset_to_param(deferred_hda)}
+        # The tool just returns the URI (or file path if it was materialized) as the output content
+        run_response = self.dataset_populator.run_tool(
+            tool_id="gx_allow_uri_if_protocol", inputs=inputs, history_id=history_id
+        )
+        output = run_response["outputs"][0]
+        output_details = self.dataset_populator.get_history_dataset_details(
+            history_id, dataset=output, wait=True, assert_ok=True
+        )
+        assert output_details["state"] == "ok"
+        output_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output)
+        assert output_content.strip() == source_uri.strip()
+
     @skip_without_tool("cat1")
     def test_run_deferred_mapping(self, history_id: str):
         elements = [
diff --git a/test/functional/tools/parameters/gx_allow_uri_if_protocol.xml b/test/functional/tools/parameters/gx_allow_uri_if_protocol.xml
new file mode 100644
index 000000000000..ab554b470f70
--- /dev/null
+++ b/test/functional/tools/parameters/gx_allow_uri_if_protocol.xml
@@ -0,0 +1,13 @@
+<tool id="gx_allow_uri_if_protocol" name="gx_allow_uri_if_protocol" version="1.0.0">
+    <command><![CDATA[
+echo '$input' > '$output'
+    ]]></command>
+    <inputs>
+        <param name="input" type="data" allow_uri_if_protocol="https"/>
+    </inputs>
+    <outputs>
+        <data name="output" format="txt"/>
+    </outputs>
+    <tests>
+    </tests>
+</tool>

From 93078d949aa902eb7668b559732fbd76d5e04568 Mon Sep 17 00:00:00 2001
From: Nicola Soranzo <nicola.soranzo@earlham.ac.uk>
Date: Tue, 29 Oct 2024 13:48:02 +0000
Subject: [PATCH 41/84] Replace poetry with uv

for lightning-fast Python dependency updates.

To update all dependencies (as done by the GitHub action):

$ make update-dependencies

To update only one package:

./lib/galaxy/dependencies/update.sh -p <PKG>
---
 .github/workflows/dependencies.yaml           |   4 +-
 doc/source/admin/framework_dependencies.rst   |  12 +-
 lib/galaxy/dependencies/dev-requirements.txt  | 376 +++++++--------
 .../dependencies/pinned-requirements.txt      | 434 +++++++++---------
 .../pinned-typecheck-requirements.txt         |  62 +--
 lib/galaxy/dependencies/update.sh             |  92 +---
 pyproject.toml                                | 363 ++++++++-------
 7 files changed, 652 insertions(+), 691 deletions(-)

diff --git a/.github/workflows/dependencies.yaml b/.github/workflows/dependencies.yaml
index 9b319d1954e1..7f3e60d54774 100644
--- a/.github/workflows/dependencies.yaml
+++ b/.github/workflows/dependencies.yaml
@@ -17,9 +17,7 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Update dependencies
-        run: | 
-          python -m venv .venv
-          make update-dependencies
+        run: make update-dependencies
       - name: Create pull request
         uses: peter-evans/create-pull-request@v6
         with:
diff --git a/doc/source/admin/framework_dependencies.rst b/doc/source/admin/framework_dependencies.rst
index 12613a31d460..c142f71cc840 100644
--- a/doc/source/admin/framework_dependencies.rst
+++ b/doc/source/admin/framework_dependencies.rst
@@ -185,7 +185,7 @@ Adding additional Galaxy dependencies
 -------------------------------------
 
 New packages can be added to Galaxy, or the versions of existing packages can be
-updated, using `poetry`_ and `Wheelforge`_.
+updated, using `uv`_ and `Wheelforge`_.
 
 If wheels already exist on PyPI for all platforms and Python versions supported
 by Galaxy, you can skip to step 3 in the process below.
@@ -195,11 +195,13 @@ by Galaxy, you can skip to step 3 in the process below.
 
 2. Submit a pull request to `Wheelforge`_.
 
-3. Add the new dependency to the `[tool.poetry.dependencies]` (or to `[tool.poetry.group.dev.dependencies]` if only needed for Galaxy development) section of `pyproject.toml` .
-4. Run `make update-dependencies` to update the requirements file in `lib/galaxy/dependencies`_.
+3. Add the new dependency to the ``dependencies`` list in the ``[project]``
+   section (or to ``dev`` list in the ``[dependency-groups]`` section if only
+   needed for Galaxy development) of `pyproject.toml`_ .
+4. Run ``make update-dependencies`` to update the requirements files in `lib/galaxy/dependencies`_.
 5. Submit a pull request to Galaxy with your changes.
 
-.. _poetry: https://python-poetry.org/
+.. _uv: https://docs.astral.sh/uv/
 .. _Wheelforge: https://github.com/galaxyproject/wheelforge/
-.. _Pull Request #4891: https://github.com/galaxyproject/galaxy/pull/4891
+.. _pyproject.toml: https://github.com/galaxyproject/galaxy/blob/dev/pyproject.toml
 .. _lib/galaxy/dependencies: https://github.com/galaxyproject/galaxy/tree/dev/lib/galaxy/dependencies
diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt
index 1258d2e754e2..a0b0f68801f0 100644
--- a/lib/galaxy/dependencies/dev-requirements.txt
+++ b/lib/galaxy/dependencies/dev-requirements.txt
@@ -1,188 +1,188 @@
---extra-index-url https://wheels.galaxyproject.org/simple
-
-aiohappyeyeballs==2.4.3 ; python_version >= "3.8" and python_version < "3.13"
-aiohttp==3.10.10 ; python_version >= "3.8" and python_version < "3.13"
-aiosignal==1.3.1 ; python_version >= "3.8" and python_version < "3.13"
-alabaster==0.7.13 ; python_version >= "3.8" and python_version < "3.13"
-anyio==4.5.2 ; python_version >= "3.8" and python_version < "3.13"
-ase==3.23.0 ; python_version >= "3.8" and python_version < "3.13"
-async-timeout==4.0.3 ; python_version >= "3.8" and python_version < "3.11"
-attrs==24.2.0 ; python_version >= "3.8" and python_version < "3.13"
-axe-selenium-python==2.1.6 ; python_version >= "3.8" and python_version < "3.13"
-babel==2.16.0 ; python_version >= "3.8" and python_version < "3.13"
-backports-tarfile==1.2.0 ; python_version >= "3.8" and python_version < "3.12"
-backports-zoneinfo==0.2.1 ; python_version >= "3.8" and python_version < "3.9"
-black==24.8.0 ; python_version >= "3.8" and python_version < "3.13"
-boto3==1.35.36 ; python_version >= "3.8" and python_version < "3.13"
-botocore==1.35.36 ; python_version >= "3.8" and python_version < "3.13"
-build==1.2.2.post1 ; python_version >= "3.8" and python_version < "3.13"
-cachecontrol[filecache]==0.14.0 ; python_version >= "3.8" and python_version < "3.13"
-certifi==2024.8.30 ; python_version >= "3.8" and python_version < "3.13"
-cffi==1.17.1 ; python_version >= "3.8" and python_version < "3.13"
-charset-normalizer==3.4.0 ; python_version >= "3.8" and python_version < "3.13"
-click==8.1.7 ; python_version >= "3.8" and python_version < "3.13"
-codespell==2.3.0 ; python_version >= "3.8" and python_version < "3.13"
-colorama==0.4.6 ; python_version >= "3.8" and python_version < "3.13" and (sys_platform == "win32" or platform_system == "Windows" or os_name == "nt")
-contourpy==1.1.1 ; python_version >= "3.8" and python_version < "3.9"
-contourpy==1.3.0 ; python_version >= "3.9" and python_version < "3.13"
-coverage[toml]==7.6.1 ; python_version >= "3.8" and python_version < "3.13"
-cryptography==43.0.3 ; python_version >= "3.8" and python_version < "3.13"
-cwltest==2.5.20240906231108 ; python_version >= "3.8" and python_version < "3.13"
-cycler==0.12.1 ; python_version >= "3.8" and python_version < "3.13"
-darker==2.1.1 ; python_version >= "3.8" and python_version < "3.13"
-darkgraylib==1.2.1 ; python_version >= "3.8" and python_version < "3.13"
-decorator==5.1.1 ; python_version >= "3.9" and python_version < "3.13"
-defusedxml==0.7.1 ; python_version >= "3.8" and python_version < "3.13"
-deprecated==1.2.14 ; python_version >= "3.8" and python_version < "3.13"
-docutils==0.20.1 ; python_version >= "3.8" and python_version < "3.13"
-dogpile-cache==1.3.3 ; python_version >= "3.9" and python_version < "3.13"
-exceptiongroup==1.2.2 ; python_version >= "3.8" and python_version < "3.11"
-filelock==3.16.1 ; python_version >= "3.8" and python_version < "3.13"
-fluent-logger==0.11.1 ; python_version >= "3.8" and python_version < "3.13"
-fonttools==4.54.1 ; python_version >= "3.8" and python_version < "3.13"
-frozenlist==1.5.0 ; python_version >= "3.8" and python_version < "3.13"
-galaxy-release-util==0.1.11 ; python_version >= "3.8" and python_version < "3.13"
-graylint==1.1.1 ; python_version >= "3.8" and python_version < "3.13"
-greenlet==3.1.1 ; python_version >= "3.8" and python_version < "3.13"
-h11==0.14.0 ; python_version >= "3.8" and python_version < "3.13"
-httpcore==1.0.6 ; python_version >= "3.8" and python_version < "3.13"
-httpx==0.27.2 ; python_version >= "3.8" and python_version < "3.13"
-idna==3.10 ; python_version >= "3.8" and python_version < "3.13"
-imagesize==1.4.1 ; python_version >= "3.8" and python_version < "3.13"
-importlib-metadata==8.5.0 ; python_version >= "3.8" and python_version < "3.13"
-importlib-resources==6.4.5 ; python_version >= "3.8" and python_version < "3.10"
-iniconfig==2.0.0 ; python_version >= "3.8" and python_version < "3.13"
-isodate==0.6.1 ; python_version >= "3.8" and python_version < "3.13"
-isort==5.13.2 ; python_version >= "3.8" and python_version < "3.13"
-jaraco-classes==3.4.0 ; python_version >= "3.8" and python_version < "3.13"
-jaraco-context==6.0.1 ; python_version >= "3.8" and python_version < "3.13"
-jaraco-functools==4.1.0 ; python_version >= "3.8" and python_version < "3.13"
-jeepney==0.8.0 ; python_version >= "3.8" and python_version < "3.13" and sys_platform == "linux"
-jinja2==3.1.4 ; python_version >= "3.8" and python_version < "3.13"
-jmespath==1.0.1 ; python_version >= "3.8" and python_version < "3.13"
-jsonschema-specifications==2023.12.1 ; python_version >= "3.9" and python_version < "3.13"
-jsonschema==4.23.0 ; python_version >= "3.9" and python_version < "3.13"
-junit-xml==1.9 ; python_version >= "3.8" and python_version < "3.13"
-keyring==25.4.1 ; python_version >= "3.8" and python_version < "3.13"
-kiwisolver==1.4.7 ; python_version >= "3.8" and python_version < "3.13"
-lxml==5.3.0 ; python_version >= "3.8" and python_version < "3.13"
-markdown-it-py==3.0.0 ; python_version >= "3.8" and python_version < "3.13"
-markdown-it-reporter==0.0.2 ; python_version >= "3.8" and python_version < "3.13"
-markupsafe==2.1.5 ; python_version >= "3.8" and python_version < "3.13"
-matplotlib==3.7.5 ; python_version >= "3.8" and python_version < "3.9"
-matplotlib==3.9.2 ; python_version >= "3.9" and python_version < "3.13"
-mdit-py-plugins==0.4.2 ; python_version >= "3.8" and python_version < "3.13"
-mdurl==0.1.2 ; python_version >= "3.8" and python_version < "3.13"
-mirakuru==2.5.2 ; python_version >= "3.8" and python_version < "3.13"
-mistune==3.0.2 ; python_version >= "3.8" and python_version < "3.13"
-more-itertools==10.5.0 ; python_version >= "3.8" and python_version < "3.13"
-msgpack==1.1.0 ; python_version >= "3.8" and python_version < "3.13"
-multidict==6.1.0 ; python_version >= "3.8" and python_version < "3.13"
-mypy-extensions==1.0.0 ; python_version >= "3.8" and python_version < "3.13"
-myst-parser==3.0.1 ; python_version >= "3.8" and python_version < "3.13"
-nh3==0.2.18 ; python_version >= "3.8" and python_version < "3.13"
-numpy==1.24.4 ; python_version >= "3.8" and python_version < "3.9"
-numpy==2.0.2 ; python_version >= "3.9" and python_version < "3.10"
-numpy==2.1.2 ; python_version >= "3.10" and python_version < "3.13"
-onedatafilerestclient==21.2.5.2 ; python_version >= "3.8" and python_version < "3.13"
-outcome==1.3.0.post0 ; python_version >= "3.8" and python_version < "3.13"
-packaging==24.1 ; python_version >= "3.8" and python_version < "3.13"
-pathspec==0.12.1 ; python_version >= "3.8" and python_version < "3.13"
-pbr==6.1.0 ; python_version >= "3.9" and python_version < "3.13"
-pillow==10.4.0 ; python_version >= "3.8" and python_version < "3.13"
-pkce==1.0.3 ; python_version >= "3.8" and python_version < "3.13"
-pkginfo==1.10.0 ; python_version >= "3.8" and python_version < "3.13"
-platformdirs==4.3.6 ; python_version >= "3.8" and python_version < "3.13"
-playwright==1.48.0 ; python_version >= "3.8" and python_version < "3.13"
-pluggy==1.5.0 ; python_version >= "3.8" and python_version < "3.13"
-port-for==0.7.3 ; python_version >= "3.8" and python_version < "3.13"
-prettytable==3.11.0 ; python_version >= "3.8" and python_version < "3.13"
-propcache==0.2.0 ; python_version >= "3.8" and python_version < "3.13"
-psutil==6.1.0 ; python_version >= "3.8" and python_version < "3.13" and sys_platform != "cygwin"
-psycopg==3.2.3 ; python_version >= "3.8" and python_version < "3.13"
-pycparser==2.22 ; python_version >= "3.8" and python_version < "3.13"
-pyee==12.0.0 ; python_version >= "3.8" and python_version < "3.13"
-pygithub==2.4.0 ; python_version >= "3.8" and python_version < "3.13"
-pygments==2.18.0 ; python_version >= "3.8" and python_version < "3.13"
-pyjwt[crypto]==2.9.0 ; python_version >= "3.8" and python_version < "3.13"
-pynacl==1.5.0 ; python_version >= "3.8" and python_version < "3.13"
-pyparsing==3.1.4 ; python_version >= "3.8" and python_version < "3.13"
-pyproject-hooks==1.2.0 ; python_version >= "3.8" and python_version < "3.13"
-pysocks==1.7.1 ; python_version >= "3.8" and python_version < "3.13"
-pytest-asyncio==0.23.8 ; python_version >= "3.8" and python_version < "3.13"
-pytest-base-url==2.1.0 ; python_version >= "3.8" and python_version < "3.13"
-pytest-cov==5.0.0 ; python_version >= "3.8" and python_version < "3.13"
-pytest-html==4.1.1 ; python_version >= "3.8" and python_version < "3.13"
-pytest-httpserver==1.1.0 ; python_version >= "3.8" and python_version < "3.13"
-pytest-json-report==1.5.0 ; python_version >= "3.8" and python_version < "3.13"
-pytest-metadata==3.1.1 ; python_version >= "3.8" and python_version < "3.13"
-pytest-mock==3.14.0 ; python_version >= "3.8" and python_version < "3.13"
-pytest-playwright==0.5.2 ; python_version >= "3.8" and python_version < "3.13"
-pytest-postgresql==6.1.1 ; python_version >= "3.8" and python_version < "3.13"
-pytest-shard==0.1.2 ; python_version >= "3.8" and python_version < "3.13"
-pytest==7.4.4 ; python_version >= "3.8" and python_version < "3.13"
-python-dateutil==2.9.0.post0 ; python_version >= "3.8" and python_version < "3.13"
-python-irodsclient==2.2.0 ; python_version >= "3.8" and python_version < "3.13"
-python-slugify==8.0.4 ; python_version >= "3.8" and python_version < "3.13"
-pytz==2024.2 ; python_version >= "3.8" and python_version < "3.9"
-pywin32-ctypes==0.2.3 ; python_version >= "3.8" and python_version < "3.13" and sys_platform == "win32"
-pyyaml==6.0.2 ; python_version >= "3.8" and python_version < "3.13"
-rdflib==6.3.2 ; python_version >= "3.8" and python_version < "3.13"
-readme-renderer==43.0 ; python_version >= "3.8" and python_version < "3.13"
-referencing==0.35.1 ; python_version >= "3.9" and python_version < "3.13"
-requests-toolbelt==1.0.0 ; python_version >= "3.8" and python_version < "3.13"
-requests==2.32.3 ; python_version >= "3.8" and python_version < "3.13"
-responses==0.25.3 ; python_version >= "3.8" and python_version < "3.13"
-rfc3986==2.0.0 ; python_version >= "3.8" and python_version < "3.13"
-rich==13.9.3 ; python_version >= "3.8" and python_version < "3.13"
-rpds-py==0.20.0 ; python_version >= "3.9" and python_version < "3.13"
-ruamel-yaml-clib==0.2.8 ; platform_python_implementation == "CPython" and python_version < "3.13" and python_version >= "3.8"
-ruamel-yaml==0.18.6 ; python_version >= "3.8" and python_version < "3.13"
-rucio-clients==35.4.1 ; python_version >= "3.9" and python_version < "3.13"
-s3transfer==0.10.3 ; python_version >= "3.8" and python_version < "3.13"
-schema-salad==8.7.20241010092723 ; python_version >= "3.8" and python_version < "3.13"
-scipy==1.10.1 ; python_version >= "3.8" and python_version < "3.9"
-scipy==1.13.1 ; python_version >= "3.9" and python_version < "3.10"
-scipy==1.14.1 ; python_version >= "3.10" and python_version < "3.13"
-secretstorage==3.3.3 ; python_version >= "3.8" and python_version < "3.13" and sys_platform == "linux"
-selenium==4.25.0 ; python_version >= "3.8" and python_version < "3.13"
-seletools==1.5.0 ; python_version >= "3.8" and python_version < "3.13"
-setuptools==75.2.0 ; python_version >= "3.8" and python_version < "3.13"
-six==1.16.0 ; python_version >= "3.8" and python_version < "3.13"
-sniffio==1.3.1 ; python_version >= "3.8" and python_version < "3.13"
-snowballstemmer==2.2.0 ; python_version >= "3.8" and python_version < "3.13"
-sortedcontainers==2.4.0 ; python_version >= "3.8" and python_version < "3.13"
-sphinx-rtd-theme==3.0.1 ; python_version >= "3.8" and python_version < "3.13"
-sphinx==7.1.2 ; python_version >= "3.8" and python_version < "3.13"
-sphinxcontrib-applehelp==1.0.4 ; python_version >= "3.8" and python_version < "3.13"
-sphinxcontrib-devhelp==1.0.2 ; python_version >= "3.8" and python_version < "3.13"
-sphinxcontrib-htmlhelp==2.0.1 ; python_version >= "3.8" and python_version < "3.13"
-sphinxcontrib-jquery==4.1 ; python_version >= "3.8" and python_version < "3.13"
-sphinxcontrib-jsmath==1.0.1 ; python_version >= "3.8" and python_version < "3.13"
-sphinxcontrib-qthelp==1.0.3 ; python_version >= "3.8" and python_version < "3.13"
-sphinxcontrib-serializinghtml==1.1.5 ; python_version >= "3.8" and python_version < "3.13"
-statsd==4.0.1 ; python_version >= "3.8" and python_version < "3.13"
-stevedore==5.3.0 ; python_version >= "3.9" and python_version < "3.13"
-tabulate==0.9.0 ; python_version >= "3.9" and python_version < "3.13"
-testfixtures==8.3.0 ; python_version >= "3.8" and python_version < "3.13"
-text-unidecode==1.3 ; python_version >= "3.8" and python_version < "3.13"
-tinydb==4.8.2 ; python_version >= "3.8" and python_version < "3.13"
-toml==0.10.2 ; python_version >= "3.8" and python_version < "3.13"
-tomli==2.0.2 ; python_version >= "3.8" and python_full_version <= "3.11.0a6"
-trio-websocket==0.11.1 ; python_version >= "3.8" and python_version < "3.13"
-trio==0.27.0 ; python_version >= "3.8" and python_version < "3.13"
-tuspy==1.0.3 ; python_version >= "3.8" and python_version < "3.13"
-twill==3.3 ; python_version >= "3.8" and python_version < "3.13"
-twine==5.1.1 ; python_version >= "3.8" and python_version < "3.13"
-typing-extensions==4.12.2 ; python_version >= "3.8" and python_version < "3.13"
-tzdata==2024.2 ; python_version >= "3.8" and python_version < "3.13" and sys_platform == "win32"
-urllib3==1.26.20 ; python_version >= "3.8" and python_version < "3.13"
-urllib3[socks]==1.26.20 ; python_version >= "3.8" and python_version < "3.13"
-watchdog==4.0.2 ; python_version >= "3.8" and python_version < "3.13"
-wcwidth==0.2.13 ; python_version >= "3.8" and python_version < "3.13"
-websocket-client==1.8.0 ; python_version >= "3.8" and python_version < "3.13"
-werkzeug==3.0.6 ; python_version >= "3.8" and python_version < "3.13"
-wrapt==1.16.0 ; python_version >= "3.8" and python_version < "3.13"
-wsproto==1.2.0 ; python_version >= "3.8" and python_version < "3.13"
-yarl==1.15.2 ; python_version >= "3.8" and python_version < "3.13"
-zipp==3.20.2 ; python_version >= "3.8" and python_version < "3.13"
+# This file was autogenerated by uv via the following command:
+#    uv export --no-hashes --only-group=dev
+aiohappyeyeballs==2.4.3
+aiohttp==3.10.10
+aiosignal==1.3.1
+alabaster==0.7.13
+anyio==4.5.2
+ase==3.23.0
+async-timeout==4.0.3 ; python_full_version < '3.11'
+attrs==24.2.0
+axe-selenium-python==2.1.6
+babel==2.16.0
+backports-tarfile==1.2.0 ; python_full_version < '3.12'
+backports-zoneinfo==0.2.1 ; python_full_version < '3.9'
+black==24.8.0
+boto3==1.35.36
+botocore==1.35.36
+build==1.2.2.post1
+cachecontrol==0.14.0
+certifi==2024.8.30
+cffi==1.17.1
+charset-normalizer==3.4.0
+click==8.1.7
+codespell==2.3.0
+colorama==0.4.6 ; (platform_system != 'Windows' and sys_platform == 'win32') or platform_system == 'Windows' or os_name == 'nt'
+contourpy==1.1.1 ; python_full_version < '3.9'
+contourpy==1.3.0 ; python_full_version >= '3.9'
+coverage==7.6.1
+cryptography==43.0.3
+cwltest==2.5.20240906231108
+cycler==0.12.1
+darker==2.1.1
+darkgraylib==1.2.1
+decorator==5.1.1 ; python_full_version >= '3.9'
+defusedxml==0.7.1
+deprecated==1.2.14
+docutils==0.20.1
+dogpile-cache==1.3.3 ; python_full_version >= '3.9'
+exceptiongroup==1.2.2 ; python_full_version < '3.11'
+filelock==3.16.1
+fluent-logger==0.11.1
+fonttools==4.54.1
+frozenlist==1.5.0
+galaxy-release-util==0.1.11
+graylint==1.1.1
+greenlet==3.1.1
+h11==0.14.0
+httpcore==1.0.6
+httpx==0.27.2
+idna==3.10
+imagesize==1.4.1
+importlib-metadata==8.5.0
+importlib-resources==6.4.5 ; python_full_version < '3.10'
+iniconfig==2.0.0
+isodate==0.6.1
+isort==5.13.2
+jaraco-classes==3.4.0
+jaraco-context==6.0.1
+jaraco-functools==4.1.0
+jeepney==0.8.0 ; sys_platform == 'linux'
+jinja2==3.1.4
+jmespath==1.0.1
+jsonschema==4.23.0 ; python_full_version >= '3.9'
+jsonschema-specifications==2023.12.1 ; python_full_version >= '3.9'
+junit-xml==1.9
+keyring==25.5.0
+kiwisolver==1.4.7
+lxml==5.3.0
+markdown-it-py==3.0.0
+markdown-it-reporter==0.0.2
+markupsafe==2.1.5
+matplotlib==3.7.5 ; python_full_version < '3.9'
+matplotlib==3.9.2 ; python_full_version >= '3.9'
+mdit-py-plugins==0.4.2
+mdurl==0.1.2
+mirakuru==2.5.2
+mistune==3.0.2
+more-itertools==10.5.0
+msgpack==1.1.0
+multidict==6.1.0
+mypy-extensions==1.0.0
+myst-parser==3.0.1
+nh3==0.2.18
+numpy==1.24.4 ; python_full_version < '3.9'
+numpy==2.0.2 ; python_full_version == '3.9.*'
+numpy==2.1.2 ; python_full_version >= '3.10'
+onedatafilerestclient==21.2.5.2
+outcome==1.3.0.post0
+packaging==24.1
+pathspec==0.12.1
+pbr==6.1.0 ; python_full_version >= '3.9'
+pillow==10.4.0
+pkce==1.0.3
+pkginfo==1.10.0
+pkgutil-resolve-name==1.3.10 ; python_version < '0'
+platformdirs==4.3.6
+playwright==1.48.0
+pluggy==1.5.0
+port-for==0.7.3
+prettytable==3.11.0
+propcache==0.2.0
+psutil==6.1.0 ; sys_platform != 'cygwin'
+psycopg==3.2.3
+pycparser==2.22
+pyee==12.0.0
+pygithub==2.4.0
+pygments==2.18.0
+pyjwt==2.9.0
+pynacl==1.5.0
+pyparsing==3.1.4
+pyproject-hooks==1.2.0
+pysocks==1.7.1
+pytest==7.4.4
+pytest-asyncio==0.23.8
+pytest-base-url==2.1.0
+pytest-cov==5.0.0
+pytest-html==4.1.1
+pytest-httpserver==1.1.0
+pytest-json-report==1.5.0
+pytest-metadata==3.1.1
+pytest-mock==3.14.0
+pytest-playwright==0.5.2
+pytest-postgresql==6.1.1
+pytest-shard==0.1.2
+python-dateutil==2.9.0.post0
+python-irodsclient==2.2.0
+python-slugify==8.0.4
+pytz==2024.2 ; python_full_version < '3.9'
+pywin32-ctypes==0.2.3 ; sys_platform == 'win32'
+pyyaml==6.0.2
+rdflib==6.3.2
+readme-renderer==43.0
+referencing==0.35.1 ; python_full_version >= '3.9'
+requests==2.32.3
+requests-toolbelt==1.0.0
+responses==0.25.3
+rfc3986==2.0.0
+rich==13.9.3
+rpds-py==0.20.0 ; python_full_version >= '3.9'
+ruamel-yaml==0.18.6
+ruamel-yaml-clib==0.2.8 ; python_full_version < '3.13' and platform_python_implementation == 'CPython'
+rucio-clients==35.4.1 ; python_full_version >= '3.9'
+s3transfer==0.10.3
+schema-salad==8.7.20241010092723
+scipy==1.10.1 ; python_full_version < '3.9'
+scipy==1.13.1 ; python_full_version == '3.9.*'
+scipy==1.14.1 ; python_full_version >= '3.10'
+secretstorage==3.3.3 ; sys_platform == 'linux'
+selenium==4.25.0
+seletools==1.5.0
+setuptools==75.2.0
+six==1.16.0
+sniffio==1.3.1
+snowballstemmer==2.2.0
+sortedcontainers==2.4.0
+sphinx==7.1.2
+sphinx-rtd-theme==3.0.1
+sphinxcontrib-applehelp==1.0.4
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==2.0.1
+sphinxcontrib-jquery==4.1
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.5
+statsd==4.0.1
+stevedore==5.3.0 ; python_full_version >= '3.9'
+tabulate==0.9.0 ; python_full_version >= '3.9'
+testfixtures==8.3.0
+text-unidecode==1.3
+tinydb==4.8.2
+toml==0.10.2
+tomli==2.0.2 ; python_full_version <= '3.11'
+trio==0.27.0
+trio-websocket==0.11.1
+tuspy==1.0.3
+twill==3.3
+twine==5.1.1
+typing-extensions==4.12.2
+tzdata==2024.2 ; sys_platform == 'win32'
+urllib3==1.26.20
+watchdog==4.0.2
+wcwidth==0.2.13
+websocket-client==1.8.0
+werkzeug==3.0.6
+wrapt==1.16.0
+wsproto==1.2.0
+yarl==1.15.2
+zipp==3.20.2
diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt
index 91752afea326..c911f9b8b305 100644
--- a/lib/galaxy/dependencies/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pinned-requirements.txt
@@ -1,218 +1,216 @@
---extra-index-url https://wheels.galaxyproject.org/simple
-
-a2wsgi==1.10.7 ; python_version >= "3.8" and python_version < "3.13"
-adal==1.2.7 ; python_version >= "3.8" and python_version < "3.13"
-aiobotocore==2.15.2 ; python_version >= "3.8" and python_version < "3.13"
-aiodataloader==0.4.0 ; python_version >= "3.8" and python_version < "3.13"
-aiofiles==24.1.0 ; python_version >= "3.8" and python_version < "3.13"
-aiohappyeyeballs==2.4.3 ; python_version >= "3.8" and python_version < "3.13"
-aiohttp==3.10.10 ; python_version >= "3.8" and python_version < "3.13"
-aioitertools==0.12.0 ; python_version >= "3.8" and python_version < "3.13"
-aiosignal==1.3.1 ; python_version >= "3.8" and python_version < "3.13"
-alembic==1.13.3 ; python_version >= "3.8" and python_version < "3.13"
-amqp==5.2.0 ; python_version >= "3.8" and python_version < "3.13"
-annotated-types==0.7.0 ; python_version >= "3.8" and python_version < "3.13"
-anyio==4.5.2 ; python_version >= "3.8" and python_version < "3.13"
-apispec==6.6.1 ; python_version >= "3.8" and python_version < "3.13"
-appdirs==1.4.4 ; python_version >= "3.8" and python_version < "3.13"
-arcp==0.2.1 ; python_version >= "3.8" and python_version < "3.13"
-argcomplete==3.5.1 ; python_version >= "3.8" and python_version < "3.13"
-async-timeout==4.0.3 ; python_version >= "3.8" and python_version < "3.11"
-attmap==0.13.2 ; python_version >= "3.8" and python_version < "3.13"
-attrs==24.2.0 ; python_version >= "3.8" and python_version < "3.13"
-babel==2.16.0 ; python_version >= "3.8" and python_version < "3.13"
-backports-zoneinfo==0.2.1 ; python_version >= "3.8" and python_version < "3.9"
-backports-zoneinfo[tzdata]==0.2.1 ; python_version >= "3.8" and python_version < "3.9"
-bagit-profile==1.3.1 ; python_version >= "3.8" and python_version < "3.13"
-bagit==1.8.1 ; python_version >= "3.8" and python_version < "3.13"
-bcrypt==4.2.0 ; python_version >= "3.8" and python_version < "3.13"
-bdbag==1.7.3 ; python_version >= "3.8" and python_version < "3.13"
-beaker==1.13.0 ; python_version >= "3.8" and python_version < "3.13"
-billiard==4.2.1 ; python_version >= "3.8" and python_version < "3.13"
-bioblend==1.3.0 ; python_version >= "3.8" and python_version < "3.13"
-bleach==6.1.0 ; python_version >= "3.8" and python_version < "3.13"
-boltons==24.0.0 ; python_version >= "3.8" and python_version < "3.13"
-boto==2.49.0 ; python_version >= "3.8" and python_version < "3.13"
-botocore==1.35.36 ; python_version >= "3.8" and python_version < "3.13"
-bx-python==0.13.0 ; python_version >= "3.8" and python_version < "3.13"
-cachecontrol[filecache]==0.14.0 ; python_version >= "3.8" and python_version < "3.13"
-celery==5.4.0 ; python_version >= "3.8" and python_version < "3.13"
-certifi==2024.8.30 ; python_version >= "3.8" and python_version < "3.13"
-cffi==1.17.1 ; python_version >= "3.8" and python_version < "3.13"
-charset-normalizer==3.4.0 ; python_version >= "3.8" and python_version < "3.13"
-cheetah3==3.2.6.post1 ; python_version >= "3.8" and python_version < "3.13"
-circus==0.18.0 ; python_version >= "3.8" and python_version < "3.13"
-click-didyoumean==0.3.1 ; python_version >= "3.8" and python_version < "3.13"
-click-plugins==1.1.1 ; python_version >= "3.8" and python_version < "3.13"
-click-repl==0.3.0 ; python_version >= "3.8" and python_version < "3.13"
-click==8.1.7 ; python_version >= "3.8" and python_version < "3.13"
-cloudauthz==0.6.0 ; python_version >= "3.8" and python_version < "3.13"
-cloudbridge==3.2.0 ; python_version >= "3.8" and python_version < "3.13"
-colorama==0.4.6 ; python_version >= "3.8" and python_version < "3.13" and platform_system == "Windows"
-coloredlogs==15.0.1 ; python_version >= "3.8" and python_version < "3.13"
-conda-package-streaming==0.11.0 ; python_version >= "3.8" and python_version < "3.13"
-cryptography==43.0.3 ; python_version >= "3.8" and python_version < "3.13"
-cwl-upgrader==1.2.11 ; python_version >= "3.8" and python_version < "3.13"
-cwl-utils==0.35 ; python_version >= "3.8" and python_version < "3.13"
-cwltool==3.1.20241007082533 ; python_version >= "3.8" and python_version < "3.13"
-defusedxml==0.7.1 ; python_version >= "3.8" and python_version < "3.13"
-deprecation==2.1.0 ; python_version >= "3.8" and python_version < "3.13"
-dictobj==0.4 ; python_version >= "3.8" and python_version < "3.13"
-dnspython==2.6.1 ; python_version >= "3.8" and python_version < "3.13"
-docopt==0.6.2 ; python_version >= "3.8" and python_version < "3.13"
-docutils==0.20.1 ; python_version >= "3.8" and python_version < "3.13"
-dparse==0.6.3 ; python_version >= "3.8" and python_version < "3.13"
-edam-ontology==1.25.2 ; python_version >= "3.8" and python_version < "3.13"
-email-validator==2.2.0 ; python_version >= "3.8" and python_version < "3.13"
-exceptiongroup==1.2.2 ; python_version >= "3.8" and python_version < "3.11"
-fastapi-slim==0.115.3 ; python_version >= "3.8" and python_version < "3.13"
-filelock==3.16.1 ; python_version >= "3.8" and python_version < "3.13"
-frozenlist==1.5.0 ; python_version >= "3.8" and python_version < "3.13"
-fs==2.4.16 ; python_version >= "3.8" and python_version < "3.13"
-fsspec==2024.10.0 ; python_version >= "3.8" and python_version < "3.13"
-future==1.0.0 ; python_version >= "3.8" and python_version < "3.13"
-galaxy2cwl==0.1.4 ; python_version >= "3.8" and python_version < "3.13"
-graphene-sqlalchemy==3.0.0rc1 ; python_version >= "3.8" and python_version < "3.13"
-graphene==3.4 ; python_version >= "3.8" and python_version < "3.13"
-graphql-core==3.2.5 ; python_version >= "3.8" and python_version < "3.13"
-graphql-relay==3.2.0 ; python_version >= "3.8" and python_version < "3.13"
-gravity==1.0.6 ; python_version >= "3.8" and python_version < "3.13"
-greenlet==3.1.1 ; python_version < "3.13" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and python_version >= "3.8"
-gunicorn==23.0.0 ; python_version >= "3.8" and python_version < "3.13"
-gxformat2==0.20.0 ; python_version >= "3.8" and python_version < "3.13"
-h11==0.14.0 ; python_version >= "3.8" and python_version < "3.13"
-h5grove==2.3.0 ; python_version >= "3.8" and python_version < "3.13"
-h5py==3.11.0 ; python_version >= "3.8" and python_version < "3.9"
-h5py==3.12.1 ; python_version >= "3.9" and python_version < "3.13"
-humanfriendly==10.0 ; python_version >= "3.8" and python_version < "3.13"
-idna==3.10 ; python_version >= "3.8" and python_version < "3.13"
-importlib-metadata==8.5.0 ; python_version >= "3.8" and python_version < "3.13"
-importlib-resources==6.4.5 ; python_version >= "3.8" and python_version < "3.9"
-isa-rwval @ git+https://github.com/nsoranzo/isa-rwval.git@3d989181058d2765a93cb0e7ca85d6955e0eb6ef ; python_version >= "3.8" and python_version < "3.13"
-isal==1.7.1 ; python_version >= "3.8" and python_version < "3.13"
-isodate==0.6.1 ; python_version >= "3.8" and python_version < "3.13"
-jinja2==3.1.4 ; python_version >= "3.8" and python_version < "3.13"
-jmespath==1.0.1 ; python_version >= "3.8" and python_version < "3.13"
-jsonref==1.1.0 ; python_version >= "3.8" and python_version < "3.13"
-jsonschema-specifications==2023.12.1 ; python_version >= "3.8" and python_version < "3.13"
-jsonschema==4.23.0 ; python_version >= "3.8" and python_version < "3.13"
-kombu==5.4.2 ; python_version >= "3.8" and python_version < "3.13"
-lagom==2.6.0 ; python_version >= "3.8" and python_version < "3.13"
-lxml==5.3.0 ; python_version >= "3.8" and python_version < "3.13"
-mako==1.3.6 ; python_version >= "3.8" and python_version < "3.13"
-markdown-it-py==3.0.0 ; python_version >= "3.8" and python_version < "3.13"
-markdown==3.7 ; python_version >= "3.8" and python_version < "3.13"
-markupsafe==2.1.5 ; python_version >= "3.8" and python_version < "3.13"
-mdurl==0.1.2 ; python_version >= "3.8" and python_version < "3.13"
-mercurial==6.8.1 ; python_version >= "3.8" and python_version < "3.13"
-mistune==3.0.2 ; python_version >= "3.8" and python_version < "3.13"
-mrcfile==1.5.3 ; python_version >= "3.8" and python_version < "3.13"
-msal==1.31.0 ; python_version >= "3.8" and python_version < "3.13"
-msgpack==1.1.0 ; python_version >= "3.8" and python_version < "3.13"
-multidict==6.1.0 ; python_version >= "3.8" and python_version < "3.13"
-mypy-extensions==1.0.0 ; python_version >= "3.8" and python_version < "3.13"
-networkx==3.1 ; python_version >= "3.8" and python_version < "3.13"
-nodeenv==1.9.1 ; python_version >= "3.8" and python_version < "3.13"
-numpy==1.24.4 ; python_version >= "3.8" and python_version < "3.9"
-numpy==2.0.2 ; python_version >= "3.9" and python_version < "3.10"
-numpy==2.1.2 ; python_version >= "3.10" and python_version < "3.13"
-oauthlib==3.2.2 ; python_version >= "3.8" and python_version < "3.13"
-orjson==3.10.10 ; python_version >= "3.8" and python_version < "3.13"
-oyaml==1.0 ; python_version >= "3.8" and python_version < "3.13"
-packaging==24.1 ; python_version >= "3.8" and python_version < "3.13"
-paramiko==3.5.0 ; python_version >= "3.8" and python_version < "3.13"
-parsley==1.3 ; python_version >= "3.8" and python_version < "3.13"
-paste==3.10.1 ; python_version >= "3.8" and python_version < "3.13"
-pastedeploy==3.1.0 ; python_version >= "3.8" and python_version < "3.13"
-pebble==5.0.7 ; python_version >= "3.8" and python_version < "3.13"
-pillow==10.4.0 ; python_version >= "3.8" and python_version < "3.13"
-pkgutil-resolve-name==1.3.10 ; python_version >= "3.8" and python_version < "3.9"
-promise==2.3 ; python_version >= "3.8" and python_version < "3.13"
-prompt-toolkit==3.0.48 ; python_version >= "3.8" and python_version < "3.13"
-propcache==0.2.0 ; python_version >= "3.8" and python_version < "3.13"
-prov==1.5.1 ; python_version >= "3.8" and python_version < "3.13"
-psutil==6.1.0 ; python_version >= "3.8" and python_version < "3.13"
-pulsar-galaxy-lib==0.15.6 ; python_version >= "3.8" and python_version < "3.13"
-pycparser==2.22 ; python_version >= "3.8" and python_version < "3.13"
-pycryptodome==3.21.0 ; python_version >= "3.8" and python_version < "3.13"
-pydantic-core==2.23.4 ; python_version >= "3.8" and python_version < "3.13"
-pydantic-tes==0.1.5 ; python_version >= "3.8" and python_version < "3.13"
-pydantic==2.9.2 ; python_version >= "3.8" and python_version < "3.13"
-pydantic[email]==2.9.2 ; python_version >= "3.8" and python_version < "3.13"
-pydot==2.0.0 ; python_version >= "3.8" and python_version < "3.13"
-pyeventsystem==0.1.0 ; python_version >= "3.8" and python_version < "3.13"
-pyfaidx==0.8.1.3 ; python_version >= "3.8" and python_version < "3.13"
-pygments==2.18.0 ; python_version >= "3.8" and python_version < "3.13"
-pyjwt==2.9.0 ; python_version >= "3.8" and python_version < "3.13"
-pyjwt[crypto]==2.9.0 ; python_version >= "3.8" and python_version < "3.13"
-pykwalify==1.8.0 ; python_version >= "3.8" and python_version < "3.13"
-pylibmagic==0.5.0 ; python_version >= "3.8" and python_version < "3.13"
-pynacl==1.5.0 ; python_version >= "3.8" and python_version < "3.13"
-pyparsing==3.1.4 ; python_version >= "3.8" and python_version < "3.13"
-pyreadline3==3.5.4 ; sys_platform == "win32" and python_version >= "3.8" and python_version < "3.13"
-pysam==0.22.1 ; python_version >= "3.8" and python_version < "3.13"
-python-dateutil==2.9.0.post0 ; python_version >= "3.8" and python_version < "3.13"
-python-magic==0.4.27 ; python_version >= "3.8" and python_version < "3.13"
-python-multipart==0.0.12 ; python_version >= "3.8" and python_version < "3.13"
-python3-openid==3.2.0 ; python_version >= "3.8" and python_version < "3.13"
-pytz==2024.2 ; python_version >= "3.8" and python_version < "3.13"
-pyyaml==6.0.2 ; python_version >= "3.8" and python_version < "3.13"
-pyzmq==26.2.0 ; python_version >= "3.8" and python_version < "3.13"
-rdflib==6.3.2 ; python_version >= "3.8" and python_version < "3.13"
-referencing==0.35.1 ; python_version >= "3.8" and python_version < "3.13"
-refgenconf==0.12.2 ; python_version >= "3.8" and python_version < "3.13"
-regex==2024.9.11 ; python_version >= "3.8" and python_version < "3.13"
-repoze-lru==0.7 ; python_version >= "3.8" and python_version < "3.13"
-requests-oauthlib==2.0.0 ; python_version >= "3.8" and python_version < "3.13"
-requests-toolbelt==1.0.0 ; python_version >= "3.8" and python_version < "3.13"
-requests-unixsocket==0.3.0 ; python_version >= "3.8" and python_version < "3.13"
-requests==2.32.3 ; python_version >= "3.8" and python_version < "3.13"
-rich==13.9.3 ; python_version >= "3.8" and python_version < "3.13"
-rocrate==0.11.0 ; python_version >= "3.8" and python_version < "3.13"
-routes==2.5.1 ; python_version >= "3.8" and python_version < "3.13"
-rpds-py==0.20.0 ; python_version >= "3.8" and python_version < "3.13"
-ruamel-yaml-clib==0.2.8 ; platform_python_implementation == "CPython" and python_version < "3.13" and python_version >= "3.8"
-ruamel-yaml==0.18.6 ; python_version >= "3.8" and python_version < "3.13"
-s3fs==2024.10.0 ; python_version >= "3.8" and python_version < "3.13"
-schema-salad==8.7.20241010092723 ; python_version >= "3.8" and python_version < "3.13"
-setuptools-scm==5.0.2 ; python_version >= "3.8" and python_version < "3.13"
-setuptools==75.2.0 ; python_version >= "3.8" and python_version < "3.13"
-six==1.16.0 ; python_version >= "3.8" and python_version < "3.13"
-sniffio==1.3.1 ; python_version >= "3.8" and python_version < "3.13"
-social-auth-core==4.5.4 ; python_version >= "3.8" and python_version < "3.13"
-sortedcontainers==2.4.0 ; python_version >= "3.8" and python_version < "3.13"
-spython==0.3.14 ; python_version >= "3.8" and python_version < "3.13"
-sqlalchemy==2.0.35 ; python_version >= "3.8" and python_version < "3.13"
-sqlitedict==2.1.0 ; python_version >= "3.8" and python_version < "3.13"
-sqlparse==0.5.1 ; python_version >= "3.8" and python_version < "3.13"
-starlette-context==0.3.6 ; python_version >= "3.8" and python_version < "3.13"
-starlette-graphene3==0.6.0 ; python_version >= "3.8" and python_version < "3.13"
-starlette==0.41.0 ; python_version >= "3.8" and python_version < "3.13"
-supervisor==4.2.5 ; python_version >= "3.8" and python_version < "3.13"
-svgwrite==1.4.3 ; python_version >= "3.8" and python_version < "3.13"
-tenacity==9.0.0 ; python_version >= "3.8" and python_version < "3.13"
-tifffile==2023.7.10 ; python_version >= "3.8" and python_version < "3.13"
-tinydb==4.8.2 ; python_version >= "3.8" and python_version < "3.13"
-tomli==2.0.2 ; python_version >= "3.8" and python_version < "3.11"
-tornado==6.4.1 ; python_version >= "3.8" and python_version < "3.13"
-tqdm==4.66.5 ; python_version >= "3.8" and python_version < "3.13"
-tuspy==1.0.3 ; python_version >= "3.8" and python_version < "3.13"
-tuswsgi==0.5.5 ; python_version >= "3.8" and python_version < "3.13"
-typing-extensions==4.12.2 ; python_version >= "3.8" and python_version < "3.13"
-tzdata==2024.2 ; python_version >= "3.8" and python_version < "3.13"
-tzlocal==5.2 ; python_version >= "3.8" and python_version < "3.13"
-ubiquerg==0.8.0 ; python_version >= "3.8" and python_version < "3.13"
-urllib3==1.26.20 ; python_version >= "3.8" and python_version < "3.13"
-uvicorn==0.32.0 ; python_version >= "3.8" and python_version < "3.13"
-uvloop==0.21.0 ; python_version >= "3.8" and python_version < "3.13"
-vine==5.1.0 ; python_version >= "3.8" and python_version < "3.13"
-wcwidth==0.2.13 ; python_version >= "3.8" and python_version < "3.13"
-webencodings==0.5.1 ; python_version >= "3.8" and python_version < "3.13"
-webob==1.8.9 ; python_version >= "3.8" and python_version < "3.13"
-whoosh==2.7.4 ; python_version >= "3.8" and python_version < "3.13"
-wrapt==1.16.0 ; python_version >= "3.8" and python_version < "3.13"
-yacman==0.9.3 ; python_version >= "3.8" and python_version < "3.13"
-yarl==1.15.2 ; python_version >= "3.8" and python_version < "3.13"
-zipp==3.20.2 ; python_version >= "3.8" and python_version < "3.13"
-zipstream-new==1.1.8 ; python_version >= "3.8" and python_version < "3.13"
-zstandard==0.23.0 ; python_version >= "3.8" and python_version < "3.13"
+# This file was autogenerated by uv via the following command:
+#    uv export --no-hashes --no-dev
+a2wsgi==1.10.7
+adal==1.2.7
+aiobotocore==2.15.2
+aiodataloader==0.4.0
+aiofiles==24.1.0
+aiohappyeyeballs==2.4.3
+aiohttp==3.10.10
+aioitertools==0.12.0
+aiosignal==1.3.1
+alembic==1.13.3
+amqp==5.2.0
+annotated-types==0.7.0
+anyio==4.5.2
+apispec==6.6.1
+appdirs==1.4.4
+arcp==0.2.1
+argcomplete==3.5.1
+async-timeout==4.0.3 ; python_full_version < '3.11'
+attmap==0.13.2
+attrs==24.2.0
+babel==2.16.0
+backports-zoneinfo==0.2.1 ; python_full_version < '3.9'
+bagit==1.8.1
+bagit-profile==1.3.1
+bcrypt==4.2.0
+bdbag==1.7.3
+beaker==1.13.0
+billiard==4.2.1
+bioblend==1.3.0
+bleach==6.1.0
+boltons==24.0.0
+boto==2.49.0
+botocore==1.35.36
+bx-python==0.13.0
+cachecontrol==0.14.0
+celery==5.4.0
+certifi==2024.8.30
+cffi==1.17.1
+charset-normalizer==3.4.0
+cheetah3==3.2.6.post1
+circus==0.18.0
+click==8.1.7
+click-didyoumean==0.3.1
+click-plugins==1.1.1
+click-repl==0.3.0
+cloudauthz==0.6.0
+cloudbridge==3.2.0
+colorama==0.4.6 ; platform_system == 'Windows'
+coloredlogs==15.0.1
+conda-package-streaming==0.11.0
+cryptography==43.0.3
+cwl-upgrader==1.2.11
+cwl-utils==0.35
+cwltool==3.1.20241007082533
+defusedxml==0.7.1
+deprecation==2.1.0
+dictobj==0.4
+dnspython==2.6.1
+docopt==0.6.2
+docutils==0.20.1
+dparse==0.6.3
+edam-ontology==1.25.2
+email-validator==2.2.0
+exceptiongroup==1.2.2 ; python_full_version < '3.11'
+fastapi-slim==0.115.4
+filelock==3.16.1
+frozenlist==1.5.0
+fs==2.4.16
+fsspec==2024.10.0
+future==1.0.0
+galaxy2cwl==0.1.4
+graphene==3.4.1
+graphene-sqlalchemy==3.0.0rc1
+graphql-core==3.2.5
+graphql-relay==3.2.0
+gravity==1.0.6
+greenlet==3.1.1 ; (python_full_version < '3.13' and platform_machine == 'AMD64') or (python_full_version < '3.13' and platform_machine == 'WIN32') or (python_full_version < '3.13' and platform_machine == 'aarch64') or (python_full_version < '3.13' and platform_machine == 'amd64') or (python_full_version < '3.13' and platform_machine == 'ppc64le') or (python_full_version < '3.13' and platform_machine == 'win32') or (python_full_version < '3.13' and platform_machine == 'x86_64')
+gunicorn==23.0.0
+gxformat2==0.20.0
+h11==0.14.0
+h5grove==2.3.0
+h5py==3.11.0 ; python_full_version < '3.9'
+h5py==3.12.1 ; python_full_version >= '3.9'
+humanfriendly==10.0
+idna==3.10
+importlib-metadata==8.5.0
+importlib-resources==6.4.5 ; python_full_version < '3.9'
+isa-rwval @ git+https://github.com/nsoranzo/isa-rwval.git@3d989181058d2765a93cb0e7ca85d6955e0eb6ef
+isal==1.7.1
+isodate==0.6.1
+jinja2==3.1.4
+jmespath==1.0.1
+jsonref==1.1.0
+jsonschema==4.23.0
+jsonschema-specifications==2023.12.1
+kombu==5.4.2
+lagom==2.6.0
+legacy-cgi==2.6.1 ; python_full_version >= '3.13'
+lxml==5.3.0
+mako==1.3.6
+markdown==3.7
+markdown-it-py==3.0.0
+markupsafe==2.1.5
+mdurl==0.1.2
+mercurial==6.8.2
+mistune==3.0.2
+mrcfile==1.5.3
+msal==1.31.0
+msgpack==1.1.0
+multidict==6.1.0
+mypy-extensions==1.0.0
+networkx==3.1
+nodeenv==1.9.1
+numpy==1.24.4 ; python_full_version < '3.9'
+numpy==2.0.2 ; python_full_version == '3.9.*'
+numpy==2.1.2 ; python_full_version >= '3.10'
+oauthlib==3.2.2
+orjson==3.10.10
+oyaml==1.0
+packaging==24.1
+paramiko==3.5.0
+parsley==1.3
+paste==3.10.1
+pastedeploy==3.1.0
+pebble==5.0.7
+pillow==10.4.0
+pkgutil-resolve-name==1.3.10 ; python_full_version < '3.9'
+promise==2.3
+prompt-toolkit==3.0.48
+propcache==0.2.0
+prov==1.5.1
+psutil==6.1.0
+pulsar-galaxy-lib==0.15.6
+pycparser==2.22
+pycryptodome==3.21.0
+pydantic==2.9.2
+pydantic-core==2.23.4
+pydantic-tes==0.1.5
+pydot==2.0.0
+pyeventsystem==0.1.0
+pyfaidx==0.8.1.3
+pygments==2.18.0
+pyjwt==2.9.0
+pykwalify==1.8.0
+pylibmagic==0.5.0
+pynacl==1.5.0
+pyparsing==3.1.4
+pyreadline3==3.5.4 ; sys_platform == 'win32'
+pysam==0.22.1
+python-dateutil==2.9.0.post0
+python-magic==0.4.27
+python-multipart==0.0.16
+python3-openid==3.2.0
+pytz==2024.2
+pyyaml==6.0.2
+pyzmq==26.2.0
+rdflib==6.3.2
+referencing==0.35.1
+refgenconf==0.12.2
+regex==2024.9.11
+repoze-lru==0.7
+requests==2.32.3
+requests-oauthlib==2.0.0
+requests-toolbelt==1.0.0
+requests-unixsocket==0.3.0
+rich==13.9.3
+rocrate==0.11.0
+routes==2.5.1
+rpds-py==0.20.0
+ruamel-yaml==0.18.6
+ruamel-yaml-clib==0.2.8 ; python_full_version < '3.13' and platform_python_implementation == 'CPython'
+s3fs==2024.10.0
+schema-salad==8.7.20241010092723
+setuptools==75.2.0
+setuptools-scm==5.0.2
+six==1.16.0
+sniffio==1.3.1
+social-auth-core==4.5.4
+sortedcontainers==2.4.0
+spython==0.3.14
+sqlalchemy==2.0.35
+sqlitedict==2.1.0
+sqlparse==0.5.1
+starlette==0.41.2
+starlette-context==0.3.6
+starlette-graphene3==0.6.0
+supervisor==4.2.5
+svgwrite==1.4.3
+tenacity==9.0.0
+tifffile==2023.7.10
+tinydb==4.8.2
+tomli==2.0.2 ; python_full_version < '3.11'
+tornado==6.4.1
+tqdm==4.66.6
+tuspy==1.0.3
+tuswsgi==0.5.5
+typing-extensions==4.12.2
+tzdata==2024.2
+tzlocal==5.2
+ubiquerg==0.8.0
+urllib3==1.26.20
+uvicorn==0.32.0
+uvloop==0.21.0
+vine==5.1.0
+wcwidth==0.2.13
+webencodings==0.5.1
+webob==1.8.9
+whoosh==2.7.4
+wrapt==1.16.0
+yacman==0.9.3
+yarl==1.15.2
+zipp==3.20.2
+zipstream-new==1.1.8
+zstandard==0.23.0
diff --git a/lib/galaxy/dependencies/pinned-typecheck-requirements.txt b/lib/galaxy/dependencies/pinned-typecheck-requirements.txt
index 30501dcbe567..a835c64276fa 100644
--- a/lib/galaxy/dependencies/pinned-typecheck-requirements.txt
+++ b/lib/galaxy/dependencies/pinned-typecheck-requirements.txt
@@ -1,30 +1,32 @@
-annotated-types==0.7.0 ; python_version >= "3.8" and python_version < "3.13"
-boto3-stubs[s3]==1.35.49 ; python_version >= "3.8" and python_version < "3.13"
-botocore-stubs==1.35.49 ; python_version >= "3.8" and python_version < "3.13"
-cffi==1.17.1 ; python_version >= "3.8" and python_version < "3.13" and platform_python_implementation != "PyPy"
-cryptography==43.0.3 ; python_version >= "3.8" and python_version < "3.13"
-lxml-stubs==0.5.1 ; python_version >= "3.8" and python_version < "3.13"
-mypy-boto3-s3==1.35.46 ; python_version >= "3.8" and python_version < "3.13"
-mypy-extensions==1.0.0 ; python_version >= "3.8" and python_version < "3.13"
-mypy==1.13.0 ; python_version >= "3.8" and python_version < "3.13"
-pycparser==2.22 ; python_version >= "3.8" and python_version < "3.13" and platform_python_implementation != "PyPy"
-pydantic-core==2.23.4 ; python_version >= "3.8" and python_version < "3.13"
-pydantic==2.9.2 ; python_version >= "3.8" and python_version < "3.13"
-tomli==2.0.2 ; python_version >= "3.8" and python_version < "3.11"
-types-awscrt==0.23.0 ; python_version >= "3.8" and python_version < "3.13"
-types-bleach==6.1.0.20240331 ; python_version >= "3.8" and python_version < "3.13"
-types-boto==2.49.18.20241019 ; python_version >= "3.8" and python_version < "3.13"
-types-contextvars==2.4.7.3 ; python_version >= "3.8" and python_version < "3.13"
-types-dataclasses==0.6.6 ; python_version >= "3.8" and python_version < "3.13"
-types-docutils==0.21.0.20241005 ; python_version >= "3.8" and python_version < "3.13"
-types-html5lib==1.1.11.20241018 ; python_version >= "3.8" and python_version < "3.13"
-types-markdown==3.7.0.20240822 ; python_version >= "3.8" and python_version < "3.13"
-types-paramiko==3.5.0.20240928 ; python_version >= "3.8" and python_version < "3.13"
-types-python-dateutil==2.9.0.20241003 ; python_version >= "3.8" and python_version < "3.13"
-types-pyyaml==6.0.12.20240917 ; python_version >= "3.8" and python_version < "3.13"
-types-requests==2.31.0.6 ; python_version >= "3.8" and python_version < "3.13"
-types-s3transfer==0.10.3 ; python_version >= "3.8" and python_version < "3.13"
-types-setuptools==75.2.0.20241025 ; python_version >= "3.8" and python_version < "3.13"
-types-six==1.16.21.20241009 ; python_version >= "3.8" and python_version < "3.13"
-types-urllib3==1.26.25.14 ; python_version >= "3.8" and python_version < "3.13"
-typing-extensions==4.12.2 ; python_version >= "3.8" and python_version < "3.13"
+# This file was autogenerated by uv via the following command:
+#    uv export --no-hashes --only-group=typecheck
+annotated-types==0.7.0
+boto3-stubs==1.35.49
+botocore-stubs==1.35.49
+cffi==1.17.1 ; platform_python_implementation != 'PyPy'
+cryptography==43.0.3
+lxml-stubs==0.5.1
+mypy==1.13.0
+mypy-boto3-s3==1.35.46
+mypy-extensions==1.0.0
+pycparser==2.22 ; platform_python_implementation != 'PyPy'
+pydantic==2.9.2
+pydantic-core==2.23.4
+tomli==2.0.2 ; python_full_version < '3.11'
+types-awscrt==0.23.0
+types-bleach==6.1.0.20240331
+types-boto==2.49.18.20241019
+types-contextvars==2.4.7.3
+types-dataclasses==0.6.6
+types-docutils==0.21.0.20241005
+types-html5lib==1.1.11.20241018
+types-markdown==3.7.0.20240822
+types-paramiko==3.5.0.20240928
+types-python-dateutil==2.9.0.20241003
+types-pyyaml==6.0.12.20240917
+types-requests==2.31.0.6
+types-s3transfer==0.10.3
+types-setuptools==75.2.0.20241025
+types-six==1.16.21.20241009
+types-urllib3==1.26.25.14
+typing-extensions==4.12.2
diff --git a/lib/galaxy/dependencies/update.sh b/lib/galaxy/dependencies/update.sh
index 8175144aba5d..2d26f6b40bef 100755
--- a/lib/galaxy/dependencies/update.sh
+++ b/lib/galaxy/dependencies/update.sh
@@ -1,25 +1,25 @@
 #!/bin/sh
 
-# This script installs poetry if it's not already installed in the user account.
-# It then runs poetry inside Galaxy's root directory, potentially updating 
-# pyproject.toml and poetry.lock, and the pinned and dev requirements files.
+# This script installs uv if it's not already installed in the user account.
+# It then runs uv inside Galaxy's root directory, potentially updating
+# pyproject.toml and uv.lock, and the pinned requirements files.
 
 set -e
 
-SUPPORTED_PYTHON_VERSIONS="3.8 3.9 3.10 3.11 3.12"
-NOT_SUPPORTED_NEXT_PYTHON_VERSION="3.13"
-
 this_directory="$(cd "$(dirname "$0")" > /dev/null && pwd)"
 
 usage() {
-    printf "Usage: %s: [-a] [pkg_spec...]\n" "${0##*/}" >&2
+    printf "Usage: %s: [-p pkg]\n" "${0##*/}" >&2
 }
 
-add=
-while getopts 'a' OPTION
+pkg=
+while getopts 'p:h' OPTION
 do
     case $OPTION in
-        a) add=1
+        p) pkg="$OPTARG"
+           ;;
+        h) usage
+           exit 0
            ;;
         ?) usage
            exit 2
@@ -28,68 +28,20 @@ do
 done
 shift $((OPTIND - 1))
 
-if [ -n "$add" ] && [ $# -eq 0 ]; then
-    printf "When adding (-a), you must provide at least one package specification.\n" >&2
-    usage
-    exit 2
-fi
-
-# Install the latest version of poetry into the user account
-curl -sSL https://install.python-poetry.org | python3 -
-poetry self add 'poetry-plugin-export!=1.7.0'  # https://github.com/python-poetry/poetry-plugin-export/issues/269
+# Create a virtual environment in a tmp directory and install uv into it
+uv_venv=$(mktemp -d "${TMPDIR:-/tmp}/uv_venv.XXXXXXXXXX")
+python3 -m venv "${uv_venv}"
+"${uv_venv}/bin/python" -m pip install uv
+uv="${uv_venv}/bin/uv"
 
-# Run poetry (this may update pyproject.toml and poetry.lock).
-if [ -z "$add" ]; then
-    poetry update -vv --lock "$@"
+# Run uv (this may update pyproject.toml and uv.lock).
+if [ -n "$pkg" ]; then
+    ${uv} lock --upgrade-package "$pkg"
 else
-    poetry add -vv --lock "$@"
+    ${uv} lock
 fi
 
 # Update pinned requirements files.
-PINNED_REQUIREMENTS_FILE=$this_directory/pinned-requirements.txt
-PINNED_DEV_REQUIREMENTS_FILE=$this_directory/dev-requirements.txt
-poetry export -f requirements.txt --without-hashes --output "$PINNED_REQUIREMENTS_FILE"
-poetry export --only dev -f requirements.txt --without-hashes --output "$PINNED_DEV_REQUIREMENTS_FILE"
-poetry export --only typecheck -f requirements.txt --without-hashes --output "$this_directory/pinned-typecheck-requirements.txt"
-
-# Fix requirements
-
-latest_package_version_for_python_version () {
-    PIP_INDEX_OUT=$(pip index versions --python-version "$2" "$1")
-    echo "$PIP_INDEX_OUT" | head -n 1 | sed -e 's/.* (\(.*\)).*/\1/'
-}
-
-split_requirement () {
-    PACKAGE_NAME=$1
-
-    if ! pip index version --help >/dev/null; then
-        echo "pip >= 21.2.1 required for the 'index' command"
-        exit 1
-    fi
-
-    NEW_REQUIREMENTS=
-    PREVIOUS_LATEST_PACKAGE_VERSION=
-    for PYTHON_VERSION in $SUPPORTED_PYTHON_VERSIONS; do
-        LATEST_PACKAGE_VERSION=$(latest_package_version_for_python_version "$PACKAGE_NAME" "$PYTHON_VERSION")
-        if [ -z "$NEW_REQUIREMENTS" ]; then
-            NEW_REQUIREMENTS="$PACKAGE_NAME==$LATEST_PACKAGE_VERSION ; python_version >= \"$PYTHON_VERSION\""
-        elif [ "$LATEST_PACKAGE_VERSION" != "$PREVIOUS_LATEST_PACKAGE_VERSION" ]; then
-            NEW_REQUIREMENTS="$NEW_REQUIREMENTS and python_version < \"$PYTHON_VERSION\"\n$PACKAGE_NAME==$LATEST_PACKAGE_VERSION ; python_version >= \"$PYTHON_VERSION\""
-        fi
-        PREVIOUS_LATEST_PACKAGE_VERSION=$LATEST_PACKAGE_VERSION
-    done
-    if [ -n "$NOT_SUPPORTED_NEXT_PYTHON_VERSION" ]; then
-        NEW_REQUIREMENTS="$NEW_REQUIREMENTS and python_version < \"$NOT_SUPPORTED_NEXT_PYTHON_VERSION\""
-    fi
-
-    sed -i.orig -e "s/^$PACKAGE_NAME==.*/$NEW_REQUIREMENTS/" "$PINNED_REQUIREMENTS_FILE" "$PINNED_DEV_REQUIREMENTS_FILE"
-}
-
-# For some packages there is no recent version that works on all Python versions
-# supported by Galaxy, so Poetry resorts to an old version. Here we replace any
-# such requirement with multiple Python-version-specific requirements.
-split_requirement contourpy
-split_requirement h5py
-split_requirement matplotlib
-split_requirement numpy
-split_requirement scipy
+${uv} export --no-hashes --no-dev > "$this_directory/pinned-requirements.txt"
+${uv} export --no-hashes --only-group=dev > "$this_directory/dev-requirements.txt"
+${uv} export --no-hashes --only-group=typecheck > "$this_directory/pinned-typecheck-requirements.txt"
diff --git a/pyproject.toml b/pyproject.toml
index 96860904bd55..131444f7c10b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,3 +1,182 @@
+[project]
+name = "galaxy"
+version = "0.1.0"  # Not the real version, which is instead defined in lib/galaxy/version.py
+description = "Galaxy is an open, web-based platform for accessible, reproducible, and transparent computational research."
+authors = [
+    {name = "Galaxy Contributors"},
+]
+maintainers = [
+    {name = "Galaxy Committers", email = "galaxy-committers@lists.galaxyproject.org"}
+]
+license = {file = "LICENSE.txt"}
+readme = "README.rst"
+requires-python = ">=3.8"
+dependencies = [
+    "a2wsgi",
+    "aiofiles",
+    "alembic",
+    "apispec",
+    "Babel",
+    "bdbag>=1.6.3",
+    "Beaker",
+    "bioblend",
+    "bleach",
+    "boltons",
+    "boto",
+    "bx-python",
+    "celery",
+    "Cheetah3",
+    "cloudauthz==0.6.0",
+    "cloudbridge",
+    "circus",
+    "conda-package-streaming",
+    "cwltool>=3.1.20230624081518",  # save time, minimum needed by cwl-1.0 branch
+    "dictobj",
+    "dnspython",
+    "docutils!=0.17,!=0.17.1",
+    "dparse",
+    "edam-ontology",
+    "fastapi-slim>=0.111.0",
+    "fs",
+    "future>=1.0.0",  # Python 3.12 support
+    "graphene",
+    "graphene-sqlalchemy>=3.0.0rc1",  # need a beta release to be compat. with starlette plugin
+    "graphql-core",
+    "gravity>=1.0.4",
+    "gunicorn",
+    "gxformat2",
+    "h5grove>=1.2.1",
+    "h5py",  # Python 3.8 support
+    "h5py>=3.12 ; python_version>='3.9'",  # Python 3.13 support
+    "importlib-resources ; python_version<'3.9'",  # for importlib.{abc.Traversable,resources.{files, Package}}
+    "isa-rwval @ git+https://github.com/nsoranzo/isa-rwval.git@master",  # https://github.com/ISA-tools/isa-rwval/pull/17
+    "isal",
+    "jinja2",
+    "kombu>=5.3",  # for importlib-metadata fix
+    "lagom",
+    "Mako",
+    "Markdown",
+    "MarkupSafe",
+    "mercurial",
+    "mrcfile",
+    "msal",
+    "nodeenv",
+    "numpy",
+    "numpy>=1.26.0 ; python_version=='3.9'",  # Python 3.12 support
+    "numpy>=2.1.0 ; python_version>='3.10'",  # Python 3.13 support
+    "packaging",
+    "paramiko!=2.9.0,!=2.9.1",  # https://github.com/paramiko/paramiko/issues/1961
+    "Parsley",
+    "Paste",
+    "pebble",
+    "pillow",
+    "psutil",
+    "pulsar-galaxy-lib>=0.15.0.dev0",
+    "pycryptodome",
+    "pydantic[email]>=2.7.4",  # https://github.com/pydantic/pydantic/pull/9639
+    "PyJWT",
+    "pykwalify",
+    "pylibmagic",
+    "pyparsing",
+    "pysam>=0.21",  # for Python 3.11 support on macOS
+    "python-dateutil",
+    "python-magic",
+    "python-multipart",  # required to support form parsing in FastAPI/Starlette
+    "PyYAML",
+    "refgenconf>=0.12.0",
+    "regex",
+    "requests",
+    "rocrate",
+    "Routes",
+    "s3fs>=2023.1.0",
+    "schema-salad!=8.3.20220721194857",  # https://github.com/common-workflow-language/schema_salad/issues/575
+    "social-auth-core>=4.5.0",  # to drop dependency on abandoned python-jose
+    "sortedcontainers",
+    "SQLAlchemy>=2.0,<2.1,!=2.0.36",  # https://github.com/sqlalchemy/sqlalchemy/issues/12019,
+    "sqlitedict",
+    "sqlparse",
+    "starlette",
+    "starlette-context",
+    "starlette-graphene3",
+    "svgwrite",
+    "tifffile",
+    "tuswsgi",
+    "typing-extensions",
+    "uvicorn!=0.28.0",  # https://github.com/galaxyproject/galaxy/issues/17669
+    "uvloop",
+    "WebOb",
+    "Whoosh",
+    "zipstream-new",
+]
+
+[project.urls]
+Homepage = "https://galaxyproject.org/"
+Documentation = "https://docs.galaxyproject.org/"
+Repository = "https://github.com/galaxyproject/galaxy"
+"Bug Tracker" = "https://github.com/galaxyproject/galaxy/issues"
+
+[dependency-groups]
+dev = [
+    "ase>=3.18.1",
+    "axe-selenium-python",
+    "black",
+    "boto3",
+    "codespell",
+    "cwltest",
+    "darker",
+    "fluent-logger",
+    "galaxy-release-util",
+    "httpx",
+    "isort",
+    "lxml!=4.2.2",
+    "markdown-it-reporter",
+    "myst-parser",
+    "onedatafilerestclient==21.2.5.2",
+    "pkce",
+    "playwright",
+    "pytest<8",  # https://github.com/galaxyproject/galaxy/issues/17561
+    "pytest-asyncio",
+    "pytest-cov",
+    "pytest-html",
+    "pytest-httpserver",
+    "python-irodsclient!=1.1.2",  # https://github.com/irods/python-irodsclient/issues/356
+    "pytest-json-report",
+    "pytest-mock",
+    "pytest-playwright",
+    "pytest-postgresql!=3.0.0",  # https://github.com/ClearcodeHQ/pytest-postgresql/issues/426
+    "pytest-shard",
+    "responses",
+    "rucio-clients>=33.6.0 ; python_version>='3.9'",  # https://github.com/galaxyproject/galaxy/pull/17156
+    "selenium",
+    "seletools",
+    "Sphinx>=4.0",  # support docutils 0.17
+    "sphinx-rtd-theme",
+    "statsd",
+    "testfixtures",
+    "tuspy",
+    "twill>=3.2.1",  # https://github.com/twill-tools/twill/issues/18
+    "watchdog",
+    "Werkzeug",
+]
+typecheck = [
+    "boto3-stubs[s3]",
+    "lxml-stubs",
+    "mypy",
+    "pydantic>=2.7.4",  # for pydantic.mypy plugin
+    "types-bleach",
+    "types-boto",
+    "types-contextvars",
+    "types-dataclasses",
+    "types-docutils",
+    "types-Markdown",
+    "types-paramiko",
+    "types-python-dateutil",
+    "types-PyYAML",
+    "types-requests",
+    "types-setuptools",
+    "types-six",
+]
+
 [tool.black]
 line-length = 120
 target-version = ['py38']
@@ -12,186 +191,10 @@ extend-exclude = '''
 [tool.darker]
 isort = true
 
-[tool.poetry]
-package-mode = false
-name = "galaxy"
-description = "Galaxy is an open, web-based platform for accessible, reproducible, and transparent computational research."
-authors = ["Galaxy Contributors"]
-license = "AFL-3.0"
-readme = "README.rst"
-homepage = "https://galaxyproject.org/"
-repository = "https://github.com/galaxyproject/galaxy/"
-documentation = "https://docs.galaxyproject.org/"
-
-[[tool.poetry.source]]
-name = "galaxyproject"
-url = "https://wheels.galaxyproject.org/simple"
-priority = "primary"
-
-[[tool.poetry.source]]
-name = "PyPI"
-priority = "primary"
-
-[tool.poetry.dependencies]
-a2wsgi = "*"
-aiofiles = "*"
-alembic = "*"
-apispec = "*"
-Babel = "*"
-bdbag = ">=1.6.3"
-Beaker = "*"
-bioblend = "*"
-bleach = "*"
-boltons = "*"
-boto = "*"
-bx-python = "*"
-celery = "*"
-Cheetah3 = "*"
-cloudauthz = "==0.6.0"
-cloudbridge = "*"
-circus = "*"
-cwltool = ">=3.1.20230624081518"  # save time, minimum needed by cwl-1.0 branch
-conda-package-streaming = "*"
-dictobj = "*"
-dnspython = "*"
-docutils = "!=0.17, !=0.17.1"
-dparse = "*"
-edam-ontology = "*"
-fastapi-slim = ">=0.111.0"
-fs = "*"
-future = ">=1.0.0"  # Python 3.12 support
-graphene = "*"
-graphene-sqlalchemy = ">=3.0.0rc1"  # need a beta release to be compat. with starlette plugin
-graphql-core = "*"
-gravity = ">=1.0.4"
-gunicorn = "*"
-gxformat2 = "*"
-h5grove = ">=1.2.1"
-h5py = "*"
-importlib-resources = { version = "*", python = "<3.9" }  # for importlib.{abc.Traversable,resources.{files, Package}}
-isa-rwval = { git = "https://github.com/nsoranzo/isa-rwval.git", branch = "master" }  # https://github.com/ISA-tools/isa-rwval/pull/17
-jinja2 = "*"
-kombu = ">=5.3"  # for importlib-metadata fix
-isal = "*"
-lagom = "*"
-Mako = "*"
-Markdown = "*"
-MarkupSafe = "*"
-Mercurial = "*"
-mrcfile = "*"
-msal = "*"
-nodeenv = "*"
-numpy = "*"
-packaging = "*"
-paramiko = "!=2.9.0, !=2.9.1"  # https://github.com/paramiko/paramiko/issues/1961
-Parsley = "*"
-Paste = "*"
-pebble = "*"
-pillow = "*"
-psutil = "*"
-pulsar-galaxy-lib = ">=0.15.0.dev0"
-pycryptodome = "*"
-pydantic = {version = ">=2.7.4", extras = ["email"]}  # https://github.com/pydantic/pydantic/pull/9639
-PyJWT = "*"
-pykwalify = "*"
-pylibmagic = "*"
-pyparsing = "*"
-pysam = ">=0.21"  # for Python 3.11 support on macOS
-python = ">=3.8,<3.13"  # Keep maximum version strict to appease numpy and scipy
-python-dateutil = "*"
-python-magic = "*"
-python-multipart = "*"  # required to support form parsing in FastAPI/Starlette
-PyYAML = "*"
-refgenconf = ">=0.12.0"
-regex = "*"
-requests = "*"
-rocrate = "*"
-Routes = "*"
-schema-salad = "!=8.3.20220721194857"  # https://github.com/common-workflow-language/schema_salad/issues/575
-social-auth-core = ">=4.5.0"  # to drop dependency on abandoned python-jose
-sortedcontainers = "*"
-SQLAlchemy = ">=2.0,<2.1,!=2.0.36"  # https://github.com/sqlalchemy/sqlalchemy/issues/12019
-sqlitedict = "*"
-sqlparse = "*"
-starlette = "*"
-starlette-context = "*"
-starlette-graphene3 = "*"
-svgwrite = "*"
-tifffile = "*"
-tuswsgi = "*"
-typing-extensions = "*"
-uvicorn = "!=0.28.0"  # https://github.com/galaxyproject/galaxy/issues/17669
-uvloop = "*"
-WebOb = "*"
-Whoosh = "*"
-zipstream-new = "*"
-s3fs = ">=2023.1.0"
-
-[tool.poetry.group.dev.dependencies]
-ase = ">=3.18.1"
-axe-selenium-python = "*"
-black = "*"
-boto3 = "*"
-codespell = "*"
-cwltest = "*"
-darker = "*"
-fluent-logger = "*"
-galaxy-release-util = "*"
-httpx = "*"
-isort = "*"
-lxml = "!=4.2.2"
-markdown-it-reporter = "*"
-myst-parser = "*"
-onedatafilerestclient = "==21.2.5.2"
-pkce = "*"
-playwright = "*"
-pytest = "*"
-pytest-asyncio = "*"
-pytest-cov = "*"
-pytest-html = "*"
-pytest-httpserver = "*"
-python-irodsclient = "!=1.1.2"  # https://github.com/irods/python-irodsclient/issues/356
-pytest-json-report = "*"
-pytest-mock = "*"
-pytest-playwright = "*"
-pytest-postgresql = "!=3.0.0"  # https://github.com/ClearcodeHQ/pytest-postgresql/issues/426
-pytest-shard = "*"
-responses = "*"
-rucio-clients = {version = ">=33.6.0", python = ">=3.9"}  # https://github.com/galaxyproject/galaxy/pull/17156
-selenium = "*"
-urllib3 = "<2"  # Necessary for poetry for resolve deps for Python 3.8+
-seletools = "*"
-Sphinx = ">=4.0"  # support docutils 0.17
-sphinx-rtd-theme = "*"
-statsd = "*"
-testfixtures = "*"
-tuspy = "*"
-twill = ">=3.2.1"  # https://github.com/twill-tools/twill/issues/18
-watchdog = "*"
-Werkzeug = "*"
-
-[tool.poetry.group.typecheck.dependencies]
-mypy = "*"
-boto3-stubs = {version = "*", extras = ["s3"]}
-lxml-stubs = "*"
-pydantic = ">=2.7.4"  # for pydantic.mypy plugin
-types-bleach = "*"
-types-boto = "*"
-types-contextvars = "*"
-types-dataclasses = "*"
-types-docutils = "*"
-types-Markdown = "*"
-types-paramiko = "*"
-types-python-dateutil = "*"
-types-PyYAML = "*"
-types-requests = "*"
-types-setuptools = "*"
-types-six = "*"
-
 [tool.ruff]
 target-version = "py38"
 exclude = [
-  "lib/tool_shed/test/test_data/repos"
+    "lib/tool_shed/test/test_data/repos"
 ]
 
 [tool.ruff.lint]
@@ -229,3 +232,9 @@ keep-runtime-typing = true
 "lib/galaxy/schema/drs/*" = ["UP006", "UP007"]
 "lib/tool_shed_client/schema/trs.py" = ["UP006", "UP007"]
 "lib/tool_shed_client/schema/trs_service_info.py" = ["UP006", "UP007"]
+
+[tool.uv]
+default-groups = []
+extra-index-url = ["https://wheels.galaxyproject.org/simple"]
+index-strategy = "unsafe-best-match"
+package = false

From 18fb0a785e03e0463c92b056500d5525b4f229a2 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Thu, 31 Oct 2024 11:49:49 +0100
Subject: [PATCH 42/84] Enhance allow_uri_if_protocol to support wildcard '*'
 for deferred datasets

---
 lib/galaxy/tool_util/xsd/galaxy.xsd | 4 ++--
 lib/galaxy/tools/evaluation.py      | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index 9de63fe28d73..376ff0e8d1e5 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -4326,8 +4326,8 @@ This is useful when the tool can handle the URI directly. Example:
 <param name="input" type="data" format="txt" allow_uri_if_protocol="http,https,s3" />
 ```
 
-You can specify multiple prefixes separated by comma. The URI will be passed to the tool
-as is if it starts with any of the specified prefixes.
+You can specify multiple prefixes separated by comma or use the wildcard '*' to always
+treat deferred datasets as URIs. The source URI will be passed to the tool as is.
 
 This attribute is only valid for `data` parameters.
          ]]></xs:documentation>
diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py
index 5b278b91d9e6..73dbeceaff39 100644
--- a/lib/galaxy/tools/evaluation.py
+++ b/lib/galaxy/tools/evaluation.py
@@ -341,7 +341,7 @@ def _should_materialize_deferred_input(self, input_name: str, input_value: Defer
         if isinstance(deferred_input, DataToolParameter) and isinstance(input_value, model.DatasetInstance):
             source_uri = input_value.sources[0].source_uri or ""
             for prefix in deferred_input.allow_uri_if_protocol:
-                if source_uri.startswith(prefix):
+                if prefix == "*" or source_uri.startswith(prefix):
                     return False
         return True
 

From 72735ebf961b56cf5dea4a4dab7b77d0462d020c Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Thu, 31 Oct 2024 11:57:47 +0100
Subject: [PATCH 43/84] Remove redundant check for missing deferred input in
 ToolEvaluator

There might be cases with additional deferred inputs outside of the input definitions see test_metadata_validator_on_deferred_input.
---
 lib/galaxy/tools/evaluation.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py
index 73dbeceaff39..8e9328c71408 100644
--- a/lib/galaxy/tools/evaluation.py
+++ b/lib/galaxy/tools/evaluation.py
@@ -334,10 +334,6 @@ def _should_materialize_deferred_input(self, input_name: str, input_value: Defer
         with a known prefix set in `allow_uri_if_protocol`.
         """
         deferred_input = self.tool.inputs.get(input_name)
-        if not deferred_input:
-            # TODO: Can this ever happen? It seems like it happens in the test suite.
-            # For example, in test_metadata_validator_on_deferred_input
-            return True
         if isinstance(deferred_input, DataToolParameter) and isinstance(input_value, model.DatasetInstance):
             source_uri = input_value.sources[0].source_uri or ""
             for prefix in deferred_input.allow_uri_if_protocol:

From f0420d70a3242fd3303b446e63cd988376f7afa3 Mon Sep 17 00:00:00 2001
From: Nicola Soranzo <nicola.soranzo@earlham.ac.uk>
Date: Fri, 6 Sep 2024 11:20:20 +0100
Subject: [PATCH 44/84] Pin minimum package versions for Python 3.13 support

Also:
- Cheetah3 package on PyPI has been renamed to CT3 since 3.3.0, see
  https://cheetahtemplate.org/news.html
---
 .../dependencies/conditional-requirements.txt  |  2 +-
 .../dependencies/pinned-requirements.txt       |  2 +-
 packages/app/setup.cfg                         |  4 ++--
 packages/data/setup.cfg                        |  3 +--
 packages/files/setup.cfg                       |  1 -
 packages/test_base/setup.cfg                   |  2 +-
 packages/tool_shed/setup.cfg                   |  2 +-
 packages/util/setup.cfg                        |  2 +-
 packages/web_apps/setup.cfg                    |  6 +++---
 pyproject.toml                                 | 18 +++++++++---------
 10 files changed, 20 insertions(+), 22 deletions(-)

diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt
index d59bd0916e69..168fea8adcd0 100644
--- a/lib/galaxy/dependencies/conditional-requirements.txt
+++ b/lib/galaxy/dependencies/conditional-requirements.txt
@@ -1,5 +1,5 @@
 # These dependencies are only required when certain config options are set
-psycopg2-binary==2.9.9
+psycopg2-binary==2.9.10
 mysqlclient
 fluent-logger
 sentry-sdk[fastapi]
diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt
index c911f9b8b305..827d67b8354d 100644
--- a/lib/galaxy/dependencies/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pinned-requirements.txt
@@ -39,7 +39,6 @@ celery==5.4.0
 certifi==2024.8.30
 cffi==1.17.1
 charset-normalizer==3.4.0
-cheetah3==3.2.6.post1
 circus==0.18.0
 click==8.1.7
 click-didyoumean==0.3.1
@@ -51,6 +50,7 @@ colorama==0.4.6 ; platform_system == 'Windows'
 coloredlogs==15.0.1
 conda-package-streaming==0.11.0
 cryptography==43.0.3
+ct3==3.3.3.post1
 cwl-upgrader==1.2.11
 cwl-utils==0.35
 cwltool==3.1.20241007082533
diff --git a/packages/app/setup.cfg b/packages/app/setup.cfg
index 8864f0c693d7..fe0dfe430c91 100644
--- a/packages/app/setup.cfg
+++ b/packages/app/setup.cfg
@@ -57,7 +57,7 @@ install_requires =
     Mako
     Markdown
     MarkupSafe
-    mercurial
+    mercurial>=6.8.2
     packaging
     paramiko!=2.9.0,!=2.9.1
     pebble
@@ -74,7 +74,7 @@ install_requires =
     starlette
     svgwrite
     typing-extensions
-    WebOb
+    WebOb>=1.8.9
     Whoosh
 packages = find:
 python_requires = >=3.8
diff --git a/packages/data/setup.cfg b/packages/data/setup.cfg
index b8916e8ce2de..01f623c7d199 100644
--- a/packages/data/setup.cfg
+++ b/packages/data/setup.cfg
@@ -46,7 +46,7 @@ install_requires =
     h5grove>=1.2.1
     h5py
     isa-rwval @ git+https://github.com/nsoranzo/isa-rwval.git@master
-    isal
+    isal>=1.7.0
     MarkupSafe
     msal
     mrcfile
@@ -62,7 +62,6 @@ install_requires =
     SQLAlchemy>=2.0,<2.1,!=2.0.36
     tifffile
     typing-extensions
-    WebOb
 packages = find:
 python_requires = >=3.8
 
diff --git a/packages/files/setup.cfg b/packages/files/setup.cfg
index 9bf5eec1d63f..81e727080f7a 100644
--- a/packages/files/setup.cfg
+++ b/packages/files/setup.cfg
@@ -34,7 +34,6 @@ include_package_data = True
 install_requires =
     galaxy-util[config_template]
     fs
-    isal
     typing-extensions
 packages = find:
 python_requires = >=3.8
diff --git a/packages/test_base/setup.cfg b/packages/test_base/setup.cfg
index 2fb287bad14c..62247a0ae264 100644
--- a/packages/test_base/setup.cfg
+++ b/packages/test_base/setup.cfg
@@ -35,7 +35,7 @@ install_requires =
     galaxy-tool-util
     galaxy-util
     bioblend
-    cwltest
+    cwltest>=2.5.20240906231108
     gxformat2
     pytest
     PyYAML
diff --git a/packages/tool_shed/setup.cfg b/packages/tool_shed/setup.cfg
index 217d4d4d2f1e..7cd328385896 100644
--- a/packages/tool_shed/setup.cfg
+++ b/packages/tool_shed/setup.cfg
@@ -48,7 +48,7 @@ install_requires =
     graphql-core
     Mako
     MarkupSafe
-    mercurial
+    mercurial>=6.8.2
     Paste
     pydantic>=2.7.4
     Routes
diff --git a/packages/util/setup.cfg b/packages/util/setup.cfg
index 064f331f621c..bb6a232cc1e2 100644
--- a/packages/util/setup.cfg
+++ b/packages/util/setup.cfg
@@ -51,7 +51,7 @@ python_requires = >=3.7
 jstree =
     dictobj
 template =
-    Cheetah3
+    CT3>=3.3.3
     future>=1.0.0
 config_template =
     Jinja2
diff --git a/packages/web_apps/setup.cfg b/packages/web_apps/setup.cfg
index bcf88a282c43..19ddb35ae0b3 100644
--- a/packages/web_apps/setup.cfg
+++ b/packages/web_apps/setup.cfg
@@ -42,7 +42,7 @@ install_requires =
     a2wsgi
     apispec
     Babel
-    Cheetah3
+    CT3>=3.3.3
     fastapi>=0.101.0
     gunicorn
     gxformat2
@@ -63,8 +63,8 @@ install_requires =
     tuswsgi
     typing-extensions
     uvicorn
-    uvloop
-    WebOb
+    uvloop>=0.21.0
+    WebOb>=1.8.9
 packages = find:
 python_requires = >=3.8
 
diff --git a/pyproject.toml b/pyproject.toml
index 131444f7c10b..145ddce07bf9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,11 +25,11 @@ dependencies = [
     "boto",
     "bx-python",
     "celery",
-    "Cheetah3",
     "cloudauthz==0.6.0",
     "cloudbridge",
     "circus",
     "conda-package-streaming",
+    "CT3>=3.3.3",  # Python 3.13 support
     "cwltool>=3.1.20230624081518",  # save time, minimum needed by cwl-1.0 branch
     "dictobj",
     "dnspython",
@@ -50,14 +50,14 @@ dependencies = [
     "h5py>=3.12 ; python_version>='3.9'",  # Python 3.13 support
     "importlib-resources ; python_version<'3.9'",  # for importlib.{abc.Traversable,resources.{files, Package}}
     "isa-rwval @ git+https://github.com/nsoranzo/isa-rwval.git@master",  # https://github.com/ISA-tools/isa-rwval/pull/17
-    "isal",
+    "isal>=1.7.0",  # Python 3.13 support
     "jinja2",
     "kombu>=5.3",  # for importlib-metadata fix
     "lagom",
     "Mako",
     "Markdown",
     "MarkupSafe",
-    "mercurial",
+    "mercurial>=6.8.2",  # Python 3.13 support
     "mrcfile",
     "msal",
     "nodeenv",
@@ -89,7 +89,7 @@ dependencies = [
     "rocrate",
     "Routes",
     "s3fs>=2023.1.0",
-    "schema-salad!=8.3.20220721194857",  # https://github.com/common-workflow-language/schema_salad/issues/575
+    "schema-salad>=8.7.20240905150001",  # Python 3.13 support
     "social-auth-core>=4.5.0",  # to drop dependency on abandoned python-jose
     "sortedcontainers",
     "SQLAlchemy>=2.0,<2.1,!=2.0.36",  # https://github.com/sqlalchemy/sqlalchemy/issues/12019,
@@ -103,8 +103,8 @@ dependencies = [
     "tuswsgi",
     "typing-extensions",
     "uvicorn!=0.28.0",  # https://github.com/galaxyproject/galaxy/issues/17669
-    "uvloop",
-    "WebOb",
+    "uvloop>=0.21.0",  # Python 3.13 support
+    "WebOb>=1.8.9",  # Python 3.13 support
     "Whoosh",
     "zipstream-new",
 ]
@@ -122,7 +122,7 @@ dev = [
     "black",
     "boto3",
     "codespell",
-    "cwltest",
+    "cwltest>=2.5.20240906231108",  # Python 3.13 support
     "darker",
     "fluent-logger",
     "galaxy-release-util",
@@ -133,7 +133,7 @@ dev = [
     "myst-parser",
     "onedatafilerestclient==21.2.5.2",
     "pkce",
-    "playwright",
+    "playwright>=1.48.0",  # Python 3.13 support
     "pytest<8",  # https://github.com/galaxyproject/galaxy/issues/17561
     "pytest-asyncio",
     "pytest-cov",
@@ -154,7 +154,7 @@ dev = [
     "statsd",
     "testfixtures",
     "tuspy",
-    "twill>=3.2.1",  # https://github.com/twill-tools/twill/issues/18
+    "twill>=3.2.5",  # Python 3.13 support
     "watchdog",
     "Werkzeug",
 ]

From 695e86eaa7e131a10532fad81531e2e312d453d4 Mon Sep 17 00:00:00 2001
From: Nicola Soranzo <nicola.soranzo@earlham.ac.uk>
Date: Fri, 28 Jun 2024 09:07:49 +0100
Subject: [PATCH 45/84] Test under Python 3.13

---
 .github/workflows/first_startup.yaml        | 2 +-
 .github/workflows/lint.yaml                 | 2 +-
 .github/workflows/lint_openapi_schema.yml   | 2 +-
 .github/workflows/osx_startup.yaml          | 2 +-
 .github/workflows/reports_startup.yaml      | 2 +-
 .github/workflows/test_galaxy_packages.yaml | 2 +-
 .github/workflows/toolshed.yaml             | 2 +-
 .github/workflows/unit.yaml                 | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/first_startup.yaml b/.github/workflows/first_startup.yaml
index e35170d063a0..05efc45367b9 100644
--- a/.github/workflows/first_startup.yaml
+++ b/.github/workflows/first_startup.yaml
@@ -23,7 +23,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.12']
+        python-version: ['3.8', '3.13']
     defaults:
       run:
         shell: bash -l {0}
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
index 130be9861afb..37d337fb95fe 100644
--- a/.github/workflows/lint.yaml
+++ b/.github/workflows/lint.yaml
@@ -22,7 +22,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.12']
+        python-version: ['3.8', '3.13']
     env:
       LINT_PATH: 'lib/galaxy/dependencies/pinned-lint-requirements.txt'
       TYPE_PATH: 'lib/galaxy/dependencies/pinned-typecheck-requirements.txt'
diff --git a/.github/workflows/lint_openapi_schema.yml b/.github/workflows/lint_openapi_schema.yml
index 2a94508ce6f2..4dc692fc2739 100644
--- a/.github/workflows/lint_openapi_schema.yml
+++ b/.github/workflows/lint_openapi_schema.yml
@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.12']
+        python-version: ['3.8', '3.13']
     steps:
       - uses: actions/checkout@v4
         with:
diff --git a/.github/workflows/osx_startup.yaml b/.github/workflows/osx_startup.yaml
index 22ffc4d7e5b6..13960eb8e85f 100644
--- a/.github/workflows/osx_startup.yaml
+++ b/.github/workflows/osx_startup.yaml
@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.12']
+        python-version: ['3.8', '3.13']
     defaults:
       run:
         shell: bash -l {0}
diff --git a/.github/workflows/reports_startup.yaml b/.github/workflows/reports_startup.yaml
index 137c1a5e1204..b7449dc5979c 100644
--- a/.github/workflows/reports_startup.yaml
+++ b/.github/workflows/reports_startup.yaml
@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.12']
+        python-version: ['3.8', '3.13']
     defaults:
       run:
         shell: bash -l {0}
diff --git a/.github/workflows/test_galaxy_packages.yaml b/.github/workflows/test_galaxy_packages.yaml
index 293e98e658bd..f38f05846e74 100644
--- a/.github/workflows/test_galaxy_packages.yaml
+++ b/.github/workflows/test_galaxy_packages.yaml
@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.12']
+        python-version: ['3.8', '3.13']
     steps:
       - uses: actions/checkout@v4
         with:
diff --git a/.github/workflows/toolshed.yaml b/.github/workflows/toolshed.yaml
index fae33b4e1225..14be278e16bc 100644
--- a/.github/workflows/toolshed.yaml
+++ b/.github/workflows/toolshed.yaml
@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.12']
+        python-version: ['3.8', '3.13']
         shed-api: ['v1', 'v2']
         test-install-client: ['galaxy_api', 'standalone']
     services:
diff --git a/.github/workflows/unit.yaml b/.github/workflows/unit.yaml
index 80f54e87a704..671dcac0162d 100644
--- a/.github/workflows/unit.yaml
+++ b/.github/workflows/unit.yaml
@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.12']
+        python-version: ['3.8', '3.13']
     steps:
       - uses: actions/checkout@v4
         with:

From ac480e7df51ecc438f9f64ea906ab1ee825408b1 Mon Sep 17 00:00:00 2001
From: Nicola Soranzo <nicola.soranzo@earlham.ac.uk>
Date: Thu, 5 Sep 2024 15:40:22 +0100
Subject: [PATCH 46/84] Drop use of deprecated imghdr standard library module

which has been removed in the upcoming Python 3.13 .
---
 lib/galaxy/util/image_util.py   |  5 -----
 packages/app/setup.cfg          |  2 +-
 packages/data/setup.cfg         |  2 +-
 packages/test.sh                |  2 +-
 packages/tool_shed/setup.cfg    |  2 +-
 packages/tool_util/setup.cfg    |  2 +-
 packages/util/setup.cfg         |  2 ++
 test/unit/util/test_checkers.py | 13 ++++++++++++-
 8 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/lib/galaxy/util/image_util.py b/lib/galaxy/util/image_util.py
index 3b11a50d2fda..b8c26cf0ff54 100644
--- a/lib/galaxy/util/image_util.py
+++ b/lib/galaxy/util/image_util.py
@@ -1,6 +1,5 @@
 """Provides utilities for working with image files."""
 
-import imghdr
 import logging
 from typing import (
     List,
@@ -22,11 +21,7 @@ def image_type(filename: str) -> Optional[str]:
             with Image.open(filename) as im:
                 fmt = im.format
         except Exception:
-            # We continue to try with imghdr, so this is a rare case of an
-            # exception we expect to happen frequently, so we're not logging
             pass
-    if not fmt:
-        fmt = imghdr.what(filename)
     if fmt:
         return fmt.upper()
     else:
diff --git a/packages/app/setup.cfg b/packages/app/setup.cfg
index fe0dfe430c91..7f3e9775529e 100644
--- a/packages/app/setup.cfg
+++ b/packages/app/setup.cfg
@@ -41,7 +41,7 @@ install_requires =
     galaxy-objectstore
     galaxy-tool-util[cwl,edam]
     galaxy-tours
-    galaxy-util
+    galaxy-util[image_util]
     galaxy-web-framework
     galaxy-web-stack
     Beaker
diff --git a/packages/data/setup.cfg b/packages/data/setup.cfg
index 01f623c7d199..2dd4e7e16b07 100644
--- a/packages/data/setup.cfg
+++ b/packages/data/setup.cfg
@@ -36,7 +36,7 @@ install_requires =
     galaxy-objectstore
     galaxy-schema
     galaxy-tool-util
-    galaxy-util[template]
+    galaxy-util[image_util,template]
     alembic
     alembic-utils
     bdbag>=1.6.3
diff --git a/packages/test.sh b/packages/test.sh
index d2ed377c352b..2120ac2b922a 100755
--- a/packages/test.sh
+++ b/packages/test.sh
@@ -49,7 +49,7 @@ while read -r package_dir || [ -n "$package_dir" ]; do  # https://stackoverflow.
 
     # Install extras (if needed)
     if [ "$package_dir" = "util" ]; then
-        pip install '.[template,jstree,config_template]'
+        pip install '.[image_util,template,jstree,config_template]'
     elif [ "$package_dir" = "tool_util" ]; then
         pip install '.[cwl,mulled,edam,extended-assertions]'
     else
diff --git a/packages/tool_shed/setup.cfg b/packages/tool_shed/setup.cfg
index 7cd328385896..0e9de75e1a4e 100644
--- a/packages/tool_shed/setup.cfg
+++ b/packages/tool_shed/setup.cfg
@@ -36,7 +36,7 @@ install_requires =
     galaxy-auth
     galaxy-config
     galaxy-data
-    galaxy-util
+    galaxy-util[image_util]
     galaxy-web-framework
     galaxy-web-stack
     galaxy-web-apps
diff --git a/packages/tool_util/setup.cfg b/packages/tool_util/setup.cfg
index eabca1a6e1b9..c0cdb4935917 100644
--- a/packages/tool_util/setup.cfg
+++ b/packages/tool_util/setup.cfg
@@ -33,7 +33,7 @@ version = 24.2.dev0
 [options]
 include_package_data = True
 install_requires =
-    galaxy-util>=22.1
+    galaxy-util[image_util]>=22.1
     conda-package-streaming
     lxml!=4.2.2
     MarkupSafe
diff --git a/packages/util/setup.cfg b/packages/util/setup.cfg
index bb6a232cc1e2..09380cd65be1 100644
--- a/packages/util/setup.cfg
+++ b/packages/util/setup.cfg
@@ -48,6 +48,8 @@ packages = find:
 python_requires = >=3.7
 
 [options.extras_require]
+image_util =
+    pillow
 jstree =
     dictobj
 template =
diff --git a/test/unit/util/test_checkers.py b/test/unit/util/test_checkers.py
index 1e30d25656cd..b474defc965a 100644
--- a/test/unit/util/test_checkers.py
+++ b/test/unit/util/test_checkers.py
@@ -1,6 +1,10 @@
+import os.path
 import tempfile
 
-from galaxy.util.checkers import check_html
+from galaxy.util.checkers import (
+    check_html,
+    check_image,
+)
 
 
 def test_check_html():
@@ -17,3 +21,10 @@ def test_check_html():
         tmpb.write(b"\x1f\x8b")
         tmpb.flush()
         assert not check_html(tmpb.name)
+
+
+def test_check_image():
+    for filename, expected in (("1.tiff", True), ("454Score.png", True), ("1.bam", False)):
+        path = f"test-data/{filename}"
+        assert os.path.exists(path)
+        assert check_image(path) is expected

From febd6f9413756f2c3e7144a65eb7417c22fd7a11 Mon Sep 17 00:00:00 2001
From: Nicola Soranzo <nicola.soranzo@earlham.ac.uk>
Date: Wed, 23 Oct 2024 11:57:55 +0100
Subject: [PATCH 47/84] Use fissix to replace lib2to3 under Python >=3.13

Add entries to `sys.modules` so that `future`/`past` can find
lib2to3 modules.
---
 .../dependencies/pinned-requirements.txt      |  1 +
 lib/galaxy/util/template.py                   | 20 +++++++++++++++++--
 packages/util/setup.cfg                       |  1 +
 pyproject.toml                                |  1 +
 4 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt
index 827d67b8354d..6227984653a6 100644
--- a/lib/galaxy/dependencies/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pinned-requirements.txt
@@ -66,6 +66,7 @@ email-validator==2.2.0
 exceptiongroup==1.2.2 ; python_full_version < '3.11'
 fastapi-slim==0.115.4
 filelock==3.16.1
+fissix==24.4.24 ; python_full_version >= '3.13'
 frozenlist==1.5.0
 fs==2.4.16
 fsspec==2024.10.0
diff --git a/lib/galaxy/util/template.py b/lib/galaxy/util/template.py
index 9b7f7965592e..01a2cd68c840 100644
--- a/lib/galaxy/util/template.py
+++ b/lib/galaxy/util/template.py
@@ -1,18 +1,34 @@
 """Entry point for the usage of Cheetah templating within Galaxy."""
 
+import sys
 import traceback
-from lib2to3.refactor import RefactoringTool
 
 from Cheetah.Compiler import Compiler
 from Cheetah.NameMapper import NotFound
 from Cheetah.Parser import ParseError
 from Cheetah.Template import Template
 from packaging.version import Version
-from past.translation import myfixes
 
 from galaxy.util.tree_dict import TreeDict
 from . import unicodify
 
+if sys.version_info >= (3, 13):
+    import fissix
+    from fissix import (
+        fixes as fissix_fixes,
+        pgen2 as fissix_pgen2,
+        refactor as fissix_refactor,
+    )
+
+    sys.modules["lib2to3"] = fissix
+    sys.modules["lib2to3.fixes"] = fissix_fixes
+    sys.modules["lib2to3.pgen2"] = fissix_pgen2
+    sys.modules["lib2to3.refactor"] = fissix_refactor
+
+from lib2to3.refactor import RefactoringTool
+
+from past.translation import myfixes
+
 # Skip libpasteurize fixers, which make sure code is py2 and py3 compatible.
 # This is not needed, we only translate code on py3.
 myfixes = [f for f in myfixes if not f.startswith("libpasteurize")]
diff --git a/packages/util/setup.cfg b/packages/util/setup.cfg
index 09380cd65be1..808606b77042 100644
--- a/packages/util/setup.cfg
+++ b/packages/util/setup.cfg
@@ -54,6 +54,7 @@ jstree =
     dictobj
 template =
     CT3>=3.3.3
+    fissix;python_version>='3.13'
     future>=1.0.0
 config_template =
     Jinja2
diff --git a/pyproject.toml b/pyproject.toml
index 145ddce07bf9..b50664156ef4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -37,6 +37,7 @@ dependencies = [
     "dparse",
     "edam-ontology",
     "fastapi-slim>=0.111.0",
+    "fissix ; python_version>='3.13'",
     "fs",
     "future>=1.0.0",  # Python 3.12 support
     "graphene",

From 8557fcb8a6327dadbdb03a6f96bffea6daf206e3 Mon Sep 17 00:00:00 2001
From: Martin Cech <marten@bx.psu.edu>
Date: Thu, 31 Oct 2024 13:28:18 +0100
Subject: [PATCH 48/84] avoid possible negative offset

---
 client/src/components/Libraries/LibraryFolder/LibraryFolder.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue b/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue
index 3d1ca0999131..e8ddce0e5a53 100644
--- a/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue
+++ b/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue
@@ -384,7 +384,7 @@ export default {
                     this.sortBy,
                     this.sortDesc,
                     this.perPage,
-                    (this.currentPage - 1) * this.perPage,
+                    (this.currentPage ? this.currentPage - 1 : 0) * this.perPage,
                     this.searchText
                 )
                 .then((response) => {

From f89c1d7185796cb21db469cf433dc39533f56d99 Mon Sep 17 00:00:00 2001
From: John Davis <jdavcs@gmail.com>
Date: Thu, 31 Oct 2024 16:49:42 -0400
Subject: [PATCH 49/84] Prevent purged users from logging in

Was possible if account was not deleted
---
 lib/galaxy/webapps/galaxy/controllers/user.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/galaxy/webapps/galaxy/controllers/user.py b/lib/galaxy/webapps/galaxy/controllers/user.py
index b0cb5ae5fe69..5ed60dbb706f 100644
--- a/lib/galaxy/webapps/galaxy/controllers/user.py
+++ b/lib/galaxy/webapps/galaxy/controllers/user.py
@@ -161,6 +161,9 @@ def __validate_login(self, trans, payload=None, **kwd):
             message, user = self.__autoregistration(trans, login, password)
             if message:
                 return self.message_exception(trans, message)
+        elif user.purged:
+            message = "This account has been permanently deleted."
+            return self.message_exception(trans, message, sanitize=False)
         elif user.deleted:
             message = (
                 "This account has been marked deleted, contact your local Galaxy administrator to restore the account."

From 92e1f199a66aab742ddb15220bbd1ad4fe9347de Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Thu, 31 Oct 2024 18:58:38 +0100
Subject: [PATCH 50/84] Add support for deferred datasets in visualizations

---
 .../History/Content/Dataset/DatasetActions.vue   |  2 +-
 .../visualization/plugins/config_parser.py       | 16 +++++++++++++++-
 lib/galaxy/visualization/plugins/registry.py     |  4 +++-
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/client/src/components/History/Content/Dataset/DatasetActions.vue b/client/src/components/History/Content/Dataset/DatasetActions.vue
index c16c88b70c2e..d4e7c6bd1174 100644
--- a/client/src/components/History/Content/Dataset/DatasetActions.vue
+++ b/client/src/components/History/Content/Dataset/DatasetActions.vue
@@ -47,7 +47,7 @@ const showRerun = computed(() => {
 });
 const showVisualizations = computed(() => {
     // TODO: Check hasViz, if visualizations are activated in the config
-    return !props.item.purged && ["ok", "failed_metadata", "error"].includes(props.item.state);
+    return !props.item.purged && ["ok", "failed_metadata", "error", "deferred"].includes(props.item.state);
 });
 const reportErrorUrl = computed(() => {
     return prependPath(props.itemUrls.reportError!);
diff --git a/lib/galaxy/visualization/plugins/config_parser.py b/lib/galaxy/visualization/plugins/config_parser.py
index 2e784abda7bc..ef6d6eb85a3d 100644
--- a/lib/galaxy/visualization/plugins/config_parser.py
+++ b/lib/galaxy/visualization/plugins/config_parser.py
@@ -301,8 +301,14 @@ def parse_tests(self, xml_tree_list):
             # result type should tell the registry how to convert the result before the test
             test_result_type = test_elem.get("result_type", "string")
 
+            # allow_deferred indicates that the visualization can work with deferred data_sources
+            # Can only be used with certain test types (e.g. isinstance)
+            allow_deferred = False
+
             # test functions should be sent an object to test, and the parsed result expected from the test
             if test_type == "isinstance":
+                allow_deferred = asbool(test_elem.get("allow_deferred", False))
+
                 # is test_attr attribute an instance of result
                 # TODO: wish we could take this further but it would mean passing in the datatypes_registry
                 def test_fn(o, result, getter=getter):
@@ -328,7 +334,15 @@ def test_fn(o, result, getter=getter):
                 def test_fn(o, result, getter=getter):
                     return str(getter(o)) == result
 
-            tests.append({"type": test_type, "result": test_result, "result_type": test_result_type, "fn": test_fn})
+            tests.append(
+                {
+                    "type": test_type,
+                    "result": test_result,
+                    "result_type": test_result_type,
+                    "fn": test_fn,
+                    "allow_deferred": allow_deferred,
+                }
+            )
 
         return tests
 
diff --git a/lib/galaxy/visualization/plugins/registry.py b/lib/galaxy/visualization/plugins/registry.py
index efc32e3f42c0..0f6690efbea7 100644
--- a/lib/galaxy/visualization/plugins/registry.py
+++ b/lib/galaxy/visualization/plugins/registry.py
@@ -263,11 +263,13 @@ def is_object_applicable(self, trans, target_object, data_source_tests):
         it can be applied to the target_object.
         """
         # log.debug( 'is_object_applicable( self, trans, %s, %s )', target_object, data_source_tests )
+        is_deferred_target = getattr(target_object, "state", None) == "deferred"
         for test in data_source_tests:
             test_type = test["type"]
             result_type = test["result_type"]
             test_result = test["result"]
             test_fn = test["fn"]
+            allow_deferred = test.get("allow_deferred", False)
             # log.debug( '%s %s: %s, %s, %s, %s', str( target_object ), 'is_object_applicable',
             #           test_type, result_type, test_result, test_fn )
 
@@ -286,7 +288,7 @@ def is_object_applicable(self, trans, target_object, data_source_tests):
                         continue
 
             # NOTE: tests are OR'd, if any test passes - the visualization can be applied
-            if test_fn(target_object, test_result):
+            if test_fn(target_object, test_result) and (not is_deferred_target or allow_deferred):
                 # log.debug( '\t test passed' )
                 return True
 

From 5a35684b288cb58f289da703721edf0e00e8fce9 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Fri, 1 Nov 2024 10:36:35 +0100
Subject: [PATCH 51/84] Refine deferred data source handling in visualization
 plugins

---
 lib/galaxy/visualization/plugins/config_parser.py | 2 +-
 lib/galaxy/visualization/plugins/registry.py      | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/visualization/plugins/config_parser.py b/lib/galaxy/visualization/plugins/config_parser.py
index ef6d6eb85a3d..199a4d9718f7 100644
--- a/lib/galaxy/visualization/plugins/config_parser.py
+++ b/lib/galaxy/visualization/plugins/config_parser.py
@@ -302,7 +302,7 @@ def parse_tests(self, xml_tree_list):
             test_result_type = test_elem.get("result_type", "string")
 
             # allow_deferred indicates that the visualization can work with deferred data_sources
-            # Can only be used with certain test types (e.g. isinstance)
+            # Can only be used with isinstance tests. By default, no visualization can work with deferred data_sources.
             allow_deferred = False
 
             # test functions should be sent an object to test, and the parsed result expected from the test
diff --git a/lib/galaxy/visualization/plugins/registry.py b/lib/galaxy/visualization/plugins/registry.py
index 0f6690efbea7..52b7f657146d 100644
--- a/lib/galaxy/visualization/plugins/registry.py
+++ b/lib/galaxy/visualization/plugins/registry.py
@@ -263,7 +263,7 @@ def is_object_applicable(self, trans, target_object, data_source_tests):
         it can be applied to the target_object.
         """
         # log.debug( 'is_object_applicable( self, trans, %s, %s )', target_object, data_source_tests )
-        is_deferred_target = getattr(target_object, "state", None) == "deferred"
+        is_deferred_target = self._is_deferred(target_object)
         for test in data_source_tests:
             test_type = test["type"]
             result_type = test["result_type"]
@@ -293,3 +293,6 @@ def is_object_applicable(self, trans, target_object, data_source_tests):
                 return True
 
         return False
+
+    def _is_deferred(self, target_object):
+        return getattr(target_object, "state", None) == "deferred"

From 0eb1e12af9a02c50f0751ea427c2f0b945aac2ab Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Fri, 1 Nov 2024 10:36:52 +0100
Subject: [PATCH 52/84] Add allow_deferred attribute to test element in
 visualization.dtd

---
 config/plugins/visualizations/visualization.dtd | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/config/plugins/visualizations/visualization.dtd b/config/plugins/visualizations/visualization.dtd
index dec57f0310ce..ed5cd7ced5b2 100644
--- a/config/plugins/visualizations/visualization.dtd
+++ b/config/plugins/visualizations/visualization.dtd
@@ -54,11 +54,17 @@
                 if result_type is 'datatype' the registry will assume the text is a datatype class name
                 and parse it into the proper class before the test (often 'isinstance') is run.
                 DEFAULT: no parsing (result should be a string)
+        allow_deferred: used in conjunction with type='isinstance' and test_attr='datatype'. If set to true,
+                the registry will allow the test to pass if the target is deferred (currently only HDAs can be deferred).
+                This is useful for visualizations that can work directly with URIs.
+                DEFAULT: false
+                
     -->
     <!ATTLIST test
-        type        CDATA #IMPLIED
-        test_attr   CDATA #IMPLIED
-        result_type CDATA #IMPLIED
+        type            CDATA #IMPLIED
+        test_attr       CDATA #IMPLIED
+        result_type     CDATA #IMPLIED
+        allow_deferred  CDATA #IMPLIED
     >
 
     <!ELEMENT to_param (#PCDATA)>

From 4be350b8e0c4763ed3e1a3228910eb8f135c8ad5 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Sat, 2 Nov 2024 13:28:26 +0100
Subject: [PATCH 53/84] Add support for URI protocols in visualization datatype
 checks

---
 .../plugins/visualizations/visualization.dtd  | 15 +++++----
 .../visualization/plugins/config_parser.py    | 17 ++++++----
 lib/galaxy/visualization/plugins/registry.py  | 33 ++++++++++++++++---
 3 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/config/plugins/visualizations/visualization.dtd b/config/plugins/visualizations/visualization.dtd
index ed5cd7ced5b2..a4640e9ca675 100644
--- a/config/plugins/visualizations/visualization.dtd
+++ b/config/plugins/visualizations/visualization.dtd
@@ -54,17 +54,18 @@
                 if result_type is 'datatype' the registry will assume the text is a datatype class name
                 and parse it into the proper class before the test (often 'isinstance') is run.
                 DEFAULT: no parsing (result should be a string)
-        allow_deferred: used in conjunction with type='isinstance' and test_attr='datatype'. If set to true,
-                the registry will allow the test to pass if the target is deferred (currently only HDAs can be deferred).
+        allow_uri_if_protocol: used in conjunction with type='isinstance' and test_attr='datatype'. Let you define
+                a list of protocols or schemes (e.g. 's3,https') that, in the case of a deferred target (e.g. currently only HDAs),
+                the registry will allow the test to pass if the the source URI has a scheme in the list.
                 This is useful for visualizations that can work directly with URIs.
-                DEFAULT: false
+                DEFAULT: []
                 
     -->
     <!ATTLIST test
-        type            CDATA #IMPLIED
-        test_attr       CDATA #IMPLIED
-        result_type     CDATA #IMPLIED
-        allow_deferred  CDATA #IMPLIED
+        type                    CDATA #IMPLIED
+        test_attr               CDATA #IMPLIED
+        result_type             CDATA #IMPLIED
+        allow_uri_if_protocol   CDATA #IMPLIED
     >
 
     <!ELEMENT to_param (#PCDATA)>
diff --git a/lib/galaxy/visualization/plugins/config_parser.py b/lib/galaxy/visualization/plugins/config_parser.py
index 199a4d9718f7..ffefbe2dfb8a 100644
--- a/lib/galaxy/visualization/plugins/config_parser.py
+++ b/lib/galaxy/visualization/plugins/config_parser.py
@@ -6,7 +6,10 @@
 )
 
 import galaxy.model
-from galaxy.util import asbool
+from galaxy.util import (
+    asbool,
+    listify,
+)
 from galaxy.util.xml_macros import load
 
 log = logging.getLogger(__name__)
@@ -301,13 +304,15 @@ def parse_tests(self, xml_tree_list):
             # result type should tell the registry how to convert the result before the test
             test_result_type = test_elem.get("result_type", "string")
 
-            # allow_deferred indicates that the visualization can work with deferred data_sources
-            # Can only be used with isinstance tests. By default, no visualization can work with deferred data_sources.
-            allow_deferred = False
+            # allow_uri_if_protocol indicates that the visualization can work with deferred data_sources which source URI
+            # matches any of the given protocols in this list. This is useful for visualizations that can work with URIs.
+            # Can only be used with isinstance tests. By default, an empty list means that the visualization doesn't support
+            # deferred data_sources.
+            allow_uri_if_protocol = []
 
             # test functions should be sent an object to test, and the parsed result expected from the test
             if test_type == "isinstance":
-                allow_deferred = asbool(test_elem.get("allow_deferred", False))
+                allow_uri_if_protocol = listify(test_elem.get("allow_uri_if_protocol"))
 
                 # is test_attr attribute an instance of result
                 # TODO: wish we could take this further but it would mean passing in the datatypes_registry
@@ -340,7 +345,7 @@ def test_fn(o, result, getter=getter):
                     "result": test_result,
                     "result_type": test_result_type,
                     "fn": test_fn,
-                    "allow_deferred": allow_deferred,
+                    "allow_uri_if_protocol": allow_uri_if_protocol,
                 }
             )
 
diff --git a/lib/galaxy/visualization/plugins/registry.py b/lib/galaxy/visualization/plugins/registry.py
index 52b7f657146d..2cb480fd15fd 100644
--- a/lib/galaxy/visualization/plugins/registry.py
+++ b/lib/galaxy/visualization/plugins/registry.py
@@ -8,6 +8,10 @@
 import logging
 import os
 import weakref
+from typing import (
+    List,
+    Optional,
+)
 
 from galaxy.exceptions import ObjectNotFound
 from galaxy.util import (
@@ -263,13 +267,12 @@ def is_object_applicable(self, trans, target_object, data_source_tests):
         it can be applied to the target_object.
         """
         # log.debug( 'is_object_applicable( self, trans, %s, %s )', target_object, data_source_tests )
-        is_deferred_target = self._is_deferred(target_object)
         for test in data_source_tests:
             test_type = test["type"]
             result_type = test["result_type"]
             test_result = test["result"]
             test_fn = test["fn"]
-            allow_deferred = test.get("allow_deferred", False)
+            supported_protocols = test.get("allow_uri_if_protocol", [])
             # log.debug( '%s %s: %s, %s, %s, %s', str( target_object ), 'is_object_applicable',
             #           test_type, result_type, test_result, test_fn )
 
@@ -288,11 +291,33 @@ def is_object_applicable(self, trans, target_object, data_source_tests):
                         continue
 
             # NOTE: tests are OR'd, if any test passes - the visualization can be applied
-            if test_fn(target_object, test_result) and (not is_deferred_target or allow_deferred):
+            if test_fn(target_object, test_result) and self._check_uri_support(target_object, supported_protocols):
                 # log.debug( '\t test passed' )
                 return True
 
         return False
 
-    def _is_deferred(self, target_object):
+    def _is_deferred(self, target_object) -> bool:
+        """Whether the target object is a deferred object."""
         return getattr(target_object, "state", None) == "deferred"
+
+    def _deferred_source_uri(self, target_object) -> Optional[str]:
+        """Get the source uri from a deferred object."""
+        sources = getattr(target_object, "sources", None)
+        if sources and sources[0]:
+            return sources[0].source_uri
+        return None
+
+    def _check_uri_support(self, target_object, supported_protocols: List[str]) -> bool:
+        """Test if the target object is deferred and has a supported protocol."""
+        if not self._is_deferred(target_object):
+            return True  # not deferred, so no uri to check
+
+        if not supported_protocols:
+            return False  # no protocols defined, means no support for deferred objects
+
+        deferred_source_uri = self._deferred_source_uri(target_object)
+        if deferred_source_uri:
+            protocol = deferred_source_uri.split("://")[0]
+            return protocol in supported_protocols
+        return False

From 688b08a15aba0e54ed2ae04158dec783b8ce9a03 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Sat, 2 Nov 2024 14:01:04 +0100
Subject: [PATCH 54/84] Add wildcard support for all protocols in visualization
 checks

---
 lib/galaxy/visualization/plugins/registry.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/galaxy/visualization/plugins/registry.py b/lib/galaxy/visualization/plugins/registry.py
index 2cb480fd15fd..bd293dca85b0 100644
--- a/lib/galaxy/visualization/plugins/registry.py
+++ b/lib/galaxy/visualization/plugins/registry.py
@@ -316,6 +316,9 @@ def _check_uri_support(self, target_object, supported_protocols: List[str]) -> b
         if not supported_protocols:
             return False  # no protocols defined, means no support for deferred objects
 
+        if "*" in supported_protocols:
+            return True  # wildcard support for all protocols
+
         deferred_source_uri = self._deferred_source_uri(target_object)
         if deferred_source_uri:
             protocol = deferred_source_uri.split("://")[0]

From 557e7bed8dcd9de8ed7aa2088a33eeeb34a4e2c4 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Sun, 3 Nov 2024 10:37:33 +0100
Subject: [PATCH 55/84] Add is_deferred property to DatasetFilenameWrapper

For improved handling of deferred data
---
 lib/galaxy/tools/wrappers.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/tools/wrappers.py b/lib/galaxy/tools/wrappers.py
index 6ad72645d7bf..eb6ef0beeee0 100644
--- a/lib/galaxy/tools/wrappers.py
+++ b/lib/galaxy/tools/wrappers.py
@@ -456,6 +456,10 @@ def serialize(self, invalid_chars: Sequence[str] = ("/",)) -> Dict[str, Any]:
     def is_collection(self) -> bool:
         return False
 
+    @property
+    def is_deferred(self) -> bool:
+        return self.unsanitized.has_deferred_data
+
     def is_of_type(self, *exts: str) -> bool:
         datatypes = []
         if not self.datatypes_registry:
@@ -474,7 +478,7 @@ def __str__(self) -> str:
         return self._path_or_uri()
 
     def _path_or_uri(self) -> str:
-        if self.unsanitized.has_deferred_data:
+        if self.is_deferred:
             return self.unsanitized.deferred_source_uri or ""
         if self.false_path is not None:
             return self.false_path

From 605389f1f2dd35d82f369c6a78d799947d30ad19 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Sun, 3 Nov 2024 11:06:25 +0100
Subject: [PATCH 56/84] Expand documentation on allow_uri_if_protocol

---
 lib/galaxy/tool_util/xsd/galaxy.xsd           | 22 +++++++++++++++++++
 .../parameters/gx_allow_uri_if_protocol.xml   | 13 ++++++++---
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index 376ff0e8d1e5..69e35696617b 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -4330,6 +4330,28 @@ You can specify multiple prefixes separated by comma or use the wildcard '*' to
 treat deferred datasets as URIs. The source URI will be passed to the tool as is.
 
 This attribute is only valid for `data` parameters.
+
+### Handling the input in the tool
+
+Since the input can be a regular file or a URI, the tool should be able to handle both cases.
+The tool should check if the input ``is_deferred`` and if so, treat it as a URI, otherwise
+it should treat it as a regular file. Please note that only deferred datasets with the specified
+protocol will be passed as URIs, the rest will be materialized as files.
+
+Here is an example command section that handles the above sample input:
+
+```python
+<command>
+  ## We should handle the case where the input must be treated as a URI with a specific protocol.
+  #if $input.is_deferred:
+      ## Here, the input is a deferred dataset which source URI has the protocol 'https', 'http' or 's3'.
+      echo '$input' > '$output' ## The ouput will be the source URI of the input.
+  #else:
+      ## Here, the input is a regular dataset or a materialized dataset in case of a 
+      ## deferred dataset which source URI has a protocol different than 'https', 'http' or 's3'.
+      cp '$input' '$output' ## The output will be a copy of the input content.
+</command>
+```
          ]]></xs:documentation>
           </xs:annotation>
         </xs:attribute>
diff --git a/test/functional/tools/parameters/gx_allow_uri_if_protocol.xml b/test/functional/tools/parameters/gx_allow_uri_if_protocol.xml
index ab554b470f70..7821b49718fb 100644
--- a/test/functional/tools/parameters/gx_allow_uri_if_protocol.xml
+++ b/test/functional/tools/parameters/gx_allow_uri_if_protocol.xml
@@ -1,6 +1,15 @@
 <tool id="gx_allow_uri_if_protocol" name="gx_allow_uri_if_protocol" version="1.0.0">
     <command><![CDATA[
-echo '$input' > '$output'
+        ## We should handle the case where the input must be treated as a URI with a specific protocol.
+        #if $input.is_deferred:
+            ## Here, the input is a deferred dataset which source URI has the protocol 'https'.
+            ## The ouput will be the source URI of the input.
+            echo '$input' > '$output'
+        #else:
+            ## Here, the input is a regular dataset or a materialized dataset in case of a 
+            ## deferred dataset which source URI has a protocol different than 'https'.
+            ## The output will be a copy of the input content.
+            cp '$input' '$output'
     ]]></command>
     <inputs>
         <param name="input" type="data" allow_uri_if_protocol="https"/>
@@ -8,6 +17,4 @@ echo '$input' > '$output'
     <outputs>
         <data name="output" format="txt"/>
     </outputs>
-    <tests>
-    </tests>
 </tool>

From 53e8a237472fd0a7afeb6d44962a03dd4337c295 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Sun, 3 Nov 2024 12:43:24 +0100
Subject: [PATCH 57/84] Refactor deferred source URI handling in
 DatasetInstance and ToolEvaluator

---
 lib/galaxy/model/__init__.py   | 2 +-
 lib/galaxy/tools/evaluation.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 7656a98fbed1..02be7ec606da 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -4688,7 +4688,7 @@ def has_deferred_data(self):
 
     @property
     def deferred_source_uri(self):
-        if self.has_deferred_data:
+        if self.has_deferred_data and self.sources:
             # Assuming the first source is the deferred source
             return self.sources[0].source_uri
         return None
diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py
index 8e9328c71408..582bd65be06e 100644
--- a/lib/galaxy/tools/evaluation.py
+++ b/lib/galaxy/tools/evaluation.py
@@ -335,7 +335,7 @@ def _should_materialize_deferred_input(self, input_name: str, input_value: Defer
         """
         deferred_input = self.tool.inputs.get(input_name)
         if isinstance(deferred_input, DataToolParameter) and isinstance(input_value, model.DatasetInstance):
-            source_uri = input_value.sources[0].source_uri or ""
+            source_uri = input_value.deferred_source_uri or ""
             for prefix in deferred_input.allow_uri_if_protocol:
                 if prefix == "*" or source_uri.startswith(prefix):
                     return False

From 2bdcec0dda2d9e862fa7a043de6b0207223e43ca Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Sun, 3 Nov 2024 12:47:15 +0100
Subject: [PATCH 58/84] Add test for gx_allow_uri_if_protocol with deferred
 collections

---
 lib/galaxy_test/api/test_tools.py             | 47 +++++++++++++++++++
 .../parameters/gx_allow_uri_if_protocol.xml   | 25 +++++-----
 2 files changed, 61 insertions(+), 11 deletions(-)

diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py
index 41d610f7b29e..5842fb6a4e9c 100644
--- a/lib/galaxy_test/api/test_tools.py
+++ b/lib/galaxy_test/api/test_tools.py
@@ -2591,6 +2591,53 @@ def test_allow_uri_if_protocol_on_deferred_input(self, history_id):
         output_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output)
         assert output_content.strip() == source_uri.strip()
 
+    @skip_without_tool("gx_allow_uri_if_protocol")
+    def test_allow_uri_if_protocol_on_collection_with_deferred(self, history_id):
+        source_uris = [
+            "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/simple_line.txt",
+            "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/simple_line_alternative.txt",
+        ]
+        elements = [
+            {
+                "src": "url",
+                "url": source_uri,
+                "deferred": True,
+                "ext": "txt",
+            }
+            for source_uri in source_uris
+        ]
+        targets = [
+            {
+                "destination": {"type": "hdca"},
+                "elements": elements,
+                "collection_type": "list",
+                "name": "deferred list",
+            }
+        ]
+        payload = {
+            "history_id": history_id,
+            "targets": json.dumps(targets),
+        }
+        fetch_response = self.dataset_populator.fetch(payload, wait=True)
+        dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
+        hdca_id = dataset_collection["id"]
+        inputs = {
+            "input1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]},
+        }
+        run_response = self.dataset_populator.run_tool(
+            tool_id="gx_allow_uri_if_protocol", inputs=inputs, history_id=history_id
+        )
+        hdca_id = run_response["implicit_collections"][0]["id"]
+        dataset_collection = self.dataset_populator.get_history_collection_details(history_id, id=hdca_id)
+        elements = dataset_collection["elements"]
+        assert len(elements) == 2
+        for element in elements:
+            object = element["object"]
+            assert isinstance(object, dict)
+            assert object["state"] == "ok"
+            output_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=object)
+            assert output_content.strip() in source_uris
+
     @skip_without_tool("cat1")
     def test_run_deferred_mapping(self, history_id: str):
         elements = [
diff --git a/test/functional/tools/parameters/gx_allow_uri_if_protocol.xml b/test/functional/tools/parameters/gx_allow_uri_if_protocol.xml
index 7821b49718fb..b6cd4d6d4205 100644
--- a/test/functional/tools/parameters/gx_allow_uri_if_protocol.xml
+++ b/test/functional/tools/parameters/gx_allow_uri_if_protocol.xml
@@ -1,18 +1,21 @@
 <tool id="gx_allow_uri_if_protocol" name="gx_allow_uri_if_protocol" version="1.0.0">
     <command><![CDATA[
-        ## We should handle the case where the input must be treated as a URI with a specific protocol.
-        #if $input.is_deferred:
-            ## Here, the input is a deferred dataset which source URI has the protocol 'https'.
-            ## The ouput will be the source URI of the input.
-            echo '$input' > '$output'
-        #else:
-            ## Here, the input is a regular dataset or a materialized dataset in case of a 
-            ## deferred dataset which source URI has a protocol different than 'https'.
-            ## The output will be a copy of the input content.
-            cp '$input' '$output'
+        #for $input in $input1:
+            ## We should handle the case where the input must be treated as a URI with a specific protocol.
+            #if $input.is_deferred:
+                ## Here, the input is a deferred dataset which source URI has the protocol 'https'.
+                ## We append the URI to the output file.
+                echo '$input' >> '$output'
+            #else:
+                ## Here, the input is a regular dataset or a materialized dataset in case of a 
+                ## deferred dataset which source URI has a protocol different than 'https'.
+                ## We append the content of the dataset to the output file.
+                cat '$input' >> '$output'
+            #end if
+        #end for
     ]]></command>
     <inputs>
-        <param name="input" type="data" allow_uri_if_protocol="https"/>
+        <param name="input1" type="data" allow_uri_if_protocol="https" multiple="true"/>
     </inputs>
     <outputs>
         <data name="output" format="txt"/>

From e5b300b9c1a20dd6f05971c3e293f99b96c1c21a Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Wed, 23 Oct 2024 12:12:28 -0400
Subject: [PATCH 59/84] Add job metrics per invocation

---
 client/src/api/schema/schema.ts               | 100 +++++++++++
 .../MetricsBoxPlots.vue                       | 169 ++++++++++++++++++
 .../WorkflowInvocationMetrics.vue             |  52 ++++++
 .../WorkflowInvocationState.vue               |   4 +
 lib/galaxy/managers/jobs.py                   |  39 +++-
 lib/galaxy/schema/schema.py                   |   4 +
 lib/galaxy/webapps/galaxy/api/workflows.py    |   9 +
 .../webapps/galaxy/services/invocations.py    |  14 ++
 lib/galaxy_test/api/test_workflows.py         |  33 ++++
 9 files changed, 423 insertions(+), 1 deletion(-)
 create mode 100644 client/src/components/WorkflowInvocationState/MetricsBoxPlots.vue
 create mode 100644 client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue

diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts
index 93d99b75fa0a..73cb76dd7af6 100644
--- a/client/src/api/schema/schema.ts
+++ b/client/src/api/schema/schema.ts
@@ -2594,6 +2594,23 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
+    "/api/invocations/{invocation_id}/metrics": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        /** Get Invocation Metrics */
+        get: operations["get_invocation_metrics_api_invocations__invocation_id__metrics_get"];
+        put?: never;
+        post?: never;
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/api/invocations/{invocation_id}/prepare_store_download": {
         parameters: {
             query?: never;
@@ -18317,6 +18334,45 @@ export interface components {
                 [key: string]: number;
             };
         };
+        /**
+         * WorkflowJobMetric
+         * @example {
+         *       "name": "start_epoch",
+         *       "plugin": "core",
+         *       "raw_value": "1614261340.0000000",
+         *       "title": "Job Start Time",
+         *       "value": "2021-02-25 14:55:40"
+         *     }
+         */
+        WorkflowJobMetric: {
+            /**
+             * Name
+             * @description The name of the metric variable.
+             */
+            name: string;
+            /**
+             * Plugin
+             * @description The instrumenter plugin that generated this metric.
+             */
+            plugin: string;
+            /**
+             * Raw Value
+             * @description The raw value of the metric as a string.
+             */
+            raw_value: string;
+            /**
+             * Title
+             * @description A descriptive title for this metric.
+             */
+            title: string;
+            /** Tool Id */
+            tool_id: string;
+            /**
+             * Value
+             * @description The textual representation of the metric value.
+             */
+            value: string;
+        };
         /** WorkflowLandingRequest */
         WorkflowLandingRequest: {
             /** Request State */
@@ -27021,6 +27077,50 @@ export interface operations {
             };
         };
     };
+    get_invocation_metrics_api_invocations__invocation_id__metrics_get: {
+        parameters: {
+            query?: never;
+            header?: {
+                /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
+                "run-as"?: string | null;
+            };
+            path: {
+                /** @description The encoded database identifier of the Invocation. */
+                invocation_id: string;
+            };
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["WorkflowJobMetric"][];
+                };
+            };
+            /** @description Request Error */
+            "4XX": {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["MessageExceptionModel"];
+                };
+            };
+            /** @description Server Error */
+            "5XX": {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["MessageExceptionModel"];
+                };
+            };
+        };
+    };
     prepare_store_download_api_invocations__invocation_id__prepare_store_download_post: {
         parameters: {
             query?: never;
diff --git a/client/src/components/WorkflowInvocationState/MetricsBoxPlots.vue b/client/src/components/WorkflowInvocationState/MetricsBoxPlots.vue
new file mode 100644
index 000000000000..1876190cc903
--- /dev/null
+++ b/client/src/components/WorkflowInvocationState/MetricsBoxPlots.vue
@@ -0,0 +1,169 @@
+<template>
+  <div ref="boxPlot"></div>
+</template>
+
+<script setup lang="ts">
+import { ref, onMounted } from 'vue';
+import * as d3 from 'd3';
+import { isTerminal } from './util';
+
+interface JobData {
+  title: string;
+  value: string;
+  plugin: string;
+  name: string;
+  raw_value: string;
+  tool_id: string;
+}
+
+
+const boxPlot = ref<HTMLElement | null>(null);
+
+// Define the job data (this can come from props or API in real scenario// Extract runtime seconds data
+// Define props
+const props = defineProps<{
+  jobData: JobData[];
+}>();
+
+// Function to render box plots for different tool_ids on the same graph
+const renderBoxPlots = () => {
+  const margin = { top: 10, right: 30, bottom: 50, left: 40 },
+    width = 800 - margin.left - margin.right,
+    height = 400 - margin.top - margin.bottom;
+
+  // Remove any previous content
+  d3.select(boxPlot.value).selectAll('*').remove();
+
+  // Group data by tool_id
+  const groupedData = d3.group(props.jobData, d => d.tool_id);
+
+  // Extract the tool_ids for the x-axis
+  const toolIds = Array.from(groupedData.keys());
+
+  // Create SVG
+  const svg = d3
+    .select(boxPlot.value)
+    .append('svg')
+    .attr('width', width + margin.left + margin.right)
+    .attr('height', height + margin.top + margin.bottom)
+    .append('g')
+    .attr('transform', `translate(${margin.left},${margin.top})`);
+
+  // X Scale: tool_ids on the x-axis
+  const x = d3.scaleBand()
+    .range([0, width])
+    .domain(toolIds)
+    .padding(0.4);
+
+  svg.append('g')
+    .attr('transform', `translate(0,${height})`)
+    .call(d3.axisBottom(x));
+
+  // Y Scale: common for all box plots
+  const allRuntimes = Array.from(props.jobData)
+    .filter(d => d.name === "runtime_seconds")
+    .map(d => parseFloat(d.raw_value));
+
+  const y = d3.scaleLinear()
+    .domain([0, d3.max(allRuntimes) as number])
+    .range([height, 0]);
+
+  svg.append('g').call(d3.axisLeft(y));
+
+  // Tooltip for individual dots
+  const tooltip = d3.select(boxPlot.value)
+    .append("div")
+    .style("position", "absolute")
+    .style("background", "lightgray")
+    .style("padding", "5px")
+    .style("border-radius", "5px")
+    .style("pointer-events", "none")
+    .style("visibility", "hidden");
+
+  // Render box plots for each tool_id
+  groupedData.forEach((data, toolId) => {
+    const runtimeData = data.map(d => parseFloat(d.raw_value));
+
+    const center = x(toolId) as number + x.bandwidth() / 2;
+    const boxWidth = x.bandwidth() / 2;
+
+    const q1 = d3.quantile(runtimeData.sort(d3.ascending), 0.25) as number;
+    const median = d3.quantile(runtimeData.sort(d3.ascending), 0.5) as number;
+    const q3 = d3.quantile(runtimeData.sort(d3.ascending), 0.75) as number;
+    const interQuantileRange = q3 - q1;
+    const min = Math.max(d3.min(runtimeData) as number, q1 - 1.5 * interQuantileRange);
+    const max = Math.min(d3.max(runtimeData) as number, q3 + 1.5 * interQuantileRange);
+
+    // Box
+    svg.append('rect')
+      .attr('x', center - boxWidth / 2)
+      .attr('y', y(q3))
+      .attr('height', y(q1) - y(q3))
+      .attr('width', boxWidth)
+      .attr('stroke', 'black')
+      .style('fill', '#69b3a2');
+
+    // Median, min, max lines
+    svg.selectAll('line')
+      .data([min, median, max])
+      .enter()
+      .append('line')
+      .attr('x1', center - boxWidth / 2)
+      .attr('x2', center + boxWidth / 2)
+      .attr('y1', d => y(d))
+      .attr('y2', d => y(d))
+      .attr('stroke', 'black');
+
+    // Vertical lines for min-max
+    svg.append('line')
+      .attr('x1', center)
+      .attr('x2', center)
+      .attr('y1', y(min))
+      .attr('y2', y(q1))
+      .attr('stroke', 'black');
+
+    svg.append('line')
+      .attr('x1', center)
+      .attr('x2', center)
+      .attr('y1', y(max))
+      .attr('y2', y(q3))
+      .attr('stroke', 'black');
+
+    // Add individual data points (dots)
+    svg.selectAll("circle")
+      .data(runtimeData)
+      .enter()
+      .append("circle")
+      .attr("cx", center) // Align horizontally to the center of the group
+      .attr("cy", d => y(d)) // Map the data value to the y-axis
+      .attr("r", 5) // Radius of the circle
+      .style("fill", "#ff5722") // Fill color for the points
+      .attr("stroke", "black")
+      .on("mouseover", function (event, d) {
+        tooltip.style("visibility", "visible")
+          .text(`Value: ${d}`);
+        d3.select(this).style("fill", "yellow");
+      })
+      .on("mousemove", function (event) {
+        tooltip.style("top", `${event.pageY - 10}px`)
+          .style("left", `${event.pageX + 10}px`);
+      })
+      .on("mouseout", function () {
+        tooltip.style("visibility", "hidden");
+        d3.select(this).style("fill", "#ff5722");
+      });
+  });
+};
+
+// Lifecycle hook to render the plot once the component is mounted
+onMounted(() => {
+  renderBoxPlots();
+});
+
+</script>
+
+<style scoped>
+svg {
+  font-family: Arial, sans-serif;
+}
+</style>
diff --git a/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue b/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue
new file mode 100644
index 000000000000..cb9aed99233f
--- /dev/null
+++ b/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue
@@ -0,0 +1,52 @@
+<script setup lang="ts">
+import { BCard } from "bootstrap-vue";
+import { GalaxyApi, type components } from "@/api";
+import { computed, ref, watch } from "vue";
+import type JobMetrics from "../JobMetrics/JobMetrics.vue";
+import { errorMessageAsString } from "@/utils/simple-error";
+import MetricsBoxPlot from "./MetricsBoxPlots.vue";
+
+const props = defineProps({
+    invocationId: {
+        type: String,
+        required: true,
+    },
+});
+
+
+const jobMetrics = ref<components["schemas"]["JobMetric"][]>()
+const fetchError = ref<string>()
+
+async function fetchMetrics() {
+    const {data, error} = await GalaxyApi().GET("/api/invocations/{invocation_id}/metrics", {
+        params: {
+            path: {
+                invocation_id: props.invocationId
+            }
+        }
+    })
+    console.log("data", data, error);
+    if (error) {
+        fetchError.value = errorMessageAsString(error)
+    } else {
+        jobMetrics.value = data
+    }
+}
+
+
+watch((props), () => fetchMetrics(), { immediate: true })
+
+const wallclock = computed(() => {
+    return jobMetrics.value?.filter((jobMetric) => jobMetric.name == "runtime_seconds")
+})
+
+</script>
+
+<template>
+    <MetricsBoxPlot v-if="wallclock" :job-data="wallclock"/>
+    <!--
+    <div>
+        {{ jobMetrics }}
+    </div>  
+    -->
+</template>
diff --git a/client/src/components/WorkflowInvocationState/WorkflowInvocationState.vue b/client/src/components/WorkflowInvocationState/WorkflowInvocationState.vue
index e035bc9b7dbe..bcf03f61b816 100644
--- a/client/src/components/WorkflowInvocationState/WorkflowInvocationState.vue
+++ b/client/src/components/WorkflowInvocationState/WorkflowInvocationState.vue
@@ -13,6 +13,7 @@ import { isTerminal, jobCount, runningCount } from "./util";
 
 import InvocationReport from "../Workflow/InvocationReport.vue";
 import WorkflowInvocationExportOptions from "./WorkflowInvocationExportOptions.vue";
+import WorkflowInvocationMetrics from "./WorkflowInvocationMetrics.vue";
 import WorkflowInvocationHeader from "./WorkflowInvocationHeader.vue";
 import WorkflowInvocationInputOutputTabs from "./WorkflowInvocationInputOutputTabs.vue";
 import WorkflowInvocationOverview from "./WorkflowInvocationOverview.vue";
@@ -185,6 +186,9 @@ function cancelWorkflowSchedulingLocal() {
                     <LoadingSpan message="Waiting to complete invocation" />
                 </BAlert>
             </BTab>
+            <BTab title="Metrics">
+                <WorkflowInvocationMetrics :invocation-id="invocation.id"></WorkflowInvocationMetrics>
+            </BTab>
         </BTabs>
     </div>
     <BAlert v-else-if="errorMessage" variant="danger" show>
diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py
index 1d4a61dac6f0..be9d0083567b 100644
--- a/lib/galaxy/managers/jobs.py
+++ b/lib/galaxy/managers/jobs.py
@@ -54,6 +54,7 @@
     ImplicitCollectionJobs,
     ImplicitCollectionJobsJobAssociation,
     Job,
+    JobMetricNumeric,
     JobParameter,
     User,
     Workflow,
@@ -729,6 +730,38 @@ def invocation_job_source_iter(sa_session, invocation_id):
             yield ("ImplicitCollectionJobs", row[1], row[2])
 
 
+def get_job_metrics_for_invocation(sa_session: galaxy_scoped_session, invocation_id: int):
+    single_job_stmnt = (
+        select(JobMetricNumeric, Job.tool_id)
+        .join(Job, JobMetricNumeric.job_id == Job.id)
+        .join(
+            WorkflowInvocationStep,
+            and_(
+                WorkflowInvocationStep.workflow_invocation_id == invocation_id, WorkflowInvocationStep.job_id == Job.id
+            ),
+        )
+    )
+    collection_job_stmnt = (
+        select(JobMetricNumeric, Job.tool_id)
+        .join(Job, JobMetricNumeric.job_id == Job.id)
+        .join(ImplicitCollectionJobsJobAssociation, Job.id == ImplicitCollectionJobsJobAssociation.job_id)
+        .join(
+            ImplicitCollectionJobs,
+            ImplicitCollectionJobs.id == ImplicitCollectionJobsJobAssociation.implicit_collection_jobs_id,
+        )
+        .join(
+            WorkflowInvocationStep,
+            and_(
+                WorkflowInvocationStep.workflow_invocation_id == invocation_id,
+                WorkflowInvocationStep.implicit_collection_jobs_id == ImplicitCollectionJobs.id,
+            ),
+        )
+    )
+    # should be sa_session.execute(single_job_stmnt.union(collection_job_stmnt)).all() but that returns
+    # columns instead of the job metrics ORM instance.
+    return list(sa_session.execute(single_job_stmnt).all()) + list(sa_session.execute(collection_job_stmnt).all())
+
+
 def fetch_job_states(sa_session, job_source_ids, job_source_types):
     assert len(job_source_ids) == len(job_source_types)
     job_ids = set()
@@ -911,6 +944,10 @@ def summarize_job_metrics(trans, job):
     Precondition: the caller has verified the job is accessible to the user
     represented by the trans parameter.
     """
+    return summarize_metrics(trans, job.metrics)
+
+
+def summarize_metrics(trans: ProvidesUserContext, job_metrics):
     safety_level = Safety.SAFE
     if trans.user_is_admin:
         safety_level = Safety.UNSAFE
@@ -922,7 +959,7 @@ def summarize_job_metrics(trans, job):
             m.metric_value,
             m.plugin,
         )
-        for m in job.metrics
+        for m in job_metrics
     ]
     dictifiable_metrics = trans.app.job_metrics.dictifiable_metrics(raw_metrics, safety_level)
     return [d.dict() for d in dictifiable_metrics]
diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py
index 0d9646316e59..88460a3f425f 100644
--- a/lib/galaxy/schema/schema.py
+++ b/lib/galaxy/schema/schema.py
@@ -2171,6 +2171,10 @@ class JobMetric(Model):
     )
 
 
+class WorkflowJobMetric(JobMetric):
+    tool_id: str
+
+
 class JobMetricCollection(RootModel):
     """Represents a collection of metrics associated with a Job."""
 
diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py
index c4abb88968c0..9348e25a977b 100644
--- a/lib/galaxy/webapps/galaxy/api/workflows.py
+++ b/lib/galaxy/webapps/galaxy/api/workflows.py
@@ -78,6 +78,7 @@
     ShareWithPayload,
     ShareWithStatus,
     SharingStatus,
+    WorkflowJobMetric,
     WorkflowLandingRequest,
     WorkflowSortByEnum,
 )
@@ -1759,3 +1760,11 @@ def workflow_invocation_jobs_summary(
     ) -> InvocationJobsResponse:
         """An alias for `GET /api/invocations/{invocation_id}/jobs_summary`. `workflow_id` is ignored."""
         return self.invocation_jobs_summary(trans=trans, invocation_id=invocation_id)
+
+    @router.get("/api/invocations/{invocation_id}/metrics")
+    def get_invocation_metrics(
+        self,
+        invocation_id: InvocationIDPathParam,
+        trans: ProvidesHistoryContext = DependsOnTrans,
+    ) -> List[WorkflowJobMetric]:
+        return self.invocations_service.show_invocation_metrics(trans=trans, invocation_id=invocation_id)
diff --git a/lib/galaxy/webapps/galaxy/services/invocations.py b/lib/galaxy/webapps/galaxy/services/invocations.py
index ce9610ac0422..4066a3dbfeaa 100644
--- a/lib/galaxy/webapps/galaxy/services/invocations.py
+++ b/lib/galaxy/webapps/galaxy/services/invocations.py
@@ -25,7 +25,9 @@
 from galaxy.managers.histories import HistoryManager
 from galaxy.managers.jobs import (
     fetch_job_states,
+    get_job_metrics_for_invocation,
     invocation_job_source_iter,
+    summarize_metrics,
 )
 from galaxy.managers.workflows import WorkflowsManager
 from galaxy.model import (
@@ -162,6 +164,18 @@ def show_invocation_step(self, trans, step_id) -> InvocationStep:
         )
         return self.serialize_workflow_invocation_step(wfi_step)
 
+    def show_invocation_metrics(self, trans: ProvidesHistoryContext, invocation_id: int):
+        extended_job_metrics = get_job_metrics_for_invocation(trans.sa_session, invocation_id)
+        job_metrics = []
+        tool_ids = []
+        for row in extended_job_metrics:
+            job_metrics.append(row[0])
+            tool_ids.append(row[1])
+        metrics_dict_list = summarize_metrics(trans, job_metrics)
+        for tool_id, metrics_dict in zip(tool_ids, metrics_dict_list):
+            metrics_dict["tool_id"] = tool_id
+        return metrics_dict_list
+
     def update_invocation_step(self, trans, step_id, action):
         wfi_step = self._workflows_manager.update_invocation_step(trans, step_id, action)
         return self.serialize_workflow_invocation_step(wfi_step)
diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py
index 174985ad51d9..f18cdd8a760d 100644
--- a/lib/galaxy_test/api/test_workflows.py
+++ b/lib/galaxy_test/api/test_workflows.py
@@ -3499,6 +3499,39 @@ def test_workflow_new_autocreated_history(self):
         invocation_id = run_workflow_dict["id"]
         self.workflow_populator.wait_for_invocation_and_jobs(new_history_id, workflow_id, invocation_id)
 
+    def test_invocation_job_metrics_simple(self):
+        with self.dataset_populator.test_history() as history_id:
+            summary = self._run_workflow(WORKFLOW_SIMPLE, test_data={"input1": "hello world"}, history_id=history_id)
+            self.workflow_populator.wait_for_invocation_and_jobs(
+                history_id=history_id, workflow_id=summary.workflow_id, invocation_id=summary.invocation_id
+            )
+            job_metrics = self._get(f"invocations/{summary.invocation_id}/metrics").json()
+            galaxy_slots = [m for m in job_metrics if m["name"] == "galaxy_slots"]
+            assert len(galaxy_slots) == 1
+
+    def test_invocation_job_metrics_map_over(self):
+        with self.dataset_populator.test_history() as history_id:
+            summary = self._run_workflow(
+                WORKFLOW_SIMPLE,
+                test_data={
+                    "input1": {
+                        "collection_type": "list",
+                        "name": "the_dataset_list",
+                        "elements": [
+                            {"identifier": "el1", "value": "1.fastq", "type": "File"},
+                            {"identifier": "el2", "value": "1.fastq", "type": "File"},
+                        ],
+                    }
+                },
+                history_id=history_id,
+            )
+            self.workflow_populator.wait_for_invocation_and_jobs(
+                history_id=history_id, workflow_id=summary.workflow_id, invocation_id=summary.invocation_id
+            )
+            job_metrics = self._get(f"invocations/{summary.invocation_id}/metrics").json()
+            galaxy_slots = [m for m in job_metrics if m["name"] == "galaxy_slots"]
+            assert len(galaxy_slots) == 2
+
     def test_workflow_output_dataset(self):
         with self.dataset_populator.test_history() as history_id:
             summary = self._run_workflow(WORKFLOW_SIMPLE, test_data={"input1": "hello world"}, history_id=history_id)

From bcf212dab41b4fede0223083536eb1ee5648f565 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Wed, 23 Oct 2024 22:16:02 -0400
Subject: [PATCH 60/84] Add basic boxplot

---
 client/package.json                           |   5 +
 .../MetricsBoxPlots.vue                       | 169 ------
 .../WorkflowInvocationState/VegaWrapper.vue   |  45 ++
 .../WorkflowInvocationMetrics.vue             | 151 +++++-
 .../WorkflowInvocationState.vue               |   4 +-
 client/yarn.lock                              | 511 +++++++++++++++++-
 6 files changed, 673 insertions(+), 212 deletions(-)
 delete mode 100644 client/src/components/WorkflowInvocationState/MetricsBoxPlots.vue
 create mode 100644 client/src/components/WorkflowInvocationState/VegaWrapper.vue

diff --git a/client/package.json b/client/package.json
index a4f2c4b695d2..b2a751ff3d94 100644
--- a/client/package.json
+++ b/client/package.json
@@ -62,6 +62,7 @@
     "dom-to-image": "^2.6.0",
     "dompurify": "^3.0.6",
     "dumpmeta-webpack-plugin": "^0.2.0",
+    "echarts": "^5.5.1",
     "elkjs": "^0.8.2",
     "file-saver": "^2.0.5",
     "flush-promises": "^1.0.2",
@@ -100,7 +101,11 @@
     "tus-js-client": "^3.1.1",
     "underscore": "^1.13.6",
     "util": "^0.12.5",
+    "vega": "^5.30.0",
+    "vega-embed": "^6.26.0",
+    "vega-lite": "^5.21.0",
     "vue": "^2.7.14",
+    "vue-echarts": "^7.0.3",
     "vue-infinite-scroll": "^2.0.2",
     "vue-multiselect": "^2.1.7",
     "vue-observe-visibility": "^1.0.0",
diff --git a/client/src/components/WorkflowInvocationState/MetricsBoxPlots.vue b/client/src/components/WorkflowInvocationState/MetricsBoxPlots.vue
deleted file mode 100644
index 1876190cc903..000000000000
--- a/client/src/components/WorkflowInvocationState/MetricsBoxPlots.vue
+++ /dev/null
@@ -1,169 +0,0 @@
-<template>
-  <div ref="boxPlot"></div>
-</template>
-
-<script setup lang="ts">
-import { ref, onMounted } from 'vue';
-import * as d3 from 'd3';
-import { isTerminal } from './util';
-
-interface JobData {
-  title: string;
-  value: string;
-  plugin: string;
-  name: string;
-  raw_value: string;
-  tool_id: string;
-}
-
-
-const boxPlot = ref<HTMLElement | null>(null);
-
-// Define the job data (this can come from props or API in real scenario// Extract runtime seconds data
-// Define props
-const props = defineProps<{
-  jobData: JobData[];
-}>();
-
-// Function to render box plots for different tool_ids on the same graph
-const renderBoxPlots = () => {
-  const margin = { top: 10, right: 30, bottom: 50, left: 40 },
-    width = 800 - margin.left - margin.right,
-    height = 400 - margin.top - margin.bottom;
-
-  // Remove any previous content
-  d3.select(boxPlot.value).selectAll('*').remove();
-
-  // Group data by tool_id
-  const groupedData = d3.group(props.jobData, d => d.tool_id);
-
-  // Extract the tool_ids for the x-axis
-  const toolIds = Array.from(groupedData.keys());
-
-  // Create SVG
-  const svg = d3
-    .select(boxPlot.value)
-    .append('svg')
-    .attr('width', width + margin.left + margin.right)
-    .attr('height', height + margin.top + margin.bottom)
-    .append('g')
-    .attr('transform', `translate(${margin.left},${margin.top})`);
-
-  // X Scale: tool_ids on the x-axis
-  const x = d3.scaleBand()
-    .range([0, width])
-    .domain(toolIds)
-    .padding(0.4);
-
-  svg.append('g')
-    .attr('transform', `translate(0,${height})`)
-    .call(d3.axisBottom(x));
-
-  // Y Scale: common for all box plots
-  const allRuntimes = Array.from(props.jobData)
-    .filter(d => d.name === "runtime_seconds")
-    .map(d => parseFloat(d.raw_value));
-
-  const y = d3.scaleLinear()
-    .domain([0, d3.max(allRuntimes) as number])
-    .range([height, 0]);
-
-  svg.append('g').call(d3.axisLeft(y));
-
-  // Tooltip for individual dots
-  const tooltip = d3.select(boxPlot.value)
-    .append("div")
-    .style("position", "absolute")
-    .style("background", "lightgray")
-    .style("padding", "5px")
-    .style("border-radius", "5px")
-    .style("pointer-events", "none")
-    .style("visibility", "hidden");
-
-  // Render box plots for each tool_id
-  groupedData.forEach((data, toolId) => {
-    const runtimeData = data.map(d => parseFloat(d.raw_value));
-
-    const center = x(toolId) as number + x.bandwidth() / 2;
-    const boxWidth = x.bandwidth() / 2;
-
-    const q1 = d3.quantile(runtimeData.sort(d3.ascending), 0.25) as number;
-    const median = d3.quantile(runtimeData.sort(d3.ascending), 0.5) as number;
-    const q3 = d3.quantile(runtimeData.sort(d3.ascending), 0.75) as number;
-    const interQuantileRange = q3 - q1;
-    const min = Math.max(d3.min(runtimeData) as number, q1 - 1.5 * interQuantileRange);
-    const max = Math.min(d3.max(runtimeData) as number, q3 + 1.5 * interQuantileRange);
-
-    // Box
-    svg.append('rect')
-      .attr('x', center - boxWidth / 2)
-      .attr('y', y(q3))
-      .attr('height', y(q1) - y(q3))
-      .attr('width', boxWidth)
-      .attr('stroke', 'black')
-      .style('fill', '#69b3a2');
-
-    // Median, min, max lines
-    svg.selectAll('line')
-      .data([min, median, max])
-      .enter()
-      .append('line')
-      .attr('x1', center - boxWidth / 2)
-      .attr('x2', center + boxWidth / 2)
-      .attr('y1', d => y(d))
-      .attr('y2', d => y(d))
-      .attr('stroke', 'black');
-
-    // Vertical lines for min-max
-    svg.append('line')
-      .attr('x1', center)
-      .attr('x2', center)
-      .attr('y1', y(min))
-      .attr('y2', y(q1))
-      .attr('stroke', 'black');
-
-    svg.append('line')
-      .attr('x1', center)
-      .attr('x2', center)
-      .attr('y1', y(max))
-      .attr('y2', y(q3))
-      .attr('stroke', 'black');
-
-    // Add individual data points (dots)
-    svg.selectAll("circle")
-      .data(runtimeData)
-      .enter()
-      .append("circle")
-      .attr("cx", center) // Align horizontally to the center of the group
-      .attr("cy", d => y(d)) // Map the data value to the y-axis
-      .attr("r", 5) // Radius of the circle
-      .style("fill", "#ff5722") // Fill color for the points
-      .attr("stroke", "black")
-      .on("mouseover", function (event, d) {
-        tooltip.style("visibility", "visible")
-          .text(`Value: ${d}`);
-        d3.select(this).style("fill", "yellow");
-      })
-      .on("mousemove", function (event) {
-        tooltip.style("top", `${event.pageY - 10}px`)
-          .style("left", `${event.pageX + 10}px`);
-      })
-      .on("mouseout", function () {
-        tooltip.style("visibility", "hidden");
-        d3.select(this).style("fill", "#ff5722");
-      });
-  });
-};
-
-// Lifecycle hook to render the plot once the component is mounted
-onMounted(() => {
-  renderBoxPlots();
-});
-
-</script>
-
-<style scoped>
-svg {
-  font-family: Arial, sans-serif;
-}
-</style>
diff --git a/client/src/components/WorkflowInvocationState/VegaWrapper.vue b/client/src/components/WorkflowInvocationState/VegaWrapper.vue
new file mode 100644
index 000000000000..586fa124cf0e
--- /dev/null
+++ b/client/src/components/WorkflowInvocationState/VegaWrapper.vue
@@ -0,0 +1,45 @@
+<template>
+    <div ref="chartContainer" class="chart"></div>
+</template>
+
+<script setup lang="ts">
+import embed, { type VisualizationSpec } from "vega-embed";
+import { onBeforeUnmount, onMounted, ref, watch } from "vue";
+
+export interface VisSpec {
+    spec: VisualizationSpec;
+}
+
+const props = defineProps<VisSpec>();
+
+const chartContainer = ref<HTMLDivElement | null>(null);
+let vegaView: any;
+
+async function embedChart() {
+    if (vegaView) {
+        vegaView.finalize();
+    }
+    if (chartContainer.value !== null) {
+        const result = await embed(chartContainer.value, props.spec, { renderer: "svg" });
+        vegaView = result.view;
+    }
+}
+
+onMounted(embedChart);
+
+watch(props, embedChart, { immediate: true, deep: true });
+
+// Cleanup the chart when the component is unmounted
+onBeforeUnmount(() => {
+    if (vegaView) {
+        vegaView.finalize();
+    }
+});
+</script>
+
+<style scoped>
+.chart {
+    width: 100%;
+    height: 100%;
+}
+</style>
diff --git a/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue b/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue
index cb9aed99233f..a924bc54aed1 100644
--- a/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue
+++ b/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue
@@ -1,10 +1,12 @@
 <script setup lang="ts">
-import { BCard } from "bootstrap-vue";
-import { GalaxyApi, type components } from "@/api";
 import { computed, ref, watch } from "vue";
-import type JobMetrics from "../JobMetrics/JobMetrics.vue";
+
+import { type components, GalaxyApi } from "@/api";
 import { errorMessageAsString } from "@/utils/simple-error";
-import MetricsBoxPlot from "./MetricsBoxPlots.vue";
+import { type ComputedRef } from "vue";
+import { type VisualizationSpec } from "vega-embed";
+
+import VegaWrapper from "./VegaWrapper.vue";
 
 const props = defineProps({
     invocationId: {
@@ -13,40 +15,141 @@ const props = defineProps({
     },
 });
 
-
-const jobMetrics = ref<components["schemas"]["JobMetric"][]>()
-const fetchError = ref<string>()
+const jobMetrics = ref<components["schemas"]["WorkflowJobMetric"][]>();
+const fetchError = ref<string>();
 
 async function fetchMetrics() {
-    const {data, error} = await GalaxyApi().GET("/api/invocations/{invocation_id}/metrics", {
+    const { data, error } = await GalaxyApi().GET("/api/invocations/{invocation_id}/metrics", {
         params: {
             path: {
-                invocation_id: props.invocationId
-            }
-        }
-    })
-    console.log("data", data, error);
+                invocation_id: props.invocationId,
+            },
+        },
+    });
     if (error) {
-        fetchError.value = errorMessageAsString(error)
+        fetchError.value = errorMessageAsString(error);
     } else {
-        jobMetrics.value = data
+        jobMetrics.value = data;
     }
 }
 
+watch(props, () => fetchMetrics(), { immediate: true });
+
+interface boxplotData {
+    x_title: string;
+    y_title: string;
+    values?: { x: string; y: Number }[];
+}
+
+const wallclock: ComputedRef<boxplotData> = computed(() => {
+    const wallclock = jobMetrics.value?.filter((jobMetric) => jobMetric.name == "runtime_seconds");
+    const values = wallclock?.map((item) => {
+        return {
+            y: parseFloat(item.raw_value),
+            x: item.tool_id,
+        };
+    });
+    return {
+        x_title: "Tool ID",
+        y_title: "Runtime (in Seconds)",
+        values,
+    };
+});
+
+const coresAllocated: ComputedRef<boxplotData> = computed(() => {
+    const coresAllocated = jobMetrics.value?.filter((jobMetric) => jobMetric.name == "galaxy_slots");
+    const values = coresAllocated?.map((item) => {
+        return {
+            y: parseFloat(item.raw_value),
+            x: item.tool_id,
+        };
+    });
+    return {
+        x_title: "Tool ID",
+        y_title: "Cores Allocated",
+        values,
+    };
+});
 
-watch((props), () => fetchMetrics(), { immediate: true })
+const memoryAllocated: ComputedRef<boxplotData> = computed(() => {
+    const memoryAllocated = jobMetrics.value?.filter((jobMetric) => jobMetric.name == "galaxy_memory_mb");
+    const values = memoryAllocated?.map((item) => {
+        return {
+            y: parseFloat(item.raw_value),
+            x: item.tool_id,
+        };
+    });
+    return {
+        x_title: "Tool ID",
+        y_title: "Memory Allocated (in MB)",
+        values,
+    };
+});
 
-const wallclock = computed(() => {
-    return jobMetrics.value?.filter((jobMetric) => jobMetric.name == "runtime_seconds")
-})
+function itemToSpec(item: boxplotData) {
+    const spec: VisualizationSpec = {
+        $schema: "https://vega.github.io/schema/vega-lite/v5.json",
+        description: "A boxplot with jittered points.",
+        data: {
+            values: item.values!,
+        },
+        transform: [
+            {
+                calculate: "random() - 0.5",
+                as: "random_jitter",
+            },
+        ],
+        layer: [
+            {
+                mark: { type: "boxplot", opacity: 0.5 },
+                encoding: {
+                    x: { field: "x", type: "nominal" },
+                    y: { field: "y", type: "quantitative" },
+                },
+                width: "container",
+            },
+            {
+                mark: {
+                    type: "point",
+                    opacity: 0.7,
+                },
+                encoding: {
+                    x: {
+                        field: "x",
+                        type: "nominal",
+                        title: item.x_title,
+                        axis: {
+                            labelAngle: -45,
+                            labelAlign: "right",
+                        },
+                    },
+                    xOffset: { field: "random_jitter", type: "quantitative", scale: { domain: [-2, 2] } },
+                    y: {
+                        field: "y",
+                        type: "quantitative",
+                        scale: { zero: false },
+                        title: item.y_title,
+                    },
+                },
+                width: "container",
+            },
+        ],
+    };
+    return spec;
+}
 
+const specs = computed(() => {
+    const items = [wallclock.value, coresAllocated.value, memoryAllocated.value].filter((item) => item.values?.length);
+    const specs = Object.fromEntries(items.map((item) => [item.y_title, itemToSpec(item)]));
+    return specs;
+});
 </script>
 
 <template>
-    <MetricsBoxPlot v-if="wallclock" :job-data="wallclock"/>
-    <!--
     <div>
-        {{ jobMetrics }}
-    </div>  
-    -->
+        <div v-for="(spec, key) in specs" :key="key">
+            <h2 class="h-l truncate text-center">{{ key }}</h2>
+            <VegaWrapper :spec="spec" />
+        </div>
+    </div>
 </template>
diff --git a/client/src/components/WorkflowInvocationState/WorkflowInvocationState.vue b/client/src/components/WorkflowInvocationState/WorkflowInvocationState.vue
index bcf03f61b816..bbc3d4a4f443 100644
--- a/client/src/components/WorkflowInvocationState/WorkflowInvocationState.vue
+++ b/client/src/components/WorkflowInvocationState/WorkflowInvocationState.vue
@@ -13,9 +13,9 @@ import { isTerminal, jobCount, runningCount } from "./util";
 
 import InvocationReport from "../Workflow/InvocationReport.vue";
 import WorkflowInvocationExportOptions from "./WorkflowInvocationExportOptions.vue";
-import WorkflowInvocationMetrics from "./WorkflowInvocationMetrics.vue";
 import WorkflowInvocationHeader from "./WorkflowInvocationHeader.vue";
 import WorkflowInvocationInputOutputTabs from "./WorkflowInvocationInputOutputTabs.vue";
+import WorkflowInvocationMetrics from "./WorkflowInvocationMetrics.vue";
 import WorkflowInvocationOverview from "./WorkflowInvocationOverview.vue";
 import LoadingSpan from "@/components/LoadingSpan.vue";
 
@@ -186,7 +186,7 @@ function cancelWorkflowSchedulingLocal() {
                     <LoadingSpan message="Waiting to complete invocation" />
                 </BAlert>
             </BTab>
-            <BTab title="Metrics">
+            <BTab title="Metrics" :lazy="true">
                 <WorkflowInvocationMetrics :invocation-id="invocation.id"></WorkflowInvocationMetrics>
             </BTab>
         </BTabs>
diff --git a/client/yarn.lock b/client/yarn.lock
index 46d7ba982257..48e01581c8b9 100644
--- a/client/yarn.lock
+++ b/client/yarn.lock
@@ -2507,6 +2507,11 @@
   dependencies:
     "@types/trusted-types" "*"
 
+"@types/estree@^1.0.0":
+  version "1.0.6"
+  resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.6.tgz#628effeeae2064a1b4e79f78e81d87b7e5fc7b50"
+  integrity sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==
+
 "@types/estree@^1.0.5":
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4"
@@ -2536,6 +2541,11 @@
   resolved "https://registry.yarnpkg.com/@types/geojson/-/geojson-7946.0.10.tgz#6dfbf5ea17142f7f9a043809f1cd4c448cb68249"
   integrity sha512-Nmh0K3iWQJzniTuPRcJn5hxXkfB1T1pgB89SBig5PlJQU5yocazeu4jATJlaA0GYFKWMqDdvYemoSnF2pXgLVA==
 
+"@types/geojson@7946.0.4":
+  version "7946.0.4"
+  resolved "https://registry.yarnpkg.com/@types/geojson/-/geojson-7946.0.4.tgz#4e049756383c3f055dd8f3d24e63fb543e98eb07"
+  integrity sha512-MHmwBtCb7OCv1DSivz2UNJXPGU/1btAWRKlqJ2saEhVJkpkvqHMMaOpKg0v4sAbDWSQekHGvPVMM8nQ+Jen03Q==
+
 "@types/graceful-fs@^4.1.3":
   version "4.1.5"
   resolved "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz"
@@ -4314,6 +4324,11 @@ combined-stream@^1.0.8:
   dependencies:
     delayed-stream "~1.0.0"
 
+commander@2, commander@^2.19.0, commander@^2.20.0:
+  version "2.20.3"
+  resolved "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz"
+  integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==
+
 commander@7, commander@^7.2.0:
   version "7.2.0"
   resolved "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz"
@@ -4324,11 +4339,6 @@ commander@^10.0.1:
   resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.1.tgz#881ee46b4f77d1c1dccc5823433aa39b022cbe06"
   integrity sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==
 
-commander@^2.19.0, commander@^2.20.0:
-  version "2.20.3"
-  resolved "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz"
-  integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==
-
 component-emitter@^1.2.1:
   version "1.3.0"
   resolved "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz"
@@ -4722,6 +4732,13 @@ custom-error-instance@2.1.1:
   resolved "https://registry.npmjs.org/custom-error-instance/-/custom-error-instance-2.1.1.tgz"
   integrity sha512-p6JFxJc3M4OTD2li2qaHkDCw9SfMw82Ldr6OC9Je1aXiGfhx2W8p3GaoeaGrPJTUN9NirTM/KTxHWMUdR1rsUg==
 
+"d3-array@1 - 3", d3-array@3.2.4, d3-array@^3.2.2:
+  version "3.2.4"
+  resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-3.2.4.tgz#15fec33b237f97ac5d7c986dc77da273a8ed0bb5"
+  integrity sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==
+  dependencies:
+    internmap "1 - 2"
+
 "d3-array@2 - 3", "d3-array@2.10.0 - 3", "d3-array@2.5.0 - 3", d3-array@3, d3-array@^3.2.0:
   version "3.2.1"
   resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-3.2.1.tgz#39331ea706f5709417d31bbb6ec152e0328b39b3"
@@ -4752,7 +4769,7 @@ d3-chord@3:
   dependencies:
     d3-path "1 - 3"
 
-"d3-color@1 - 3", d3-color@3:
+"d3-color@1 - 3", d3-color@3, d3-color@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-3.1.0.tgz#395b2833dfac71507f12ac2f7af23bf819de24e2"
   integrity sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==
@@ -4771,6 +4788,13 @@ d3-delaunay@6:
   dependencies:
     delaunator "5"
 
+d3-delaunay@^6.0.2:
+  version "6.0.4"
+  resolved "https://registry.yarnpkg.com/d3-delaunay/-/d3-delaunay-6.0.4.tgz#98169038733a0a5babbeda55054f795bb9e4a58b"
+  integrity sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==
+  dependencies:
+    delaunator "5"
+
 "d3-dispatch@1 - 3", d3-dispatch@3:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/d3-dispatch/-/d3-dispatch-3.0.1.tgz#5fc75284e9c2375c36c839411a0cf550cbfc4d5e"
@@ -4784,7 +4808,7 @@ d3-delaunay@6:
     d3-dispatch "1 - 3"
     d3-selection "3"
 
-"d3-dsv@1 - 3", d3-dsv@3:
+"d3-dsv@1 - 3", d3-dsv@3, d3-dsv@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/d3-dsv/-/d3-dsv-3.0.1.tgz#c63af978f4d6a0d084a52a673922be2160789b73"
   integrity sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==
@@ -4805,7 +4829,7 @@ d3-fetch@3:
   dependencies:
     d3-dsv "1 - 3"
 
-d3-force@3:
+d3-force@3, d3-force@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/d3-force/-/d3-force-3.0.0.tgz#3e2ba1a61e70888fe3d9194e30d6d14eece155c4"
   integrity sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==
@@ -4814,11 +4838,27 @@ d3-force@3:
     d3-quadtree "1 - 3"
     d3-timer "1 - 3"
 
-"d3-format@1 - 3", d3-format@3:
+"d3-format@1 - 3", d3-format@3, d3-format@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-3.1.0.tgz#9260e23a28ea5cb109e93b21a06e24e2ebd55641"
   integrity sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==
 
+d3-geo-projection@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/d3-geo-projection/-/d3-geo-projection-4.0.0.tgz#dc229e5ead78d31869a4e87cf1f45bd2716c48ca"
+  integrity sha512-p0bK60CEzph1iqmnxut7d/1kyTmm3UWtPlwdkM31AU+LW+BXazd5zJdoCn7VFxNCHXRngPHRnsNn5uGjLRGndg==
+  dependencies:
+    commander "7"
+    d3-array "1 - 3"
+    d3-geo "1.12.0 - 3"
+
+"d3-geo@1.12.0 - 3", d3-geo@^3.1.0:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/d3-geo/-/d3-geo-3.1.1.tgz#6027cf51246f9b2ebd64f99e01dc7c3364033a4d"
+  integrity sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q==
+  dependencies:
+    d3-array "2.5.0 - 3"
+
 d3-geo@3:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/d3-geo/-/d3-geo-3.1.0.tgz#74fd54e1f4cebd5185ac2039217a98d39b0a4c0e"
@@ -4826,12 +4866,12 @@ d3-geo@3:
   dependencies:
     d3-array "2.5.0 - 3"
 
-d3-hierarchy@3:
+d3-hierarchy@3, d3-hierarchy@^3.1.2:
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz#b01cd42c1eed3d46db77a5966cf726f8c09160c6"
   integrity sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==
 
-"d3-interpolate@1 - 3", "d3-interpolate@1.2.0 - 3", d3-interpolate@3:
+"d3-interpolate@1 - 3", "d3-interpolate@1.2.0 - 3", d3-interpolate@3, d3-interpolate@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/d3-interpolate/-/d3-interpolate-3.0.1.tgz#3c47aa5b32c5b3dfb56ef3fd4342078a632b400d"
   integrity sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==
@@ -4866,7 +4906,15 @@ d3-scale-chromatic@3:
     d3-color "1 - 3"
     d3-interpolate "1 - 3"
 
-d3-scale@4:
+d3-scale-chromatic@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz#34c39da298b23c20e02f1a4b239bd0f22e7f1314"
+  integrity sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==
+  dependencies:
+    d3-color "1 - 3"
+    d3-interpolate "1 - 3"
+
+d3-scale@4, d3-scale@^4.0.2:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/d3-scale/-/d3-scale-4.0.2.tgz#82b38e8e8ff7080764f8dcec77bd4be393689396"
   integrity sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==
@@ -4882,28 +4930,28 @@ d3-scale@4:
   resolved "https://registry.yarnpkg.com/d3-selection/-/d3-selection-3.0.0.tgz#c25338207efa72cc5b9bd1458a1a41901f1e1b31"
   integrity sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==
 
-d3-shape@3:
+d3-shape@3, d3-shape@^3.2.0:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/d3-shape/-/d3-shape-3.2.0.tgz#a1a839cbd9ba45f28674c69d7f855bcf91dfc6a5"
   integrity sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==
   dependencies:
     d3-path "^3.1.0"
 
-"d3-time-format@2 - 4", d3-time-format@4:
+"d3-time-format@2 - 4", d3-time-format@4, d3-time-format@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/d3-time-format/-/d3-time-format-4.1.0.tgz#7ab5257a5041d11ecb4fe70a5c7d16a195bb408a"
   integrity sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==
   dependencies:
     d3-time "1 - 3"
 
-"d3-time@1 - 3", "d3-time@2.1.1 - 3", d3-time@3:
+"d3-time@1 - 3", "d3-time@2.1.1 - 3", d3-time@3, d3-time@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-3.1.0.tgz#9310db56e992e3c0175e1ef385e545e48a9bb5c7"
   integrity sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==
   dependencies:
     d3-array "2 - 3"
 
-"d3-timer@1 - 3", d3-timer@3:
+"d3-timer@1 - 3", d3-timer@3, d3-timer@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/d3-timer/-/d3-timer-3.0.1.tgz#6284d2a2708285b1abb7e201eda4380af35e63b0"
   integrity sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==
@@ -5327,6 +5375,14 @@ eastasianwidth@^0.2.0:
   resolved "https://registry.yarnpkg.com/eastasianwidth/-/eastasianwidth-0.2.0.tgz#696ce2ec0aa0e6ea93a397ffcf24aa7840c827cb"
   integrity sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==
 
+echarts@^5.5.1:
+  version "5.5.1"
+  resolved "https://registry.yarnpkg.com/echarts/-/echarts-5.5.1.tgz#8dc9c68d0c548934bedcb5f633db07ed1dd2101c"
+  integrity sha512-Fce8upazaAXUVUVsjgV6mBnGuqgO+JNDlcgF79Dksy4+wgGpQB2lmYoO4TSweFg/mZITdpGHomw/cNBJZj1icA==
+  dependencies:
+    tslib "2.3.0"
+    zrender "5.6.0"
+
 editorconfig@^0.15.3:
   version "0.15.3"
   resolved "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz"
@@ -6071,6 +6127,11 @@ fast-glob@^3.3.0:
     merge2 "^1.3.0"
     micromatch "^4.0.4"
 
+fast-json-patch@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/fast-json-patch/-/fast-json-patch-3.1.1.tgz#85064ea1b1ebf97a3f7ad01e23f9337e72c66947"
+  integrity sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ==
+
 fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0, fast-json-stable-stringify@^2.1.0:
   version "2.1.0"
   resolved "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz"
@@ -8129,6 +8190,11 @@ json-stable-stringify-without-jsonify@^1.0.1:
   resolved "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz"
   integrity sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==
 
+json-stringify-pretty-compact@^3.0.0, json-stringify-pretty-compact@~3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/json-stringify-pretty-compact/-/json-stringify-pretty-compact-3.0.0.tgz#f71ef9d82ef16483a407869556588e91b681d9ab"
+  integrity sha512-Rc2suX5meI0S3bfdZuA7JMFBGkJ875ApfVyq2WHELjBiiG22My/l7/8zPpH/CfFVQHuVLd8NLR0nv6vi0BYYKA==
+
 json5@^1.0.1:
   version "1.0.1"
   resolved "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz"
@@ -10526,6 +10592,11 @@ semver@^7.3.4, semver@^7.3.6, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4:
   dependencies:
     lru-cache "^6.0.0"
 
+semver@^7.6.2:
+  version "7.6.3"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
+  integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
+
 send@0.19.0:
   version "0.19.0"
   resolved "https://registry.yarnpkg.com/send/-/send-0.19.0.tgz#bbc5a388c8ea6c048967049dbeac0e4a3f09d7f8"
@@ -11367,6 +11438,13 @@ toidentifier@1.0.1:
   resolved "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz"
   integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==
 
+topojson-client@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/topojson-client/-/topojson-client-3.1.0.tgz#22e8b1ed08a2b922feeb4af6f53b6ef09a467b99"
+  integrity sha512-605uxS6bcYxGXw9qi62XyrV6Q3xwbndjachmNxu8HWTtVPxZfEJN9fd/SZS1Q54Sn2y0TMyMxFj/cJINqGHrKw==
+  dependencies:
+    commander "2"
+
 tough-cookie@^4.1.2:
   version "4.1.3"
   resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf"
@@ -11468,6 +11546,11 @@ tsconfig@^7.0.0:
     strip-bom "^3.0.0"
     strip-json-comments "^2.0.0"
 
+tslib@2.3.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.0.tgz#803b8cdab3e12ba581a4ca41c8839bbb0dacb09e"
+  integrity sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==
+
 tslib@^1.9.0:
   version "1.14.1"
   resolved "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz"
@@ -11488,6 +11571,16 @@ tslib@^2.4.0:
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae"
   integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==
 
+tslib@^2.6.3:
+  version "2.8.0"
+  resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.0.tgz#d124c86c3c05a40a91e6fdea4021bd31d377971b"
+  integrity sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA==
+
+tslib@~2.6.3:
+  version "2.6.3"
+  resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.3.tgz#0438f810ad7a9edcde7a241c3d80db693c8cbfe0"
+  integrity sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==
+
 tus-js-client@^3.1.1:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/tus-js-client/-/tus-js-client-3.1.1.tgz#87cb72e528d274d0a8ff62e9c18165f1e901ce9e"
@@ -11837,6 +11930,371 @@ vary@~1.1.2:
   resolved "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz"
   integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==
 
+vega-canvas@^1.2.7:
+  version "1.2.7"
+  resolved "https://registry.yarnpkg.com/vega-canvas/-/vega-canvas-1.2.7.tgz#cf62169518f5dcd91d24ad352998c2248f8974fb"
+  integrity sha512-OkJ9CACVcN9R5Pi9uF6MZBF06pO6qFpDYHWSKBJsdHP5o724KrsgR6UvbnXFH82FdsiTOff/HqjuaG8C7FL+9Q==
+
+vega-crossfilter@~4.1.2:
+  version "4.1.2"
+  resolved "https://registry.yarnpkg.com/vega-crossfilter/-/vega-crossfilter-4.1.2.tgz#810281c279b3592310f12814bc61206dd42ca61d"
+  integrity sha512-J7KVEXkpfRJBfRvwLxn5vNCzQCNkrnzmDvkvwhuiwT4gPm5sk7MK5TuUP8GCl/iKYw+kWeVXEtrVHwWtug+bcQ==
+  dependencies:
+    d3-array "^3.2.2"
+    vega-dataflow "^5.7.6"
+    vega-util "^1.17.2"
+
+vega-dataflow@^5.7.6, vega-dataflow@~5.7.6:
+  version "5.7.6"
+  resolved "https://registry.yarnpkg.com/vega-dataflow/-/vega-dataflow-5.7.6.tgz#21dfad9120cb18d9aeaed578658670839d1adc95"
+  integrity sha512-9Md8+5iUC1MVKPKDyZ7pCEHk6I9am+DgaMzZqo/27O/KI4f23/WQXPyuI8jbNmc/mkm340P0TKREmzL5M7+2Dg==
+  dependencies:
+    vega-format "^1.1.2"
+    vega-loader "^4.5.2"
+    vega-util "^1.17.2"
+
+vega-embed@^6.26.0:
+  version "6.26.0"
+  resolved "https://registry.yarnpkg.com/vega-embed/-/vega-embed-6.26.0.tgz#25ca51783b2819adf6e6330ae6dd5771e8da8653"
+  integrity sha512-AZCTdKHDAuhp6TFZRQOOs332tStCwZr/5e4uZMNEuJL69A57cT66NNZJdNiCP6u66REzIToYtMJhMTL9wl5B3A==
+  dependencies:
+    fast-json-patch "^3.1.1"
+    json-stringify-pretty-compact "^3.0.0"
+    semver "^7.6.2"
+    tslib "^2.6.3"
+    vega-interpreter "^1.0.5"
+    vega-schema-url-parser "^2.2.0"
+    vega-themes "^2.15.0"
+    vega-tooltip "^0.34.0"
+
+vega-encode@~4.10.1:
+  version "4.10.1"
+  resolved "https://registry.yarnpkg.com/vega-encode/-/vega-encode-4.10.1.tgz#1656e20396db99c414f495704ef3d9cff99631df"
+  integrity sha512-d25nVKZDrg109rC65M8uxE+7iUrTxktaqgK4fU3XZBgpWlh1K4UbU5nDag7kiHVVN4tKqwgd+synEotra9TiVQ==
+  dependencies:
+    d3-array "^3.2.2"
+    d3-interpolate "^3.0.1"
+    vega-dataflow "^5.7.6"
+    vega-scale "^7.4.1"
+    vega-util "^1.17.2"
+
+vega-event-selector@^3.0.1, vega-event-selector@~3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/vega-event-selector/-/vega-event-selector-3.0.1.tgz#b99e92147b338158f8079d81b28b2e7199c2e259"
+  integrity sha512-K5zd7s5tjr1LiOOkjGpcVls8GsH/f2CWCrWcpKy74gTCp+llCdwz0Enqo013ZlGaRNjfgD/o1caJRt3GSaec4A==
+
+vega-expression@^5.0.1, vega-expression@^5.1.1, vega-expression@~5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/vega-expression/-/vega-expression-5.1.1.tgz#9b2d287a1f34d990577c9798ae68ec88453815ef"
+  integrity sha512-zv9L1Hm0KHE9M7mldHyz8sXbGu3KmC0Cdk7qfHkcTNS75Jpsem6jkbu6ZAwx5cNUeW91AxUQOu77r4mygq2wUQ==
+  dependencies:
+    "@types/estree" "^1.0.0"
+    vega-util "^1.17.2"
+
+vega-force@~4.2.1:
+  version "4.2.1"
+  resolved "https://registry.yarnpkg.com/vega-force/-/vega-force-4.2.1.tgz#bdce6ec8572867b4ff2fb7e09d2894798c5358ec"
+  integrity sha512-2BcuuqFr77vcCyKfcpedNFeYMxi+XEFCrlgLWNx7YV0PI8pdP5y/yPkzyuE9Tb894+KkRAvfQHZRAshcnFNcMw==
+  dependencies:
+    d3-force "^3.0.0"
+    vega-dataflow "^5.7.6"
+    vega-util "^1.17.2"
+
+vega-format@^1.1.2, vega-format@~1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/vega-format/-/vega-format-1.1.2.tgz#d344ba8a2680144e92127459c149a4181e9e7f84"
+  integrity sha512-0kUfAj0dg0U6GcEY0Kp6LiSTCZ8l8jl1qVdQyToMyKmtZg/q56qsiJQZy3WWRr1MtWkTIZL71xSJXgjwjeUaAw==
+  dependencies:
+    d3-array "^3.2.2"
+    d3-format "^3.1.0"
+    d3-time-format "^4.1.0"
+    vega-time "^2.1.2"
+    vega-util "^1.17.2"
+
+vega-functions@^5.15.0, vega-functions@~5.15.0:
+  version "5.15.0"
+  resolved "https://registry.yarnpkg.com/vega-functions/-/vega-functions-5.15.0.tgz#a7905e1dd6457efe265dbf954cbc0a5721c484b0"
+  integrity sha512-pCqmm5efd+3M65jrJGxEy3UGuRksmK6DnWijoSNocnxdCBxez+yqUUVX9o2pN8VxMe3648vZnR9/Vk5CXqRvIQ==
+  dependencies:
+    d3-array "^3.2.2"
+    d3-color "^3.1.0"
+    d3-geo "^3.1.0"
+    vega-dataflow "^5.7.6"
+    vega-expression "^5.1.1"
+    vega-scale "^7.4.1"
+    vega-scenegraph "^4.13.0"
+    vega-selections "^5.4.2"
+    vega-statistics "^1.9.0"
+    vega-time "^2.1.2"
+    vega-util "^1.17.2"
+
+vega-geo@~4.4.2:
+  version "4.4.2"
+  resolved "https://registry.yarnpkg.com/vega-geo/-/vega-geo-4.4.2.tgz#da4a08ee39c9488bfc4fe6493779f584dd8bb412"
+  integrity sha512-unuV/UxUHf6UJu6GYxMZonC3SZlMfFXYLOkgEsRSvmsMPt3+CVv8FmG88dXNRUJUrdROrJepgecqx0jOwMSnGA==
+  dependencies:
+    d3-array "^3.2.2"
+    d3-color "^3.1.0"
+    d3-geo "^3.1.0"
+    vega-canvas "^1.2.7"
+    vega-dataflow "^5.7.6"
+    vega-projection "^1.6.1"
+    vega-statistics "^1.9.0"
+    vega-util "^1.17.2"
+
+vega-hierarchy@~4.1.2:
+  version "4.1.2"
+  resolved "https://registry.yarnpkg.com/vega-hierarchy/-/vega-hierarchy-4.1.2.tgz#e42938c42527b392b110b1e3bf89eaa456dba1b8"
+  integrity sha512-m+xDtT5092YPSnV0rdTLW+AWmoCb+A54JQ66MUJwiDBpKxvfKnTiQeuiWDU2YudjUoXZN9EBOcI6QHF8H2Lu2A==
+  dependencies:
+    d3-hierarchy "^3.1.2"
+    vega-dataflow "^5.7.6"
+    vega-util "^1.17.2"
+
+vega-interpreter@^1.0.5:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/vega-interpreter/-/vega-interpreter-1.0.5.tgz#19e1d1b5f84a4ea9cb25c4e90a05ce16cd058484"
+  integrity sha512-po6oTOmeQqr1tzTCdD15tYxAQLeUnOVirAysgVEemzl+vfmvcEP7jQmlc51jz0jMA+WsbmE6oJywisQPu/H0Bg==
+
+vega-label@~1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/vega-label/-/vega-label-1.3.0.tgz#21b3e5ef40e63f51ac987a449d183068c4961503"
+  integrity sha512-EfSFSCWAwVPsklM5g0gUEuohALgryuGC/SKMmsOH7dYT/bywmLBZhLVbrE+IHJAUauoGrMhYw1mqnXL/0giJBg==
+  dependencies:
+    vega-canvas "^1.2.7"
+    vega-dataflow "^5.7.6"
+    vega-scenegraph "^4.13.0"
+    vega-util "^1.17.2"
+
+vega-lite@^5.21.0:
+  version "5.21.0"
+  resolved "https://registry.yarnpkg.com/vega-lite/-/vega-lite-5.21.0.tgz#21ce8b905a02ba364b7b1d7ef471497ba3e12e93"
+  integrity sha512-hNxM9nuMqpI1vkUOhEx6ewEf23WWLmJxSFJ4TA86AW43ixJyqcLV+iSCO0NipuVTE0rlDcc2e8joSewWyOlEwA==
+  dependencies:
+    json-stringify-pretty-compact "~3.0.0"
+    tslib "~2.6.3"
+    vega-event-selector "~3.0.1"
+    vega-expression "~5.1.1"
+    vega-util "~1.17.2"
+    yargs "~17.7.2"
+
+vega-loader@^4.5.2, vega-loader@~4.5.2:
+  version "4.5.2"
+  resolved "https://registry.yarnpkg.com/vega-loader/-/vega-loader-4.5.2.tgz#7212f093c397b153f69f7e6cfef47817c17c5c01"
+  integrity sha512-ktIdGz3DRIS3XfTP9lJ6oMT5cKwC86nQkjUbXZbOtwXQFVNE2xVWBuH13GP6FKUZxg5hJCMtb5v/e/fwTvhKsQ==
+  dependencies:
+    d3-dsv "^3.0.1"
+    node-fetch "^2.6.7"
+    topojson-client "^3.1.0"
+    vega-format "^1.1.2"
+    vega-util "^1.17.2"
+
+vega-parser@~6.4.0:
+  version "6.4.0"
+  resolved "https://registry.yarnpkg.com/vega-parser/-/vega-parser-6.4.0.tgz#6a12f07f0f9178492a17842efe7e1f51a2d36bed"
+  integrity sha512-/hFIJs0yITxfvLIfhhcpUrcbKvu4UZYoMGmly5PSsbgo60oAsVQW8ZbX2Ji3iNFqZJh1ifoX/P0j+9wep1OISw==
+  dependencies:
+    vega-dataflow "^5.7.6"
+    vega-event-selector "^3.0.1"
+    vega-functions "^5.15.0"
+    vega-scale "^7.4.1"
+    vega-util "^1.17.2"
+
+vega-projection@^1.6.1, vega-projection@~1.6.1:
+  version "1.6.1"
+  resolved "https://registry.yarnpkg.com/vega-projection/-/vega-projection-1.6.1.tgz#da687abc60f4a93bb888385beb23e0a1000f8b57"
+  integrity sha512-sqfnAAHumU7MWU1tQN3b6HNgKGF3legek0uLHhjLKcDJQxEc7kwcD18txFz2ffQks6d5j+AUhBiq4GARWf0DEQ==
+  dependencies:
+    d3-geo "^3.1.0"
+    d3-geo-projection "^4.0.0"
+    vega-scale "^7.4.1"
+
+vega-regression@~1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/vega-regression/-/vega-regression-1.3.0.tgz#3e68e234fa9460041fac082c6a3469c896d436a8"
+  integrity sha512-gxOQfmV7Ft/MYKpXDEo09WZyBuKOBqxqDRWay9KtfGq/E0Y4vbTPsWLv2cB1ToPJdKE6XSN6Re9tCIw5M/yMUg==
+  dependencies:
+    d3-array "^3.2.2"
+    vega-dataflow "^5.7.6"
+    vega-statistics "^1.9.0"
+    vega-util "^1.17.2"
+
+vega-runtime@^6.2.0, vega-runtime@~6.2.0:
+  version "6.2.0"
+  resolved "https://registry.yarnpkg.com/vega-runtime/-/vega-runtime-6.2.0.tgz#10f435089fff11d8e1b49cb0cbab8041731e6f06"
+  integrity sha512-30UXbujWjKNd5aeP+oeHuwFmzuyVYlBj4aDy9+AjfWLECu8wJt4K01vwegcaGPdCWcPLVIv4Oa9Lob4mcXn5KQ==
+  dependencies:
+    vega-dataflow "^5.7.6"
+    vega-util "^1.17.2"
+
+vega-scale@^7.4.1, vega-scale@~7.4.1:
+  version "7.4.1"
+  resolved "https://registry.yarnpkg.com/vega-scale/-/vega-scale-7.4.1.tgz#2dcd3e39ebb00269b03a8be86e44c7b48c67442a"
+  integrity sha512-dArA28DbV/M92O2QvswnzCmQ4bq9WwLKUoyhqFYWCltmDwkmvX7yhqiFLFMWPItIm7mi4Qyoygby6r4DKd1X2A==
+  dependencies:
+    d3-array "^3.2.2"
+    d3-interpolate "^3.0.1"
+    d3-scale "^4.0.2"
+    d3-scale-chromatic "^3.1.0"
+    vega-time "^2.1.2"
+    vega-util "^1.17.2"
+
+vega-scenegraph@^4.13.0, vega-scenegraph@~4.13.0:
+  version "4.13.0"
+  resolved "https://registry.yarnpkg.com/vega-scenegraph/-/vega-scenegraph-4.13.0.tgz#c4fa5c82773f6244a9ca8b01a44e380adf03fabd"
+  integrity sha512-nfl45XtuqB5CxyIZJ+bbJ+dofzosPCRlmF+eUQo+0J23NkNXsTzur+1krJDSdhcw0SOYs4sbYRoMz1cpuOM4+Q==
+  dependencies:
+    d3-path "^3.1.0"
+    d3-shape "^3.2.0"
+    vega-canvas "^1.2.7"
+    vega-loader "^4.5.2"
+    vega-scale "^7.4.1"
+    vega-util "^1.17.2"
+
+vega-schema-url-parser@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/vega-schema-url-parser/-/vega-schema-url-parser-2.2.0.tgz#a0d1e02915adfbfcb1fd517c8c2ebe2419985c1e"
+  integrity sha512-yAtdBnfYOhECv9YC70H2gEiqfIbVkq09aaE4y/9V/ovEFmH9gPKaEgzIZqgT7PSPQjKhsNkb6jk6XvSoboxOBw==
+
+vega-selections@^5.4.2:
+  version "5.4.2"
+  resolved "https://registry.yarnpkg.com/vega-selections/-/vega-selections-5.4.2.tgz#cb4f41f5d4c0ee924ebf131b8dbd43e7885bcad4"
+  integrity sha512-99FUhYmg0jOJr2/K4TcEURmJRkuibrCDc8KBUX7qcQEITzrZ5R6a4QE+sarCvbb3hi8aA9GV2oyST6MQeA9mgQ==
+  dependencies:
+    d3-array "3.2.4"
+    vega-expression "^5.0.1"
+    vega-util "^1.17.1"
+
+vega-statistics@^1.9.0, vega-statistics@~1.9.0:
+  version "1.9.0"
+  resolved "https://registry.yarnpkg.com/vega-statistics/-/vega-statistics-1.9.0.tgz#7d6139cea496b22d60decfa6abd73346f70206f9"
+  integrity sha512-GAqS7mkatpXcMCQKWtFu1eMUKLUymjInU0O8kXshWaQrVWjPIO2lllZ1VNhdgE0qGj4oOIRRS11kzuijLshGXQ==
+  dependencies:
+    d3-array "^3.2.2"
+
+vega-themes@^2.15.0:
+  version "2.15.0"
+  resolved "https://registry.yarnpkg.com/vega-themes/-/vega-themes-2.15.0.tgz#cf7592efb45406957e9beb67d7033ee5f7b7a511"
+  integrity sha512-DicRAKG9z+23A+rH/3w3QjJvKnlGhSbbUXGjBvYGseZ1lvj9KQ0BXZ2NS/+MKns59LNpFNHGi9us/wMlci4TOA==
+
+vega-time@^2.1.2, vega-time@~2.1.2:
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/vega-time/-/vega-time-2.1.2.tgz#0c414e74780613d6d3234fb97f19b50c0ebd9f49"
+  integrity sha512-6rXc6JdDt8MnCRy6UzUCsa6EeFycPDmvioMddLfKw38OYCV8pRQC5nw44gyddOwXgUTJLiCtn/sp53P0iA542A==
+  dependencies:
+    d3-array "^3.2.2"
+    d3-time "^3.1.0"
+    vega-util "^1.17.2"
+
+vega-tooltip@^0.34.0:
+  version "0.34.0"
+  resolved "https://registry.yarnpkg.com/vega-tooltip/-/vega-tooltip-0.34.0.tgz#e0aa4d9c9bcf155e257650ba7e670fad7b1ff5ab"
+  integrity sha512-TtxwkcLZ5aWQTvKGlfWDou8tISGuxmqAW1AgGZjrDpf75qsXvgtbPdRAAls2LZMqDxpr5T1kMEZs9XbSpiI8yw==
+  dependencies:
+    vega-util "^1.17.2"
+
+vega-transforms@~4.12.0:
+  version "4.12.0"
+  resolved "https://registry.yarnpkg.com/vega-transforms/-/vega-transforms-4.12.0.tgz#6a69e0b67934b0c0a40a6f607fdb543bf749955e"
+  integrity sha512-bh/2Qbj85O70mjfLRgPKAsABArgSUP0k+GjmaY54zukIRxoGxKju+85nigeX/aR/INpEqNWif+5lL+NvmyWA5w==
+  dependencies:
+    d3-array "^3.2.2"
+    vega-dataflow "^5.7.6"
+    vega-statistics "^1.9.0"
+    vega-time "^2.1.2"
+    vega-util "^1.17.2"
+
+vega-typings@~1.3.1:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/vega-typings/-/vega-typings-1.3.1.tgz#025a6031505794b44d9b6e2c49d4551b8918d4ae"
+  integrity sha512-j9Sdgmvowz09jkMgTFGVfiv7ycuRP/TQkdHRPXIYwt3RDgPQn7inyFcJ8C8ABFt4MiMWdjOwbneF6KWW8TRXIw==
+  dependencies:
+    "@types/geojson" "7946.0.4"
+    vega-event-selector "^3.0.1"
+    vega-expression "^5.1.1"
+    vega-util "^1.17.2"
+
+vega-util@^1.17.1, vega-util@^1.17.2, vega-util@~1.17.2:
+  version "1.17.2"
+  resolved "https://registry.yarnpkg.com/vega-util/-/vega-util-1.17.2.tgz#f69aa09fd5d6110c19c4a0f0af9e35945b99987d"
+  integrity sha512-omNmGiZBdjm/jnHjZlywyYqafscDdHaELHx1q96n5UOz/FlO9JO99P4B3jZg391EFG8dqhWjQilSf2JH6F1mIw==
+
+vega-view-transforms@~4.6.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/vega-view-transforms/-/vega-view-transforms-4.6.0.tgz#829d56ca3c8116b0dded4ec0502f4ac70253de9a"
+  integrity sha512-z3z66aJTA3ZRo4oBY4iBXnn+A4KqBGZT/UrlKDbm+7Ec+Ip+hK2tF8Kmhp/WNcMsDZoUWFqLJgR2VgOgvJk9RA==
+  dependencies:
+    vega-dataflow "^5.7.6"
+    vega-scenegraph "^4.13.0"
+    vega-util "^1.17.2"
+
+vega-view@~5.13.0:
+  version "5.13.0"
+  resolved "https://registry.yarnpkg.com/vega-view/-/vega-view-5.13.0.tgz#8ea96da9fcdf42fe7c0e95fe6258933477524745"
+  integrity sha512-ZPAAQ3iYz6YrQjJoDT+0bcxJkXt9PKF5v4OO7Omw8PFhkIv++jFXeKlQTW1bBtyQ92dkdGGHv5lYY67Djqjf3A==
+  dependencies:
+    d3-array "^3.2.2"
+    d3-timer "^3.0.1"
+    vega-dataflow "^5.7.6"
+    vega-format "^1.1.2"
+    vega-functions "^5.15.0"
+    vega-runtime "^6.2.0"
+    vega-scenegraph "^4.13.0"
+    vega-util "^1.17.2"
+
+vega-voronoi@~4.2.3:
+  version "4.2.3"
+  resolved "https://registry.yarnpkg.com/vega-voronoi/-/vega-voronoi-4.2.3.tgz#54c4bb96b9b94c3fa0160bee24695dcb9d583fe1"
+  integrity sha512-aYYYM+3UGqwsOx+TkVtF1IZfguy0H7AN79dR8H0nONRIc+vhk/lbnlkgwY2nSzEu0EZ4b5wZxeGoDBEVmdDEcg==
+  dependencies:
+    d3-delaunay "^6.0.2"
+    vega-dataflow "^5.7.6"
+    vega-util "^1.17.2"
+
+vega-wordcloud@~4.1.5:
+  version "4.1.5"
+  resolved "https://registry.yarnpkg.com/vega-wordcloud/-/vega-wordcloud-4.1.5.tgz#789c9e67225c77f3f35a6fc052beec1c2bdc8b5e"
+  integrity sha512-p+qXU3cb9VeWzJ/HEdax0TX2mqDJcSbrCIfo2d/EalOXGkvfSLKobsmMQ8DxPbtVp0uhnpvfCGDyMJw+AzcI2A==
+  dependencies:
+    vega-canvas "^1.2.7"
+    vega-dataflow "^5.7.6"
+    vega-scale "^7.4.1"
+    vega-statistics "^1.9.0"
+    vega-util "^1.17.2"
+
+vega@^5.30.0:
+  version "5.30.0"
+  resolved "https://registry.yarnpkg.com/vega/-/vega-5.30.0.tgz#d12350c829878b481453ab28ce10855a954df06d"
+  integrity sha512-ZGoC8LdfEUV0LlXIuz7hup9jxuQYhSaWek2M7r9dEHAPbPrzSQvKXZ0BbsJbrarM100TGRpTVN/l1AFxCwDkWw==
+  dependencies:
+    vega-crossfilter "~4.1.2"
+    vega-dataflow "~5.7.6"
+    vega-encode "~4.10.1"
+    vega-event-selector "~3.0.1"
+    vega-expression "~5.1.1"
+    vega-force "~4.2.1"
+    vega-format "~1.1.2"
+    vega-functions "~5.15.0"
+    vega-geo "~4.4.2"
+    vega-hierarchy "~4.1.2"
+    vega-label "~1.3.0"
+    vega-loader "~4.5.2"
+    vega-parser "~6.4.0"
+    vega-projection "~1.6.1"
+    vega-regression "~1.3.0"
+    vega-runtime "~6.2.0"
+    vega-scale "~7.4.1"
+    vega-scenegraph "~4.13.0"
+    vega-statistics "~1.9.0"
+    vega-time "~2.1.2"
+    vega-transforms "~4.12.0"
+    vega-typings "~1.3.1"
+    vega-util "~1.17.2"
+    vega-view "~5.13.0"
+    vega-view-transforms "~4.6.0"
+    vega-voronoi "~4.2.3"
+    vega-wordcloud "~4.1.5"
+
 vinyl-fs@^3.0.0:
   version "3.0.3"
   resolved "https://registry.npmjs.org/vinyl-fs/-/vinyl-fs-3.0.3.tgz"
@@ -11895,6 +12353,18 @@ vue-demi@>=0.14.7:
   resolved "https://registry.yarnpkg.com/vue-demi/-/vue-demi-0.14.7.tgz#8317536b3ef74c5b09f268f7782e70194567d8f2"
   integrity sha512-EOG8KXDQNwkJILkx/gPcoL/7vH+hORoBaKgGe+6W7VFMvCYJfmF2dGbvgDroVnI8LU7/kTu8mbjRZGBU1z9NTA==
 
+vue-demi@^0.13.11:
+  version "0.13.11"
+  resolved "https://registry.yarnpkg.com/vue-demi/-/vue-demi-0.13.11.tgz#7d90369bdae8974d87b1973564ad390182410d99"
+  integrity sha512-IR8HoEEGM65YY3ZJYAjMlKygDQn25D5ajNFNoKh9RSDMQtlzCxtfQjdQgv9jjK+m3377SsJXY8ysq8kLCZL25A==
+
+vue-echarts@^7.0.3:
+  version "7.0.3"
+  resolved "https://registry.yarnpkg.com/vue-echarts/-/vue-echarts-7.0.3.tgz#bf79f7ee0144bbdc6aee5610e8443fed91f6abbe"
+  integrity sha512-/jSxNwOsw5+dYAUcwSfkLwKPuzTQ0Cepz1LxCOpj2QcHrrmUa/Ql0eQqMmc1rTPQVrh2JQ29n2dhq75ZcHvRDw==
+  dependencies:
+    vue-demi "^0.13.11"
+
 vue-eslint-parser@^9.0.1:
   version "9.0.3"
   resolved "https://registry.npmjs.org/vue-eslint-parser/-/vue-eslint-parser-9.0.3.tgz"
@@ -12467,7 +12937,7 @@ yargs@^17.3.1:
     y18n "^5.0.5"
     yargs-parser "^21.1.1"
 
-yargs@^17.7.2:
+yargs@^17.7.2, yargs@~17.7.2:
   version "17.7.2"
   resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269"
   integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==
@@ -12508,3 +12978,10 @@ yoctocolors-cjs@^2.1.2:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/yoctocolors-cjs/-/yoctocolors-cjs-2.1.2.tgz#f4b905a840a37506813a7acaa28febe97767a242"
   integrity sha512-cYVsTjKl8b+FrnidjibDWskAv7UKOfcwaVZdp/it9n1s9fU3IkgDbhdIRKCW4JDsAlECJY0ytoVPT3sK6kideA==
+
+zrender@5.6.0:
+  version "5.6.0"
+  resolved "https://registry.yarnpkg.com/zrender/-/zrender-5.6.0.tgz#01325b0bb38332dd5e87a8dbee7336cafc0f4a5b"
+  integrity sha512-uzgraf4njmmHAbEUxMJ8Oxg+P3fT04O+9p7gY+wJRVxo8Ge+KmYv0WJev945EH4wFuc4OY2NLXz46FZrWS9xJg==
+  dependencies:
+    tslib "2.3.0"

From 6b94267a882c1656a2d4423db9f126432168ef75 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 4 Nov 2024 13:44:15 +0100
Subject: [PATCH 61/84] Enable grouping by workflow step

---
 client/src/api/schema/schema.ts               |  4 ++
 .../WorkflowInvocationMetrics.vue             | 48 ++++++++++++++-----
 lib/galaxy/managers/jobs.py                   | 12 +++--
 lib/galaxy/schema/schema.py                   |  2 +
 .../webapps/galaxy/services/invocations.py    | 12 ++++-
 5 files changed, 62 insertions(+), 16 deletions(-)

diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts
index 73cb76dd7af6..348c9cda9fab 100644
--- a/client/src/api/schema/schema.ts
+++ b/client/src/api/schema/schema.ts
@@ -18360,6 +18360,10 @@ export interface components {
              * @description The raw value of the metric as a string.
              */
             raw_value: string;
+            /** Step Index */
+            step_index: number;
+            /** Step Label */
+            step_label: string | null;
             /**
              * Title
              * @description A descriptive title for this metric.
diff --git a/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue b/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue
index a924bc54aed1..d372ede7f3a5 100644
--- a/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue
+++ b/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue
@@ -1,10 +1,10 @@
 <script setup lang="ts">
+import { type VisualizationSpec } from "vega-embed";
 import { computed, ref, watch } from "vue";
+import { type ComputedRef } from "vue";
 
 import { type components, GalaxyApi } from "@/api";
 import { errorMessageAsString } from "@/utils/simple-error";
-import { type ComputedRef } from "vue";
-import { type VisualizationSpec } from "vega-embed";
 
 import VegaWrapper from "./VegaWrapper.vue";
 
@@ -15,9 +15,15 @@ const props = defineProps({
     },
 });
 
+const groupBy = ref<"tool_id" | "step_id">("tool_id");
 const jobMetrics = ref<components["schemas"]["WorkflowJobMetric"][]>();
 const fetchError = ref<string>();
 
+const attributeToLabel = {
+    tool_id: "Tool ID",
+    step_id: "Step",
+};
+
 async function fetchMetrics() {
     const { data, error } = await GalaxyApi().GET("/api/invocations/{invocation_id}/metrics", {
         params: {
@@ -35,6 +41,16 @@ async function fetchMetrics() {
 
 watch(props, () => fetchMetrics(), { immediate: true });
 
+function itemToX(item: components["schemas"]["WorkflowJobMetric"]) {
+    if (groupBy.value === "tool_id") {
+        return item.tool_id;
+    } else if (groupBy.value === "step_id") {
+        return `${item.step_index + 1}: ${item.step_label || item.tool_id}`;
+    } else {
+        throw Error("Cannot happen");
+    }
+}
+
 interface boxplotData {
     x_title: string;
     y_title: string;
@@ -46,11 +62,11 @@ const wallclock: ComputedRef<boxplotData> = computed(() => {
     const values = wallclock?.map((item) => {
         return {
             y: parseFloat(item.raw_value),
-            x: item.tool_id,
+            x: itemToX(item),
         };
     });
     return {
-        x_title: "Tool ID",
+        x_title: attributeToLabel[groupBy.value],
         y_title: "Runtime (in Seconds)",
         values,
     };
@@ -61,11 +77,11 @@ const coresAllocated: ComputedRef<boxplotData> = computed(() => {
     const values = coresAllocated?.map((item) => {
         return {
             y: parseFloat(item.raw_value),
-            x: item.tool_id,
+            x: itemToX(item),
         };
     });
     return {
-        x_title: "Tool ID",
+        x_title: attributeToLabel[groupBy.value],
         y_title: "Cores Allocated",
         values,
     };
@@ -76,7 +92,7 @@ const memoryAllocated: ComputedRef<boxplotData> = computed(() => {
     const values = memoryAllocated?.map((item) => {
         return {
             y: parseFloat(item.raw_value),
-            x: item.tool_id,
+            x: itemToX(item),
         };
     });
     return {
@@ -147,9 +163,19 @@ const specs = computed(() => {
 
 <template>
     <div>
-        <div v-for="(spec, key) in specs" :key="key">
-            <h2 class="h-l truncate text-center">{{ key }}</h2>
-            <VegaWrapper :spec="spec" />
-        </div>
+        <b-tabs lazy>
+            <b-tab title="Summary by Tool" @click="groupBy = 'tool_id'">
+                <div v-for="(spec, key) in specs" :key="key">
+                    <h2 class="h-l truncate text-center">{{ key }}</h2>
+                    <VegaWrapper :spec="spec" />
+                </div>
+            </b-tab>
+            <b-tab title="Summary by Workflow Step" @click="groupBy = 'step_id'">
+                <div v-for="(spec, key) in specs" :key="key">
+                    <h2 class="h-l truncate text-center">{{ key }}</h2>
+                    <VegaWrapper :spec="spec" />
+                </div>
+            </b-tab>
+        </b-tabs>
     </div>
 </template>
diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py
index be9d0083567b..09406182e7aa 100644
--- a/lib/galaxy/managers/jobs.py
+++ b/lib/galaxy/managers/jobs.py
@@ -60,6 +60,7 @@
     Workflow,
     WorkflowInvocation,
     WorkflowInvocationStep,
+    WorkflowStep,
     YIELD_PER_ROWS,
 )
 from galaxy.model.base import transaction
@@ -732,7 +733,7 @@ def invocation_job_source_iter(sa_session, invocation_id):
 
 def get_job_metrics_for_invocation(sa_session: galaxy_scoped_session, invocation_id: int):
     single_job_stmnt = (
-        select(JobMetricNumeric, Job.tool_id)
+        select(WorkflowStep.order_index, Job.tool_id, WorkflowStep.label, JobMetricNumeric)
         .join(Job, JobMetricNumeric.job_id == Job.id)
         .join(
             WorkflowInvocationStep,
@@ -740,9 +741,10 @@ def get_job_metrics_for_invocation(sa_session: galaxy_scoped_session, invocation
                 WorkflowInvocationStep.workflow_invocation_id == invocation_id, WorkflowInvocationStep.job_id == Job.id
             ),
         )
+        .join(WorkflowStep, WorkflowStep.id == WorkflowInvocationStep.workflow_step_id)
     )
     collection_job_stmnt = (
-        select(JobMetricNumeric, Job.tool_id)
+        select(WorkflowStep.order_index, Job.tool_id, WorkflowStep.label, JobMetricNumeric)
         .join(Job, JobMetricNumeric.job_id == Job.id)
         .join(ImplicitCollectionJobsJobAssociation, Job.id == ImplicitCollectionJobsJobAssociation.job_id)
         .join(
@@ -756,10 +758,14 @@ def get_job_metrics_for_invocation(sa_session: galaxy_scoped_session, invocation
                 WorkflowInvocationStep.implicit_collection_jobs_id == ImplicitCollectionJobs.id,
             ),
         )
+        .join(WorkflowStep, WorkflowStep.id == WorkflowInvocationStep.workflow_step_id)
     )
     # should be sa_session.execute(single_job_stmnt.union(collection_job_stmnt)).all() but that returns
     # columns instead of the job metrics ORM instance.
-    return list(sa_session.execute(single_job_stmnt).all()) + list(sa_session.execute(collection_job_stmnt).all())
+    return sorted(
+        (*sa_session.execute(single_job_stmnt).all(), *sa_session.execute(collection_job_stmnt).all()),
+        key=lambda row: row[0],
+    )
 
 
 def fetch_job_states(sa_session, job_source_ids, job_source_types):
diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py
index 88460a3f425f..e97a743e53b3 100644
--- a/lib/galaxy/schema/schema.py
+++ b/lib/galaxy/schema/schema.py
@@ -2173,6 +2173,8 @@ class JobMetric(Model):
 
 class WorkflowJobMetric(JobMetric):
     tool_id: str
+    step_index: int
+    step_label: Optional[str]
 
 
 class JobMetricCollection(RootModel):
diff --git a/lib/galaxy/webapps/galaxy/services/invocations.py b/lib/galaxy/webapps/galaxy/services/invocations.py
index 4066a3dbfeaa..4db9a0d5f1a0 100644
--- a/lib/galaxy/webapps/galaxy/services/invocations.py
+++ b/lib/galaxy/webapps/galaxy/services/invocations.py
@@ -168,12 +168,20 @@ def show_invocation_metrics(self, trans: ProvidesHistoryContext, invocation_id:
         extended_job_metrics = get_job_metrics_for_invocation(trans.sa_session, invocation_id)
         job_metrics = []
         tool_ids = []
+        step_indexes = []
+        step_labels = []
         for row in extended_job_metrics:
-            job_metrics.append(row[0])
+            step_indexes.append(row[0])
             tool_ids.append(row[1])
+            step_labels.append(row[2])
+            job_metrics.append(row[3])
         metrics_dict_list = summarize_metrics(trans, job_metrics)
-        for tool_id, metrics_dict in zip(tool_ids, metrics_dict_list):
+        for tool_id, step_index, step_label, metrics_dict in zip(
+            tool_ids, step_indexes, step_labels, metrics_dict_list
+        ):
             metrics_dict["tool_id"] = tool_id
+            metrics_dict["step_index"] = step_index
+            metrics_dict["step_label"] = step_label
         return metrics_dict_list
 
     def update_invocation_step(self, trans, step_id, action):

From 39a7ed26694aa757bf3e5d8342f9994ad3bfab44 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 4 Nov 2024 18:20:49 +0100
Subject: [PATCH 62/84] Decrease value parsing repetition, add peakMemory

---
 .../WorkflowInvocationMetrics.vue             | 51 +++++++++----------
 1 file changed, 23 insertions(+), 28 deletions(-)

diff --git a/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue b/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue
index d372ede7f3a5..4da3248864f1 100644
--- a/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue
+++ b/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue
@@ -57,49 +57,44 @@ interface boxplotData {
     values?: { x: string; y: Number }[];
 }
 
-const wallclock: ComputedRef<boxplotData> = computed(() => {
-    const wallclock = jobMetrics.value?.filter((jobMetric) => jobMetric.name == "runtime_seconds");
+function metricToSpecData(
+    jobMetrics: components["schemas"]["WorkflowJobMetric"][] | undefined,
+    metricName: string,
+    yTitle: string,
+    transform?: (param: number) => number
+) {
+    const wallclock = jobMetrics?.filter((jobMetric) => jobMetric.name == metricName);
     const values = wallclock?.map((item) => {
+        let y = parseFloat(item.raw_value);
+        if (transform !== undefined) {
+            y = transform(y);
+        }
         return {
-            y: parseFloat(item.raw_value),
+            y,
             x: itemToX(item),
         };
     });
     return {
         x_title: attributeToLabel[groupBy.value],
-        y_title: "Runtime (in Seconds)",
+        y_title: yTitle,
         values,
     };
+}
+
+const wallclock: ComputedRef<boxplotData> = computed(() => {
+    return metricToSpecData(jobMetrics.value, "runtime_seconds", "Runtime (in Seconds)");
 });
 
 const coresAllocated: ComputedRef<boxplotData> = computed(() => {
-    const coresAllocated = jobMetrics.value?.filter((jobMetric) => jobMetric.name == "galaxy_slots");
-    const values = coresAllocated?.map((item) => {
-        return {
-            y: parseFloat(item.raw_value),
-            x: itemToX(item),
-        };
-    });
-    return {
-        x_title: attributeToLabel[groupBy.value],
-        y_title: "Cores Allocated",
-        values,
-    };
+    return metricToSpecData(jobMetrics.value, "galaxy_slots", "Cores Allocated");
 });
 
 const memoryAllocated: ComputedRef<boxplotData> = computed(() => {
-    const memoryAllocated = jobMetrics.value?.filter((jobMetric) => jobMetric.name == "galaxy_memory_mb");
-    const values = memoryAllocated?.map((item) => {
-        return {
-            y: parseFloat(item.raw_value),
-            x: itemToX(item),
-        };
-    });
-    return {
-        x_title: "Tool ID",
-        y_title: "Memory Allocated (in MB)",
-        values,
-    };
+    return metricToSpecData(jobMetrics.value, "galaxy_memory_mb", "Memory Allocated (in MB)");
+});
+
+const peakMemory: ComputedRef<boxplotData> = computed(() => {
+    return metricToSpecData(jobMetrics.value, "memory.peak", "Max memory usage recorded (in MB)", (v) => v / 1024 ** 2);
 });
 
 function itemToSpec(item: boxplotData) {

From 1fadd2f658c508972fd36eea6976066482db7a9e Mon Sep 17 00:00:00 2001
From: Nicola Soranzo <nicola.soranzo@earlham.ac.uk>
Date: Sun, 3 Nov 2024 18:04:54 +0000
Subject: [PATCH 63/84] Do not recalculate dependencies when exporting

Saves a few milliseconds :D
---
 lib/galaxy/dependencies/dev-requirements.txt              | 2 +-
 lib/galaxy/dependencies/pinned-requirements.txt           | 2 +-
 lib/galaxy/dependencies/pinned-typecheck-requirements.txt | 2 +-
 lib/galaxy/dependencies/update.sh                         | 6 +++---
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt
index a0b0f68801f0..685b1149cd08 100644
--- a/lib/galaxy/dependencies/dev-requirements.txt
+++ b/lib/galaxy/dependencies/dev-requirements.txt
@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv export --no-hashes --only-group=dev
+#    uv export --frozen --no-hashes --only-group=dev
 aiohappyeyeballs==2.4.3
 aiohttp==3.10.10
 aiosignal==1.3.1
diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt
index 6227984653a6..6f4c5cc99496 100644
--- a/lib/galaxy/dependencies/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pinned-requirements.txt
@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv export --no-hashes --no-dev
+#    uv export --frozen --no-hashes --no-dev
 a2wsgi==1.10.7
 adal==1.2.7
 aiobotocore==2.15.2
diff --git a/lib/galaxy/dependencies/pinned-typecheck-requirements.txt b/lib/galaxy/dependencies/pinned-typecheck-requirements.txt
index a835c64276fa..316819f75cd7 100644
--- a/lib/galaxy/dependencies/pinned-typecheck-requirements.txt
+++ b/lib/galaxy/dependencies/pinned-typecheck-requirements.txt
@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv export --no-hashes --only-group=typecheck
+#    uv export --frozen --no-hashes --only-group=typecheck
 annotated-types==0.7.0
 boto3-stubs==1.35.49
 botocore-stubs==1.35.49
diff --git a/lib/galaxy/dependencies/update.sh b/lib/galaxy/dependencies/update.sh
index 2d26f6b40bef..12078ddbed3d 100755
--- a/lib/galaxy/dependencies/update.sh
+++ b/lib/galaxy/dependencies/update.sh
@@ -42,6 +42,6 @@ else
 fi
 
 # Update pinned requirements files.
-${uv} export --no-hashes --no-dev > "$this_directory/pinned-requirements.txt"
-${uv} export --no-hashes --only-group=dev > "$this_directory/dev-requirements.txt"
-${uv} export --no-hashes --only-group=typecheck > "$this_directory/pinned-typecheck-requirements.txt"
+${uv} export --frozen --no-hashes --no-dev > "$this_directory/pinned-requirements.txt"
+${uv} export --frozen --no-hashes --only-group=dev > "$this_directory/dev-requirements.txt"
+${uv} export --frozen --no-hashes --only-group=typecheck > "$this_directory/pinned-typecheck-requirements.txt"

From 71e499bb7864acf9ca381a02ed923aa9535d62bb Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 5 Nov 2024 11:34:42 +0100
Subject: [PATCH 64/84] Optimize/fix sqlite hid update statement

`./run_tests.sh --skip-common-startup -api lib/galaxy_test/api -- -s -k
cache` would fail on sqlite with database locked.

Not using a separate connection means that we expire loaded attributes
in the sqlite case, but I don't think we need to be concerned about
performance if you're using sqlite.
---
 lib/galaxy/model/__init__.py | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 582da171d315..12d2a554512d 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -91,7 +91,10 @@
     update,
     VARCHAR,
 )
-from sqlalchemy.exc import OperationalError
+from sqlalchemy.exc import (
+    CompileError,
+    OperationalError,
+)
 from sqlalchemy.ext import hybrid
 from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.ext.orderinglist import ordering_list
@@ -3120,16 +3123,21 @@ def _next_hid(self, n=1):
         table = self.__table__
         history_id = cached_id(self)
         update_stmt = update(table).where(table.c.id == history_id).values(hid_counter=table.c.hid_counter + n)
+        update_returning_stmnt = update_stmt.returning(table.c.hid_counter)
 
-        with engine.begin() as conn:
-            if engine.name in ["postgres", "postgresql"]:
-                stmt = update_stmt.returning(table.c.hid_counter)
-                updated_hid = conn.execute(stmt).scalar()
-                hid = updated_hid - n
-            else:
-                select_stmt = select(table.c.hid_counter).where(table.c.id == history_id).with_for_update()
-                hid = conn.execute(select_stmt).scalar()
-                conn.execute(update_stmt)
+        if engine.name != "sqlite":
+            with engine.begin() as conn:
+                hid = conn.execute(update_returning_stmnt).scalar() - n
+        else:
+            try:
+                hid = session.execute(update_returning_stmnt).scalar() - n
+            except (CompileError, OperationalError):
+                # The RETURNING clause was added to SQLite in version 3.35.0.
+                # Not using FOR UPDATE either, since that was added in 3.45.0,
+                # and anyway does a whole-table lock
+                select_stmt = select(table.c.hid_counter).where(table.c.id == history_id)
+                hid = session.execute(select_stmt).scalar()
+                session.execute(update_stmt)
 
         session.expire(self, ["hid_counter"])
         return hid

From 15ad84636ae9751b4a326f878581aa4f24a2b476 Mon Sep 17 00:00:00 2001
From: Arash <arash77.kad@gmail.com>
Date: Mon, 14 Oct 2024 11:11:16 +0200
Subject: [PATCH 65/84] Fix issue with generating slug for sharing history

---
 lib/galaxy/managers/sharable.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/galaxy/managers/sharable.py b/lib/galaxy/managers/sharable.py
index 1f607f5d2589..9b0234b04144 100644
--- a/lib/galaxy/managers/sharable.py
+++ b/lib/galaxy/managers/sharable.py
@@ -324,6 +324,8 @@ def _slugify(self, start_with):
         # Remove trailing '-'.
         if slug_base.endswith("-"):
             slug_base = slug_base[:-1]
+        if not slug_base:
+            slug_base = "unnamed"
         return slug_base
 
     def _default_slug_base(self, item):

From 217ee881ecbea1d13d6c34e829d47d46b67b52e9 Mon Sep 17 00:00:00 2001
From: Arash <arash77.kad@gmail.com>
Date: Mon, 14 Oct 2024 11:39:47 +0200
Subject: [PATCH 66/84] Add test for sharing non-alphanumeric history

---
 lib/galaxy_test/api/test_histories.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib/galaxy_test/api/test_histories.py b/lib/galaxy_test/api/test_histories.py
index 6d3ba17d87e0..4e9accce2f55 100644
--- a/lib/galaxy_test/api/test_histories.py
+++ b/lib/galaxy_test/api/test_histories.py
@@ -544,6 +544,12 @@ def test_anonymous_can_import_published(self):
             }
             self.dataset_populator.import_history(import_data)
 
+    def test_publish_non_alphanumeric(self):
+        history_name = f"تاریخچه"
+        history_id = self.dataset_populator.new_history(name=history_name)
+        response = self.dataset_populator.make_public(history_id)
+        assert response["username_and_slug"]
+
     def test_immutable_history_update_fails(self):
         history_id = self._create_history("TestHistoryForImmutability")["id"]
 

From 6f91b0f68ea5677675d064a058895fcafab7d28b Mon Sep 17 00:00:00 2001
From: Arash <arash77.kad@gmail.com>
Date: Mon, 14 Oct 2024 11:44:46 +0200
Subject: [PATCH 67/84] Fix python lint

---
 lib/galaxy_test/api/test_histories.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy_test/api/test_histories.py b/lib/galaxy_test/api/test_histories.py
index 4e9accce2f55..13648aee583f 100644
--- a/lib/galaxy_test/api/test_histories.py
+++ b/lib/galaxy_test/api/test_histories.py
@@ -545,7 +545,7 @@ def test_anonymous_can_import_published(self):
             self.dataset_populator.import_history(import_data)
 
     def test_publish_non_alphanumeric(self):
-        history_name = f"تاریخچه"
+        history_name = "تاریخچه"
         history_id = self.dataset_populator.new_history(name=history_name)
         response = self.dataset_populator.make_public(history_id)
         assert response["username_and_slug"]

From 17c962a66ef29843914dc8f64e671426a93b8b85 Mon Sep 17 00:00:00 2001
From: Arash <arash77.kad@gmail.com>
Date: Mon, 14 Oct 2024 16:34:31 +0200
Subject: [PATCH 68/84] Refactor valid slug to just exclude `/:?#` characters

---
 client/src/components/Sharing/SlugInput.vue       |  2 +-
 lib/galaxy/managers/base.py                       |  4 ++--
 lib/galaxy/managers/sharable.py                   | 15 +++------------
 lib/galaxy/schema/schema.py                       |  2 +-
 lib/galaxy/webapps/base/controller.py             |  5 +++++
 lib/galaxy/webapps/galaxy/controllers/dataset.py  |  3 +--
 lib/galaxy/webapps/galaxy/controllers/history.py  |  3 +--
 lib/galaxy/webapps/galaxy/controllers/page.py     |  3 +--
 .../webapps/galaxy/controllers/visualization.py   |  3 +--
 lib/galaxy/webapps/galaxy/controllers/workflow.py |  3 +--
 10 files changed, 17 insertions(+), 26 deletions(-)

diff --git a/client/src/components/Sharing/SlugInput.vue b/client/src/components/Sharing/SlugInput.vue
index fc356dd4ad8f..291cb06641fd 100644
--- a/client/src/components/Sharing/SlugInput.vue
+++ b/client/src/components/Sharing/SlugInput.vue
@@ -15,7 +15,7 @@ const initialSlug = props.slug;
 function onChange(value: string) {
     const slugFormatted = value
         .replace(/\s+/g, "-")
-        .replace(/[^a-zA-Z0-9-]/g, "")
+        .replace(/[\/:?#]/g, "")
         .toLowerCase();
 
     emit("change", slugFormatted);
diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py
index 2a0b4765c0de..cc09af0059ea 100644
--- a/lib/galaxy/managers/base.py
+++ b/lib/galaxy/managers/base.py
@@ -1295,9 +1295,9 @@ def raise_filter_err(attr, op, val, msg):
 
 
 def is_valid_slug(slug):
-    """Returns true iff slug is valid."""
+    """Returns true if slug is valid."""
 
-    VALID_SLUG_RE = re.compile(r"^[a-z0-9\-]+$")
+    VALID_SLUG_RE = re.compile(r"^[^/:?#]+$")
     return VALID_SLUG_RE.match(slug)
 
 
diff --git a/lib/galaxy/managers/sharable.py b/lib/galaxy/managers/sharable.py
index 9b0234b04144..9318dd2b559a 100644
--- a/lib/galaxy/managers/sharable.py
+++ b/lib/galaxy/managers/sharable.py
@@ -291,7 +291,7 @@ def set_slug(self, item, new_slug, user, flush=True):
         Validate and set the new slug for `item`.
         """
         # precondition: has been validated
-        if not self.is_valid_slug(new_slug):
+        if not base.is_valid_slug(new_slug):
             raise exceptions.RequestParameterInvalidException("Invalid slug", slug=new_slug)
 
         if item.slug == new_slug:
@@ -309,23 +309,14 @@ def set_slug(self, item, new_slug, user, flush=True):
                 session.commit()
         return item
 
-    def is_valid_slug(self, slug):
-        """
-        Returns true if `slug` is valid.
-        """
-        VALID_SLUG_RE = re.compile(r"^[a-z0-9\-]+$")
-        return VALID_SLUG_RE.match(slug)
-
     def _slugify(self, start_with):
         # Replace whitespace with '-'
         slug_base = re.sub(r"\s+", "-", start_with)
-        # Remove all non-alphanumeric characters.
-        slug_base = re.sub(r"[^a-zA-Z0-9\-]", "", slug_base)
+        # Remove all /:?# characters.
+        slug_base = re.sub(r"[/:?#]", "", slug_base)
         # Remove trailing '-'.
         if slug_base.endswith("-"):
             slug_base = slug_base[:-1]
-        if not slug_base:
-            slug_base = "unnamed"
         return slug_base
 
     def _default_slug_base(self, item):
diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py
index 0d9646316e59..10920de2d6d2 100644
--- a/lib/galaxy/schema/schema.py
+++ b/lib/galaxy/schema/schema.py
@@ -3678,7 +3678,7 @@ class PageSummaryBase(Model):
         ...,  # Required
         title="Identifier",
         description="The title slug for the page URL, must be unique.",
-        pattern=r"^[a-z0-9\-]+$",
+        pattern=r"^[^/:?#]+$",
     )
 
 
diff --git a/lib/galaxy/webapps/base/controller.py b/lib/galaxy/webapps/base/controller.py
index 9fdfa1079f6e..0cff38f5c862 100644
--- a/lib/galaxy/webapps/base/controller.py
+++ b/lib/galaxy/webapps/base/controller.py
@@ -1123,6 +1123,11 @@ def share(self, trans, id=None, email="", **kwd):
     @web.expose
     def display_by_username_and_slug(self, trans, username, slug, **kwargs):
         """Display item by username and slug."""
+        # Ensure slug is in the correct format.
+        slug = slug.encode("latin1").decode("utf-8")
+        self._display_by_username_and_slug(trans, username, slug, **kwargs)
+
+    def _display_by_username_and_slug(self, trans, username, slug, **kwargs):
         raise NotImplementedError()
 
     def get_item(self, trans, id):
diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py
index c11661edb6b7..6d98060692da 100644
--- a/lib/galaxy/webapps/galaxy/controllers/dataset.py
+++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py
@@ -431,8 +431,7 @@ def _get_dataset_for_edit(self, trans, dataset_id):
             trans.app.security_agent.set_dataset_permission(data.dataset, permissions)
         return data, None
 
-    @web.expose
-    def display_by_username_and_slug(self, trans, username, slug, filename=None, preview=True, **kwargs):
+    def _display_by_username_and_slug(self, trans, username, slug, filename=None, preview=True, **kwargs):
         """Display dataset by username and slug; because datasets do not yet have slugs, the slug is the dataset's id."""
         dataset = self._check_dataset(trans, slug)
         # Filename used for composite types.
diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py
index ce4112871279..03739a9e4016 100644
--- a/lib/galaxy/webapps/galaxy/controllers/history.py
+++ b/lib/galaxy/webapps/galaxy/controllers/history.py
@@ -88,8 +88,7 @@ def view(self, trans, id=None, show_deleted=False, show_hidden=False, use_panels
             "allow_user_dataset_purge": trans.app.config.allow_user_dataset_purge,
         }
 
-    @web.expose
-    def display_by_username_and_slug(self, trans, username, slug, **kwargs):
+    def _display_by_username_and_slug(self, trans, username, slug, **kwargs):
         """
         Display history based on a username and slug.
         """
diff --git a/lib/galaxy/webapps/galaxy/controllers/page.py b/lib/galaxy/webapps/galaxy/controllers/page.py
index 047fa98e28fb..fac24d92648d 100644
--- a/lib/galaxy/webapps/galaxy/controllers/page.py
+++ b/lib/galaxy/webapps/galaxy/controllers/page.py
@@ -170,8 +170,7 @@ def display(self, trans, id, **kwargs):
             raise web.httpexceptions.HTTPNotFound()
         return self.display_by_username_and_slug(trans, page.user.username, page.slug)
 
-    @web.expose
-    def display_by_username_and_slug(self, trans, username, slug, **kwargs):
+    def _display_by_username_and_slug(self, trans, username, slug, **kwargs):
         """Display page based on a username and slug."""
 
         # Get page.
diff --git a/lib/galaxy/webapps/galaxy/controllers/visualization.py b/lib/galaxy/webapps/galaxy/controllers/visualization.py
index 622a896ab2b9..a530b7769978 100644
--- a/lib/galaxy/webapps/galaxy/controllers/visualization.py
+++ b/lib/galaxy/webapps/galaxy/controllers/visualization.py
@@ -111,8 +111,7 @@ def imp(self, trans, id, **kwargs):
                 use_panels=True,
             )
 
-    @web.expose
-    def display_by_username_and_slug(self, trans, username, slug, **kwargs):
+    def _display_by_username_and_slug(self, trans, username, slug, **kwargs):
         """Display visualization based on a username and slug."""
 
         # Get visualization.
diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py
index 5928e925df58..186a95b56116 100644
--- a/lib/galaxy/webapps/galaxy/controllers/workflow.py
+++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py
@@ -43,8 +43,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
     def __init__(self, app: StructuredApp):
         super().__init__(app)
 
-    @web.expose
-    def display_by_username_and_slug(self, trans, username, slug, format="html", **kwargs):
+    def _display_by_username_and_slug(self, trans, username, slug, format="html", **kwargs):
         """
         Display workflow based on a username and slug. Format can be html, json, or json-download.
         """

From 35db3c33db959215f2152da878da99712e336be9 Mon Sep 17 00:00:00 2001
From: Arash <arash77.kad@gmail.com>
Date: Mon, 14 Oct 2024 17:12:02 +0200
Subject: [PATCH 69/84] No escape needed for '/' in slug input

---
 client/src/components/Sharing/SlugInput.vue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/components/Sharing/SlugInput.vue b/client/src/components/Sharing/SlugInput.vue
index 291cb06641fd..8c9751ccca62 100644
--- a/client/src/components/Sharing/SlugInput.vue
+++ b/client/src/components/Sharing/SlugInput.vue
@@ -15,7 +15,7 @@ const initialSlug = props.slug;
 function onChange(value: string) {
     const slugFormatted = value
         .replace(/\s+/g, "-")
-        .replace(/[\/:?#]/g, "")
+        .replace(/[/:?#]/g, "")
         .toLowerCase();
 
     emit("change", slugFormatted);

From 29479cd00a1c942d84b282e8ffbf7450de0824f9 Mon Sep 17 00:00:00 2001
From: Arash <arash77.kad@gmail.com>
Date: Tue, 15 Oct 2024 15:19:29 +0200
Subject: [PATCH 70/84] Refactor slug generation to use slugify library

---
 lib/galaxy/managers/base.py     |  5 ++---
 lib/galaxy/managers/sharable.py | 13 ++-----------
 2 files changed, 4 insertions(+), 14 deletions(-)

diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py
index cc09af0059ea..c5ea85ab5e94 100644
--- a/lib/galaxy/managers/base.py
+++ b/lib/galaxy/managers/base.py
@@ -46,6 +46,7 @@
 )
 
 import sqlalchemy
+from slugify import slugify
 from sqlalchemy.orm import Query
 from sqlalchemy.orm.scoping import scoped_session
 from typing_extensions import Protocol
@@ -1296,9 +1297,7 @@ def raise_filter_err(attr, op, val, msg):
 
 def is_valid_slug(slug):
     """Returns true if slug is valid."""
-
-    VALID_SLUG_RE = re.compile(r"^[^/:?#]+$")
-    return VALID_SLUG_RE.match(slug)
+    return slugify(slug, allow_unicode=True) == slug
 
 
 class SortableManager:
diff --git a/lib/galaxy/managers/sharable.py b/lib/galaxy/managers/sharable.py
index 9318dd2b559a..cf7ff0a2a5c5 100644
--- a/lib/galaxy/managers/sharable.py
+++ b/lib/galaxy/managers/sharable.py
@@ -20,6 +20,7 @@
     Type,
 )
 
+from slugify import slugify
 from sqlalchemy import (
     exists,
     false,
@@ -309,16 +310,6 @@ def set_slug(self, item, new_slug, user, flush=True):
                 session.commit()
         return item
 
-    def _slugify(self, start_with):
-        # Replace whitespace with '-'
-        slug_base = re.sub(r"\s+", "-", start_with)
-        # Remove all /:?# characters.
-        slug_base = re.sub(r"[/:?#]", "", slug_base)
-        # Remove trailing '-'.
-        if slug_base.endswith("-"):
-            slug_base = slug_base[:-1]
-        return slug_base
-
     def _default_slug_base(self, item):
         # override in subclasses
         if hasattr(item, "title"):
@@ -334,7 +325,7 @@ def get_unique_slug(self, item):
 
         # Setup slug base.
         if cur_slug is None or cur_slug == "":
-            slug_base = self._slugify(self._default_slug_base(item))
+            slug_base = slugify(self._default_slug_base(item), allow_unicode=True)
         else:
             slug_base = cur_slug
 

From 49a22a62d51a08ec01ec8ab628355d5f3d68906f Mon Sep 17 00:00:00 2001
From: Arash <arash77.kad@gmail.com>
Date: Tue, 15 Oct 2024 15:26:17 +0200
Subject: [PATCH 71/84] remove unused import

---
 lib/galaxy/managers/sharable.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/galaxy/managers/sharable.py b/lib/galaxy/managers/sharable.py
index cf7ff0a2a5c5..04e00357f518 100644
--- a/lib/galaxy/managers/sharable.py
+++ b/lib/galaxy/managers/sharable.py
@@ -11,7 +11,6 @@
 """
 
 import logging
-import re
 from typing import (
     Any,
     List,

From 2740abe4c02814ffc0f6ea429096a764c59561b7 Mon Sep 17 00:00:00 2001
From: Arash <arash77.kad@gmail.com>
Date: Tue, 15 Oct 2024 15:46:26 +0200
Subject: [PATCH 72/84] Add types-python-slugify library for mypy type check

---
 lib/galaxy/dependencies/pinned-typecheck-requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/galaxy/dependencies/pinned-typecheck-requirements.txt b/lib/galaxy/dependencies/pinned-typecheck-requirements.txt
index 316819f75cd7..4272d276e9a1 100644
--- a/lib/galaxy/dependencies/pinned-typecheck-requirements.txt
+++ b/lib/galaxy/dependencies/pinned-typecheck-requirements.txt
@@ -30,3 +30,4 @@ types-setuptools==75.2.0.20241025
 types-six==1.16.21.20241009
 types-urllib3==1.26.25.14
 typing-extensions==4.12.2
+types-python-slugify==8.0.2.20240310

From 9acb8a1fd00ca084f0277f0f95f0a73e286fcd84 Mon Sep 17 00:00:00 2001
From: Arash <arash77.kad@gmail.com>
Date: Tue, 15 Oct 2024 17:42:40 +0200
Subject: [PATCH 73/84] order alphabetically

---
 lib/galaxy/dependencies/pinned-typecheck-requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/dependencies/pinned-typecheck-requirements.txt b/lib/galaxy/dependencies/pinned-typecheck-requirements.txt
index 4272d276e9a1..947da8733d03 100644
--- a/lib/galaxy/dependencies/pinned-typecheck-requirements.txt
+++ b/lib/galaxy/dependencies/pinned-typecheck-requirements.txt
@@ -23,6 +23,7 @@ types-html5lib==1.1.11.20241018
 types-markdown==3.7.0.20240822
 types-paramiko==3.5.0.20240928
 types-python-dateutil==2.9.0.20241003
+types-python-slugify==8.0.2.20240310
 types-pyyaml==6.0.12.20240917
 types-requests==2.31.0.6
 types-s3transfer==0.10.3
@@ -30,4 +31,3 @@ types-setuptools==75.2.0.20241025
 types-six==1.16.21.20241009
 types-urllib3==1.26.25.14
 typing-extensions==4.12.2
-types-python-slugify==8.0.2.20240310

From 0d21224657766fe55aea2c1eb834e0efb35f2349 Mon Sep 17 00:00:00 2001
From: Arash <arash77.kad@gmail.com>
Date: Tue, 15 Oct 2024 17:47:08 +0200
Subject: [PATCH 74/84] Add python-slugify library to app/setup.cfg

---
 packages/app/setup.cfg | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/app/setup.cfg b/packages/app/setup.cfg
index 7f3e9775529e..1e6b7ca57d3f 100644
--- a/packages/app/setup.cfg
+++ b/packages/app/setup.cfg
@@ -64,6 +64,7 @@ install_requires =
     pulsar-galaxy-lib>=0.15.0.dev0
     pydantic>=2.7.4
     pysam>=0.21
+    python-slugify
     PyJWT
     PyYAML
     refgenconf>=0.12.0

From 4247a5cc44ef0a8cf59d23acd01f3eb8018073c9 Mon Sep 17 00:00:00 2001
From: Arash <arash77.kad@gmail.com>
Date: Thu, 17 Oct 2024 15:54:33 +0200
Subject: [PATCH 75/84] Assert slug in response

---
 lib/galaxy_test/api/test_histories.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy_test/api/test_histories.py b/lib/galaxy_test/api/test_histories.py
index 13648aee583f..42fbe529ef16 100644
--- a/lib/galaxy_test/api/test_histories.py
+++ b/lib/galaxy_test/api/test_histories.py
@@ -548,7 +548,7 @@ def test_publish_non_alphanumeric(self):
         history_name = "تاریخچه"
         history_id = self.dataset_populator.new_history(name=history_name)
         response = self.dataset_populator.make_public(history_id)
-        assert response["username_and_slug"]
+        assert history_name in response["username_and_slug"]
 
     def test_immutable_history_update_fails(self):
         history_id = self._create_history("TestHistoryForImmutability")["id"]

From 5d9a867dbe9ffe8e9bf9a4786b3a809c19a5f858 Mon Sep 17 00:00:00 2001
From: Arash <arash77.kad@gmail.com>
Date: Thu, 24 Oct 2024 14:09:15 +0200
Subject: [PATCH 76/84] Move is_valid_slug function into SlugBuilder class as
 classmethod

---
 lib/galaxy/managers/base.py                          | 6 ------
 lib/galaxy/managers/pages.py                         | 2 +-
 lib/galaxy/managers/sharable.py                      | 7 ++++++-
 lib/galaxy/webapps/base/controller.py                | 2 +-
 lib/galaxy/webapps/galaxy/services/visualizations.py | 7 ++-----
 5 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py
index c5ea85ab5e94..135616fbed5a 100644
--- a/lib/galaxy/managers/base.py
+++ b/lib/galaxy/managers/base.py
@@ -46,7 +46,6 @@
 )
 
 import sqlalchemy
-from slugify import slugify
 from sqlalchemy.orm import Query
 from sqlalchemy.orm.scoping import scoped_session
 from typing_extensions import Protocol
@@ -1295,11 +1294,6 @@ def raise_filter_err(attr, op, val, msg):
     raise exceptions.RequestParameterInvalidException(msg, column=attr, operation=op, val=val)
 
 
-def is_valid_slug(slug):
-    """Returns true if slug is valid."""
-    return slugify(slug, allow_unicode=True) == slug
-
-
 class SortableManager:
     """A manager interface for parsing order_by strings into actual 'order by' queries."""
 
diff --git a/lib/galaxy/managers/pages.py b/lib/galaxy/managers/pages.py
index 581b5b93562b..101d785c688e 100644
--- a/lib/galaxy/managers/pages.py
+++ b/lib/galaxy/managers/pages.py
@@ -248,7 +248,7 @@ def create_page(self, trans, payload: CreatePagePayload):
             raise exceptions.ObjectAttributeMissingException("Page name is required")
         elif not payload.slug:
             raise exceptions.ObjectAttributeMissingException("Page id is required")
-        elif not base.is_valid_slug(payload.slug):
+        elif not sharable.SlugBuilder.is_valid_slug(payload.slug):
             raise exceptions.ObjectAttributeInvalidException(
                 "Page identifier must consist of only lowercase letters, numbers, and the '-' character"
             )
diff --git a/lib/galaxy/managers/sharable.py b/lib/galaxy/managers/sharable.py
index 04e00357f518..cfdcda372e97 100644
--- a/lib/galaxy/managers/sharable.py
+++ b/lib/galaxy/managers/sharable.py
@@ -291,7 +291,7 @@ def set_slug(self, item, new_slug, user, flush=True):
         Validate and set the new slug for `item`.
         """
         # precondition: has been validated
-        if not base.is_valid_slug(new_slug):
+        if not SlugBuilder.is_valid_slug(new_slug):
             raise exceptions.RequestParameterInvalidException("Invalid slug", slug=new_slug)
 
         if item.slug == new_slug:
@@ -563,6 +563,11 @@ def create_item_slug(self, sa_session, item) -> bool:
         item.slug = new_slug
         return item.slug == cur_slug
 
+    @classmethod
+    def is_valid_slug(self, slug):
+        """Returns true if slug is valid."""
+        return slugify(slug, allow_unicode=True) == slug
+
 
 def slug_exists(session, model_class, user, slug, ignore_deleted=False):
     stmt = select(exists().where(model_class.user == user).where(model_class.slug == slug))
diff --git a/lib/galaxy/webapps/base/controller.py b/lib/galaxy/webapps/base/controller.py
index 0cff38f5c862..9ee829bfae28 100644
--- a/lib/galaxy/webapps/base/controller.py
+++ b/lib/galaxy/webapps/base/controller.py
@@ -1090,7 +1090,7 @@ class SharableMixin:
 
     def _is_valid_slug(self, slug):
         """Returns true if slug is valid."""
-        return managers_base.is_valid_slug(slug)
+        return SlugBuilder.is_valid_slug(slug)
 
     @web.expose
     @web.require_login("modify Galaxy items")
diff --git a/lib/galaxy/webapps/galaxy/services/visualizations.py b/lib/galaxy/webapps/galaxy/services/visualizations.py
index 220db64d20dd..d2b7e6819519 100644
--- a/lib/galaxy/webapps/galaxy/services/visualizations.py
+++ b/lib/galaxy/webapps/galaxy/services/visualizations.py
@@ -8,10 +8,7 @@
 )
 
 from galaxy import exceptions
-from galaxy.managers.base import (
-    is_valid_slug,
-    security_check,
-)
+from galaxy.managers.base import security_check
 from galaxy.managers.context import ProvidesUserContext
 from galaxy.managers.sharable import (
     slug_exists,
@@ -302,7 +299,7 @@ def _create_visualization(
         # Error checking.
         if slug:
             slug_err = ""
-            if not is_valid_slug(slug):
+            if not SlugBuilder.is_valid_slug(slug):
                 slug_err = (
                     "visualization identifier must consist of only lowercase letters, numbers, and the '-' character"
                 )

From b97ede779c1ab72a1c714016dd58b705f001eb7b Mon Sep 17 00:00:00 2001
From: Nicola Soranzo <nicola.soranzo@earlham.ac.uk>
Date: Tue, 5 Nov 2024 15:28:02 +0000
Subject: [PATCH 77/84] Add type annotations

---
 lib/galaxy/tools/data_fetch.py | 39 +++++++++++++++++++---------------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/lib/galaxy/tools/data_fetch.py b/lib/galaxy/tools/data_fetch.py
index fe1419aa1434..01d43461f509 100644
--- a/lib/galaxy/tools/data_fetch.py
+++ b/lib/galaxy/tools/data_fetch.py
@@ -34,6 +34,7 @@
 from galaxy.util.compression_utils import CompressedFile
 from galaxy.util.hash_util import (
     HASH_NAMES,
+    HashFunctionNameEnum,
     verify_hash,
 )
 
@@ -85,7 +86,7 @@ def _request_to_galaxy_json(upload_config: "UploadConfig", request):
     return {"__unnamed_outputs": fetched_targets}
 
 
-def _fetch_target(upload_config: "UploadConfig", target):
+def _fetch_target(upload_config: "UploadConfig", target: Dict[str, Any]):
     destination = target.get("destination", None)
     assert destination, "No destination defined."
 
@@ -254,14 +255,15 @@ def _resolve_item_with_primary(item):
             hash_value = item.get(hash_function)
             if hash_value:
                 hashes.append({"hash_function": hash_function, "hash_value": hash_value})
-        for hash_dict in hashes:
-            hash_function = hash_dict.get("hash_function")
-            hash_value = hash_dict.get("hash_value")
-            try:
-                _handle_hash_validation(upload_config, hash_function, hash_value, path)
-            except Exception as e:
-                error_message = str(e)
-                item["error_message"] = error_message
+        if path:
+            for hash_dict in hashes:
+                hash_function = hash_dict.get("hash_function")
+                hash_value = hash_dict.get("hash_value")
+                try:
+                    _handle_hash_validation(upload_config, hash_function, hash_value, path)
+                except Exception as e:
+                    error_message = str(e)
+                    item["error_message"] = error_message
 
         dbkey = item.get("dbkey", "?")
         link_data_only = upload_config.link_data_only
@@ -422,7 +424,7 @@ def _bagit_to_items(directory):
     return items
 
 
-def _decompress_target(upload_config: "UploadConfig", target):
+def _decompress_target(upload_config: "UploadConfig", target: Dict[str, Any]):
     elements_from_name, elements_from_path = _has_src_to_path(upload_config, target, is_dataset=False)
     # by default Galaxy will check for a directory with a single file and interpret that
     # as the new root for expansion, this is a good user experience for uploading single
@@ -481,12 +483,13 @@ def _has_src_to_name(item) -> Optional[str]:
     return name
 
 
-def _has_src_to_path(upload_config, item, is_dataset=False) -> Tuple[str, str]:
+def _has_src_to_path(upload_config: "UploadConfig", item: Dict[str, Any], is_dataset: bool = False) -> Tuple[str, str]:
     assert "src" in item, item
     src = item.get("src")
     name = item.get("name")
     if src == "url":
         url = item.get("url")
+        assert url, "url cannot be empty"
         try:
             path = stream_url_to_file(url, file_sources=upload_config.file_sources, dir=upload_config.working_directory)
         except Exception as e:
@@ -513,7 +516,9 @@ def _has_src_to_path(upload_config, item, is_dataset=False) -> Tuple[str, str]:
     return name, path
 
 
-def _handle_hash_validation(upload_config, hash_function, hash_value, path):
+def _handle_hash_validation(
+    upload_config: "UploadConfig", hash_function: HashFunctionNameEnum, hash_value: str, path: str
+):
     if upload_config.validate_hashes:
         verify_hash(path, hash_func_name=hash_function, hash_value=hash_value, what="upload")
 
@@ -548,11 +553,11 @@ def get_file_sources(working_directory, file_sources_as_dict=None):
 class UploadConfig:
     def __init__(
         self,
-        request,
-        registry,
-        working_directory,
-        allow_failed_collections,
-        file_sources_dict=None,
+        request: Dict[str, Any],
+        registry: Registry,
+        working_directory: str,
+        allow_failed_collections: bool,
+        file_sources_dict: Optional[Dict] = None,
     ):
         self.registry = registry
         self.working_directory = working_directory

From 3c601ffd385f9f3a48a8ea6e0623c4b2a9f43b5d Mon Sep 17 00:00:00 2001
From: Arash <arash77.kad@gmail.com>
Date: Tue, 5 Nov 2024 16:33:29 +0100
Subject: [PATCH 78/84] Add python-slugify and types-python-slugify to
 dependencies and typecheck

---
 pyproject.toml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pyproject.toml b/pyproject.toml
index b50664156ef4..674528fdc0ce 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -83,6 +83,7 @@ dependencies = [
     "python-dateutil",
     "python-magic",
     "python-multipart",  # required to support form parsing in FastAPI/Starlette
+    "python-slugify",
     "PyYAML",
     "refgenconf>=0.12.0",
     "regex",
@@ -172,6 +173,7 @@ typecheck = [
     "types-Markdown",
     "types-paramiko",
     "types-python-dateutil",
+    "types-python-slugify",
     "types-PyYAML",
     "types-requests",
     "types-setuptools",

From 1f223d0dc501a5345f33a9cf8015f8e04fe36404 Mon Sep 17 00:00:00 2001
From: Nicola Soranzo <nicola.soranzo@earlham.ac.uk>
Date: Tue, 5 Nov 2024 15:33:40 +0000
Subject: [PATCH 79/84] Test hash validation also for upload by path

---
 test/unit/app/tools/test_data_fetch.py | 36 +++++++++++++++++++++++---
 1 file changed, 33 insertions(+), 3 deletions(-)

diff --git a/test/unit/app/tools/test_data_fetch.py b/test/unit/app/tools/test_data_fetch.py
index ee7cdd61536f..8c1a64318de6 100644
--- a/test/unit/app/tools/test_data_fetch.py
+++ b/test/unit/app/tools/test_data_fetch.py
@@ -5,6 +5,9 @@
 from contextlib import contextmanager
 from shutil import rmtree
 from tempfile import mkdtemp
+from typing import Optional
+
+import pytest
 
 from galaxy.tools.data_fetch import main
 from galaxy.util.unittest_utils import skip_if_github_down
@@ -13,7 +16,17 @@
 URI_FOR_1_2_3 = f"base64://{B64_FOR_1_2_3}"
 
 
-def test_simple_path_get():
+@pytest.mark.parametrize(
+    "hash_value, error_message",
+    [
+        ("471ddd37fc297fba09b893b88739ece9", None),
+        (
+            "thisisbad",
+            "Failed to validate upload with [MD5] - expected [thisisbad] got [471ddd37fc297fba09b893b88739ece9]",
+        ),
+    ],
+)
+def test_simple_path_get(hash_value: str, error_message: Optional[str]):
     with _execute_context() as execute_context:
         job_directory = execute_context.job_directory
         example_path = os.path.join(job_directory, "example_file")
@@ -25,13 +38,30 @@ def test_simple_path_get():
                     "destination": {
                         "type": "hdas",
                     },
-                    "elements": [{"src": "path", "path": example_path}],
+                    "elements": [
+                        {
+                            "src": "path",
+                            "path": example_path,
+                            "hashes": [
+                                {
+                                    "hash_function": "MD5",
+                                    "hash_value": hash_value,
+                                }
+                            ],
+                        }
+                    ],
                 }
-            ]
+            ],
+            "validate_hashes": True,
         }
         execute_context.execute_request(request)
         output = _unnamed_output(execute_context)
         assert output
+        hda_result = output["elements"][0]
+        if error_message is not None:
+            assert hda_result["error_message"] == error_message
+        else:
+            assert "error_message" not in hda_result
 
 
 @skip_if_github_down

From b1691c56f8933872c05adacdc391578b63002b31 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 4 Nov 2024 18:32:33 +0100
Subject: [PATCH 80/84] Rerender on resize

---
 client/src/components/WorkflowInvocationState/VegaWrapper.vue | 4 ++++
 .../WorkflowInvocationState/WorkflowInvocationMetrics.vue     | 4 +++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/client/src/components/WorkflowInvocationState/VegaWrapper.vue b/client/src/components/WorkflowInvocationState/VegaWrapper.vue
index 586fa124cf0e..e5f4a8f49ba8 100644
--- a/client/src/components/WorkflowInvocationState/VegaWrapper.vue
+++ b/client/src/components/WorkflowInvocationState/VegaWrapper.vue
@@ -3,6 +3,7 @@
 </template>
 
 <script setup lang="ts">
+import { useResizeObserver } from "@vueuse/core";
 import embed, { type VisualizationSpec } from "vega-embed";
 import { onBeforeUnmount, onMounted, ref, watch } from "vue";
 
@@ -28,6 +29,9 @@ async function embedChart() {
 onMounted(embedChart);
 
 watch(props, embedChart, { immediate: true, deep: true });
+useResizeObserver(chartContainer, () => {
+    embedChart();
+});
 
 // Cleanup the chart when the component is unmounted
 onBeforeUnmount(() => {
diff --git a/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue b/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue
index 4da3248864f1..f848d9f1d6b9 100644
--- a/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue
+++ b/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue
@@ -150,7 +150,9 @@ function itemToSpec(item: boxplotData) {
 }
 
 const specs = computed(() => {
-    const items = [wallclock.value, coresAllocated.value, memoryAllocated.value].filter((item) => item.values?.length);
+    const items = [wallclock.value, coresAllocated.value, memoryAllocated.value, peakMemory.value].filter(
+        (item) => item.values?.length
+    );
     const specs = Object.fromEntries(items.map((item) => [item.y_title, itemToSpec(item)]));
     return specs;
 });

From 84742ee1412832115ebfd0a915a0bbe4cdc06387 Mon Sep 17 00:00:00 2001
From: Nicola Soranzo <nicola.soranzo@earlham.ac.uk>
Date: Tue, 5 Nov 2024 23:44:04 +0000
Subject: [PATCH 81/84] Run ``uv lock`` under Python 3.9
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fix:

```
Using CPython 3.8.18 interpreter at: /opt/hostedtoolcache/Python/3.8.18/x64/bin/python
  × No solution found when resolving dependencies for split
  │ (python_full_version == '3.9.*'):
  ╰─▶ Because only the following versions of rucio-clients{python_full_version >= '3.9'} are available:
          rucio-clients{python_full_version >= '3.9'}<=33.6.1
          rucio-clients{python_full_version >= '3.9'}>=34.0.0,<=34.6.0
          rucio-clients{python_full_version >= '3.9'}>=35.0.0
      and all of:
          rucio-clients{python_full_version >= '3.9'}>=33.6.0,<=33.6.1
          rucio-clients{python_full_version >= '3.9'}>=34.0.0,<=34.6.0
          rucio-clients{python_full_version >= '3.9'}>=35.0.0
      require Python >=3.9, <4, we can conclude that all of:
          rucio-clients{python_full_version >= '3.9'}>=33.6.0,<33.6.1
          ...
          rucio-clients{python_full_version >= '3.9'}>35.4.1,<35.5.0
          rucio-clients{python_full_version >= '3.9'}>35.5.0
       are incompatible.
      And because galaxy:dev depends on rucio-clients{python_full_version >=
      '3.9'}>=33.6.0 and your project depends on galaxy:dev, we can conclude
      that your project's requirements are unsatisfiable.

      hint: The source distribution for rucio-clients{python_full_version >=
      '3.9'}==35.5.0 does not include static metadata. Generating metadata for
      this package requires Python >=3.9, <4, but Python 3.8.18 is installed.
```

See https://github.com/galaxyproject/galaxy/actions/runs/11684632042/job/32536277008
---
 .github/workflows/dependencies.yaml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/dependencies.yaml b/.github/workflows/dependencies.yaml
index 7f3e60d54774..2401107eb325 100644
--- a/.github/workflows/dependencies.yaml
+++ b/.github/workflows/dependencies.yaml
@@ -8,14 +8,15 @@ jobs:
     name: Update dependencies
     if: github.repository_owner == 'galaxyproject'
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ['3.8']
     steps:
       - uses: actions/checkout@v4
+      # Install Python 3.8 for update_lint_requirements.sh
+      # Install Python 3.9 (as default) to allow `uv lock` to generate metadata for rucio-clients
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: |
+            3.8
+            3.9
       - name: Update dependencies
         run: make update-dependencies
       - name: Create pull request

From 3a6fdf24fb64269e8bf82dfaf93c7b5da46e9284 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 5 Nov 2024 11:37:52 +0100
Subject: [PATCH 82/84] Allow searching jobs in public histories for job cache

---
 client/src/api/schema/schema.ts   | 10 +++++++
 lib/galaxy/managers/jobs.py       | 17 ++++++++----
 lib/galaxy/model/__init__.py      |  1 +
 lib/galaxy/schema/jobs.py         |  1 +
 lib/galaxy_test/api/test_tools.py | 43 ++++++++++++++++++++++++++-----
 lib/galaxy_test/base/api.py       |  3 ++-
 6 files changed, 63 insertions(+), 12 deletions(-)

diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts
index 93d99b75fa0a..409b4bd01bd2 100644
--- a/client/src/api/schema/schema.ts
+++ b/client/src/api/schema/schema.ts
@@ -8753,6 +8753,11 @@ export interface components {
              * @description The email of the user that owns this job. Only the owner of the job and administrators can see this value.
              */
             user_email?: string | null;
+            /**
+             * User Id
+             * @description User ID of user that ran ran job
+             */
+            user_id?: string | null;
         };
         /** EncodedJobParameterHistoryItem */
         EncodedJobParameterHistoryItem: {
@@ -16177,6 +16182,11 @@ export interface components {
              * @description The email of the user that owns this job. Only the owner of the job and administrators can see this value.
              */
             user_email?: string | null;
+            /**
+             * User Id
+             * @description User ID of user that ran ran job
+             */
+            user_id?: string | null;
         };
         /**
          * Src
diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py
index 1d4a61dac6f0..4b214f5b8af8 100644
--- a/lib/galaxy/managers/jobs.py
+++ b/lib/galaxy/managers/jobs.py
@@ -493,11 +493,18 @@ def _build_job_subquery(
         self, tool_id: str, user_id: int, tool_version: Optional[str], job_state, wildcard_param_dump
     ):
         """Build subquery that selects a job with correct job parameters."""
-        stmt = select(model.Job.id).where(
-            and_(
-                model.Job.tool_id == tool_id,
-                model.Job.user_id == user_id,
-                model.Job.copied_from_job_id.is_(None),  # Always pick original job
+        stmt = (
+            select(model.Job.id)
+            .join(model.History, model.Job.history_id == model.History.id)
+            .where(
+                and_(
+                    model.Job.tool_id == tool_id,
+                    or_(
+                        model.Job.user_id == user_id,
+                        model.History.published == true(),
+                    ),
+                    model.Job.copied_from_job_id.is_(None),  # Always pick original job
+                )
             )
         )
         if tool_version:
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index ad657b8cc717..f1015a284738 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -1549,6 +1549,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable):
         "galaxy_version",
         "command_version",
         "copied_from_job_id",
+        "user_id",
     ]
 
     _numeric_metric = JobMetricNumeric
diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py
index ec7a971a3614..6b785b904093 100644
--- a/lib/galaxy/schema/jobs.py
+++ b/lib/galaxy/schema/jobs.py
@@ -191,6 +191,7 @@ class EncodedJobDetails(JobSummary):
         title="Output collections",
         description="",
     )
+    user_id: Optional[EncodedDatabaseIdField] = Field(default=None, description="User ID of user that ran ran job")
 
 
 class JobDestinationParams(Model):
diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py
index 75c3998d8f30..6be4ea256490 100644
--- a/lib/galaxy_test/api/test_tools.py
+++ b/lib/galaxy_test/api/test_tools.py
@@ -8,7 +8,9 @@
     Any,
     Dict,
     List,
+    Optional,
 )
+from uuid import uuid4
 
 import pytest
 from requests import (
@@ -974,15 +976,19 @@ def test_run_cat1(self):
             output1_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output1)
             assert output1_content.strip() == "Cat1Test"
 
+    def _get_cat1_inputs(self, history_id):
+        new_dataset = self.dataset_populator.new_dataset(history_id, content="Cat1Test")
+        inputs = dict(
+            input1=dataset_to_param(new_dataset),
+        )
+        return inputs
+
     @skip_without_tool("cat1")
     @requires_new_history
     def test_run_cat1_use_cached_job(self):
         with self.dataset_populator.test_history_for(self.test_run_cat1_use_cached_job) as history_id:
             # Run simple non-upload tool with an input data parameter.
-            new_dataset = self.dataset_populator.new_dataset(history_id, content="Cat1Test")
-            inputs = dict(
-                input1=dataset_to_param(new_dataset),
-            )
+            inputs = self._get_cat1_inputs(history_id)
             outputs_one = self._run_cat1(history_id, inputs=inputs, assert_ok=True, wait_for_job=True)
             outputs_two = self._run_cat1(
                 history_id, inputs=inputs, use_cached_job=False, assert_ok=True, wait_for_job=True
@@ -998,6 +1004,31 @@ def test_run_cat1_use_cached_job(self):
             assert len(filenames) == 3, filenames
             assert len(set(filenames)) <= 2, filenames
 
+    @skip_without_tool("cat1")
+    @requires_new_history
+    def test_run_cat1_use_cached_job_from_public_history(self):
+        with self.dataset_populator.test_history_for(self.test_run_cat1_use_cached_job) as history_id:
+            # Run simple non-upload tool with an input data parameter.
+            inputs = self._get_cat1_inputs(history_id)
+            original_output = self._run_cat1(history_id, inputs=inputs, assert_ok=True, wait_for_job=True)
+            original_job = self.dataset_populator.get_job_details(original_output["jobs"][0]["id"], full=True).json()
+
+            def run_again(user_email):
+                with self._different_user_and_history(user_email=user_email) as different_history_id:
+                    cached_output = self._run_cat1(
+                        different_history_id, inputs=inputs, use_cached_job=True, assert_ok=True, wait_for_job=True
+                    )
+                    return self.dataset_populator.get_job_details(cached_output["jobs"][0]["id"], full=True).json()
+
+            job = run_again(f"{uuid4()}@test.com")
+            assert job["user_id"] != original_job["user_id"]
+            assert not job["copied_from_job_id"]
+            # publish history, now we can use cached job
+            self.dataset_populator.make_public(history_id=history_id)
+            cached_job = run_again(f"{uuid4()}@test.com")
+            assert cached_job["user_id"] != original_job["user_id"]
+            assert cached_job["copied_from_job_id"] == original_output["jobs"][0]["id"]
+
     @skip_without_tool("cat1")
     def test_run_cat1_listified_param(self):
         with self.dataset_populator.test_history_for(self.test_run_cat1_listified_param) as history_id:
@@ -2809,8 +2840,8 @@ def _assert_dataset_permission_denied_response(self, response):
         # assert "User does not have permission to use a dataset" in err_message, err_message
 
     @contextlib.contextmanager
-    def _different_user_and_history(self):
-        with self._different_user():
+    def _different_user_and_history(self, user_email: Optional[str] = None):
+        with self._different_user(email=user_email):
             with self.dataset_populator.test_history() as other_history_id:
                 yield other_history_id
 
diff --git a/lib/galaxy_test/base/api.py b/lib/galaxy_test/base/api.py
index 63e33dd5e90a..057d0bf3000d 100644
--- a/lib/galaxy_test/base/api.py
+++ b/lib/galaxy_test/base/api.py
@@ -147,7 +147,7 @@ def _setup_user_get_key(self, email, password=None):
         return user, self._post(f"users/{user['id']}/api_key", admin=True).json()
 
     @contextmanager
-    def _different_user(self, email=OTHER_USER, anon=False):
+    def _different_user(self, email: Optional[str] = OTHER_USER, anon=False):
         """Use in test cases to switch get/post operations to act as new user
 
         ..code-block:: python
@@ -156,6 +156,7 @@ def _different_user(self, email=OTHER_USER, anon=False):
                 self._get("histories")  # Gets other_user@bx.psu.edu histories.
 
         """
+        email = OTHER_USER if email is None else email
         original_api_key = self.user_api_key
         original_interactor_key = self.galaxy_interactor.api_key
         original_cookies = self.galaxy_interactor.cookies

From 360b72a82a3b7b2fe9dd626760c201928542e876 Mon Sep 17 00:00:00 2001
From: Marius van den Beek <m.vandenbeek@gmail.com>
Date: Wed, 6 Nov 2024 11:19:52 +0100
Subject: [PATCH 83/84] Improve user_id description, set email where needed

---
 client/src/api/schema/schema.ts | 4 ++--
 lib/galaxy/schema/jobs.py       | 2 +-
 lib/galaxy_test/base/api.py     | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts
index 409b4bd01bd2..0b3550f4bbf8 100644
--- a/client/src/api/schema/schema.ts
+++ b/client/src/api/schema/schema.ts
@@ -8755,7 +8755,7 @@ export interface components {
             user_email?: string | null;
             /**
              * User Id
-             * @description User ID of user that ran ran job
+             * @description User ID of user that ran this job
              */
             user_id?: string | null;
         };
@@ -16184,7 +16184,7 @@ export interface components {
             user_email?: string | null;
             /**
              * User Id
-             * @description User ID of user that ran ran job
+             * @description User ID of user that ran this job
              */
             user_id?: string | null;
         };
diff --git a/lib/galaxy/schema/jobs.py b/lib/galaxy/schema/jobs.py
index 6b785b904093..44cc5061619c 100644
--- a/lib/galaxy/schema/jobs.py
+++ b/lib/galaxy/schema/jobs.py
@@ -191,7 +191,7 @@ class EncodedJobDetails(JobSummary):
         title="Output collections",
         description="",
     )
-    user_id: Optional[EncodedDatabaseIdField] = Field(default=None, description="User ID of user that ran ran job")
+    user_id: Optional[EncodedDatabaseIdField] = Field(default=None, description="User ID of user that ran this job")
 
 
 class JobDestinationParams(Model):
diff --git a/lib/galaxy_test/base/api.py b/lib/galaxy_test/base/api.py
index 057d0bf3000d..b1881f917b63 100644
--- a/lib/galaxy_test/base/api.py
+++ b/lib/galaxy_test/base/api.py
@@ -147,7 +147,7 @@ def _setup_user_get_key(self, email, password=None):
         return user, self._post(f"users/{user['id']}/api_key", admin=True).json()
 
     @contextmanager
-    def _different_user(self, email: Optional[str] = OTHER_USER, anon=False):
+    def _different_user(self, email: Optional[str] = None, anon=False):
         """Use in test cases to switch get/post operations to act as new user
 
         ..code-block:: python
@@ -156,7 +156,6 @@ def _different_user(self, email: Optional[str] = OTHER_USER, anon=False):
                 self._get("histories")  # Gets other_user@bx.psu.edu histories.
 
         """
-        email = OTHER_USER if email is None else email
         original_api_key = self.user_api_key
         original_interactor_key = self.galaxy_interactor.api_key
         original_cookies = self.galaxy_interactor.cookies
@@ -165,6 +164,7 @@ def _different_user(self, email: Optional[str] = OTHER_USER, anon=False):
             self.galaxy_interactor.cookies = cookies
             new_key = None
         else:
+            email = OTHER_USER if email is None else email
             _, new_key = self._setup_user_get_key(email)
         try:
             self.user_api_key = new_key

From 946d2ffa31673ce01f0f10efa7b8bbce483a6b2b Mon Sep 17 00:00:00 2001
From: Laila Los <44241786+ElectronicBlueberry@users.noreply.github.com>
Date: Wed, 6 Nov 2024 17:26:01 +0100
Subject: [PATCH 84/84] split vega related components into lazy loaded bundle

---
 .../WorkflowInvocationState/WorkflowInvocationMetrics.vue     | 4 ++--
 client/webpack.config.js                                      | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue b/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue
index f848d9f1d6b9..91782651b709 100644
--- a/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue
+++ b/client/src/components/WorkflowInvocationState/WorkflowInvocationMetrics.vue
@@ -1,12 +1,12 @@
 <script setup lang="ts">
-import { type VisualizationSpec } from "vega-embed";
+import type { VisualizationSpec } from "vega-embed";
 import { computed, ref, watch } from "vue";
 import { type ComputedRef } from "vue";
 
 import { type components, GalaxyApi } from "@/api";
 import { errorMessageAsString } from "@/utils/simple-error";
 
-import VegaWrapper from "./VegaWrapper.vue";
+const VegaWrapper = () => import("./VegaWrapper.vue");
 
 const props = defineProps({
     invocationId: {
diff --git a/client/webpack.config.js b/client/webpack.config.js
index 6ca6b748151d..43699d475e3a 100644
--- a/client/webpack.config.js
+++ b/client/webpack.config.js
@@ -25,6 +25,9 @@ const modulesExcludedFromLibs = [
     "elkjs",
     "@citation-js",
     "citeproc",
+    "vega",
+    "vega-embed",
+    "vega-lite",
 ].join("|");
 
 const buildDate = new Date();