Merge branch 'main' into integrate-notification-system

.github/workflows/deploy.yaml

@@ -4,7 +4,7 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
           python-version: '3.12'

.github/workflows/lint.yaml

@@ -10,11 +10,11 @@ jobs:
       matrix:
         python-version: ['3.8', '3.12']
     steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install tox
-      run: python -m pip install 'tox>=1.8.0'
-    - name: Lint
-      run: tox -e lint
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install tox
+        run: python -m pip install 'tox>=1.8.0'
+      - name: Lint
+        run: tox -e lint

.github/workflows/test.yaml

@@ -33,6 +33,7 @@ jobs:
         tox_env: [py38]
         galaxy_version:
           - dev
+          - release_24.1
           - release_24.0
           - release_23.2
           - release_23.1
@@ -59,7 +60,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Cache pip dir
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: pip-cache-${{ matrix.tox_env }}-${{ matrix.galaxy_version }}

CHANGELOG.md

@@ -1,8 +1,31 @@
 ### BioBlend v
 
+* Added support for Galaxy release 24.1.
+
+### BioBlend v1.3.0 - 2024-05-12
+
 * Dropped support for Python 3.7. Added support for Python 3.12. Added support
   for Galaxy releases 23.2 and 24.0.
 
+* Added ``copy_elements`` parameter to
+  ``HistoryClient.create_dataset_collection()`` and BioBlend.objects
+  ``History.create_dataset_collection()`` methods.
+
+* Added ``wait`` parameter to ``HistoryClient.delete_dataset()`` and
+  BioBlend.objects ``HistoryDatasetAssociation.delete()`` methods.
+
+* Added ``create_time_min``, ``create_time_max``, ``update_time_min``,
+  ``update_time_max``, ``view``, ``keys``, ``limit`` and ``offset`` parameters
+  to ``HistoryClient.get_histories()`` (thanks to
+  [cat-bro](https://github.com/cat-bro)).
+
+* Added ``create_time_min``, ``create_time_max``, ``update_time_min`` and
+  ``update_time_max`` parameters to ``HistoryClient.get_published_histories()``
+  (thanks to [cat-bro](https://github.com/cat-bro)).
+
+* Added ``keys`` parameter to ``HistoryClient.show_history()`` (thanks to
+  [cat-bro](https://github.com/cat-bro)).
+
 * Dropped broken ``deleted`` parameter of ``DatasetClient.show_dataset()``.
 
 * Parameters after ``password`` in the ``__init__()`` method of the
@@ -12,8 +35,17 @@
 * Classes defined in ``bioblend.galaxy.objects.wrappers`` are no more
   re-exported by ``bioblend.galaxy.objects``.
 
+* ``DatasetTimeoutException`` and ``DatasetCollectionTimeoutException`` are now
+  aliases for ``TimeoutException`` instead of subclasses.
+
 * Added support for the new "cancelling" invocation state.
 
+* Fixed ``InvocationClient.get_invocation_biocompute_object()`` method on
+  upcoming Galaxy 24.1 .
+
+* Improvements to linting and tests (thanks to
+  [Matthias Bernt](https://github.com/bernt-matthias)).
+
 ### BioBlend v1.2.0 - 2023-06-30
 
 * Dropped support for Galaxy releases 17.09-19.01. Added support for Galaxy

bioblend/__init__.py

@@ -2,8 +2,11 @@
 import logging
 import logging.config
 import os
+import time
 from typing import (
+    Callable,
     Optional,
+    TypeVar,
     Union,
 )
 
@@ -13,7 +16,7 @@
 )
 
 # Current version of the library
-__version__ = "1.2.0"
+__version__ = "1.3.0"
 
 # default chunk size (in bytes) for reading remote data
 try:
@@ -116,3 +119,38 @@ def __str__(self) -> str:
 
 class TimeoutException(Exception):
     pass
+
+
+class NotReady(Exception):
+    pass
+
+
+T = TypeVar("T")
+
+
+def wait_on(func: Callable[[], T], maxwait: float = 60, interval: float = 3) -> T:
+    """
+    Wait until a function returns without raising a NotReady exception
+
+    :param func: function to wait on. It should accept no parameters.
+
+    :param maxwait: Total time (in seconds) to wait for the function to return
+      without raising a NotReady exception. After this time, a
+      ``TimeoutException`` will be raised.
+
+    :param interval: Time (in seconds) to wait between 2 consecutive checks.
+    """
+    assert maxwait >= 0
+    assert interval > 0
+
+    time_left = maxwait
+    while True:
+        try:
+            return func()
+        except NotReady as e:
+            if time_left > 0:
+                log.info("%s. Will wait %s more s", e, time_left)
+                time.sleep(min(time_left, interval))
+                time_left -= interval
+            else:
+                raise TimeoutException(f"{e} after {maxwait} s")

bioblend/_tests/TestGalaxyDatasets.py

@@ -24,7 +24,6 @@ def setUp(self):
     def tearDown(self):
         self.gi.histories.delete_history(self.history_id, purge=True)
 
-    @test_util.skip_unless_galaxy("release_19.05")
     def test_show_nonexistent_dataset(self):
         with pytest.raises(ConnectionError):
             self.gi.datasets.show_dataset("nonexistent_id")
@@ -65,25 +64,21 @@ def test_download_dataset(self):
             f.flush()
             assert f.read() == expected_contents
 
-    @test_util.skip_unless_galaxy("release_19.05")
     def test_get_datasets(self):
         datasets = self.gi.datasets.get_datasets()
         dataset_ids = [dataset["id"] for dataset in datasets]
         assert self.dataset_id in dataset_ids
 
-    @test_util.skip_unless_galaxy("release_19.05")
     def test_get_datasets_history(self):
         datasets = self.gi.datasets.get_datasets(history_id=self.history_id)
         assert len(datasets) == 1
 
-    @test_util.skip_unless_galaxy("release_19.05")
     def test_get_datasets_limit_offset(self):
         datasets = self.gi.datasets.get_datasets(limit=1)
         assert len(datasets) == 1
         datasets = self.gi.datasets.get_datasets(history_id=self.history_id, offset=1)
         assert datasets == []
 
-    @test_util.skip_unless_galaxy("release_19.05")
     def test_get_datasets_name(self):
         datasets = self.gi.datasets.get_datasets(history_id=self.history_id, name="Pasted Entry")
         assert len(datasets) == 1
@@ -143,7 +138,6 @@ def test_get_datasets_visible(self):
         datasets = self.gi.datasets.get_datasets(history_id=self.history_id, visible=False)
         assert len(datasets) == 0
 
-    @test_util.skip_unless_galaxy("release_19.05")
     def test_get_datasets_ordering(self):
         self.dataset_id2 = self._test_dataset(self.history_id, contents=self.dataset_contents)
         self.gi.datasets.wait_for_dataset(self.dataset_id2)
@@ -156,7 +150,6 @@ def test_get_datasets_ordering(self):
         datasets = self.gi.datasets.get_datasets(history_id=self.history_id, order="hid-asc")
         assert datasets[0]["id"] == self.dataset_id
 
-    @test_util.skip_unless_galaxy("release_19.05")
     def test_get_datasets_deleted(self):
         deleted_datasets = self.gi.datasets.get_datasets(history_id=self.history_id, deleted=True)
         assert deleted_datasets == []
@@ -165,11 +158,10 @@ def test_get_datasets_deleted(self):
         assert len(deleted_datasets) == 1
         purged_datasets = self.gi.datasets.get_datasets(history_id=self.history_id, purged=True)
         assert purged_datasets == []
-        self.gi.histories.delete_dataset(self.history_id, self.dataset_id, purge=True)
+        self.gi.histories.delete_dataset(self.history_id, self.dataset_id, purge=True, wait=True)
         purged_datasets = self.gi.datasets.get_datasets(history_id=self.history_id, purged=True)
         assert len(purged_datasets) == 1
 
-    @test_util.skip_unless_galaxy("release_19.05")
     def test_get_datasets_tool_id_and_tag(self):
         cat1_datasets = self.gi.datasets.get_datasets(history_id=self.history_id, tool_id="cat1")
         assert cat1_datasets == []
@@ -189,7 +181,6 @@ def test_wait_for_dataset(self):
 
         self.gi.histories.delete_history(history_id, purge=True)
 
-    @test_util.skip_unless_galaxy("release_19.05")
     def test_dataset_permissions(self):
         admin_user_id = self.gi.users.get_current_user()["id"]
         username = test_util.random_string()

bioblend/_tests/TestGalaxyHistories.py

@@ -70,6 +70,15 @@ def test_get_histories(self):
         all_histories = self.gi.histories.get_histories()
         assert len(all_histories) > 0
 
+        # Test limit and offset
+        first = self.gi.histories.get_histories(limit=1)
+        others = self.gi.histories.get_histories(offset=1)
+        assert len(first) == 1
+        assert [h["id"] for h in all_histories] == [h["id"] for h in first] + [h["id"] for h in others]
+
+        out_of_limit = self.gi.histories.get_histories(offset=1000000)
+        assert out_of_limit == []
+
         # Check whether id is present, when searched by name
         histories = self.gi.histories.get_histories(name=self.default_history_name)
         assert len([h for h in histories if h["id"] == self.history["id"]]) == 1
@@ -91,7 +100,7 @@ def test_get_histories(self):
 
         # Test keys: check that fields requested are returned
         histories_with_keys = self.gi.histories.get_histories(keys=["id", "user_id", "size"])
-        assert {key for key in histories_with_keys[0]} >= {"id", "user_id", "size"}
+        assert set(histories_with_keys[0]) >= {"id", "user_id", "size"}
 
         # TODO: check whether deleted history is returned correctly
         # At the moment, get_histories() returns only not-deleted histories
@@ -194,7 +203,7 @@ def test_delete_dataset(self):
     def test_purge_dataset(self):
         history_id = self.history["id"]
         dataset1_id = self._test_dataset(history_id)
-        self.gi.histories.delete_dataset(history_id, dataset1_id, purge=True)
+        self.gi.histories.delete_dataset(history_id, dataset1_id, purge=True, wait=True)
         dataset = self.gi.histories.show_dataset(history_id, dataset1_id)
         assert dataset["deleted"]
         assert dataset["purged"]

bioblend/_tests/TestGalaxyJobs.py

@@ -144,7 +144,6 @@ def test_rerun_and_remap(self):
         assert last_dataset["id"] == history_contents[2]["id"]
         self._wait_and_verify_dataset(last_dataset["id"], b"line 1\tline 1\n")
 
-    @test_util.skip_unless_galaxy("release_19.05")
     @test_util.skip_unless_tool("random_lines1")
     def test_get_common_problems(self):
         job_id = self._run_tool()["jobs"][0]["id"]

bioblend/_tests/TestGalaxyObjects.py

@@ -966,7 +966,7 @@ def test_dataset_delete(self):
         assert not self.ds.purged
 
     def test_dataset_purge(self):
-        self.ds.delete(purge=True)
+        self.ds.delete(purge=True, wait=True)
         assert self.ds.deleted
         assert self.ds.purged
 

bioblend/_tests/TestGalaxyTools.py

@@ -125,7 +125,6 @@ def test_run_cat1(self):
         # TODO: Wait for results and verify it has 3 lines - 1 2 3, 4 5 6,
         # and 7 8 9.
 
-    @test_util.skip_unless_galaxy("release_19.05")
     @test_util.skip_unless_tool("CONVERTER_fasta_to_bowtie_color_index")
     def test_tool_dependency_install(self):
         installed_dependencies = self.gi.tools.install_dependencies("CONVERTER_fasta_to_bowtie_color_index")

bioblend/_tests/TestGalaxyUsers.py

@@ -11,6 +11,7 @@ def test_get_users(self):
         for user in users:
             assert user["id"] is not None
             assert user["email"] is not None
+            assert user["deleted"] is not None
 
     def test_show_user(self):
         current_user = self.gi.users.get_current_user()
@@ -23,10 +24,9 @@ def test_show_user(self):
     #        assert user["nice_total_disk_usage"] == current_user["nice_total_disk_usage"]
     #        assert user["total_disk_usage"] == current_user["total_disk_usage"]
 
+    @test_util.skip_unless_galaxy("release_19.09")  # for user purging
     def test_create_remote_user(self):
         # WARNING: only admins can create users!
-        # WARNING: Users cannot be purged through the Galaxy API, so execute
-        # this test only on a disposable Galaxy instance!
         if not self.gi.config.get_config()["use_remote_user"]:
             self.skipTest("This Galaxy instance is not configured to use remote users")
         new_user_email = "[email protected]"
@@ -36,11 +36,17 @@ def test_create_remote_user(self):
             deleted_user = self.gi.users.delete_user(user["id"])
             assert deleted_user["email"] == new_user_email
             assert deleted_user["deleted"]
+            assert not deleted_user["purged"]
 
+            purged_user = self.gi.users.delete_user(user["id"], purge=True)
+            # email is redacted when purging a user
+            assert purged_user["email"] != new_user_email
+            assert purged_user["deleted"]
+            assert purged_user["purged"]
+
+    @test_util.skip_unless_galaxy("release_19.09")  # for user purging
     def test_create_local_user(self):
         # WARNING: only admins can create users!
-        # WARNING: Users cannot be purged through the Galaxy API, so execute
-        # this test only on a disposable Galaxy instance!
         if self.gi.config.get_config()["use_remote_user"]:
             self.skipTest("This Galaxy instance is not configured to use local users")
         new_username = test_util.random_string()
@@ -52,11 +58,20 @@ def test_create_local_user(self):
         # test a BioBlend GalaxyInstance can be created using username+password
         user_gi = bioblend.galaxy.GalaxyInstance(url=self.gi.base_url, email=new_user_email, password=password)
         assert user_gi.users.get_current_user()["email"] == new_user_email
-        # test deletion
+        # test deletion and purging
         if self.gi.config.get_config()["allow_user_deletion"]:
             deleted_user = self.gi.users.delete_user(new_user["id"])
+            assert deleted_user["username"] == new_username
             assert deleted_user["email"] == new_user_email
             assert deleted_user["deleted"]
+            assert not deleted_user["purged"]
+
+            purged_user = self.gi.users.delete_user(new_user["id"], purge=True)
+            # username and email are redacted when purging a user
+            assert purged_user["username"] != new_username
+            assert purged_user["email"] != new_user_email
+            assert purged_user["deleted"]
+            assert purged_user["purged"]
 
     def test_get_current_user(self):
         user = self.gi.users.get_current_user()
@@ -66,10 +81,9 @@ def test_get_current_user(self):
         assert user["nice_total_disk_usage"] is not None
         assert user["total_disk_usage"] is not None
 
+    @test_util.skip_unless_galaxy("release_19.09")  # for user purging
     def test_update_user(self):
         # WARNING: only admins can create users!
-        # WARNING: Users cannot be purged through the Galaxy API, so execute
-        # this test only on a disposable Galaxy instance!
         if self.gi.config.get_config()["use_remote_user"]:
             self.skipTest("This Galaxy instance is not configured to use local users")
         new_username = test_util.random_string()
@@ -84,8 +98,12 @@ def test_update_user(self):
         assert updated_user["username"] == updated_username
         assert updated_user["email"] == updated_user_email
 
+        # delete and purge user after test (if possile)
         if self.gi.config.get_config()["allow_user_deletion"]:
             self.gi.users.delete_user(new_user_id)
+            purged_user = self.gi.users.delete_user(new_user_id, purge=True)
+            assert purged_user["deleted"]
+            assert purged_user["purged"]
 
     def test_get_user_apikey(self):
         # Test getting the API key of the current user, which surely has one

bioblend/_tests/template_galaxy.yml

@@ -17,4 +17,5 @@ galaxy:
   master_api_key: $BIOBLEND_GALAXY_MASTER_API_KEY
   enable_quotas: true
   cleanup_job: onsuccess
-  enable_notification_system: true
+  enable_notification_system: true
+  enable_celery_tasks: true