From 6158eb581e0a7568acaf0c4e48162eb022a9db40 Mon Sep 17 00:00:00 2001 From: Alex MacLean Date: Tue, 5 Jan 2016 15:09:56 -0500 Subject: [PATCH 01/12] Add ability to create dataset collections to galaxy.objects --- bioblend/galaxy/objects/wrappers.py | 72 +++++++++++++++++++++++++++++ tests/TestGalaxyObjects.py | 23 ++++++++- 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/bioblend/galaxy/objects/wrappers.py b/bioblend/galaxy/objects/wrappers.py index d95fe972a..0e4118fb2 100644 --- a/bioblend/galaxy/objects/wrappers.py +++ b/bioblend/galaxy/objects/wrappers.py @@ -626,6 +626,50 @@ def delete(self): self.refresh() +@six.add_metaclass(abc.ABCMeta) +class DatasetCollection(Wrapper): + """ + Abstract base class for Galaxy dataset collections. + """ + BASE_ATTRS = Wrapper.BASE_ATTRS + ('deleted') + POLLING_INTERVAL = 1 # for state monitoring + + @abc.abstractmethod + def __init__(self, dsc_dict, container, gi=None): + super(DatasetCollection, self).__init__(dsc_dict, gi=gi) + object.__setattr__(self, 'container', container) + + +class HistoryDatasetCollectionAssociation(DatasetCollection): + """ + Maps to a Galaxy ``HistoryDatasetCollectionAssociation``. + """ + BASE_ATTRS = DatasetCollection.BASE_ATTRS + ('tags', 'visible', 'elements') + SRC = 'hdca' + + def __init__(self, dsc_dict, container, gi=None): + super(HistoryDatasetCollectionAssociation, self).__init__( + dsc_dict, container, gi=gi) + + @property + def gi_module(self): + return self.gi.histories + + @property + def _stream_url(self): + base_url = self.gi.gi._make_url( + self.gi.gi.histories, module_id=self.container.id, contents=True) + return "%s/%s/display" % (base_url, self.id) + + def delete(self): + """ + Delete this dataset collection. + """ + self.gi.gi.histories.delete_dataset_collection(self.container.id, self.id) + self.container.refresh() + self.refresh() + + class LibRelatedDataset(Dataset): """ Base class for LibraryDatasetDatasetAssociation and LibraryDataset classes. @@ -807,6 +851,7 @@ class History(DatasetContainer): """ BASE_ATTRS = DatasetContainer.BASE_ATTRS + ('annotation', 'state', 'state_ids', 'state_details', 'tags') DS_TYPE = HistoryDatasetAssociation + DSC_TYPE = HistoryDatasetCollectionAssociation CONTENT_INFO_TYPE = HistoryContentInfo API_MODULE = 'histories' @@ -965,6 +1010,33 @@ def download(self, jeha_id, outf, chunk_size=bioblend.CHUNK_SIZE): return self.gi.gi.histories.download_history( self.id, jeha_id, outf, chunk_size=chunk_size) + def new_dataset_collection(self, hist_id, collection_description): + """ + Create a new dataset collection in the history by providing the dataset ids. + + :type collection_description: bioblend.galaxy.dataset_collections.CollectionDescription + :param collection_description: a description of the dataset collection + + :rtype: class:`~.HistoryDatasetCollectionAssociation` + :return: the new dataset collection + """ + dataset_collection = self.gi.gi.histories.create_dataset_collection(hist_id, collection_description) + self.refresh() + return self.get_dataset_collection(dataset_collection['id']) + + def get_dataset_collection(self, dsc_id): + """ + Retrieve the dataset collection corresponding to the given id. + + :type dsc_id: str + :param dsc_id: dataset collection id + + :rtype: :class:`~.HistoryDatasetCollectionAssociation` + :return: the dataset collection corresponding to ``dsc_id`` + """ + dsc_dict = self.gi.gi.histories.show_dataset_collection(self.id, dsc_id) + return self.DSC_TYPE(dsc_dict, self, gi=self.gi) + class Library(DatasetContainer): """ diff --git a/tests/TestGalaxyObjects.py b/tests/TestGalaxyObjects.py index e59791788..e0853a5d5 100644 --- a/tests/TestGalaxyObjects.py +++ b/tests/TestGalaxyObjects.py @@ -16,10 +16,10 @@ import bioblend.galaxy.objects.wrappers as wrappers import bioblend.galaxy.objects.galaxy_instance as galaxy_instance from bioblend import ConnectionError +from bioblend.galaxy import dataset_collections as collections import test_util from test_util import unittest - bioblend.set_stream_logger('test', level='INFO') socket.setdefaulttimeout(10.0) THIS_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -558,6 +558,15 @@ def __check_dataset(self, hda): self.assertEqual(len(self.hist.dataset_ids), 1) self.assertEqual(self.hist.dataset_ids[0], hda.id) + def __check_dataset_collection(self, hdca): + self.assertIsInstance(hdca, wrappers.HistoryDatasetCollectionAssociation) + self.assertIs(hdca.container, self.hist) + self.assertEqual(len(self.hist.dataset_ids), 2) + dataset1 = self.hist.get_dataset(self.hist.dataset_ids[0]) + dataset2 = self.hist.get_dataset(self.hist.dataset_ids[1]) + self.assertEqual(dataset1.id, hdca.elements[0]['object']['id']) + self.assertEqual(dataset2.id, hdca.elements[1]['object']['id']) + def test_import_dataset(self): lib = self.gi.libraries.create('test_%s' % uuid.uuid4().hex) lds = lib.upload_data(FOO_DATA) @@ -617,6 +626,18 @@ def test_update(self): self.assertEqual(self.hist.annotation, new_annotation) self.assertEqual(self.hist.tags, new_tags) + def test_new_dataset_collection(self): + dataset1 = self.hist.paste_content(FOO_DATA) + dataset2 = self.hist.paste_content(FOO_DATA_2) + collection_description=collections.CollectionDescription( + name="MyDatasetList", + elements=[ + collections.HistoryDatasetElement(name="sample1", id=dataset1.id), + collections.HistoryDatasetElement(name="sample2", id=dataset2.id), + ] + ) + collection_response = self.hist.new_dataset_collection(self.hist.id, collection_description) + self.__check_dataset_collection(collection_response) @test_util.skip_unless_galaxy() class TestHDAContents(GalaxyObjectsTestBase): From 5a8b382cebd5325f3c31628f3193360dbdde39e0 Mon Sep 17 00:00:00 2001 From: Alex MacLean Date: Tue, 5 Jan 2016 16:05:30 -0500 Subject: [PATCH 02/12] Small syntax fix for data collection attributes --- bioblend/galaxy/objects/wrappers.py | 4 +++- tests/TestGalaxyObjects.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bioblend/galaxy/objects/wrappers.py b/bioblend/galaxy/objects/wrappers.py index 0e4118fb2..60e1f1991 100644 --- a/bioblend/galaxy/objects/wrappers.py +++ b/bioblend/galaxy/objects/wrappers.py @@ -631,7 +631,9 @@ class DatasetCollection(Wrapper): """ Abstract base class for Galaxy dataset collections. """ - BASE_ATTRS = Wrapper.BASE_ATTRS + ('deleted') + BASE_ATTRS = Wrapper.BASE_ATTRS + ( + 'state', 'deleted' + ) POLLING_INTERVAL = 1 # for state monitoring @abc.abstractmethod diff --git a/tests/TestGalaxyObjects.py b/tests/TestGalaxyObjects.py index e0853a5d5..879d1f1e0 100644 --- a/tests/TestGalaxyObjects.py +++ b/tests/TestGalaxyObjects.py @@ -639,6 +639,7 @@ def test_new_dataset_collection(self): collection_response = self.hist.new_dataset_collection(self.hist.id, collection_description) self.__check_dataset_collection(collection_response) + @test_util.skip_unless_galaxy() class TestHDAContents(GalaxyObjectsTestBase): From f4a1264d32d13711cd52a786835a37c0397aed18 Mon Sep 17 00:00:00 2001 From: Alex MacLean Date: Thu, 7 Jan 2016 09:38:18 -0500 Subject: [PATCH 03/12] Add delete dataset collection method and fix whitespace error --- bioblend/galaxy/objects/wrappers.py | 11 +++++++++++ tests/TestGalaxyObjects.py | 17 +++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/bioblend/galaxy/objects/wrappers.py b/bioblend/galaxy/objects/wrappers.py index 60e1f1991..89525db8d 100644 --- a/bioblend/galaxy/objects/wrappers.py +++ b/bioblend/galaxy/objects/wrappers.py @@ -641,6 +641,17 @@ def __init__(self, dsc_dict, container, gi=None): super(DatasetCollection, self).__init__(dsc_dict, gi=gi) object.__setattr__(self, 'container', container) + def refresh(self): + """ + Re-fetch the attributes pertaining to this object. + + Returns: self + """ + gi_client = getattr(self.gi.gi, self.container.API_MODULE) + dsc_dict = gi_client.show_dataset_collection(self.container.id, self.id) + self.__init__(dsc_dict, self.container, self.gi) + return self + class HistoryDatasetCollectionAssociation(DatasetCollection): """ diff --git a/tests/TestGalaxyObjects.py b/tests/TestGalaxyObjects.py index 879d1f1e0..76cb4bfac 100644 --- a/tests/TestGalaxyObjects.py +++ b/tests/TestGalaxyObjects.py @@ -627,18 +627,27 @@ def test_update(self): self.assertEqual(self.hist.tags, new_tags) def test_new_dataset_collection(self): + collection_description = self._create_collection_description() + collection_response = self.hist.new_dataset_collection(self.hist.id, collection_description) + self.__check_dataset_collection(collection_response) + + def test_delete_dataset_collection(self): + collection_description = self._create_collection_description() + dataset_collection = self.hist.new_dataset_collection(self.hist.id, collection_description) + dataset_collection.delete() + self.assertTrue(dataset_collection.deleted) + + def _create_collection_description(self): dataset1 = self.hist.paste_content(FOO_DATA) dataset2 = self.hist.paste_content(FOO_DATA_2) - collection_description=collections.CollectionDescription( + collection_description = collections.CollectionDescription( name="MyDatasetList", elements=[ collections.HistoryDatasetElement(name="sample1", id=dataset1.id), collections.HistoryDatasetElement(name="sample2", id=dataset2.id), ] ) - collection_response = self.hist.new_dataset_collection(self.hist.id, collection_description) - self.__check_dataset_collection(collection_response) - + return collection_description @test_util.skip_unless_galaxy() class TestHDAContents(GalaxyObjectsTestBase): From 784d01310fa5de6ae8cedb3b3e19fd33714f72c4 Mon Sep 17 00:00:00 2001 From: Alex MacLean Date: Thu, 7 Jan 2016 13:24:46 -0500 Subject: [PATCH 04/12] Add test for running workflow with dataset collection as an input --- bioblend/galaxy/objects/wrappers.py | 5 +-- tests/TestGalaxyObjects.py | 25 +++++++++-- tests/data/dataset_collection_run.ga | 65 ++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 6 deletions(-) create mode 100644 tests/data/dataset_collection_run.ga diff --git a/bioblend/galaxy/objects/wrappers.py b/bioblend/galaxy/objects/wrappers.py index 89525db8d..c158a5d40 100644 --- a/bioblend/galaxy/objects/wrappers.py +++ b/bioblend/galaxy/objects/wrappers.py @@ -447,7 +447,6 @@ def run(self, input_map=None, history='', params=None, import_inputs=False, res = self.gi.gi.workflows.run_workflow(self.id, **kwargs) # res structure: {'history': HIST_ID, 'outputs': [DS_ID, DS_ID, ...]} out_hist = self.gi.histories.get(res['history']) - assert set(res['outputs']).issubset(out_hist.dataset_ids) outputs = [out_hist.get_dataset(_) for _ in res['outputs']] if wait: @@ -1023,7 +1022,7 @@ def download(self, jeha_id, outf, chunk_size=bioblend.CHUNK_SIZE): return self.gi.gi.histories.download_history( self.id, jeha_id, outf, chunk_size=chunk_size) - def new_dataset_collection(self, hist_id, collection_description): + def new_dataset_collection(self, collection_description): """ Create a new dataset collection in the history by providing the dataset ids. @@ -1033,7 +1032,7 @@ def new_dataset_collection(self, hist_id, collection_description): :rtype: class:`~.HistoryDatasetCollectionAssociation` :return: the new dataset collection """ - dataset_collection = self.gi.gi.histories.create_dataset_collection(hist_id, collection_description) + dataset_collection = self.gi.gi.histories.create_dataset_collection(self.id, collection_description) self.refresh() return self.get_dataset_collection(dataset_collection['id']) diff --git a/tests/TestGalaxyObjects.py b/tests/TestGalaxyObjects.py index 76cb4bfac..260b045a0 100644 --- a/tests/TestGalaxyObjects.py +++ b/tests/TestGalaxyObjects.py @@ -20,6 +20,7 @@ import test_util from test_util import unittest + bioblend.set_stream_logger('test', level='INFO') socket.setdefaulttimeout(10.0) THIS_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -628,12 +629,12 @@ def test_update(self): def test_new_dataset_collection(self): collection_description = self._create_collection_description() - collection_response = self.hist.new_dataset_collection(self.hist.id, collection_description) - self.__check_dataset_collection(collection_response) + dataset_collection = self.hist.new_dataset_collection(collection_description) + self.__check_dataset_collection(dataset_collection) def test_delete_dataset_collection(self): collection_description = self._create_collection_description() - dataset_collection = self.hist.new_dataset_collection(self.hist.id, collection_description) + dataset_collection = self.hist.new_dataset_collection(collection_description) dataset_collection.delete() self.assertTrue(dataset_collection.deleted) @@ -735,6 +736,24 @@ def test_new_history(self): def test_params(self): self.__test(params=True) + def test_run_workflow_with_dataset_collection(self): + wf_file = os.path.join(THIS_DIR, 'data', 'dataset_collection_run.ga') + with open(wf_file) as f: + wf = self.gi.workflows.import_new(f.read()) + history_name = "Run Workflow With Dataset Collection" + outputhist = self.gi.histories.create(history_name) + collection_description = collections.CollectionDescription( + name="MyDatasetList", + elements=[ + collections.HistoryDatasetElement(name="sample1", id=self.inputs[0].id), + collections.HistoryDatasetElement(name="sample2", id=self.inputs[1].id), + ] + ) + dataset_collection = outputhist.new_dataset_collection(collection_description) + input_map = {"Input Dataset Collection": dataset_collection} + outputs = wf.run(input_map, history_name) + self.assertEqual(len(outputs), 2) + @test_util.skip_unless_galaxy() class TestJob(GalaxyObjectsTestBase): diff --git a/tests/data/dataset_collection_run.ga b/tests/data/dataset_collection_run.ga new file mode 100644 index 000000000..a23d759dd --- /dev/null +++ b/tests/data/dataset_collection_run.ga @@ -0,0 +1,65 @@ +{ + "a_galaxy_workflow": "true", + "annotation": "", + "format-version": "0.1", + "name": "DC Run", + "steps": { + "0": { + "annotation": "", + "id": 0, + "input_connections": {}, + "inputs": [ + { + "description": "", + "name": "Input Dataset Collection" + } + ], + "label": null, + "name": "Input dataset collection", + "outputs": [], + "position": { + "left": 338, + "top": 306 + }, + "tool_errors": null, + "tool_id": null, + "tool_state": "{\"collection_type\": \"list\", \"name\": \"Input Dataset Collection\"}", + "tool_version": null, + "type": "data_collection_input", + "user_outputs": [], + "uuid": "4a497708-240c-420e-9c7c-a2dc0200476e" + }, + "1": { + "annotation": "", + "id": 1, + "input_connections": { + "input": { + "id": 0, + "output_name": "output" + } + }, + "inputs": [], + "label": null, + "name": "Sort", + "outputs": [ + { + "name": "out_file1", + "type": "input" + } + ], + "position": { + "left": 676, + "top": 286 + }, + "post_job_actions": {}, + "tool_errors": null, + "tool_id": "sort1", + "tool_state": "{\"__page__\": 0, \"style\": \"\\\"num\\\"\", \"column\": \"\\\"1\\\"\", \"__rerun_remap_job_id__\": null, \"order\": \"\\\"DESC\\\"\", \"input\": \"null\", \"column_set\": \"[]\"}", + "tool_version": "1.0.3", + "type": "tool", + "user_outputs": [], + "uuid": "3c33333f-08ce-43bf-bc09-fa0b5776845b" + } + }, + "uuid": "fd34f048-e17a-4e06-85e8-ed391192d284" +} \ No newline at end of file From 0327da6332c709f756d18ac5bfb12d40da860a4e Mon Sep 17 00:00:00 2001 From: Alex MacLean Date: Fri, 8 Jan 2016 09:58:35 -0500 Subject: [PATCH 05/12] Add blank line for flake8 --- tests/TestGalaxyObjects.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/TestGalaxyObjects.py b/tests/TestGalaxyObjects.py index 260b045a0..cd3c0aa6e 100644 --- a/tests/TestGalaxyObjects.py +++ b/tests/TestGalaxyObjects.py @@ -650,6 +650,7 @@ def _create_collection_description(self): ) return collection_description + @test_util.skip_unless_galaxy() class TestHDAContents(GalaxyObjectsTestBase): From 4778b3db16bba5c1c8383f7eec93fcfe3adb1452 Mon Sep 17 00:00:00 2001 From: Alex MacLean Date: Fri, 8 Jan 2016 10:49:49 -0500 Subject: [PATCH 06/12] Dataset collection test requirements and remove import renaming --- tests/TestGalaxyObjects.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/TestGalaxyObjects.py b/tests/TestGalaxyObjects.py index cd3c0aa6e..726c0bf32 100644 --- a/tests/TestGalaxyObjects.py +++ b/tests/TestGalaxyObjects.py @@ -16,7 +16,7 @@ import bioblend.galaxy.objects.wrappers as wrappers import bioblend.galaxy.objects.galaxy_instance as galaxy_instance from bioblend import ConnectionError -from bioblend.galaxy import dataset_collections as collections +from bioblend.galaxy import dataset_collections import test_util from test_util import unittest @@ -627,11 +627,13 @@ def test_update(self): self.assertEqual(self.hist.annotation, new_annotation) self.assertEqual(self.hist.tags, new_tags) + @test_util.skip_unless_galaxy('release_14.06') def test_new_dataset_collection(self): collection_description = self._create_collection_description() dataset_collection = self.hist.new_dataset_collection(collection_description) self.__check_dataset_collection(dataset_collection) + @test_util.skip_unless_galaxy('release_14.06') def test_delete_dataset_collection(self): collection_description = self._create_collection_description() dataset_collection = self.hist.new_dataset_collection(collection_description) @@ -641,11 +643,11 @@ def test_delete_dataset_collection(self): def _create_collection_description(self): dataset1 = self.hist.paste_content(FOO_DATA) dataset2 = self.hist.paste_content(FOO_DATA_2) - collection_description = collections.CollectionDescription( + collection_description = dataset_collections.CollectionDescription( name="MyDatasetList", elements=[ - collections.HistoryDatasetElement(name="sample1", id=dataset1.id), - collections.HistoryDatasetElement(name="sample2", id=dataset2.id), + dataset_collections.HistoryDatasetElement(name="sample1", id=dataset1.id), + dataset_collections.HistoryDatasetElement(name="sample2", id=dataset2.id), ] ) return collection_description @@ -737,17 +739,18 @@ def test_new_history(self): def test_params(self): self.__test(params=True) + @test_util.skip_unless_galaxy('release_14.06') def test_run_workflow_with_dataset_collection(self): wf_file = os.path.join(THIS_DIR, 'data', 'dataset_collection_run.ga') with open(wf_file) as f: wf = self.gi.workflows.import_new(f.read()) history_name = "Run Workflow With Dataset Collection" outputhist = self.gi.histories.create(history_name) - collection_description = collections.CollectionDescription( + collection_description = dataset_collections.CollectionDescription( name="MyDatasetList", elements=[ - collections.HistoryDatasetElement(name="sample1", id=self.inputs[0].id), - collections.HistoryDatasetElement(name="sample2", id=self.inputs[1].id), + dataset_collections.HistoryDatasetElement(name="sample1", id=self.inputs[0].id), + dataset_collections.HistoryDatasetElement(name="sample2", id=self.inputs[1].id), ] ) dataset_collection = outputhist.new_dataset_collection(collection_description) From b77dcd9567c88eae8f83ca32f246458e070205ae Mon Sep 17 00:00:00 2001 From: Alex MacLean Date: Fri, 8 Jan 2016 11:27:29 -0500 Subject: [PATCH 07/12] Additional minor fixes --- bioblend/galaxy/objects/wrappers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bioblend/galaxy/objects/wrappers.py b/bioblend/galaxy/objects/wrappers.py index c158a5d40..2b403cbea 100644 --- a/bioblend/galaxy/objects/wrappers.py +++ b/bioblend/galaxy/objects/wrappers.py @@ -26,6 +26,8 @@ 'Folder', 'Dataset', 'HistoryDatasetAssociation', + 'DatasetCollection', + 'HistoryDatasetCollectionAssociation', 'LibraryDatasetDatasetAssociation', 'LibraryDataset', 'Tool', @@ -633,7 +635,6 @@ class DatasetCollection(Wrapper): BASE_ATTRS = Wrapper.BASE_ATTRS + ( 'state', 'deleted' ) - POLLING_INTERVAL = 1 # for state monitoring @abc.abstractmethod def __init__(self, dsc_dict, container, gi=None): From 9ed67b3358cc962f9a16bb3d327c3157899fd819 Mon Sep 17 00:00:00 2001 From: Alex MacLean Date: Mon, 11 Jan 2016 11:21:20 -0500 Subject: [PATCH 08/12] Renaming, remove stream_url(), and fix test but it's failing --- bioblend/galaxy/objects/wrappers.py | 8 +------ tests/TestGalaxyObjects.py | 36 ++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/bioblend/galaxy/objects/wrappers.py b/bioblend/galaxy/objects/wrappers.py index 2b403cbea..2d6bf63b8 100644 --- a/bioblend/galaxy/objects/wrappers.py +++ b/bioblend/galaxy/objects/wrappers.py @@ -668,12 +668,6 @@ def __init__(self, dsc_dict, container, gi=None): def gi_module(self): return self.gi.histories - @property - def _stream_url(self): - base_url = self.gi.gi._make_url( - self.gi.gi.histories, module_id=self.container.id, contents=True) - return "%s/%s/display" % (base_url, self.id) - def delete(self): """ Delete this dataset collection. @@ -1023,7 +1017,7 @@ def download(self, jeha_id, outf, chunk_size=bioblend.CHUNK_SIZE): return self.gi.gi.histories.download_history( self.id, jeha_id, outf, chunk_size=chunk_size) - def new_dataset_collection(self, collection_description): + def create_dataset_collection(self, collection_description): """ Create a new dataset collection in the history by providing the dataset ids. diff --git a/tests/TestGalaxyObjects.py b/tests/TestGalaxyObjects.py index 726c0bf32..72121e445 100644 --- a/tests/TestGalaxyObjects.py +++ b/tests/TestGalaxyObjects.py @@ -628,7 +628,7 @@ def test_update(self): self.assertEqual(self.hist.tags, new_tags) @test_util.skip_unless_galaxy('release_14.06') - def test_new_dataset_collection(self): + def test_create_dataset_collection(self): collection_description = self._create_collection_description() dataset_collection = self.hist.new_dataset_collection(collection_description) self.__check_dataset_collection(dataset_collection) @@ -739,24 +739,38 @@ def test_new_history(self): def test_params(self): self.__test(params=True) - @test_util.skip_unless_galaxy('release_14.06') - def test_run_workflow_with_dataset_collection(self): +@test_util.skip_unless_galaxy() +class TestRunDatasetCollectionWorkflow(GalaxyObjectsTestBase): + + def setUp(self): + super(TestRunDatasetCollectionWorkflow, self).setUp() wf_file = os.path.join(THIS_DIR, 'data', 'dataset_collection_run.ga') with open(wf_file) as f: - wf = self.gi.workflows.import_new(f.read()) - history_name = "Run Workflow With Dataset Collection" - outputhist = self.gi.histories.create(history_name) + self.wf = self.gi.workflows.import_new(f.read()) + self.hist_name = 'test_%s' % uuid.uuid4().hex + self.hist = self.gi.histories.create(self.hist_name) + + def tearDown(self): + self.wf.delete() + + @test_util.skip_unless_galaxy('release_14.06') + def test_run_workflow_with_dataset_collection(self): + dataset1 = self.hist.paste_content('foo\nbar\n') + dataset2 = self.hist.paste_content('foo2\nbar2\n') collection_description = dataset_collections.CollectionDescription( name="MyDatasetList", elements=[ - dataset_collections.HistoryDatasetElement(name="sample1", id=self.inputs[0].id), - dataset_collections.HistoryDatasetElement(name="sample2", id=self.inputs[1].id), + dataset_collections.HistoryDatasetElement(name="sample1", id=dataset1.id), + dataset_collections.HistoryDatasetElement(name="sample2", id=dataset2.id), ] ) - dataset_collection = outputhist.new_dataset_collection(collection_description) + dataset_collection = self.hist.create_dataset_collection(collection_description) input_map = {"Input Dataset Collection": dataset_collection} - outputs = wf.run(input_map, history_name) - self.assertEqual(len(outputs), 2) + outputs, out_hist = self.wf.run(input_map, self.hist, wait=True) + self.assertIsInstance(outputs[0], wrappers.HistoryDatasetCollectionAssociation) + self.assertEqual(len(outputs), 1) + self.asssertEqual(out_hist.id, self.hist.id) + out_hist.delete(purge=True) @test_util.skip_unless_galaxy() From 0b861e03a8b4c175948bdbdf6f2558720bae2027 Mon Sep 17 00:00:00 2001 From: Alex MacLean Date: Mon, 11 Jan 2016 11:22:38 -0500 Subject: [PATCH 09/12] Fix typo --- tests/TestGalaxyObjects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/TestGalaxyObjects.py b/tests/TestGalaxyObjects.py index 72121e445..5e230e6c6 100644 --- a/tests/TestGalaxyObjects.py +++ b/tests/TestGalaxyObjects.py @@ -769,7 +769,7 @@ def test_run_workflow_with_dataset_collection(self): outputs, out_hist = self.wf.run(input_map, self.hist, wait=True) self.assertIsInstance(outputs[0], wrappers.HistoryDatasetCollectionAssociation) self.assertEqual(len(outputs), 1) - self.asssertEqual(out_hist.id, self.hist.id) + self.assertEqual(out_hist.id, self.hist.id) out_hist.delete(purge=True) From 747c5e2f3d66fce7200a47e273d121d75b243409 Mon Sep 17 00:00:00 2001 From: Alex MacLean Date: Mon, 11 Jan 2016 11:40:26 -0500 Subject: [PATCH 10/12] Whitespace fix --- tests/TestGalaxyObjects.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/TestGalaxyObjects.py b/tests/TestGalaxyObjects.py index 5e230e6c6..89c0f765a 100644 --- a/tests/TestGalaxyObjects.py +++ b/tests/TestGalaxyObjects.py @@ -739,6 +739,7 @@ def test_new_history(self): def test_params(self): self.__test(params=True) + @test_util.skip_unless_galaxy() class TestRunDatasetCollectionWorkflow(GalaxyObjectsTestBase): From 48b256400cb90a83622fd461d65264e12cceda22 Mon Sep 17 00:00:00 2001 From: Alex MacLean Date: Mon, 11 Jan 2016 12:03:53 -0500 Subject: [PATCH 11/12] Method call fixes --- tests/TestGalaxyObjects.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/TestGalaxyObjects.py b/tests/TestGalaxyObjects.py index 89c0f765a..b08d5c087 100644 --- a/tests/TestGalaxyObjects.py +++ b/tests/TestGalaxyObjects.py @@ -630,13 +630,13 @@ def test_update(self): @test_util.skip_unless_galaxy('release_14.06') def test_create_dataset_collection(self): collection_description = self._create_collection_description() - dataset_collection = self.hist.new_dataset_collection(collection_description) + dataset_collection = self.hist.create_dataset_collection(collection_description) self.__check_dataset_collection(dataset_collection) @test_util.skip_unless_galaxy('release_14.06') def test_delete_dataset_collection(self): collection_description = self._create_collection_description() - dataset_collection = self.hist.new_dataset_collection(collection_description) + dataset_collection = self.hist.create_dataset_collection(collection_description) dataset_collection.delete() self.assertTrue(dataset_collection.deleted) From a23270cd8c91de9163bb09ad817eee110862703d Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 12 Jan 2016 20:07:53 +0000 Subject: [PATCH 12/12] Add dataset collections to the outputs of Workflow.run() . Also: - Expose collection_type attribute of DatasetCollection. - Rework new tests. --- bioblend/galaxy/objects/wrappers.py | 14 ++++-- tests/TestGalaxyObjects.py | 62 +++++++++++++------------- tests/data/dataset_collection_run.ga | 65 ---------------------------- 3 files changed, 40 insertions(+), 101 deletions(-) delete mode 100644 tests/data/dataset_collection_run.ga diff --git a/bioblend/galaxy/objects/wrappers.py b/bioblend/galaxy/objects/wrappers.py index 2d6bf63b8..c5681117f 100644 --- a/bioblend/galaxy/objects/wrappers.py +++ b/bioblend/galaxy/objects/wrappers.py @@ -447,9 +447,17 @@ def run(self, input_map=None, history='', params=None, import_inputs=False, raise TypeError( 'history must be either a history wrapper or a string') res = self.gi.gi.workflows.run_workflow(self.id, **kwargs) - # res structure: {'history': HIST_ID, 'outputs': [DS_ID, DS_ID, ...]} + # res structure: {'history': HIST_ID, 'outputs': [CI_ID, CI_ID, ...]} out_hist = self.gi.histories.get(res['history']) - outputs = [out_hist.get_dataset(_) for _ in res['outputs']] + content_infos_dict = dict() + for ci in out_hist.content_infos: + content_infos_dict[ci.id] = ci + outputs = [] + for output_id in res['outputs']: + if content_infos_dict[output_id].type == 'file': + outputs.append(out_hist.get_dataset(output_id)) + elif content_infos_dict[output_id].type == 'collection': + outputs.append(out_hist.get_dataset_collection(output_id)) if wait: self.gi._wait_datasets(outputs, polling_interval=polling_interval, @@ -633,7 +641,7 @@ class DatasetCollection(Wrapper): Abstract base class for Galaxy dataset collections. """ BASE_ATTRS = Wrapper.BASE_ATTRS + ( - 'state', 'deleted' + 'state', 'deleted', 'collection_type' ) @abc.abstractmethod diff --git a/tests/TestGalaxyObjects.py b/tests/TestGalaxyObjects.py index b08d5c087..8e3cc3bee 100644 --- a/tests/TestGalaxyObjects.py +++ b/tests/TestGalaxyObjects.py @@ -559,15 +559,6 @@ def __check_dataset(self, hda): self.assertEqual(len(self.hist.dataset_ids), 1) self.assertEqual(self.hist.dataset_ids[0], hda.id) - def __check_dataset_collection(self, hdca): - self.assertIsInstance(hdca, wrappers.HistoryDatasetCollectionAssociation) - self.assertIs(hdca.container, self.hist) - self.assertEqual(len(self.hist.dataset_ids), 2) - dataset1 = self.hist.get_dataset(self.hist.dataset_ids[0]) - dataset2 = self.hist.get_dataset(self.hist.dataset_ids[1]) - self.assertEqual(dataset1.id, hdca.elements[0]['object']['id']) - self.assertEqual(dataset2.id, hdca.elements[1]['object']['id']) - def test_import_dataset(self): lib = self.gi.libraries.create('test_%s' % uuid.uuid4().hex) lds = lib.upload_data(FOO_DATA) @@ -629,28 +620,32 @@ def test_update(self): @test_util.skip_unless_galaxy('release_14.06') def test_create_dataset_collection(self): - collection_description = self._create_collection_description() - dataset_collection = self.hist.create_dataset_collection(collection_description) - self.__check_dataset_collection(dataset_collection) + self._create_collection_description() + hdca = self.hist.create_dataset_collection(self.collection_description) + self.assertIsInstance(hdca, wrappers.HistoryDatasetCollectionAssociation) + self.assertEqual(hdca.collection_type, 'list') + self.assertIs(hdca.container, self.hist) + self.assertEqual(len(hdca.elements), 2) + self.assertEqual(self.dataset1.id, hdca.elements[0]['object']['id']) + self.assertEqual(self.dataset2.id, hdca.elements[1]['object']['id']) @test_util.skip_unless_galaxy('release_14.06') def test_delete_dataset_collection(self): - collection_description = self._create_collection_description() - dataset_collection = self.hist.create_dataset_collection(collection_description) - dataset_collection.delete() - self.assertTrue(dataset_collection.deleted) + self._create_collection_description() + hdca = self.hist.create_dataset_collection(self.collection_description) + hdca.delete() + self.assertTrue(hdca.deleted) def _create_collection_description(self): - dataset1 = self.hist.paste_content(FOO_DATA) - dataset2 = self.hist.paste_content(FOO_DATA_2) - collection_description = dataset_collections.CollectionDescription( + self.dataset1 = self.hist.paste_content(FOO_DATA) + self.dataset2 = self.hist.paste_content(FOO_DATA_2) + self.collection_description = dataset_collections.CollectionDescription( name="MyDatasetList", elements=[ - dataset_collections.HistoryDatasetElement(name="sample1", id=dataset1.id), - dataset_collections.HistoryDatasetElement(name="sample2", id=dataset2.id), + dataset_collections.HistoryDatasetElement(name="sample1", id=self.dataset1.id), + dataset_collections.HistoryDatasetElement(name="sample2", id=self.dataset2.id), ] ) - return collection_description @test_util.skip_unless_galaxy() @@ -740,24 +735,22 @@ def test_params(self): self.__test(params=True) -@test_util.skip_unless_galaxy() +@test_util.skip_unless_galaxy('release_14.08') class TestRunDatasetCollectionWorkflow(GalaxyObjectsTestBase): def setUp(self): super(TestRunDatasetCollectionWorkflow, self).setUp() - wf_file = os.path.join(THIS_DIR, 'data', 'dataset_collection_run.ga') - with open(wf_file) as f: + with open(SAMPLE_WF_COLL_FN) as f: self.wf = self.gi.workflows.import_new(f.read()) - self.hist_name = 'test_%s' % uuid.uuid4().hex - self.hist = self.gi.histories.create(self.hist_name) + self.hist = self.gi.histories.create('test_%s' % uuid.uuid4().hex) def tearDown(self): self.wf.delete() + self.hist.delete(purge=True) - @test_util.skip_unless_galaxy('release_14.06') def test_run_workflow_with_dataset_collection(self): - dataset1 = self.hist.paste_content('foo\nbar\n') - dataset2 = self.hist.paste_content('foo2\nbar2\n') + dataset1 = self.hist.paste_content(FOO_DATA) + dataset2 = self.hist.paste_content(FOO_DATA_2) collection_description = dataset_collections.CollectionDescription( name="MyDatasetList", elements=[ @@ -766,12 +759,15 @@ def test_run_workflow_with_dataset_collection(self): ] ) dataset_collection = self.hist.create_dataset_collection(collection_description) - input_map = {"Input Dataset Collection": dataset_collection} + input_map = {"Input Dataset Collection": dataset_collection, + "Input 2": dataset1} outputs, out_hist = self.wf.run(input_map, self.hist, wait=True) - self.assertIsInstance(outputs[0], wrappers.HistoryDatasetCollectionAssociation) self.assertEqual(len(outputs), 1) + out_hdca = outputs[0] + self.assertIsInstance(out_hdca, wrappers.HistoryDatasetCollectionAssociation) + self.assertEqual(out_hdca.collection_type, 'list') + self.assertEqual(len(out_hdca.elements), 2) self.assertEqual(out_hist.id, self.hist.id) - out_hist.delete(purge=True) @test_util.skip_unless_galaxy() diff --git a/tests/data/dataset_collection_run.ga b/tests/data/dataset_collection_run.ga deleted file mode 100644 index a23d759dd..000000000 --- a/tests/data/dataset_collection_run.ga +++ /dev/null @@ -1,65 +0,0 @@ -{ - "a_galaxy_workflow": "true", - "annotation": "", - "format-version": "0.1", - "name": "DC Run", - "steps": { - "0": { - "annotation": "", - "id": 0, - "input_connections": {}, - "inputs": [ - { - "description": "", - "name": "Input Dataset Collection" - } - ], - "label": null, - "name": "Input dataset collection", - "outputs": [], - "position": { - "left": 338, - "top": 306 - }, - "tool_errors": null, - "tool_id": null, - "tool_state": "{\"collection_type\": \"list\", \"name\": \"Input Dataset Collection\"}", - "tool_version": null, - "type": "data_collection_input", - "user_outputs": [], - "uuid": "4a497708-240c-420e-9c7c-a2dc0200476e" - }, - "1": { - "annotation": "", - "id": 1, - "input_connections": { - "input": { - "id": 0, - "output_name": "output" - } - }, - "inputs": [], - "label": null, - "name": "Sort", - "outputs": [ - { - "name": "out_file1", - "type": "input" - } - ], - "position": { - "left": 676, - "top": 286 - }, - "post_job_actions": {}, - "tool_errors": null, - "tool_id": "sort1", - "tool_state": "{\"__page__\": 0, \"style\": \"\\\"num\\\"\", \"column\": \"\\\"1\\\"\", \"__rerun_remap_job_id__\": null, \"order\": \"\\\"DESC\\\"\", \"input\": \"null\", \"column_set\": \"[]\"}", - "tool_version": "1.0.3", - "type": "tool", - "user_outputs": [], - "uuid": "3c33333f-08ce-43bf-bc09-fa0b5776845b" - } - }, - "uuid": "fd34f048-e17a-4e06-85e8-ed391192d284" -} \ No newline at end of file