From 5e216ea4ad7ea63d32eb43421a7ece5f24467e8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Mon, 16 Dec 2024 08:38:02 +0100 Subject: [PATCH] Revert "Support listing file sources asynchronously" This reverts commit 668437aff7bce0a475e55879bbe29e314dfcd8f1. --- lib/galaxy/files/sources/__init__.py | 8 +- lib/galaxy/files/sources/_pyfilesystem2.py | 2 +- lib/galaxy/files/sources/invenio.py | 2 +- lib/galaxy/files/sources/posix.py | 2 +- lib/galaxy/files/sources/s3fs.py | 2 +- lib/galaxy/managers/file_source_instances.py | 34 ++++----- lib/galaxy/managers/remote_files.py | 4 +- lib/galaxy/util/unittest.py | 23 ------ lib/galaxy/webapps/galaxy/api/file_sources.py | 12 +-- lib/galaxy/webapps/galaxy/api/remote_files.py | 4 +- packages/app/test-requirements.txt | 1 - packages/files/test-requirements.txt | 1 - test/unit/app/managers/base.py | 15 +--- .../app/managers/test_user_file_sources.py | 44 +++++------ test/unit/files/_util.py | 12 +-- test/unit/files/test_dropbox.py | 5 +- test/unit/files/test_gcsfs.py | 5 +- test/unit/files/test_googledrive.py | 5 +- test/unit/files/test_onedata.py | 5 +- test/unit/files/test_posix.py | 75 ++++++++----------- test/unit/files/test_s3.py | 5 +- test/unit/files/test_temp.py | 66 +++++++--------- test/unit/files/test_webdav.py | 19 ++--- 23 files changed, 144 insertions(+), 207 deletions(-) diff --git a/lib/galaxy/files/sources/__init__.py b/lib/galaxy/files/sources/__init__.py index dff9ad097fab..c00160cc4102 100644 --- a/lib/galaxy/files/sources/__init__.py +++ b/lib/galaxy/files/sources/__init__.py @@ -299,7 +299,7 @@ def get_uri_root(self) -> str: """Return a prefix for the root (e.g. gxfiles://prefix/).""" @abc.abstractmethod - async def list( + def list( self, path="/", recursive=False, @@ -446,7 +446,7 @@ def _serialization_props(self, user_context: "OptionalUserContext" = None) -> Fi Used in to_dict method if for_serialization is True. """ - async def list( + def list( self, path="/", recursive=False, @@ -470,9 +470,9 @@ async def list( if offset is not None and offset < 0: raise RequestParameterInvalidException("Offset must be greater than or equal to 0.") - return await self._list(path, recursive, user_context, opts, limit, offset, query) + return self._list(path, recursive, user_context, opts, limit, offset, query) - async def _list( + def _list( self, path="/", recursive=False, diff --git a/lib/galaxy/files/sources/_pyfilesystem2.py b/lib/galaxy/files/sources/_pyfilesystem2.py index 76993ededbe9..5b3add6e6119 100644 --- a/lib/galaxy/files/sources/_pyfilesystem2.py +++ b/lib/galaxy/files/sources/_pyfilesystem2.py @@ -50,7 +50,7 @@ def __init__(self, **kwd: Unpack[FilesSourceProperties]): def _open_fs(self, user_context: OptionalUserContext = None, opts: Optional[FilesSourceOptions] = None) -> FS: """Subclasses must instantiate a PyFilesystem2 handle for this file system.""" - async def _list( + def _list( self, path="/", recursive=False, diff --git a/lib/galaxy/files/sources/invenio.py b/lib/galaxy/files/sources/invenio.py index 7ba366a9b440..146d63d0b641 100644 --- a/lib/galaxy/files/sources/invenio.py +++ b/lib/galaxy/files/sources/invenio.py @@ -146,7 +146,7 @@ def to_relative_path(self, url: str) -> str: def get_repository_interactor(self, repository_url: str) -> RDMRepositoryInteractor: return InvenioRepositoryInteractor(repository_url, self) - async def _list( + def _list( self, path="/", recursive=True, diff --git a/lib/galaxy/files/sources/posix.py b/lib/galaxy/files/sources/posix.py index cac3a8fbee47..258d5ef20a69 100644 --- a/lib/galaxy/files/sources/posix.py +++ b/lib/galaxy/files/sources/posix.py @@ -60,7 +60,7 @@ def __init__(self, **kwd: Unpack[PosixFilesSourceProperties]): def prefer_links(self) -> bool: return self._prefer_links - async def _list( + def _list( self, path="/", recursive=True, diff --git a/lib/galaxy/files/sources/s3fs.py b/lib/galaxy/files/sources/s3fs.py index d1453965bf5e..c6a7dedc96d3 100644 --- a/lib/galaxy/files/sources/s3fs.py +++ b/lib/galaxy/files/sources/s3fs.py @@ -68,7 +68,7 @@ def __init__(self, **kwd: Unpack[S3FsFilesSourceProperties]): if self._endpoint_url: self._props.update({"client_kwargs": {"endpoint_url": self._endpoint_url}}) - async def _list( + def _list( self, path="/", recursive=True, diff --git a/lib/galaxy/managers/file_source_instances.py b/lib/galaxy/managers/file_source_instances.py index 5936cecd7acd..4572eafc8c66 100644 --- a/lib/galaxy/managers/file_source_instances.py +++ b/lib/galaxy/managers/file_source_instances.py @@ -334,45 +334,45 @@ def create_instance(self, trans: ProvidesUserContext, payload: CreateInstancePay self._save(persisted_file_source) return self._to_model(trans, persisted_file_source) - async def test_modify_instance( + def test_modify_instance( self, trans: ProvidesUserContext, id: UUID4, payload: TestModifyInstancePayload ) -> PluginStatus: persisted_file_source = self._get(trans, id) if isinstance(payload, TestUpgradeInstancePayload): - return await self._plugin_status_for_upgrade(trans, payload, persisted_file_source) + return self._plugin_status_for_upgrade(trans, payload, persisted_file_source) else: assert isinstance(payload, TestUpdateInstancePayload) - return await self._plugin_status_for_update(trans, payload, persisted_file_source) + return self._plugin_status_for_update(trans, payload, persisted_file_source) - async def _plugin_status_for_update( + def _plugin_status_for_update( self, trans: ProvidesUserContext, payload: TestUpdateInstancePayload, persisted_file_source: UserFileSource ) -> PluginStatus: template = self._get_template(persisted_file_source) target = UpdateTestTarget(persisted_file_source, payload) - return await self._plugin_status_for_template(trans, target, template) + return self._plugin_status_for_template(trans, target, template) - async def _plugin_status_for_upgrade( + def _plugin_status_for_upgrade( self, trans: ProvidesUserContext, payload: TestUpgradeInstancePayload, persisted_file_source: UserFileSource ) -> PluginStatus: template = self._get_and_validate_target_upgrade_template(persisted_file_source, payload) target = UpgradeTestTarget(persisted_file_source, payload) - return await self._plugin_status_for_template(trans, target, template) + return self._plugin_status_for_template(trans, target, template) - async def plugin_status_for_instance(self, trans: ProvidesUserContext, id: UUID4): + def plugin_status_for_instance(self, trans: ProvidesUserContext, id: UUID4): persisted_file_source = self._get(trans, id) - return await self._plugin_status(trans, persisted_file_source, to_template_reference(persisted_file_source)) + return self._plugin_status(trans, persisted_file_source, to_template_reference(persisted_file_source)) - async def plugin_status(self, trans: ProvidesUserContext, payload: CreateInstancePayload) -> PluginStatus: + def plugin_status(self, trans: ProvidesUserContext, payload: CreateInstancePayload) -> PluginStatus: target = CreateTestTarget(payload, UserFileSource) - return await self._plugin_status(trans, target, payload) + return self._plugin_status(trans, target, payload) - async def _plugin_status( + def _plugin_status( self, trans: ProvidesUserContext, target: CanTestPluginStatus, template_reference: TemplateReference ): template = self._catalog.find_template(template_reference) - return await self._plugin_status_for_template(trans, target, template) + return self._plugin_status_for_template(trans, target, template) - async def _plugin_status_for_template( + def _plugin_status_for_template( self, trans: ProvidesUserContext, payload: CanTestPluginStatus, template: FileSourceTemplate ): template_definition_status = status_template_definition(template) @@ -396,7 +396,7 @@ async def _plugin_status_for_template( if template_settings_status.is_not_ok: return PluginStatus(**status_kwds) assert configuration - file_source, connection_status = await self._connection_status(trans, payload, configuration) + file_source, connection_status = self._connection_status(trans, payload, configuration) status_kwds["connection"] = connection_status if connection_status.is_not_ok: return PluginStatus(**status_kwds) @@ -443,7 +443,7 @@ def _template_settings_status( exception = e return configuration, settings_exception_to_status(exception) - async def _connection_status( + def _connection_status( self, trans: ProvidesUserContext, target: CanTestPluginStatus, configuration: FileSourceConfiguration ) -> Tuple[Optional[BaseFilesSource], PluginAspectStatus]: file_source = None @@ -471,7 +471,7 @@ async def _connection_status( # a connection problem if we cannot browsable_file_source = cast(SupportsBrowsing, file_source) user_context = ProvidesFileSourcesUserContext(trans) - await browsable_file_source.list("/", recursive=False, user_context=user_context) + browsable_file_source.list("/", recursive=False, user_context=user_context) except Exception as e: exception = e return file_source, connection_exception_to_status("file source", exception) diff --git a/lib/galaxy/managers/remote_files.py b/lib/galaxy/managers/remote_files.py index 2f5dd0779d0e..165efe3c95b2 100644 --- a/lib/galaxy/managers/remote_files.py +++ b/lib/galaxy/managers/remote_files.py @@ -43,7 +43,7 @@ class RemoteFilesManager: def __init__(self, app: MinimalManagerApp): self._app = app - async def index( + def index( self, user_ctx: ProvidesUserContext, target: str, @@ -93,7 +93,7 @@ async def index( opts = FilesSourceOptions() opts.writeable = writeable or False try: - index, count = await file_source.list( + index, count = file_source.list( file_source_path.path, recursive=recursive, user_context=user_file_source_context, diff --git a/lib/galaxy/util/unittest.py b/lib/galaxy/util/unittest.py index 9c2252281db0..8794f34a6daf 100644 --- a/lib/galaxy/util/unittest.py +++ b/lib/galaxy/util/unittest.py @@ -1,5 +1,3 @@ -import inspect - import pytest @@ -45,24 +43,3 @@ def assertRaises(self, exception): def assertRaisesRegex(self, exception, regex): return pytest.raises(exception, match=regex) - - -class MarkAsyncMeta(type): - """ - Metaclass that marks all asynchronous methods of a class as async tests. - - Methods that are not recognized by pytest as tests will simply be ignored, despite having been marked as async - tests. - """ - - def __new__(cls, name, bases, dict_): - for attribute_name, attribute_value in dict_.items(): - if inspect.iscoroutinefunction(attribute_value): - dict_[attribute_name] = pytest.mark.asyncio(attribute_value) - return super().__new__(cls, name, bases, dict_) - - -class IsolatedAsyncioTestCase(TestCase, metaclass=MarkAsyncMeta): - """ - Partial re-implementation of standard library `unittest.IsolatedAsyncioTestCase` using pytest methods. - """ diff --git a/lib/galaxy/webapps/galaxy/api/file_sources.py b/lib/galaxy/webapps/galaxy/api/file_sources.py index 62f7d63a5165..4546e7b88ebc 100644 --- a/lib/galaxy/webapps/galaxy/api/file_sources.py +++ b/lib/galaxy/webapps/galaxy/api/file_sources.py @@ -96,12 +96,12 @@ def create( summary="Test payload for creating user-bound file source.", operation_id="file_sources__test_new_instance_configuration", ) - async def test_instance_configuration( + def test_instance_configuration( self, trans: ProvidesUserContext = DependsOnTrans, payload: CreateInstancePayload = Body(...), ) -> PluginStatus: - return await self.file_source_instances_manager.plugin_status(trans, payload) + return self.file_source_instances_manager.plugin_status(trans, payload) @router.get( "/api/file_source_instances", @@ -131,12 +131,12 @@ def instances_show( summary="Test a file source instance and return status.", operation_id="file_sources__instances_test_instance", ) - async def instance_test( + def instance_test( self, trans: ProvidesUserContext = DependsOnTrans, uuid: UUID4 = UserFileSourceIdPathParam, ) -> PluginStatus: - return await self.file_source_instances_manager.plugin_status_for_instance(trans, uuid) + return self.file_source_instances_manager.plugin_status_for_instance(trans, uuid) @router.put( "/api/file_source_instances/{uuid}", @@ -156,13 +156,13 @@ def update_instance( summary="Test updating or upgrading user file source instance.", operation_id="file_sources__test_instances_update", ) - async def test_update_instance( + def test_update_instance( self, trans: ProvidesUserContext = DependsOnTrans, uuid: UUID4 = UserFileSourceIdPathParam, payload: TestModifyInstancePayload = Body(...), ) -> PluginStatus: - return await self.file_source_instances_manager.test_modify_instance(trans, uuid, payload) + return self.file_source_instances_manager.test_modify_instance(trans, uuid, payload) @router.delete( "/api/file_source_instances/{uuid}", diff --git a/lib/galaxy/webapps/galaxy/api/remote_files.py b/lib/galaxy/webapps/galaxy/api/remote_files.py index ef0c9e5a2238..c5a708d748bf 100644 --- a/lib/galaxy/webapps/galaxy/api/remote_files.py +++ b/lib/galaxy/webapps/galaxy/api/remote_files.py @@ -128,7 +128,7 @@ class FastAPIRemoteFiles: deprecated=True, summary="Displays remote files available to the user. Please use /api/remote_files instead.", ) - async def index( + def index( self, response: Response, user_ctx: ProvidesUserContext = DependsOnTrans, @@ -146,7 +146,7 @@ async def index( The total count of files and directories is returned in the 'total_matches' header. """ - result, count = await self.manager.index( + result, count = self.manager.index( user_ctx, target, format, recursive, disable, writeable, limit, offset, query, sort_by ) response.headers["total_matches"] = str(count) diff --git a/packages/app/test-requirements.txt b/packages/app/test-requirements.txt index da161030b1c0..7075b49702b9 100644 --- a/packages/app/test-requirements.txt +++ b/packages/app/test-requirements.txt @@ -2,5 +2,4 @@ mock-ssh-server pkce pykwalify pytest -pytest-asyncio testfixtures diff --git a/packages/files/test-requirements.txt b/packages/files/test-requirements.txt index e5abfc9b7a80..ad572b378cec 100644 --- a/packages/files/test-requirements.txt +++ b/packages/files/test-requirements.txt @@ -1,4 +1,3 @@ pytest -pytest-asyncio fs-gcsfs s3fs>=2023.1.0,<2024 diff --git a/test/unit/app/managers/base.py b/test/unit/app/managers/base.py index 57aa1a41e7c7..5eee85e4e022 100644 --- a/test/unit/app/managers/base.py +++ b/test/unit/app/managers/base.py @@ -8,14 +8,9 @@ from galaxy.app_unittest_utils import galaxy_mock from galaxy.managers.users import UserManager -from galaxy.util.unittest import ( - IsolatedAsyncioTestCase, - TestCase, -) +from galaxy.util.unittest import TestCase from galaxy.work.context import SessionRequestContext -__all__ = ("BaseIsolatedAsyncioTestCase", "BaseTestCase", "CreatesCollectionsMixin") - # ============================================================================= admin_email = "admin@admin.admin" admin_users = admin_email @@ -109,14 +104,6 @@ def assertIsJsonifyable(self, item): assert isinstance(json.dumps(item), str) -class BaseIsolatedAsyncioTestCase(BaseTestCase, IsolatedAsyncioTestCase): - """ - Asynchronous version of `BaseTestCase`. - - Can run sync tests too. - """ - - class CreatesCollectionsMixin: trans: SessionRequestContext diff --git a/test/unit/app/managers/test_user_file_sources.py b/test/unit/app/managers/test_user_file_sources.py index 7ebe61447cf0..72ca2c358edd 100644 --- a/test/unit/app/managers/test_user_file_sources.py +++ b/test/unit/app/managers/test_user_file_sources.py @@ -46,7 +46,7 @@ from galaxy.schema.schema import OAuth2State from galaxy.util import config_templates from galaxy.util.config_templates import RawTemplateConfig -from .base import BaseIsolatedAsyncioTestCase +from .base import BaseTestCase SIMPLE_FILE_SOURCE_NAME = "myfilesource" SIMPLE_FILE_SOURCE_DESCRIPTION = "a description of my file source" @@ -205,7 +205,7 @@ def simple_vault_template(tmp_path): ) -class TestFileSourcesTestCase(BaseIsolatedAsyncioTestCase): +class TestFileSourcesTestCase(BaseTestCase): manager: FileSourceInstancesManager file_sources: UserDefinedFileSourcesImpl @@ -282,7 +282,7 @@ def mock_get_token_from_code_raw( user_object = self._create_instance(create_payload) assert get_uuid(user_object.uuid) == get_uuid(uuid) - async def test_oauth2_access_token_injection_during_verify(self, tmp_path, monkeypatch): + def test_oauth2_access_token_injection_during_verify(self, tmp_path, monkeypatch): if DropboxFS is None: raise SkipTest("Optional dropbpox dependency not available") self._init_dropbox_env(tmp_path, monkeypatch) @@ -320,14 +320,14 @@ def __init__(self, **kwd): monkeypatch.setattr(config_templates, "get_token_from_refresh_raw", mock_get_token_from_refresh_raw) monkeypatch.setattr(dropbox, "DropboxFS", MockDropboxFS) - status = await self.manager.plugin_status(self.trans, create_payload) + status = self.manager.plugin_status(self.trans, create_payload) assert status.oauth2_access_token_generation assert not status.oauth2_access_token_generation.is_not_ok assert status.connection assert not status.connection.is_not_ok assert pyfilesystem_fs_init_kwd["access_token"] == "my_test_access_token" - async def test_report_oauth2_access_token_generation_failure(self, tmp_path, monkeypatch): + def test_report_oauth2_access_token_generation_failure(self, tmp_path, monkeypatch): self._init_dropbox_env(tmp_path, monkeypatch) uuid = uuid4().hex @@ -350,7 +350,7 @@ def mock_get_token_from_refresh_raw(refresh_token, client_pair, config): return MockExceptionResponse(excepton_message) monkeypatch.setattr(config_templates, "get_token_from_refresh_raw", mock_get_token_from_refresh_raw) - status = await self.manager.plugin_status(self.trans, create_payload) + status = self.manager.plugin_status(self.trans, create_payload) assert status.oauth2_access_token_generation assert status.oauth2_access_token_generation.is_not_ok assert excepton_message in status.oauth2_access_token_generation.message @@ -605,7 +605,7 @@ def test_upgrade_fails_if_new_secrets_absent(self, tmp_path): self._assert_modify_throws_exception(user_object_store, upgrade_to_1, RequestParameterMissingException) - async def test_status_valid(self, tmp_path): + def test_status_valid(self, tmp_path): self.init_user_in_database() self._init_managers(tmp_path) (tmp_path / self.trans.user.username).mkdir() @@ -617,14 +617,14 @@ async def test_status_valid(self, tmp_path): variables={}, secrets={}, ) - status = await self.manager.plugin_status(self.trans, create_payload) + status = self.manager.plugin_status(self.trans, create_payload) assert status.connection assert not status.connection.is_not_ok assert not status.template_definition.is_not_ok assert status.template_settings assert not status.template_settings.is_not_ok - async def test_status_invalid_connection(self, tmp_path): + def test_status_invalid_connection(self, tmp_path): self.init_user_in_database() self._init_managers(tmp_path) # We don't make the directory like above so it doesn't exist @@ -637,7 +637,7 @@ async def test_status_invalid_connection(self, tmp_path): variables={}, secrets={}, ) - status = await self.manager.plugin_status(self.trans, create_payload) + status = self.manager.plugin_status(self.trans, create_payload) assert not status.template_definition.is_not_ok assert status.template_settings assert not status.template_settings.is_not_ok @@ -647,7 +647,7 @@ async def test_status_invalid_connection(self, tmp_path): assert status.connection assert status.connection.is_not_ok - async def test_status_invalid_settings_undefined_variable(self, tmp_path): + def test_status_invalid_settings_undefined_variable(self, tmp_path): self.init_user_in_database() self._init_managers(tmp_path, config_dict=invalid_home_directory_template(tmp_path)) create_payload = CreateInstancePayload( @@ -658,7 +658,7 @@ async def test_status_invalid_settings_undefined_variable(self, tmp_path): variables={}, secrets={}, ) - status = await self.manager.plugin_status(self.trans, create_payload) + status = self.manager.plugin_status(self.trans, create_payload) assert not status.template_definition.is_not_ok assert status.template_settings assert status.template_settings.is_not_ok @@ -667,7 +667,7 @@ async def test_status_invalid_settings_undefined_variable(self, tmp_path): ) assert status.connection is None - async def test_status_invalid_settings_configuration_validation(self, tmp_path): + def test_status_invalid_settings_configuration_validation(self, tmp_path): self.init_user_in_database() self._init_managers(tmp_path, config_dict=invalid_home_directory_template_type_error(tmp_path)) create_payload = CreateInstancePayload( @@ -678,14 +678,14 @@ async def test_status_invalid_settings_configuration_validation(self, tmp_path): variables={}, secrets={}, ) - status = await self.manager.plugin_status(self.trans, create_payload) + status = self.manager.plugin_status(self.trans, create_payload) assert not status.template_definition.is_not_ok assert status.template_settings assert status.template_settings.is_not_ok assert "Input should be a valid boolean" in status.template_settings.message assert status.connection is None - async def test_status_existing_valid(self, tmp_path): + def test_status_existing_valid(self, tmp_path): self.init_user_in_database() self._init_managers(tmp_path) (tmp_path / self.trans.user.username).mkdir() @@ -698,14 +698,14 @@ async def test_status_existing_valid(self, tmp_path): secrets={}, ) user_file_source = self._create_instance(create_payload) - status = await self.manager.plugin_status_for_instance(self.trans, user_file_source.uuid) + status = self.manager.plugin_status_for_instance(self.trans, user_file_source.uuid) assert status.connection assert not status.connection.is_not_ok assert not status.template_definition.is_not_ok assert status.template_settings assert not status.template_settings.is_not_ok - async def test_status_update_valid(self, tmp_path): + def test_status_update_valid(self, tmp_path): self._init_managers(tmp_path, config_dict=simple_variable_template(tmp_path)) create_payload = CreateInstancePayload( name=SIMPLE_FILE_SOURCE_NAME, @@ -723,14 +723,14 @@ async def test_status_update_valid(self, tmp_path): } ) - status = await self.manager.test_modify_instance(self.trans, user_file_source.uuid, update) + status = self.manager.test_modify_instance(self.trans, user_file_source.uuid, update) assert not status.template_definition.is_not_ok assert status.template_settings assert not status.template_settings.is_not_ok assert status.connection assert status.connection.is_not_ok - async def test_status_upgrade_valid(self, tmp_path): + def test_status_upgrade_valid(self, tmp_path): user_file_source = self._init_upgrade_test_case(tmp_path) assert "sec1" in user_file_source.secrets assert "sec2" not in user_file_source.secrets @@ -745,21 +745,21 @@ async def test_status_upgrade_valid(self, tmp_path): }, variables={}, ) - status = await self.manager.test_modify_instance(self.trans, user_file_source.uuid, upgrade_to_1) + status = self.manager.test_modify_instance(self.trans, user_file_source.uuid, upgrade_to_1) assert not status.template_definition.is_not_ok assert status.template_settings assert not status.template_settings.is_not_ok assert status.connection assert status.connection.is_not_ok - async def test_status_upgrade_invalid(self, tmp_path): + def test_status_upgrade_invalid(self, tmp_path): user_file_source = self._init_invalid_upgrade_test_case(tmp_path) upgrade_to_1 = TestUpgradeInstancePayload( template_version=1, secrets={}, variables={}, ) - status = await self.manager.test_modify_instance(self.trans, user_file_source.uuid, upgrade_to_1) + status = self.manager.test_modify_instance(self.trans, user_file_source.uuid, upgrade_to_1) assert not status.template_definition.is_not_ok assert status.template_settings assert status.template_settings.is_not_ok diff --git a/test/unit/files/_util.py b/test/unit/files/_util.py index 9c53d4811f21..0a4a77ee14ff 100644 --- a/test/unit/files/_util.py +++ b/test/unit/files/_util.py @@ -36,7 +36,7 @@ def find(dir_list, class_=None, name=None): return None -async def list_root( +def list_root( file_sources: ConfiguredFileSources, uri: str, recursive: bool, @@ -44,11 +44,11 @@ async def list_root( ): file_source_pair = file_sources.get_file_source_path(uri) file_source = file_source_pair.file_source - res, _ = await file_source.list("/", recursive=recursive, user_context=user_context) + res, _ = file_source.list("/", recursive=recursive, user_context=user_context) return res -async def list_dir( +def list_dir( file_sources: ConfiguredFileSources, uri: str, recursive: bool, @@ -58,7 +58,7 @@ async def list_dir( file_source = file_source_pair.file_source print(file_source_pair.path) print(uri) - res, _ = await file_source.list(file_source_pair.path, recursive=recursive, user_context=user_context) + res, _ = file_source.list(file_source_pair.path, recursive=recursive, user_context=user_context) return res @@ -186,14 +186,14 @@ def assert_can_write_and_read_to_conf(conf: dict): ) -async def assert_simple_file_realize(conf_file, recursive=False, filename="a", contents="a\n", contains=False): +def assert_simple_file_realize(conf_file, recursive=False, filename="a", contents="a\n", contains=False): user_context = user_context_fixture() file_sources = configured_file_sources(conf_file) file_source_pair = file_sources.get_file_source_path("gxfiles://test1") assert file_source_pair.path == "/" file_source = file_source_pair.file_source - res, _ = await file_source.list("/", recursive=recursive, user_context=user_context) + res, _ = file_source.list("/", recursive=recursive, user_context=user_context) a_file = find(res, class_="File", name=filename) assert a_file diff --git a/test/unit/files/test_dropbox.py b/test/unit/files/test_dropbox.py index 8605f4c12530..111612ea4956 100644 --- a/test/unit/files/test_dropbox.py +++ b/test/unit/files/test_dropbox.py @@ -13,6 +13,5 @@ @skip_if_no_dropbox_access_token -@pytest.mark.asyncio -async def test_file_source(): - await assert_simple_file_realize(FILE_SOURCES_CONF, recursive=True) +def test_file_source(): + assert_simple_file_realize(FILE_SOURCES_CONF, recursive=True) diff --git a/test/unit/files/test_gcsfs.py b/test/unit/files/test_gcsfs.py index 215c79dd8044..94a113717835 100644 --- a/test/unit/files/test_gcsfs.py +++ b/test/unit/files/test_gcsfs.py @@ -19,8 +19,7 @@ @skip_if_no_gcsfs_libs -@pytest.mark.asyncio -async def test_file_source(): - await assert_simple_file_realize( +def test_file_source(): + assert_simple_file_realize( FILE_SOURCES_CONF, recursive=False, filename="README", contents="1000genomes", contains=True ) diff --git a/test/unit/files/test_googledrive.py b/test/unit/files/test_googledrive.py index c177d11fbcda..2661713ee629 100644 --- a/test/unit/files/test_googledrive.py +++ b/test/unit/files/test_googledrive.py @@ -15,6 +15,5 @@ @skip_if_no_google_drive_access_token -@pytest.mark.asyncio -async def test_file_source(): - await assert_simple_file_realize(FILE_SOURCES_CONF) +def test_file_source(): + assert_simple_file_realize(FILE_SOURCES_CONF) diff --git a/test/unit/files/test_onedata.py b/test/unit/files/test_onedata.py index 524027a244f0..b4a5813699fb 100644 --- a/test/unit/files/test_onedata.py +++ b/test/unit/files/test_onedata.py @@ -14,6 +14,5 @@ @skip_if_no_onedata_access_token -@pytest.mark.asyncio -async def test_file_source(): - await assert_simple_file_realize(FILE_SOURCES_CONF) +def test_file_source(): + assert_simple_file_realize(FILE_SOURCES_CONF) diff --git a/test/unit/files/test_posix.py b/test/unit/files/test_posix.py index 153a2415555e..a5749500c597 100644 --- a/test/unit/files/test_posix.py +++ b/test/unit/files/test_posix.py @@ -37,8 +37,7 @@ EMAIL = "alice@galaxyproject.org" -@pytest.mark.asyncio -async def test_posix(): +def test_posix(): file_sources = _configured_file_sources() as_dict = file_sources.to_dict() assert len(as_dict["file_sources"]) == 1 @@ -47,7 +46,7 @@ async def test_posix(): _download_and_check_file(file_sources) - res = await list_root(file_sources, "gxfiles://test1", recursive=False) + res = list_root(file_sources, "gxfiles://test1", recursive=False) file_a = find_file_a(res) assert file_a assert file_a["uri"] == "gxfiles://test1/a" @@ -57,7 +56,7 @@ async def test_posix(): assert subdir1["class"] == "Directory" assert subdir1["uri"] == "gxfiles://test1/subdir1" - res = await list_dir(file_sources, "gxfiles://test1/subdir1", recursive=False) + res = list_dir(file_sources, "gxfiles://test1/subdir1", recursive=False) subdir2 = find(res, name="subdir2") assert subdir2, res assert subdir2["uri"] == "gxfiles://test1/subdir1/subdir2" @@ -66,7 +65,7 @@ async def test_posix(): assert file_c, res assert file_c["uri"] == "gxfiles://test1/subdir1/c" - res = await list_root(file_sources, "gxfiles://test1", recursive=True) + res = list_root(file_sources, "gxfiles://test1", recursive=True) subdir1 = find(res, name="subdir1") subdir2 = find(res, name="subdir2") assert subdir1["class"] == "Directory" @@ -121,28 +120,26 @@ def test_posix_nonexistent_parent_write(): assert "Parent" in str(e) -@pytest.mark.asyncio -async def test_posix_per_user(): +def test_posix_per_user(): file_sources = _configured_file_sources(per_user=True) user_context = user_context_fixture() assert_realizes_as(file_sources, "gxfiles://test1/a", "a\n", user_context=user_context) - res = await list_root(file_sources, "gxfiles://test1", recursive=False, user_context=user_context) + res = list_root(file_sources, "gxfiles://test1", recursive=False, user_context=user_context) assert find_file_a(res) -@pytest.mark.asyncio -async def test_posix_per_user_writable(): +def test_posix_per_user_writable(): file_sources = _configured_file_sources(per_user=True, writable=True) user_context = user_context_fixture() - res = await list_root(file_sources, "gxfiles://test1", recursive=False, user_context=user_context) + res = list_root(file_sources, "gxfiles://test1", recursive=False, user_context=user_context) b = find(res, name="b") assert b is None write_from(file_sources, "gxfiles://test1/b", "my test content", user_context=user_context) - res = await list_root(file_sources, "gxfiles://test1", recursive=False, user_context=user_context) + res = list_root(file_sources, "gxfiles://test1", recursive=False, user_context=user_context) b = find(res, name="b") assert b is not None, b @@ -272,8 +269,7 @@ def test_user_import_dir_implicit_config(): assert_realizes_as(file_sources, "gxuserimport://a", "a\n", user_context=user_context) -@pytest.mark.asyncio -async def test_posix_user_access_requires_role(): +def test_posix_user_access_requires_role(): allowed_role_name = "role1" plugin_extra_config = { "requires_roles": allowed_role_name, @@ -281,13 +277,13 @@ async def test_posix_user_access_requires_role(): file_sources = _configured_file_sources(writable=True, plugin_extra_config=plugin_extra_config) user_context = user_context_fixture() - await _assert_user_access_prohibited(file_sources, user_context) + _assert_user_access_prohibited(file_sources, user_context) user_context = user_context_fixture(role_names={allowed_role_name}) - await _assert_user_access_granted(file_sources, user_context) + _assert_user_access_granted(file_sources, user_context) -async def test_posix_user_access_requires_group(): +def test_posix_user_access_requires_group(): allowed_group_name = "group1" plugin_extra_config = { "requires_groups": allowed_group_name, @@ -295,14 +291,13 @@ async def test_posix_user_access_requires_group(): file_sources = _configured_file_sources(writable=True, plugin_extra_config=plugin_extra_config) user_context = user_context_fixture() - await _assert_user_access_prohibited(file_sources, user_context) + _assert_user_access_prohibited(file_sources, user_context) user_context = user_context_fixture(group_names={allowed_group_name}) - await _assert_user_access_granted(file_sources, user_context) + _assert_user_access_granted(file_sources, user_context) -@pytest.mark.asyncio -async def test_posix_admin_user_has_access(): +def test_posix_admin_user_has_access(): plugin_extra_config = { "requires_roles": "role1", "requires_groups": "group1", @@ -310,14 +305,13 @@ async def test_posix_admin_user_has_access(): file_sources = _configured_file_sources(writable=True, plugin_extra_config=plugin_extra_config) user_context = user_context_fixture() - await _assert_user_access_prohibited(file_sources, user_context) + _assert_user_access_prohibited(file_sources, user_context) user_context = user_context_fixture(is_admin=True) - await _assert_user_access_granted(file_sources, user_context) + _assert_user_access_granted(file_sources, user_context) -@pytest.mark.asyncio -async def test_posix_user_access_requires_role_and_group(): +def test_posix_user_access_requires_role_and_group(): allowed_group_name = "group1" allowed_role_name = "role1" plugin_extra_config = { @@ -327,17 +321,16 @@ async def test_posix_user_access_requires_role_and_group(): file_sources = _configured_file_sources(writable=True, plugin_extra_config=plugin_extra_config) user_context = user_context_fixture(group_names={allowed_group_name}) - await _assert_user_access_prohibited(file_sources, user_context) + _assert_user_access_prohibited(file_sources, user_context) user_context = user_context_fixture(role_names={allowed_role_name}) - await _assert_user_access_prohibited(file_sources, user_context) + _assert_user_access_prohibited(file_sources, user_context) user_context = user_context_fixture(role_names={allowed_role_name}, group_names={allowed_group_name}) - await _assert_user_access_granted(file_sources, user_context) + _assert_user_access_granted(file_sources, user_context) -@pytest.mark.asyncio -async def test_posix_user_access_using_boolean_rules(): +def test_posix_user_access_using_boolean_rules(): plugin_extra_config = { "requires_roles": "role1 and (role2 or role3)", "requires_groups": "group1 and group2 and not group3", @@ -345,19 +338,19 @@ async def test_posix_user_access_using_boolean_rules(): file_sources = _configured_file_sources(writable=True, plugin_extra_config=plugin_extra_config) user_context = user_context_fixture(role_names=set(), group_names=set()) - await _assert_user_access_prohibited(file_sources, user_context) + _assert_user_access_prohibited(file_sources, user_context) user_context = user_context_fixture(role_names={"role1"}, group_names={"group1", "group2"}) - await _assert_user_access_prohibited(file_sources, user_context) + _assert_user_access_prohibited(file_sources, user_context) user_context = user_context_fixture(role_names={"role1", "role3"}, group_names={"group1", "group2", "group3"}) - await _assert_user_access_prohibited(file_sources, user_context) + _assert_user_access_prohibited(file_sources, user_context) user_context = user_context_fixture(role_names={"role1", "role2"}, group_names={"group3", "group5"}) - await _assert_user_access_prohibited(file_sources, user_context) + _assert_user_access_prohibited(file_sources, user_context) user_context = user_context_fixture(role_names={"role1", "role3"}, group_names={"group1", "group2"}) - await _assert_user_access_granted(file_sources, user_context) + _assert_user_access_granted(file_sources, user_context) def test_posix_file_url_only_mode_non_admin_cannot_retrieve(): @@ -454,10 +447,9 @@ def test_posix_file_url_disallowed_root(): assert_realizes_as(file_sources, test_url, "some content\n") -@pytest.mark.asyncio -async def _assert_user_access_prohibited(file_sources, user_context): +def _assert_user_access_prohibited(file_sources, user_context): with pytest.raises(ItemAccessibilityException): - await list_root(file_sources, "gxfiles://test1", recursive=False, user_context=user_context) + list_root(file_sources, "gxfiles://test1", recursive=False, user_context=user_context) with pytest.raises(ItemAccessibilityException): write_from(file_sources, "gxfiles://test1/b", "my test content", user_context=user_context) @@ -466,14 +458,13 @@ async def _assert_user_access_prohibited(file_sources, user_context): assert_realizes_as(file_sources, "gxfiles://test1/a", "a\n", user_context=user_context) -@pytest.mark.asyncio -async def _assert_user_access_granted(file_sources, user_context): - res = await list_root(file_sources, "gxfiles://test1", recursive=False, user_context=user_context) +def _assert_user_access_granted(file_sources, user_context): + res = list_root(file_sources, "gxfiles://test1", recursive=False, user_context=user_context) assert res write_from(file_sources, "gxfiles://test1/b", "my test content", user_context=user_context) - res = await list_root(file_sources, "gxfiles://test1", recursive=False, user_context=user_context) + res = list_root(file_sources, "gxfiles://test1", recursive=False, user_context=user_context) b = find(res, name="b") assert b is not None, b diff --git a/test/unit/files/test_s3.py b/test/unit/files/test_s3.py index c1b12ebf8d8f..7529959a5488 100644 --- a/test/unit/files/test_s3.py +++ b/test/unit/files/test_s3.py @@ -17,9 +17,8 @@ FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "s3_file_sources_conf.yml") -@pytest.mark.asyncio -async def test_file_source(): - await assert_simple_file_realize( +def test_file_source(): + assert_simple_file_realize( FILE_SOURCES_CONF, recursive=False, filename="data_use_policies.txt", diff --git a/test/unit/files/test_temp.py b/test/unit/files/test_temp.py index eae44e90c57b..2d51a359916b 100644 --- a/test/unit/files/test_temp.py +++ b/test/unit/files/test_temp.py @@ -56,35 +56,34 @@ def test_list_recursive(temp_file_source: TempFilesSource): assert_list_names(temp_file_source, "/", recursive=True, expected_names=expected_names) -@pytest.mark.asyncio -async def test_pagination(temp_file_source: TempFilesSource): +def test_pagination(temp_file_source: TempFilesSource): # Pagination is only supported for non-recursive listings. recursive = False - root_lvl_entries, count = await temp_file_source.list("/", recursive=recursive) + root_lvl_entries, count = temp_file_source.list("/", recursive=recursive) assert count == 4 assert len(root_lvl_entries) == 4 # Get first entry - result, count = await temp_file_source.list("/", recursive=recursive, limit=1, offset=0) + result, count = temp_file_source.list("/", recursive=recursive, limit=1, offset=0) assert count == 4 assert len(result) == 1 assert result[0] == root_lvl_entries[0] # Get second entry - result, count = await temp_file_source.list("/", recursive=recursive, limit=1, offset=1) + result, count = temp_file_source.list("/", recursive=recursive, limit=1, offset=1) assert count == 4 assert len(result) == 1 assert result[0] == root_lvl_entries[1] # Get second and third entry - result, count = await temp_file_source.list("/", recursive=recursive, limit=2, offset=1) + result, count = temp_file_source.list("/", recursive=recursive, limit=2, offset=1) assert count == 4 assert len(result) == 2 assert result[0] == root_lvl_entries[1] assert result[1] == root_lvl_entries[2] # Get last three entries - result, count = await temp_file_source.list("/", recursive=recursive, limit=3, offset=1) + result, count = temp_file_source.list("/", recursive=recursive, limit=3, offset=1) assert count == 4 assert len(result) == 3 assert result[0] == root_lvl_entries[1] @@ -92,101 +91,95 @@ async def test_pagination(temp_file_source: TempFilesSource): assert result[2] == root_lvl_entries[3] -@pytest.mark.asyncio -async def test_search(temp_file_source: TempFilesSource): +def test_search(temp_file_source: TempFilesSource): # Search is only supported for non-recursive listings. recursive = False - root_lvl_entries, count = await temp_file_source.list("/", recursive=recursive) + root_lvl_entries, count = temp_file_source.list("/", recursive=recursive) assert count == 4 assert len(root_lvl_entries) == 4 - result, count = await temp_file_source.list("/", recursive=recursive, query="a") + result, count = temp_file_source.list("/", recursive=recursive, query="a") assert count == 1 assert len(result) == 1 assert result[0]["name"] == "a" - result, count = await temp_file_source.list("/", recursive=recursive, query="b") + result, count = temp_file_source.list("/", recursive=recursive, query="b") assert count == 1 assert len(result) == 1 assert result[0]["name"] == "b" - result, count = await temp_file_source.list("/", recursive=recursive, query="c") + result, count = temp_file_source.list("/", recursive=recursive, query="c") assert count == 1 assert len(result) == 1 assert result[0]["name"] == "c" # Searching for 'd' at root level should return the directory 'dir1' but not the file 'd' # as it is not a direct child of the root. - result, count = await temp_file_source.list("/", recursive=recursive, query="d") + result, count = temp_file_source.list("/", recursive=recursive, query="d") assert count == 1 assert len(result) == 1 assert result[0]["name"] == "dir1" # Searching for 'e' at root level should not return anything. - result, count = await temp_file_source.list("/", recursive=recursive, query="e") + result, count = temp_file_source.list("/", recursive=recursive, query="e") assert count == 0 assert len(result) == 0 - result, count = await temp_file_source.list("/dir1", recursive=recursive, query="e") + result, count = temp_file_source.list("/dir1", recursive=recursive, query="e") assert count == 1 assert len(result) == 1 assert result[0]["name"] == "e" -@pytest.mark.asyncio -async def test_query_with_empty_string(temp_file_source: TempFilesSource): +def test_query_with_empty_string(temp_file_source: TempFilesSource): recursive = False - root_lvl_entries, count = await temp_file_source.list("/", recursive=recursive) + root_lvl_entries, count = temp_file_source.list("/", recursive=recursive) assert count == 4 assert len(root_lvl_entries) == 4 - result, count = await temp_file_source.list("/", recursive=recursive, query="") + result, count = temp_file_source.list("/", recursive=recursive, query="") assert count == 4 assert len(result) == 4 assert result == root_lvl_entries -@pytest.mark.asyncio -async def test_pagination_not_supported_raises(temp_file_source: TempFilesSource): +def test_pagination_not_supported_raises(temp_file_source: TempFilesSource): TempFilesSource.supports_pagination = False recursive = False with pytest.raises(RequestParameterInvalidException) as exc_info: - await temp_file_source.list("/", recursive=recursive, limit=1, offset=0) + temp_file_source.list("/", recursive=recursive, limit=1, offset=0) assert "Pagination is not supported" in str(exc_info.value) TempFilesSource.supports_pagination = True -@pytest.mark.asyncio -async def test_pagination_parameters_non_negative(temp_file_source: TempFilesSource): +def test_pagination_parameters_non_negative(temp_file_source: TempFilesSource): recursive = False with pytest.raises(RequestParameterInvalidException) as exc_info: - await temp_file_source.list("/", recursive=recursive, limit=-1, offset=0) + temp_file_source.list("/", recursive=recursive, limit=-1, offset=0) assert "Limit must be greater than 0" in str(exc_info.value) with pytest.raises(RequestParameterInvalidException) as exc_info: - await temp_file_source.list("/", recursive=recursive, limit=0, offset=0) + temp_file_source.list("/", recursive=recursive, limit=0, offset=0) assert "Limit must be greater than 0" in str(exc_info.value) with pytest.raises(RequestParameterInvalidException) as exc_info: - await temp_file_source.list("/", recursive=recursive, limit=1, offset=-1) + temp_file_source.list("/", recursive=recursive, limit=1, offset=-1) assert "Offset must be greater than or equal to 0" in str(exc_info.value) -@pytest.mark.asyncio -async def test_search_not_supported_raises(temp_file_source: TempFilesSource): +def test_search_not_supported_raises(temp_file_source: TempFilesSource): TempFilesSource.supports_search = False recursive = False with pytest.raises(RequestParameterInvalidException) as exc_info: - await temp_file_source.list("/", recursive=recursive, query="a") + temp_file_source.list("/", recursive=recursive, query="a") assert "Server-side search is not supported by this file source" in str(exc_info.value) TempFilesSource.supports_search = True -@pytest.mark.asyncio -async def test_sorting_not_supported_raises(temp_file_source: TempFilesSource): +def test_sorting_not_supported_raises(temp_file_source: TempFilesSource): recursive = False with pytest.raises(RequestParameterInvalidException) as exc_info: - await temp_file_source.list("/", recursive=recursive, sort_by="name") + temp_file_source.list("/", recursive=recursive, sort_by="name") assert "Server-side sorting is not supported by this file source" in str(exc_info.value) @@ -209,9 +202,8 @@ def _upload_to(file_source: TempFilesSource, target_uri: str, content: str, user file_source.write_from(target_uri, f.name, user_context=user_context) -@pytest.mark.asyncio -async def assert_list_names(file_source: TempFilesSource, uri: str, recursive: bool, expected_names: List[str]): - result, count = await file_source.list(uri, recursive=recursive) +def assert_list_names(file_source: TempFilesSource, uri: str, recursive: bool, expected_names: List[str]): + result, count = file_source.list(uri, recursive=recursive) assert count == len(expected_names) assert sorted([entry["name"] for entry in result]) == sorted(expected_names) return result diff --git a/test/unit/files/test_webdav.py b/test/unit/files/test_webdav.py index 771f54989d75..9294881ec247 100644 --- a/test/unit/files/test_webdav.py +++ b/test/unit/files/test_webdav.py @@ -25,8 +25,7 @@ @skip_if_no_webdav -@pytest.mark.asyncio -async def test_file_source(): +def test_file_source(): file_sources = configured_file_sources(FILE_SOURCES_CONF) file_source_pair = file_sources.get_file_source_path("gxfiles://test1") @@ -47,7 +46,7 @@ async def test_file_source(): assert subdir1["class"] == "Directory" assert subdir1["uri"] == "gxfiles://test1/subdir1" - res = await list_dir(file_sources, "gxfiles://test1/subdir1", recursive=False) + res = list_dir(file_sources, "gxfiles://test1/subdir1", recursive=False) subdir2 = find(res, name="subdir2") assert subdir2, res assert subdir2["uri"] == "gxfiles://test1/subdir1/subdir2" @@ -64,18 +63,17 @@ def test_sniff_to_tmp(): @skip_if_no_webdav -@pytest.mark.asyncio -async def test_serialization(): +def test_serialization(): configs = [FILE_SOURCES_CONF_NO_USE_TEMP_FILES, FILE_SOURCES_CONF] for config in configs: # serialize the configured file sources and rematerialize them, # ensure they still function. This is needed for uploading files. file_sources = serialize_and_recover(configured_file_sources(config)) - res = await list_root(file_sources, "gxfiles://test1", recursive=True) + res = list_root(file_sources, "gxfiles://test1", recursive=True) assert find_file_a(res) - res = await list_root(file_sources, "gxfiles://test1", recursive=False) + res = list_root(file_sources, "gxfiles://test1", recursive=False) assert find_file_a(res) _download_and_check_file(file_sources) @@ -103,14 +101,13 @@ def test_config_options(): @skip_if_no_webdav -@pytest.mark.asyncio -async def test_serialization_user(): +def test_serialization_user(): file_sources_o = configured_file_sources(USER_FILE_SOURCES_CONF) user_context = user_context_fixture() - res = await list_root(file_sources_o, "gxfiles://test1", recursive=True, user_context=user_context) + res = list_root(file_sources_o, "gxfiles://test1", recursive=True, user_context=user_context) assert find_file_a(res) file_sources = serialize_and_recover(file_sources_o, user_context=user_context) - res = await list_root(file_sources, "gxfiles://test1", recursive=True, user_context=None) + res = list_root(file_sources, "gxfiles://test1", recursive=True, user_context=None) assert find_file_a(res)