diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 7f3159b131e7..95228b803f7a 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -29,6 +29,7 @@ from typing import ( Any, cast, + ClassVar, Dict, Generic, Iterable, @@ -159,7 +160,6 @@ ) from galaxy.model.orm.now import now from galaxy.model.orm.util import add_object_to_object_session -from galaxy.objectstore import ObjectStorePopulator from galaxy.objectstore.templates import ( ObjectStoreConfiguration, ObjectStoreTemplate, @@ -219,6 +219,10 @@ from galaxy.util.sanitize_html import sanitize_html if TYPE_CHECKING: + from galaxy.objectstore import ( + BaseObjectStore, + ObjectStorePopulator, + ) from galaxy.schema.invocation import InvocationMessageUnion log = logging.getLogger(__name__) @@ -4041,7 +4045,7 @@ def flush(self): sa_session.commit() -def setup_global_object_store_for_models(object_store): +def setup_global_object_store_for_models(object_store: "BaseObjectStore") -> None: Dataset.object_store = object_store @@ -4133,7 +4137,9 @@ class conversion_messages(str, Enum): permitted_actions = get_permitted_actions(filter="DATASET") file_path = "/tmp/" - object_store = None # This get initialized in mapping.py (method init) by app.py + object_store: ClassVar[Optional["BaseObjectStore"]] = ( + None # This get initialized in mapping.py (method init) by app.py + ) engine = None def __init__( @@ -4283,7 +4289,7 @@ def _calculate_size(self) -> int: except OSError: return 0 assert self.object_store - return self.object_store.size(self) # type:ignore[unreachable] + return self.object_store.size(self) @overload def get_size(self, nice_size: Literal[False], calculate_size: bool = True) -> int: ... @@ -4663,7 +4669,7 @@ def get_quota_source_label(self): quota_source_label = property(get_quota_source_label) - def set_skipped(self, object_store_populator: ObjectStorePopulator): + def set_skipped(self, object_store_populator: "ObjectStorePopulator") -> None: assert self.dataset object_store_populator.set_object_store_id(self) self.extension = "expression.json" diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 053e177edd9f..e1d975e5be5a 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -3,6 +3,7 @@ from typing import ( Optional, Type, + TYPE_CHECKING, ) from galaxy import model @@ -16,6 +17,9 @@ from galaxy.model.security import GalaxyRBACAgent from galaxy.model.triggers.update_audit_table import install as install_timestamp_triggers +if TYPE_CHECKING: + from galaxy.objectstore import BaseObjectStore + log = logging.getLogger(__name__) metadata = mapper_registry.metadata @@ -99,8 +103,11 @@ def _build_model_mapping(engine, map_install_models, thread_local_log) -> Galaxy def init_models_from_config( - config: GalaxyAppConfiguration, map_install_models=False, object_store=None, trace_logger=None -): + config: GalaxyAppConfiguration, + map_install_models: bool = False, + object_store: Optional["BaseObjectStore"] = None, + trace_logger=None, +) -> GalaxyModelMapping: model = init( config.file_path, config.database_connection, diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 4a4d639bf927..75fd174cbb75 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -235,7 +235,15 @@ def get_data( @abc.abstractmethod def get_filename( - self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False + self, + obj, + base_dir=None, + dir_only=False, + extra_dir=None, + extra_dir_at_root=False, + alt_name=None, + obj_dir=False, + sync_cache: bool = True, ): """ Get the expected filename with absolute path for object with id `obj.id`. @@ -450,35 +458,161 @@ def _get_object_id(self, obj): def _invoke(self, delegate, obj=None, **kwargs): return self.__getattribute__(f"_{delegate}")(obj=obj, **kwargs) - def exists(self, obj, **kwargs): - return self._invoke("exists", obj, **kwargs) + def exists(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None): + return self._invoke( + "exists", + obj, + base_dir=base_dir, + dir_only=dir_only, + extra_dir=extra_dir, + extra_dir_at_root=extra_dir_at_root, + alt_name=alt_name, + ) - def construct_path(self, obj, **kwargs): - return self._invoke("construct_path", obj, **kwargs) + def construct_path( + self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None + ): + return self._invoke( + "construct_path", + obj, + base_dir=base_dir, + dir_only=dir_only, + extra_dir=extra_dir, + extra_dir_at_root=extra_dir_at_root, + alt_name=alt_name, + ) - def create(self, obj, **kwargs): - return self._invoke("create", obj, **kwargs) + def create( + self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False + ): + return self._invoke( + "create", + obj, + base_dir=base_dir, + dir_only=dir_only, + extra_dir=extra_dir, + extra_dir_at_root=extra_dir_at_root, + alt_name=alt_name, + obj_dir=obj_dir, + ) - def empty(self, obj, **kwargs): - return self._invoke("empty", obj, **kwargs) + def empty(self, obj, base_dir=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): + return self._invoke( + "empty", + obj, + base_dir=base_dir, + extra_dir=extra_dir, + extra_dir_at_root=extra_dir_at_root, + alt_name=alt_name, + obj_dir=obj_dir, + ) - def size(self, obj, **kwargs): - return self._invoke("size", obj, **kwargs) + def size(self, obj, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False) -> int: + return self._invoke( + "size", obj, extra_dir=extra_dir, extra_dir_at_root=extra_dir_at_root, alt_name=alt_name, obj_dir=obj_dir + ) - def delete(self, obj, **kwargs): - return self._invoke("delete", obj, **kwargs) + def delete( + self, + obj, + entire_dir=False, + base_dir=None, + extra_dir=None, + extra_dir_at_root=False, + alt_name=None, + obj_dir=False, + ): + return self._invoke( + "delete", + obj, + entire_dir=entire_dir, + base_dir=base_dir, + extra_dir=extra_dir, + extra_dir_at_root=extra_dir_at_root, + alt_name=alt_name, + obj_dir=obj_dir, + ) - def get_data(self, obj, **kwargs): - return self._invoke("get_data", obj, **kwargs) + def get_data( + self, + obj, + start=0, + count=-1, + base_dir=None, + extra_dir=None, + extra_dir_at_root=False, + alt_name=None, + obj_dir=False, + ): + return self._invoke( + "get_data", + obj, + start=start, + count=count, + base_dir=base_dir, + extra_dir=extra_dir, + extra_dir_at_root=extra_dir_at_root, + alt_name=alt_name, + obj_dir=obj_dir, + ) - def get_filename(self, obj, **kwargs): - return self._invoke("get_filename", obj, **kwargs) + def get_filename( + self, + obj, + base_dir=None, + dir_only=False, + extra_dir=None, + extra_dir_at_root=False, + alt_name=None, + obj_dir=False, + sync_cache: bool = True, + ): + return self._invoke( + "get_filename", + obj, + base_dir=base_dir, + dir_only=dir_only, + extra_dir=extra_dir, + extra_dir_at_root=extra_dir_at_root, + alt_name=alt_name, + obj_dir=obj_dir, + sync_cache=sync_cache, + ) - def update_from_file(self, obj, **kwargs): - return self._invoke("update_from_file", obj, **kwargs) + def update_from_file( + self, + obj, + base_dir=None, + extra_dir=None, + extra_dir_at_root=False, + alt_name=None, + obj_dir=False, + file_name=None, + create=False, + preserve_symlinks=False, + ): + return self._invoke( + "update_from_file", + obj, + base_dir=base_dir, + extra_dir=extra_dir, + extra_dir_at_root=extra_dir_at_root, + alt_name=alt_name, + obj_dir=obj_dir, + file_name=file_name, + create=create, + preserve_symlinks=preserve_symlinks, + ) - def get_object_url(self, obj, **kwargs): - return self._invoke("get_object_url", obj, **kwargs) + def get_object_url(self, obj, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False): + return self._invoke( + "get_object_url", + obj, + extra_dir=extra_dir, + extra_dir_at_root=extra_dir_at_root, + alt_name=alt_name, + obj_dir=obj_dir, + ) def get_concrete_store_name(self, obj): return self._invoke("get_concrete_store_name", obj) @@ -843,7 +977,7 @@ def _create(self, obj, **kwargs): def _empty(self, obj, **kwargs): """Override `ObjectStore`'s stub by checking file size on disk.""" - return self.size(obj, **kwargs) == 0 + return self._size(obj, **kwargs) == 0 def _size(self, obj, **kwargs) -> int: """Override `ObjectStore`'s stub by return file size on disk. @@ -1045,7 +1179,7 @@ def _repr_object_for_exception(self, obj): def _call_method(self, method, obj, default, default_is_exception, **kwargs): """Check all children object stores for the first one with the dataset.""" for store in self.backends.values(): - if store.exists(obj, **kwargs): + if store._exists(obj, **kwargs): return store.__getattribute__(method)(obj, **kwargs) if default_is_exception: raise default( @@ -1109,22 +1243,22 @@ def __init__( user_selection_allowed = [] for backend_def in config_dict["backends"]: - backened_id = backend_def["id"] + backend_id = backend_def["id"] maxpctfull = backend_def.get("max_percent_full", 0) weight = backend_def["weight"] allow_selection = backend_def.get("allow_selection") if allow_selection: - user_selection_allowed.append(backened_id) + user_selection_allowed.append(backend_id) backend = build_object_store_from_config(config, config_dict=backend_def, fsmon=fsmon) - self.backends[backened_id] = backend - self.max_percent_full[backened_id] = maxpctfull + self.backends[backend_id] = backend + self.max_percent_full[backend_id] = maxpctfull for _ in range(0, weight): # The simplest way to do weighting: add backend ids to a # sequence the number of times equalling weight, then randomly # choose a backend from that sequence at creation - self.weighted_backend_ids.append(backened_id) + self.weighted_backend_ids.append(backend_id) self.original_weighted_backend_ids = self.weighted_backend_ids self.user_object_store_resolver = user_object_store_resolver @@ -1340,7 +1474,7 @@ def __get_store_id_for(self, obj, **kwargs): # distributed object store, or if the object's store id is invalid, # try to locate the object for id, store in self.backends.items(): - if store.exists(obj, **kwargs): + if store._exists(obj, **kwargs): log.warning( f"{obj.__class__.__name__} object with ID {obj.id} found in backend object store with ID {id}" ) @@ -1401,7 +1535,7 @@ def __init__(self, config, config_dict, fsmon=False): """The default constructor. Extends `NestedObjectStore`.""" super().__init__(config, config_dict) - backends: Dict[int, ObjectStore] = {} + backends: Dict[int, BaseObjectStore] = {} is_private = config_dict.get("private", DEFAULT_PRIVATE) for order, backend_def in enumerate(config_dict["backends"]): backend_is_private = backend_def.get("private") diff --git a/test/unit/app/tools/test_metadata.py b/test/unit/app/tools/test_metadata.py index 7e1784a84caa..8f83c6b90ef6 100644 --- a/test/unit/app/tools/test_metadata.py +++ b/test/unit/app/tools/test_metadata.py @@ -18,7 +18,7 @@ class TestMetadata(TestCase, tools_support.UsesTools): def setUp(self): super().setUp() self.setup_app() - model.Dataset.object_store = self.app.object_store # type: ignore[assignment] + model.Dataset.object_store = self.app.object_store job = model.Job() sa_session = self.app.model.session sa_session.add(job)