From 216cab7fc60a46285fce0437a6b18a1af3bd1d7c Mon Sep 17 00:00:00 2001 From: pauldg Date: Thu, 10 Oct 2024 17:18:38 +0200 Subject: [PATCH 1/9] added ssl options using json file --- lib/galaxy/objectstore/irods.py | 36 ++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/objectstore/irods.py b/lib/galaxy/objectstore/irods.py index 727bf9c87b14..b1fc1edbb90a 100644 --- a/lib/galaxy/objectstore/irods.py +++ b/lib/galaxy/objectstore/irods.py @@ -2,9 +2,11 @@ Object Store plugin for the Integrated Rule-Oriented Data System (iRODS) """ +import json import logging import os import shutil +import ssl import threading from datetime import datetime from pathlib import Path @@ -40,7 +42,7 @@ def _config_xml_error(tag): def _config_dict_error(key): - msg = "No {key} key in config dictionary".forma(key=key) + msg = "No {key} key in config dictionary".format(key=key) raise Exception(msg) @@ -51,6 +53,7 @@ def parse_config_xml(config_xml): _config_xml_error("auth") username = a_xml[0].get("username") password = a_xml[0].get("password") + sslfile = a_xml[0].get("sslfile", None) r_xml = config_xml.findall("resource") if not r_xml: @@ -88,6 +91,7 @@ def parse_config_xml(config_xml): "auth": { "username": username, "password": password, + "sslfile": sslfile, }, "resource": { "name": resource_name, @@ -138,6 +142,7 @@ def __init__(self, config, config_dict): self.password = auth_dict.get("password") if self.password is None: _config_dict_error("auth->password") + self.sslfile = auth_dict.get("sslfile") resource_dict = config_dict["resource"] if resource_dict is None: @@ -193,15 +198,27 @@ def __init__(self, config, config_dict): if irods is None: raise Exception(IRODS_IMPORT_MESSAGE) + + + session_params = { + 'host': self.host, + 'port': self.port, + 'user': self.username, + 'password': self.password, + 'zone': self.zone, + 'refresh_time': self.refresh_time, + } + + # Add ssl parameters only if self.sslfile is not None + if self.sslfile is not None: + with open(self.sslfile, "r") as file: + ssl_settings = json.load(file) + + ssl_settings['ssl_context'] = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH) + session_params.update(ssl_settings) + + self.session = iRODSSession(**session_params) - self.session = iRODSSession( - host=self.host, - port=self.port, - user=self.username, - password=self.password, - zone=self.zone, - refresh_time=self.refresh_time, - ) # Set connection timeout self.session.connection_timeout = self.timeout @@ -284,6 +301,7 @@ def _config_to_dict(self): "auth": { "username": self.username, "password": self.password, + "sslfile": self.sslfile, }, "resource": { "name": self.resource, From f0c7801f15d8c0dc3d01b9564be496e30b497be3 Mon Sep 17 00:00:00 2001 From: pauldg Date: Tue, 15 Oct 2024 10:56:28 +0200 Subject: [PATCH 2/9] changes to add ssl settings directly to object store config --- lib/galaxy/objectstore/irods.py | 69 +++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/objectstore/irods.py b/lib/galaxy/objectstore/irods.py index b1fc1edbb90a..4fef8bb6dcf1 100644 --- a/lib/galaxy/objectstore/irods.py +++ b/lib/galaxy/objectstore/irods.py @@ -53,7 +53,15 @@ def parse_config_xml(config_xml): _config_xml_error("auth") username = a_xml[0].get("username") password = a_xml[0].get("password") - sslfile = a_xml[0].get("sslfile", None) + # sslfile = a_xml[0].get("sslfile", None) + client_server_negotiation = a_xml[0].get("client_server_negotiation", None) + client_server_policy = a_xml[0].get("client_server_policy", None) + encryption_algorithm = a_xml[0].get("encryption_algorithm", None) + encryption_key_size = int(a_xml[0].get("encryption_key_size", None)) + encryption_num_hash_rounds = int(a_xml[0].get("encryption_num_hash_rounds", None)) + encryption_salt_size = int(a_xml[0].get("encryption_salt_size", None)) + ssl_verify_server = a_xml[0].get("ssl_verify_server", None) + ssl_ca_certificate_file = a_xml[0].get("ssl_ca_certificate_file", None) r_xml = config_xml.findall("resource") if not r_xml: @@ -91,7 +99,15 @@ def parse_config_xml(config_xml): "auth": { "username": username, "password": password, - "sslfile": sslfile, + # "sslfile": sslfile, + "client_server_negotiation": client_server_negotiation, + "client_server_policy": client_server_policy, + "encryption_algorithm": encryption_algorithm, + "encryption_key_size": encryption_key_size, + "encryption_num_hash_rounds": encryption_num_hash_rounds, + "encryption_salt_size": encryption_salt_size, + "ssl_verify_server": ssl_verify_server, + "ssl_ca_certificate_file": ssl_ca_certificate_file, }, "resource": { "name": resource_name, @@ -142,7 +158,15 @@ def __init__(self, config, config_dict): self.password = auth_dict.get("password") if self.password is None: _config_dict_error("auth->password") - self.sslfile = auth_dict.get("sslfile") + # self.sslfile = auth_dict.get("sslfile") + self.client_server_negotiation = auth_dict.get("client_server_negotiation") + self.client_server_policy = auth_dict.get("client_server_policy") + self.encryption_algorithm = auth_dict.get("encryption_algorithm") + self.encryption_key_size = auth_dict.get("encryption_key_size") + self.encryption_num_hash_rounds = auth_dict.get("encryption_num_hash_rounds") + self.encryption_salt_size = auth_dict.get("encryption_salt_size") + self.ssl_verify_server = auth_dict.get("ssl_verify_server") + self.ssl_ca_certificate_file = auth_dict.get("ssl_ca_certificate_file") resource_dict = config_dict["resource"] if resource_dict is None: @@ -194,11 +218,11 @@ def __init__(self, config, config_dict): if irods is None: raise Exception(IRODS_IMPORT_MESSAGE) - self.home = f"/{self.zone}/home/{self.username}" + # self.home = f"/{self.zone}/home/{self.username}" + self.home = "/vsc_galaxy/home/t1_data_2024_04/ingress/dev-paul" if irods is None: raise Exception(IRODS_IMPORT_MESSAGE) - session_params = { 'host': self.host, @@ -207,18 +231,33 @@ def __init__(self, config, config_dict): 'password': self.password, 'zone': self.zone, 'refresh_time': self.refresh_time, + 'client_server_negotiation': self.client_server_negotiation, + 'client_server_policy': self.client_server_policy, + 'encryption_algorithm': self.encryption_algorithm, + 'encryption_key_size': self.encryption_key_size, + 'encryption_num_hash_rounds': self.encryption_num_hash_rounds, + 'encryption_salt_size': self.encryption_salt_size, + 'ssl_verify_server': self.ssl_verify_server, + 'ssl_ca_certificate_file': self.ssl_ca_certificate_file, + 'ssl_context': ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH) } # Add ssl parameters only if self.sslfile is not None - if self.sslfile is not None: - with open(self.sslfile, "r") as file: - ssl_settings = json.load(file) + # if self.sslfile is not None: + # with open(self.sslfile, "r") as file: + # ssl_settings = json.load(file) - ssl_settings['ssl_context'] = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH) - session_params.update(ssl_settings) + # ssl_settings['ssl_context'] = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH) + # session_params.update(ssl_settings) self.session = iRODSSession(**session_params) + log.debug("SESSION PARAMS: %s", session_params) + + coll = self.session.collections.get("/vsc_galaxy/") + for col in coll.subcollections: + log.debug("COLLECTION: %s", col) + # Set connection timeout self.session.connection_timeout = self.timeout @@ -301,7 +340,15 @@ def _config_to_dict(self): "auth": { "username": self.username, "password": self.password, - "sslfile": self.sslfile, + # "sslfile": self.sslfile, + "client_server_negotiation": self.client_server_negotiation, + "client_server_policy": self.client_server_policy, + "encryption_algorithm": self.encryption_algorithm, + "encryption_key_size": self.encryption_key_size, + "encryption_num_hash_rounds": self.encryption_num_hash_rounds, + "encryption_salt_size": self.encryption_salt_size, + "ssl_verify_server": self.ssl_verify_server, + "ssl_ca_certificate_file": self.ssl_ca_certificate_file, }, "resource": { "name": self.resource, From 6d8a2d63c0560cb57857b824a48d2f10c17e7b90 Mon Sep 17 00:00:00 2001 From: pauldg Date: Wed, 16 Oct 2024 22:31:02 +0200 Subject: [PATCH 3/9] xml ssl tags and replace static self.home with self.logical_path --- lib/galaxy/objectstore/irods.py | 122 +++++++++++++++++--------------- 1 file changed, 63 insertions(+), 59 deletions(-) diff --git a/lib/galaxy/objectstore/irods.py b/lib/galaxy/objectstore/irods.py index 4fef8bb6dcf1..9e3bfdae7190 100644 --- a/lib/galaxy/objectstore/irods.py +++ b/lib/galaxy/objectstore/irods.py @@ -53,15 +53,16 @@ def parse_config_xml(config_xml): _config_xml_error("auth") username = a_xml[0].get("username") password = a_xml[0].get("password") - # sslfile = a_xml[0].get("sslfile", None) - client_server_negotiation = a_xml[0].get("client_server_negotiation", None) - client_server_policy = a_xml[0].get("client_server_policy", None) - encryption_algorithm = a_xml[0].get("encryption_algorithm", None) - encryption_key_size = int(a_xml[0].get("encryption_key_size", None)) - encryption_num_hash_rounds = int(a_xml[0].get("encryption_num_hash_rounds", None)) - encryption_salt_size = int(a_xml[0].get("encryption_salt_size", None)) - ssl_verify_server = a_xml[0].get("ssl_verify_server", None) - ssl_ca_certificate_file = a_xml[0].get("ssl_ca_certificate_file", None) + + s_xml = config_xml.findall("ssl") + client_server_negotiation = s_xml[0].get("client_server_negotiation", None) + client_server_policy = s_xml[0].get("client_server_policy", None) + encryption_algorithm = s_xml[0].get("encryption_algorithm", None) + encryption_key_size = int(s_xml[0].get("encryption_key_size", None)) + encryption_num_hash_rounds = int(s_xml[0].get("encryption_num_hash_rounds", None)) + encryption_salt_size = int(s_xml[0].get("encryption_salt_size", None)) + ssl_verify_server = s_xml[0].get("ssl_verify_server", None) + ssl_ca_certificate_file = s_xml[0].get("ssl_ca_certificate_file", None) r_xml = config_xml.findall("resource") if not r_xml: @@ -82,6 +83,11 @@ def parse_config_xml(config_xml): refresh_time = int(c_xml[0].get("refresh_time", 300)) connection_pool_monitor_interval = int(c_xml[0].get("connection_pool_monitor_interval", -1)) + l_xml = config_xml.findall("logical") + if not l_xml: + _config_xml_error("logical") + logical_path = l_xml[0].get("path", None) + c_xml = config_xml.findall("cache") if not c_xml: _config_xml_error("cache") @@ -99,7 +105,8 @@ def parse_config_xml(config_xml): "auth": { "username": username, "password": password, - # "sslfile": sslfile, + }, + "ssl": { "client_server_negotiation": client_server_negotiation, "client_server_policy": client_server_policy, "encryption_algorithm": encryption_algorithm, @@ -122,6 +129,9 @@ def parse_config_xml(config_xml): "refresh_time": refresh_time, "connection_pool_monitor_interval": connection_pool_monitor_interval, }, + "logical": { + "path": logical_path, + }, "cache": { "size": cache_size, "path": staging_path, @@ -158,15 +168,17 @@ def __init__(self, config, config_dict): self.password = auth_dict.get("password") if self.password is None: _config_dict_error("auth->password") - # self.sslfile = auth_dict.get("sslfile") - self.client_server_negotiation = auth_dict.get("client_server_negotiation") - self.client_server_policy = auth_dict.get("client_server_policy") - self.encryption_algorithm = auth_dict.get("encryption_algorithm") - self.encryption_key_size = auth_dict.get("encryption_key_size") - self.encryption_num_hash_rounds = auth_dict.get("encryption_num_hash_rounds") - self.encryption_salt_size = auth_dict.get("encryption_salt_size") - self.ssl_verify_server = auth_dict.get("ssl_verify_server") - self.ssl_ca_certificate_file = auth_dict.get("ssl_ca_certificate_file") + + ssl_dict = config_dict.get("ssl") or {} + + self.client_server_negotiation = ssl_dict.get("client_server_negotiation") + self.client_server_policy = ssl_dict.get("client_server_policy") + self.encryption_algorithm = ssl_dict.get("encryption_algorithm") + self.encryption_key_size = ssl_dict.get("encryption_key_size") + self.encryption_num_hash_rounds = ssl_dict.get("encryption_num_hash_rounds") + self.encryption_salt_size = ssl_dict.get("encryption_salt_size") + self.ssl_verify_server = ssl_dict.get("ssl_verify_server") + self.ssl_ca_certificate_file = ssl_dict.get("ssl_ca_certificate_file") resource_dict = config_dict["resource"] if resource_dict is None: @@ -201,6 +213,11 @@ def __init__(self, config, config_dict): if self.connection_pool_monitor_interval is None: _config_dict_error("connection->connection_pool_monitor_interval") + logical_dict = config_dict.get("logical") or {} + self.logical_path = logical_dict.get("path") or f"/{self.zone}/home/{self.username}" + if self.logical_path is None: + _config_dict_error("logical->path") + cache_dict = config_dict.get("cache") or {} self.cache_size = cache_dict.get("size") or self.config.object_store_cache_path if self.cache_size is None: @@ -218,46 +235,29 @@ def __init__(self, config, config_dict): if irods is None: raise Exception(IRODS_IMPORT_MESSAGE) - # self.home = f"/{self.zone}/home/{self.username}" - self.home = "/vsc_galaxy/home/t1_data_2024_04/ingress/dev-paul" - if irods is None: raise Exception(IRODS_IMPORT_MESSAGE) session_params = { - 'host': self.host, - 'port': self.port, - 'user': self.username, - 'password': self.password, - 'zone': self.zone, - 'refresh_time': self.refresh_time, - 'client_server_negotiation': self.client_server_negotiation, - 'client_server_policy': self.client_server_policy, - 'encryption_algorithm': self.encryption_algorithm, - 'encryption_key_size': self.encryption_key_size, - 'encryption_num_hash_rounds': self.encryption_num_hash_rounds, - 'encryption_salt_size': self.encryption_salt_size, - 'ssl_verify_server': self.ssl_verify_server, - 'ssl_ca_certificate_file': self.ssl_ca_certificate_file, - 'ssl_context': ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH) + "host": self.host, + "port": self.port, + "user": self.username, + "password": self.password, + "zone": self.zone, + "refresh_time": self.refresh_time, + "client_server_negotiation": self.client_server_negotiation, + "client_server_policy": self.client_server_policy, + "encryption_algorithm": self.encryption_algorithm, + "encryption_key_size": self.encryption_key_size, + "encryption_num_hash_rounds": self.encryption_num_hash_rounds, + "encryption_salt_size": self.encryption_salt_size, + "ssl_verify_server": self.ssl_verify_server, + "ssl_ca_certificate_file": self.ssl_ca_certificate_file, + "ssl_context": ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH), } - # Add ssl parameters only if self.sslfile is not None - # if self.sslfile is not None: - # with open(self.sslfile, "r") as file: - # ssl_settings = json.load(file) - - # ssl_settings['ssl_context'] = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH) - # session_params.update(ssl_settings) - self.session = iRODSSession(**session_params) - log.debug("SESSION PARAMS: %s", session_params) - - coll = self.session.collections.get("/vsc_galaxy/") - for col in coll.subcollections: - log.debug("COLLECTION: %s", col) - # Set connection timeout self.session.connection_timeout = self.timeout @@ -340,7 +340,8 @@ def _config_to_dict(self): "auth": { "username": self.username, "password": self.password, - # "sslfile": self.sslfile, + }, + "ssl": { "client_server_negotiation": self.client_server_negotiation, "client_server_policy": self.client_server_policy, "encryption_algorithm": self.encryption_algorithm, @@ -363,6 +364,9 @@ def _config_to_dict(self): "refresh_time": self.refresh_time, "connection_pool_monitor_interval": self.connection_pool_monitor_interval, }, + "logical": { + "path": self.logical_path, + }, "cache": { "size": self.cache_size, "path": self.staging_path, @@ -377,7 +381,7 @@ def _get_remote_size(self, rel_path): data_object_name = p.stem + p.suffix subcollection_name = p.parent - collection_path = f"{self.home}/{subcollection_name}" + collection_path = f"{self.logical_path}/{subcollection_name}" data_object_path = f"{collection_path}/{data_object_name}" options = {kw.DEST_RESC_NAME_KW: self.resource} @@ -397,7 +401,7 @@ def _exists_remotely(self, rel_path): data_object_name = p.stem + p.suffix subcollection_name = p.parent - collection_path = f"{self.home}/{subcollection_name}" + collection_path = f"{self.logical_path}/{subcollection_name}" data_object_path = f"{collection_path}/{data_object_name}" options = {kw.DEST_RESC_NAME_KW: self.resource} @@ -419,7 +423,7 @@ def _download(self, rel_path): data_object_name = p.stem + p.suffix subcollection_name = p.parent - collection_path = f"{self.home}/{subcollection_name}" + collection_path = f"{self.logical_path}/{subcollection_name}" data_object_path = f"{collection_path}/{data_object_name}" # we need to allow irods to override already existing zero-size output files created # in object store cache during job setup (see also https://github.com/galaxyproject/galaxy/pull/17025#discussion_r1394517033) @@ -460,7 +464,7 @@ def _push_to_storage(self, rel_path, source_file=None, from_string=None): return False # Check if the data object exists in iRODS - collection_path = f"{self.home}/{subcollection_name}" + collection_path = f"{self.logical_path}/{subcollection_name}" data_object_path = f"{collection_path}/{data_object_name}" exists = False @@ -538,7 +542,7 @@ def _delete(self, obj, entire_dir: bool = False, **kwargs) -> bool: if entire_dir and extra_dir: shutil.rmtree(self._get_cache_path(rel_path), ignore_errors=True) - col_path = f"{self.home}/{rel_path}" + col_path = f"{self.logical_path}/{rel_path}" col = None try: col = self.session.collections.get(col_path) @@ -566,7 +570,7 @@ def _delete(self, obj, entire_dir: bool = False, **kwargs) -> bool: data_object_name = p.stem + p.suffix subcollection_name = p.parent - collection_path = f"{self.home}/{subcollection_name}" + collection_path = f"{self.logical_path}/{subcollection_name}" data_object_path = f"{collection_path}/{data_object_name}" try: @@ -592,7 +596,7 @@ def _get_object_url(self, obj, **kwargs): data_object_name = p.stem + p.suffix subcollection_name = p.parent - collection_path = f"{self.home}/{subcollection_name}" + collection_path = f"{self.logical_path}/{subcollection_name}" data_object_path = f"{collection_path}/{data_object_name}" return data_object_path From 2665e04e390389763e385324db67be4de755daa4 Mon Sep 17 00:00:00 2001 From: pauldg Date: Wed, 16 Oct 2024 22:37:14 +0200 Subject: [PATCH 4/9] updated sample --- lib/galaxy/config/sample/object_store_conf.xml.sample | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/galaxy/config/sample/object_store_conf.xml.sample b/lib/galaxy/config/sample/object_store_conf.xml.sample index db4271765203..c7518d2738e0 100644 --- a/lib/galaxy/config/sample/object_store_conf.xml.sample +++ b/lib/galaxy/config/sample/object_store_conf.xml.sample @@ -202,6 +202,10 @@ + + From c206f7512ddaaf15e6aefa375b656ab1ea794f38 Mon Sep 17 00:00:00 2001 From: pauldg Date: Thu, 17 Oct 2024 13:56:35 +0200 Subject: [PATCH 5/9] update sample files to demonstrate ssl and logical path --- .../config/sample/object_store_conf.sample.yml | 14 ++++++++++++++ .../config/sample/object_store_conf.xml.sample | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/lib/galaxy/config/sample/object_store_conf.sample.yml b/lib/galaxy/config/sample/object_store_conf.sample.yml index 90fcf2fcccd1..4ea83f3f2d7e 100644 --- a/lib/galaxy/config/sample/object_store_conf.sample.yml +++ b/lib/galaxy/config/sample/object_store_conf.sample.yml @@ -264,6 +264,20 @@ connection: timeout: 30 refresh_time: 300 connection_pool_monitor_interval: 3600 +# ssl section is optional +ssl: + client_server_negotiation: request_server_negotiation + client_server_policy: CS_NEG_REQUIRE + encryption_algorithm: AES-256-CBC + encryption_key_size: 32 + encryption_num_hash_rounds: 16 + encryption_salt_size: 8 + ssl_verify_server: cert + ssl_ca_certificate_file: /etc/irods/ssl/irods.crt +# The logical section is optional and is meant for defining the remote path +# if not defined the default path is: /zone_name/home/username +logical: + path: /tempZone/home/rods cache: path: database/object_store_cache_irods size: 1000 diff --git a/lib/galaxy/config/sample/object_store_conf.xml.sample b/lib/galaxy/config/sample/object_store_conf.xml.sample index c7518d2738e0..f8746ff7cee6 100644 --- a/lib/galaxy/config/sample/object_store_conf.xml.sample +++ b/lib/galaxy/config/sample/object_store_conf.xml.sample @@ -195,6 +195,10 @@