Merge pull request galaxyproject#17654 from nsoranzo/ruff_C4_pyupgrade

Code cleanups from ruff and pyupgrade

Makefile

@@ -53,9 +53,9 @@ remove-unused-imports:  ## Remove unused imports in Python code base
 	$(IN_VENV) autoflake --in-place --remove-all-unused-imports --recursive --verbose lib/ test/
 
 pyupgrade:  ## Convert older code patterns to Python3.7/3.8 idiomatic ones
-	ack --type=python -f | grep -v '^lib/galaxy/schema/bco/\|^lib/galaxy/schema/drs/\|^lib/tool_shed_client/schema/trs\|^tools/\|^.venv/\|^.tox/\|^lib/galaxy/files/sources/\|^lib/galaxy/job_metrics/\|^lib/galaxy/objectstore/\|^lib/galaxy/tool_util/\|^lib/galaxy/util/\|^test/functional/tools/cwl_tools/' | xargs pyupgrade --py38-plus
-	ack --type=python -f | grep -v '^lib/galaxy/schema/bco/\|^lib/galaxy/schema/drs/\|^lib/tool_shed_client/schema/trs\|^tools/\|^.venv/\|^.tox/\|^lib/galaxy/files/sources/\|^lib/galaxy/job_metrics/\|^lib/galaxy/objectstore/\|^lib/galaxy/tool_util/\|^lib/galaxy/util/\|^test/functional/tools/cwl_tools/' | xargs auto-walrus
-	ack --type=python -f lib/galaxy/files/sources/ lib/galaxy/job_metrics/ lib/galaxy/objectstore/ lib/galaxy/tool_util/ lib/galaxy/util/ | xargs pyupgrade --py37-plus
+	ack --type=python -f | grep -v '^lib/galaxy/schema/bco/\|^lib/galaxy/schema/drs/\|^lib/tool_shed_client/schema/trs\|^tools/\|^.venv/\|^.tox/\|^lib/galaxy/exceptions/\|^lib/galaxy/job_metrics/\|^lib/galaxy/objectstore/\|^lib/galaxy/tool_util/\|^lib/galaxy/util/\|^test/functional/tools/cwl_tools/' | xargs pyupgrade --py38-plus
+	ack --type=python -f | grep -v '^lib/galaxy/schema/bco/\|^lib/galaxy/schema/drs/\|^lib/tool_shed_client/schema/trs\|^tools/\|^.venv/\|^.tox/\|^lib/galaxy/exceptions/\|^lib/galaxy/job_metrics/\|^lib/galaxy/objectstore/\|^lib/galaxy/tool_util/\|^lib/galaxy/util/\|^test/functional/tools/cwl_tools/' | xargs auto-walrus
+	ack --type=python -f lib/galaxy/exceptions/ lib/galaxy/job_metrics/ lib/galaxy/objectstore/ lib/galaxy/tool_util/ lib/galaxy/util/ | xargs pyupgrade --py37-plus
 
 docs-slides-ready:
 	test -f plantuml.jar ||  wget http://jaist.dl.sourceforge.net/project/plantuml/plantuml.jar

lib/galaxy/authnz/managers.py

@@ -193,7 +193,7 @@ def _parse_custos_config(self, config_xml):
         if config_xml.find("well_known_oidc_config_uri") is not None:
             rtv["well_known_oidc_config_uri"] = config_xml.find("well_known_oidc_config_uri").text
         if config_xml.findall("allowed_idp") is not None:
-            self.allowed_idps = list(map(lambda idp: idp.text, config_xml.findall("allowed_idp")))
+            self.allowed_idps = [idp.text for idp in config_xml.findall("allowed_idp")]
         if config_xml.find("ca_bundle") is not None:
             rtv["ca_bundle"] = config_xml.find("ca_bundle").text
         if config_xml.find("icon") is not None:

lib/galaxy/datatypes/anvio.py

@@ -45,7 +45,7 @@ def generate_primary_file(self, dataset: HasExtraFilesAndMetadata) -> str:
                     missing_text = " (missing)"
                 rval.append(f'<li><a href="{composite_name}">{composite_name}</a>{opt_text}{missing_text}</li>')
             rval.append("</ul>")
-        defined_files = map(lambda x: x[0], defined_files)
+        defined_files = (x[0] for x in defined_files)
         extra_files = []
         for dirpath, _dirnames, filenames in os.walk(dataset.extra_files_path, followlinks=True):
             for filename in filenames:

lib/galaxy/datatypes/binary.py

@@ -2210,8 +2210,8 @@ def set_meta(self, dataset: DatasetProtocol, overwrite: bool = True, **kwd) -> N
         try:
             with h5py.File(dataset.get_file_name(), "r") as mat_file:
                 dataset.metadata.materials = list(mat_file.keys())
-                sgn = dict()
-                lp = dict()
+                sgn = {}
+                lp = {}
                 for m in mat_file.keys():
                     if "SpaceGroupNumber" in mat_file[m] and len(mat_file[m]["SpaceGroupNumber"]) > 0:
                         sgn[m] = mat_file[m]["SpaceGroupNumber"][0].item()
@@ -2401,8 +2401,8 @@ def init_meta(self, dataset: HasMetadata, copy_from: Optional[HasMetadata] = Non
     def set_meta(self, dataset: DatasetProtocol, overwrite: bool = True, **kwd) -> None:
         try:
             tables = []
-            columns = dict()
-            rowcounts = dict()
+            columns = {}
+            rowcounts = {}
             conn = sqlite.connect(dataset.get_file_name())
             c = conn.cursor()
             tables_query = "SELECT name,sql FROM sqlite_master WHERE type='table' ORDER BY name"

lib/galaxy/datatypes/dataproviders/base.py

@@ -128,7 +128,7 @@ def writelines(self, sequence):
     # def readline( self ):
     #    return self.next()
     def readlines(self):
-        return [line for line in self]
+        return list(self)
 
     # iterator interface
     def __iter__(self):

lib/galaxy/datatypes/dataproviders/dataset.py

@@ -749,7 +749,7 @@ def __iter__(self):
             for i, row in enumerate(results):
                 if i >= self.limit:
                     break
-                yield [val for val in row]
+                yield list(row)
         else:
             yield
 

lib/galaxy/datatypes/dataproviders/line.py

@@ -249,7 +249,7 @@ def assemble_current_block(self):
         Called per block (just before providing).
         """
         # empty block_lines and assemble block
-        return list(self.block_lines.popleft() for i in range(len(self.block_lines)))
+        return [self.block_lines.popleft() for i in range(len(self.block_lines))]
 
     def filter_block(self, block):
         """

lib/galaxy/datatypes/goldenpath.py

@@ -84,7 +84,7 @@ def sniff_prefix(self, file_prefix: FilePrefix) -> bool:
                         assert line[8] in ["+", "-", "?", "0", "na"]
                     if line[4] == "U":
                         assert int(line[5]) == 100
-                    assert all(map(lambda x: str(x).isnumeric() and int(x) > 0, ostensible_numbers))
+                    assert all(str(x).isnumeric() and int(x) > 0 for x in ostensible_numbers)
                     found_non_comment_lines = True
         except Exception:
             return False

lib/galaxy/datatypes/text.py

@@ -471,7 +471,7 @@ def _transform_dict_list_ids(dict_list):
                                     for k, v in column["metadata"].items():
                                         if v is not None:
                                             keep_columns.add(k)
-                            final_list = sorted(list(keep_columns))
+                            final_list = sorted(keep_columns)
                             dataset.metadata.table_column_metadata_headers = final_list
                         if b_name in b_transform:
                             metadata_value = b_transform[b_name](metadata_value)

lib/galaxy/datatypes/util/maf_utilities.py

@@ -183,7 +183,7 @@ def get_sequence(self, species):
 
     # returns the reverse complement of the sequence for a species
     def get_sequence_reverse_complement(self, species):
-        complement = [base for base in self.get_sequence(species).translate(self.DNA_COMPLEMENT)]
+        complement = list(self.get_sequence(species).translate(self.DNA_COMPLEMENT))
         complement.reverse()
         return "".join(complement)
 
@@ -274,7 +274,7 @@ def get_sequence(self, species):
 
     # returns the reverse complement of the sequence for a species
     def get_sequence_reverse_complement(self, species):
-        complement = [base for base in self.get_sequence(species).translate(self.DNA_COMPLEMENT)]
+        complement = list(self.get_sequence(species).translate(self.DNA_COMPLEMENT))
         complement.reverse()
         return "".join(complement)
 
@@ -683,7 +683,7 @@ def iter_components_by_src(block, src):
 
 
 def get_components_by_src(block, src):
-    return [value for value in iter_components_by_src(block, src)]
+    return list(iter_components_by_src(block, src))
 
 
 def iter_components_by_src_start(block, src):
@@ -693,7 +693,7 @@ def iter_components_by_src_start(block, src):
 
 
 def get_components_by_src_start(block, src):
-    return [value for value in iter_components_by_src_start(block, src)]
+    return list(iter_components_by_src_start(block, src))
 
 
 def sort_block_components_by_block(block1, block2):

lib/galaxy/files/sources/__init__.py

@@ -468,8 +468,7 @@ def _get_error_msg_for(rule_name: str) -> str:
 
 def uri_join(*args):
     # url_join doesn't work with non-standard scheme
-    arg0 = args[0]
-    if "://" in arg0:
+    if "://" in (arg0 := args[0]):
         scheme, path = arg0.split("://", 1)
         rval = f"{scheme}://{slash_join(path, *args[1:]) if path else slash_join(*args[1:])}"
     else:

lib/galaxy/files/sources/drs.py

@@ -8,13 +8,13 @@
 
 from typing_extensions import Unpack
 
-from galaxy.util.drs import fetch_drs_to_file
 from . import (
     BaseFilesSource,
     FilesSourceOptions,
     FilesSourceProperties,
     PluginKind,
 )
+from .util import fetch_drs_to_file
 
 log = logging.getLogger(__name__)
 

lib/galaxy/files/sources/http.py

@@ -85,8 +85,7 @@ def _serialization_props(self, user_context=None) -> HTTPFilesSourceProperties:
         return cast(HTTPFilesSourceProperties, effective_props)
 
     def score_url_match(self, url: str):
-        match = self._url_regex.match(url)
-        if match:
+        if match := self._url_regex.match(url):
             return match.span()[1]
         else:
             return 0

lib/galaxy/jobs/__init__.py

@@ -122,7 +122,7 @@ def __init__(self, **kwds):
         self["env"] = []
         self["resubmit"] = []
         # dict is appropriate (rather than a bunch) since keys may not be valid as attributes
-        self["params"] = dict()
+        self["params"] = {}
 
         # Use the values persisted in an existing job
         if "from_job" in kwds and kwds["from_job"].destination_id is not None:
@@ -143,7 +143,7 @@ class JobToolConfiguration(Bunch):
     def __init__(self, **kwds):
         self["handler"] = None
         self["destination"] = None
-        self["params"] = dict()
+        self["params"] = {}
         super().__init__(**kwds)
 
     def get_resource_group(self):
@@ -448,7 +448,7 @@ def _configure_from_dict(self, job_config_dict):
         execution_dict = job_config_dict.get("execution", {})
         environments = execution_dict.get("environments", [])
         enviroment_iter = (
-            map(lambda e: (e["id"], e), environments) if isinstance(environments, list) else environments.items()
+            ((e["id"], e) for e in environments) if isinstance(environments, list) else environments.items()
         )
         for environment_id, environment_dict in enviroment_iter:
             metrics = environment_dict.get("metrics")
@@ -520,11 +520,11 @@ def _configure_from_dict(self, job_config_dict):
                 assert tool_class is None
                 tool_id = raw_tool_id.lower().rstrip("/")
                 if tool_id not in self.tools:
-                    self.tools[tool_id] = list()
+                    self.tools[tool_id] = []
             else:
                 assert tool_class in VALID_TOOL_CLASSES, tool_class
                 if tool_class not in self.tool_classes:
-                    self.tool_classes[tool_class] = list()
+                    self.tool_classes[tool_class] = []
 
             params = tool.get("params")
             if params is None:
@@ -663,7 +663,7 @@ def get_params(config, parent):
             key = param.get("id")
             if key in ["container", "container_override"]:
                 containers = map(requirements.container_from_element, param.findall("container"))
-                param_value = list(map(lambda c: c.to_dict(), containers))
+                param_value = [c.to_dict() for c in containers]
             else:
                 param_value = param.text
 
@@ -2273,7 +2273,7 @@ def setup_external_metadata(
         if set_extension:
             for output_dataset_assoc in job.output_datasets:
                 if output_dataset_assoc.dataset.ext == "auto":
-                    context = self.get_dataset_finish_context(dict(), output_dataset_assoc)
+                    context = self.get_dataset_finish_context({}, output_dataset_assoc)
                     output_dataset_assoc.dataset.extension = context.get("ext", "data")
             with transaction(self.sa_session):
                 self.sa_session.commit()

lib/galaxy/jobs/runners/condor.py

@@ -227,7 +227,7 @@ def stop_job(self, job_wrapper):
             try:
                 log.info(f"stop_job(): {job.id}: trying to stop container .... ({external_id})")
                 # self.watched = [cjs for cjs in self.watched if cjs.job_id != external_id]
-                new_watch_list = list()
+                new_watch_list = []
                 cjs = None
                 for tcjs in self.watched:
                     if tcjs.job_id != external_id:

lib/galaxy/jobs/runners/drmaa.py

@@ -51,7 +51,7 @@ def __init__(self, app, nworkers, **kwargs):
             runner_param_specs[f"{retry_exception}_retries"] = dict(map=int, valid=lambda x: int(x) >= 0, default=0)
 
         if "runner_param_specs" not in kwargs:
-            kwargs["runner_param_specs"] = dict()
+            kwargs["runner_param_specs"] = {}
         kwargs["runner_param_specs"].update(runner_param_specs)
 
         super().__init__(app, nworkers, **kwargs)

lib/galaxy/jobs/runners/godocker.py

@@ -133,7 +133,7 @@ def __init__(self, app, nworkers, **kwargs):
             godocker_master=dict(map=str), user=dict(map=str), key=dict(map=str), godocker_project=dict(map=str)
         )
         if "runner_param_specs" not in kwargs:
-            kwargs["runner_param_specs"] = dict()
+            kwargs["runner_param_specs"] = {}
         kwargs["runner_param_specs"].update(runner_param_specs)
 
         # Start the job runner parent object
@@ -379,7 +379,7 @@ def post_task(self, job_wrapper):
                 volume = job_destination.params["godocker_volumes"]
                 volume = volume.split(",")
                 for i in volume:
-                    temp = dict({"name": i})
+                    temp = {"name": i}
                     volumes.append(temp)
             except Exception:
                 log.debug("godocker_volume not set, using default.")

lib/galaxy/jobs/runners/kubernetes.py

@@ -112,7 +112,7 @@ def __init__(self, app, nworkers, **kwargs):
         )
 
         if "runner_param_specs" not in kwargs:
-            kwargs["runner_param_specs"] = dict()
+            kwargs["runner_param_specs"] = {}
         kwargs["runner_param_specs"].update(runner_param_specs)
 
         # Start the job runner parent object
@@ -972,8 +972,7 @@ def __cleanup_k8s_guest_ports(self, job_wrapper, k8s_job):
         else:
             log.debug(f"No service found for job with k8s_job_name {k8s_job_name}")
         # remove the interactive environment entrypoints
-        eps = job_wrapper.get_job().interactivetool_entry_points
-        if eps:
+        if eps := job_wrapper.get_job().interactivetool_entry_points:
             log.debug(f"Removing entry points for job with ID {job_wrapper.get_id_tag()}")
             self.app.interactivetool_manager.remove_entry_points(eps)
 

lib/galaxy/jobs/runners/univa.py

@@ -72,7 +72,7 @@ def check_watched_item(self, ajs, new_watched):
         return state
 
     def _complete_terminal_job(self, ajs, drmaa_state, **kwargs):
-        extinfo = dict()
+        extinfo = {}
         # get state with job_info/qstat + wait/qacct
         state = self._get_drmaa_state(ajs.job_id, self.ds, True, extinfo)
         # log.debug("UnivaJobRunner:_complete_terminal_job ({jobid}) -> state {state} info {info}".format(jobid=ajs.job_id, state=self.drmaa_job_state_strings[state], info=extinfo))
@@ -214,7 +214,7 @@ def _get_drmaa_state_qacct(self, job_id, extinfo):
         # log.debug("UnivaJobRunner._get_drmaa_state_qacct ({jobid})".format(jobid=job_id))
         signals = {
             k: v
-            for v, k in reversed(sorted(signal.__dict__.items()))
+            for v, k in sorted(signal.__dict__.items(), reverse=True)
             if v.startswith("SIG") and not v.startswith("SIG_")
         }
         cmd = ["qacct", "-j", job_id]
@@ -235,7 +235,7 @@ def _get_drmaa_state_qacct(self, job_id, extinfo):
                     return self.drmaa.JobState.UNDETERMINED
             else:
                 break
-        qacct = dict()
+        qacct = {}
         for line in stdout.split("\n"):
             # remove header
             if line.startswith("=") or line == "":

lib/galaxy/managers/collections.py

@@ -350,7 +350,7 @@ def get_converters_for_collection(
                 suitable_converters = suitable_converters.intersection(set_of_new_converters)
                 if suitable_converters:
                     most_recent_datatype = datatype
-        suitable_tool_ids = list()
+        suitable_tool_ids = []
         for tool in suitable_converters:
             tool_info = {
                 "tool_id": tool[1].id,

lib/galaxy/managers/datasets.py

@@ -164,8 +164,7 @@ def update_object_store_id(self, trans, dataset, object_store_id: str):
             # has been shared.
             raise exceptions.InsufficientPermissionsException("Cannot change dataset permissions...")
 
-        quota_source_map = self.app.object_store.get_quota_source_map()
-        if quota_source_map:
+        if quota_source_map := self.app.object_store.get_quota_source_map():
             old_label = quota_source_map.get_quota_source_label(old_object_store_id)
             new_label = quota_source_map.get_quota_source_label(new_object_store_id)
             if old_label != new_label:

lib/galaxy/managers/datatypes.py

@@ -25,7 +25,7 @@ def view_index(
         if upload_only:
             return datatypes_registry.upload_file_formats
         else:
-            return [ext for ext in datatypes_registry.datatypes_by_extension]
+            return list(datatypes_registry.datatypes_by_extension)
     else:
         rval = []
         for datatype_info_dict in datatypes_registry.datatype_info_dicts:

lib/galaxy/managers/history_contents.py

@@ -236,9 +236,7 @@ def _union_of_contents(self, container, expand_models=True, **kwargs):
             return contents_results
 
         # partition ids into a map of { component_class names -> list of ids } from the above union query
-        id_map: Dict[str, List[int]] = dict(
-            [(self.contained_class_type_name, []), (self.subcontainer_class_type_name, [])]
-        )
+        id_map: Dict[str, List[int]] = {self.contained_class_type_name: [], self.subcontainer_class_type_name: []}
         for result in contents_results:
             result_type = self._get_union_type(result)
             contents_id = self._get_union_id(result)

lib/galaxy/managers/libraries.py

@@ -155,7 +155,7 @@ def list(self, trans, deleted: Optional[bool] = False) -> Tuple[Query, Dict[str,
         """
         is_admin = trans.user_is_admin
         library_access_action = trans.app.security_agent.permitted_actions.LIBRARY_ACCESS.action
-        restricted_library_ids = {id for id in get_library_ids(trans.sa_session, library_access_action)}
+        restricted_library_ids = set(get_library_ids(trans.sa_session, library_access_action))
         prefetched_ids = {"restricted_library_ids": restricted_library_ids}
 
         if is_admin:

lib/galaxy/managers/users.py

@@ -406,7 +406,7 @@ def user_can_do_run_as(self, user) -> bool:
 
     # ---- preferences
     def preferences(self, user):
-        return {key: value for key, value in user.preferences.items()}
+        return dict(user.preferences.items())
 
     # ---- roles and permissions
     def private_role(self, user):