New feature: update exts list versions to latest

.gitignore

@@ -10,7 +10,7 @@ dist/
 *egg-info/
 *.swp
 .mypy_cache/
-
+.venv*
 Dockerfile.*
 Singularity.*
 test-reports/

easybuild/framework/easyblock.py

@@ -63,7 +63,7 @@
 from easybuild.framework.easyconfig.format.format import SANITY_CHECK_PATHS_DIRS, SANITY_CHECK_PATHS_FILES
 from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig
 from easybuild.framework.easyconfig.style import MAX_LINE_LENGTH
-from easybuild.framework.easyconfig.tools import dump_env_easyblock, get_paths_for
+from easybuild.framework.easyconfig.tools import dump_env_easyblock, get_paths_for, get_pkg_metadata, format_metadata_as_extension
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict
 from easybuild.framework.extension import Extension, resolve_exts_filter_template
 from easybuild.tools import LooseVersion, config, run
@@ -72,7 +72,7 @@
 from easybuild.tools.build_log import print_error, print_msg, print_warning
 from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES
 from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES
-from easybuild.tools.config import EASYBUILD_SOURCES_URL # noqa
+from easybuild.tools.config import EASYBUILD_SOURCES_URL  # noqa
 from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
 from easybuild.tools.config import install_path, log_path, package_path, source_paths
 from easybuild.tools.environment import restore_env, sanitize_env
@@ -4849,3 +4849,276 @@ def make_checksum_lines(checksums, indent_level):
             ectxt = regex.sub('\n'.join(exts_list_lines), ectxt)
 
         write_file(ec['spec'], ectxt)
+
+
+def get_updated_exts_list(exts_list, exts_defaultclass, bioconductor_version=None):
+    """
+    Get the list of all extensions in exts_list updated to their latest version.
+
+    :param exts_defaultclass: default class for the extensions ('RPackage', 'PythonPackage')
+    :param exts_list: list of extensions to be updated.
+    :param bioconductor_version: bioconductor's version to use (if any)
+
+    :return: list with extensions updated to their latest versions.
+    """
+
+    # check if the exts_list is empty
+    if not exts_list:
+        raise EasyBuildError("No exts_list provided for updating")
+
+    # check if the exts_defaultclass is empty
+    if not exts_defaultclass:
+        raise EasyBuildError("No exts_defaultclass provided for updating")
+
+    # init variables
+    updated_exts_list = []
+
+    # offsets for printing the package information
+    PKG_NAME_OFFSET = 25
+    PKG_VERSION_OFFSET = 10
+    INFO_OFFSET = 20
+
+    # aesthetic terminal print
+    print()
+
+    # loop over all extensions and update their version
+    for ext in exts_list:
+
+        if isinstance(ext, str):
+            # if the extension is a string, then store it as is and skip further processing
+            updated_exts_list.append({"name": ext, "version": None,  "options": None})
+
+            # print message to the user
+            print_msg(
+                f"Package {ext:<{PKG_NAME_OFFSET}} v{('---'):<{PKG_VERSION_OFFSET}} {'letf as is':<{INFO_OFFSET}}", log=_log)
+
+            continue
+
+        elif isinstance(ext, tuple):
+            # get the values of the exts_list extension
+            ext_name, ext_version, ext_options = ext
+
+        else:
+            raise EasyBuildError("Invalid extension format")
+
+        # get metadata of the latest version of the extension
+        metadata = get_pkg_metadata(pkg_class=exts_defaultclass,
+                                    pkg_name=ext_name,
+                                    pkg_version=None,
+                                    bioc_version=bioconductor_version)
+
+        if metadata:
+            # process the metadata and format it as an extension
+            updated_ext = format_metadata_as_extension(exts_defaultclass, metadata, bioconductor_version)
+
+            # print message to the user
+            if ext_version == updated_ext['version']:
+                print_msg(
+                    f"Package {ext_name:<{PKG_NAME_OFFSET}} v{('_' if ext_version is None else ext_version):<{PKG_VERSION_OFFSET}} {'up-to-date':<{INFO_OFFSET}}", log=_log)
+            else:
+                print_msg(
+                    f"Package {ext_name:<{PKG_NAME_OFFSET}} v{('_' if ext_version is None else ext_version):<{PKG_VERSION_OFFSET}} updated to v{updated_ext['version']:<{INFO_OFFSET}}", log=_log)
+
+        else:
+            # no metadata found, therefore store the original extension
+            updated_ext = {"name": ext_name, "version": ext_version,  "options": ext_options}
+
+            # print message to the user
+            print_msg(
+                f"Package {ext_name:<{PKG_NAME_OFFSET}} v{('_' if ext_version is None else ext_version):<{PKG_VERSION_OFFSET}} {'info not found':<{INFO_OFFSET}}", log=_log)
+
+        # store the updated extension
+        updated_exts_list.append(updated_ext)
+
+    # aesthetic terminal print
+    print()
+
+    return updated_exts_list
+
+
+def get_updated_easyconfig(ec, update_param, update_data):
+    """
+    Get a new Easyconfig with the updated given data.
+
+    :param ec: EasyConfig instance to update.
+    :param update_param: parameter to update in the EasyConfig.
+    :param update_data: data to update in the EasyConfig.
+
+    :return: new EasyConfig instance with the updated data.
+    """
+
+    if not ec:
+        raise EasyBuildError("No EasyConfig instance provided to udpate Easyconfig")
+
+    if not update_param:
+        raise EasyBuildError("No parameter provided to update Easyconfig")
+
+    if not update_data:
+        raise EasyBuildError("No data provided to update Easyconfig")
+
+    if update_param == "exts_list":
+        # format the new exts_list to be written to the easyconfig file
+        exts_list_formatted = ['exts_list = [']
+
+        # iterate over the new extensions list and format them
+        for ext in update_data:
+
+            if ext['version'] is None:
+                exts_list_formatted.append("%s'%s'," % (INDENT_4SPACES, ext['name']))
+            else:
+                # append name and version
+                exts_list_formatted.append("%s('%s', '%s', {" % (INDENT_4SPACES, ext['name'], ext['version']))
+
+                # iterate over the options and format them
+                for key, value in ext['options'].items():
+                    # if value is a string, then add quotes so they are printed correctly
+                    if isinstance(value, str):
+                        value = "'%s'" % value
+
+                    # append the key and value of the option
+                    exts_list_formatted.append("%s'%s': %s," % (INDENT_4SPACES * 2, key, value))
+
+                # close the extension
+                exts_list_formatted.append('%s}),' % (INDENT_4SPACES,))
+
+        # close the exts_list
+        exts_list_formatted.append(']\n')
+
+        # read the easyconfig file and replace the exts_list with the new one
+        regex = re.compile(r'^exts_list(.|\n)*?\n\]\s*$', re.M)
+        new_ec = regex.sub('\n'.join(exts_list_formatted), read_file(ec['spec']))
+    else:
+        raise EasyBuildError("Invalid parameter to update Easyconfig")
+
+    return new_ec
+
+
+def get_exts_list(ec):
+    """
+    Get the extension list from the given EasyConfig instance.
+
+    :param ec: EasyConfig instance.
+
+    :return: list of extensions from the given EasyConfig instance.
+    """
+
+    if not ec:
+        raise EasyBuildError("No EasyConfig instance provided to retrieve extensions from")
+
+    # get the extension list from the easyconfig file
+    exts_list = ec.get('ec', {}).get('exts_list', [])
+
+    if exts_list:
+        print_msg("Found %s extensions..." % len(exts_list), log=_log)
+    else:
+        print_warning("No extensions found in easyconfig...", log=_log)
+
+    return exts_list
+
+
+def get_exts_list_class(ec):
+    """
+    Get the exts_defaultclass or deduce it from the given EasyConfig instance.
+
+    :param ec: EasyConfig instance.
+
+    :return: the class of the extensions from the given EasyConfig instance.
+    """
+
+    if not ec:
+        raise EasyBuildError("No EasyConfig instance provided to retrieve extensions from")
+
+    # get the extension list class from the easyconfig file
+    exts_list_class = ec.get('ec', {}).get('exts_defaultclass', None)
+
+    # if no exts_defaultclass is found, try to deduce it from the EasyConfig name
+    if not exts_list_class:
+
+        # get the name of the EasyConfig
+        name = ec.get('ec', {}).get('name', None)
+
+        if name:
+            if name == 'R' or name.startswith('R-'):
+                exts_list_class = 'RPackage'
+            if name == 'Python' or name.startswith('Python-'):
+                exts_list_class = 'PythonPackage'
+
+    if exts_list_class:
+        print_msg("Found extension list class: %s..." % exts_list_class, log=_log)
+    else:
+        print_warning("No extension list class found in Easyconfig...", log=_log)
+
+    return exts_list_class
+
+
+def get_bioconductor_version(ec):
+    """
+    Get the Bioconductor version stored in the local_biocver parameter from the given EasyConfig instance.
+
+    :param ec: EasyConfig instance.
+
+    :return: The Bioconductor version of the given EasyConfig instance (if any).
+    """
+
+    if not ec:
+        raise EasyBuildError("No EasyConfig instance provided to retrieve extensions from")
+
+    # get the Bioconductor version from the easyconfig file
+    # assume that the Bioconductor version is stored in the 'local_biocver' parameter
+    # as this is not a standard parameter we need to parse the raw text
+    rawtxt = getattr(ec['ec'], 'rawtxt', '')
+    match = re.search(r'local_biocver\s*=\s*([0-9.]+)', rawtxt)
+
+    if match:
+        bioconductor_version = match.group(1)
+        print_msg("Using Bioconductor v%s..." % (bioconductor_version), log=_log)
+    else:
+        bioconductor_version = None
+        print_msg("'local_biocver' parameter not set in easyconfig. Bioconductor packages will not be considered...", log=_log)
+
+    return bioconductor_version
+
+
+def update_exts_list(ecs):
+    """
+    Write a new EasyConfig recipe with all extensions in exts_list updated to the latest version.
+
+    :param ecs: list of EasyConfig instances to complete dependencies for
+    """
+
+    for ec in ecs:
+
+        # welcome message
+        print()
+        print_msg("UPDATING EASYCONFIG %s" % ec['spec'], log=_log)
+
+        # get the extension list
+        print_msg("Getting extension list...", log=_log)
+        exts_list = get_exts_list(ec)
+
+        # get the extension's list class
+        print_msg("Getting extension's list class...", log=_log)
+        exts_defaultclass = get_exts_list_class(ec)
+
+        # get the Bioconductor version
+        print_msg("Getting Bioconductor version (if any)...", log=_log)
+        bioconductor_version = get_bioconductor_version(ec)
+
+        # get a new exts_list with all extensions to their latest version.
+        print_msg("Updating extension list...", log=_log)
+        updated_exts_list = get_updated_exts_list(exts_list, exts_defaultclass, bioconductor_version)
+
+        # get new easyconfig file with the updated extensions list
+        print_msg('Updating Easyconfig instance...', log=_log)
+        updated_easyconfig = get_updated_easyconfig(ec, "exts_list", updated_exts_list)
+
+        # back up the original easyconfig file
+        ec_backup = back_up_file(ec['spec'], backup_extension='bak_update')
+        print_msg("Backing up EasyConfig file at %s" % ec_backup, log=_log)
+
+        # write the new easyconfig file
+        print_msg('Writing updated EasyConfig file...', log=_log)
+        write_file(ec['spec'], updated_easyconfig)
+
+        # success message
+        print_msg('EASYCONFIG SUCCESSFULLY UPDATED!\n', log=_log)