Initial work to demonstrate an actor config framework.

repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py

@@ -1,8 +1,10 @@
 from leapp.actors import Actor
+from leapp.configs import Transaction_ToInstall, Transaction_ToKeep, Transaction_ToRemove
 from leapp.libraries.actor.rpmtransactionconfigtaskscollector import load_tasks
 from leapp.models import DistributionSignedRPM, RpmTransactionTasks
 from leapp.tags import FactsPhaseTag, IPUWorkflowTag
 
+
 CONFIGURATION_BASE_PATH = '/etc/leapp/transaction'
 
 
@@ -13,11 +15,11 @@ class RpmTransactionConfigTasksCollector(Actor):
     After collecting task data from /etc/leapp/transaction directory, a message with relevant data
     will be produced.
     """
-
+    configs = (Transaction_ToInstall, Transaction_ToKeep, Transaction_ToRemove)
     name = 'rpm_transaction_config_tasks_collector'
     consumes = (DistributionSignedRPM,)
     produces = (RpmTransactionTasks,)
     tags = (FactsPhaseTag, IPUWorkflowTag)
 
     def process(self):
-        self.produce(load_tasks(CONFIGURATION_BASE_PATH, self.log))
+        self.produce(load_tasks(self.config, self.log))

repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py

@@ -18,11 +18,11 @@ def load_tasks_file(path, logger):
     return []
 
 
-def load_tasks(base_dir, logger):
+def load_tasks(config, logger):
     # Loads configuration files to_install, to_keep, and to_remove from the given base directory
     rpms = next(api.consume(DistributionSignedRPM))
     rpm_names = [rpm.name for rpm in rpms.items]
-    to_install = load_tasks_file(os.path.join(base_dir, 'to_install'), logger)
+    to_install = config['transaction']['to_install']
     # we do not want to put into rpm transaction what is already installed (it will go to "to_upgrade" bucket)
     to_install_filtered = [pkg for pkg in to_install if pkg not in rpm_names]
 
@@ -34,5 +34,5 @@ def load_tasks(base_dir, logger):
 
     return RpmTransactionTasks(
         to_install=to_install_filtered,
-        to_keep=load_tasks_file(os.path.join(base_dir, 'to_keep'), logger),
-        to_remove=load_tasks_file(os.path.join(base_dir, 'to_remove'), logger))
+        to_keep=config['transaction']['to_keep'],
+        to_remove=config['transaction']['to_remove'])

repos/system_upgrade/common/configs/__init__.py

@@ -0,0 +1,190 @@
+# This is code that needs to go in the leapp framework.
+# Putting it here for now so that everything is in one git repo.
+"""
+Config file format:
+    yaml file like this:
+
+---
+# Note: have to add a fields.Map type before we can use yaml mappings.
+section_name:
+    field1_name: value
+    field2_name:
+        - listitem1
+        - listitem2
+section2_name:
+    field3_name: value
+
+Config files are any yaml files in /etc/leapp/actor_config.d/
+(This is settable in /etc/leapp/leapp.conf)
+
+"""
+__metaclass__ = type
+
+import abc
+import importlib
+import os
+import pkgutil
+
+import six
+import yaml
+
+try:
+    # Compiled versions if available, for speed
+    from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
+except ImportError:
+    from yaml import SafeLoader, SafeDumper
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Config:
+    @abc.abstractproperty
+    def section():
+        pass
+
+    @abc.abstractproperty
+    def name():
+        pass
+
+    @abc.abstractproperty
+    def type_():
+        pass
+
+    @abc.abstractproperty
+    def description():
+        pass
+
+    @classmethod
+    def to_dict(cls):
+        """
+        Return a dictionary representation of the config item that would be suitable for putting
+        into a config file.
+        """
+        representation = {
+            cls.section: {
+                '{0}_description__'.format(cls.name): cls.description
+            }
+        }
+        ### TODO: Retrieve the default values from the type field.
+        # representation[cls.section][cls.name] = cls.type_.get_default()
+
+        return representation
+
+
+def parse_system_config_files(config_dir='/etc/leapp/config.d'):
+    """
+    Read all configuration files from the config_dir and return a dict with their values.
+    """
+    potential_config_files = []
+    for root, dirs, files in os.walk(config_dir):
+        for file in files:
+            potential_config_fiels.append(os.path.join(root, file))
+
+    potential_config_files.sort()
+    all_config = {}
+    for filename in potential_config_files:
+        try:
+            cfg = SafeLoader(filename)
+        except Exception as e:
+            if filename.endswith('.yml') or filename.endswith('.yaml'):
+                ### TODO: Should use a logger once we know where this will live
+                print("Warning: unparsable yaml file {0} in the config directory."
+                      " Error: {1}".format(filename, str(e)))
+            continue
+
+        ### TODO: Should we check whether we are overwriting any already defined keys?
+        all_configs.extend(cfg)
+
+    return all_config
+
+
+def all_repository_config_schemas():
+    """
+    Return all the configuration items present in all repositories.
+    """
+    # Need to loop through all the configuration defined in all the
+    # repositories and then return them.
+    # POC Code
+    config_path = "/srv/leapp/leapp-repository-git/actor-config/repos"
+    config_mod_prefix = "system_upgrade.common.configs"
+    configs = set()
+
+    # In Python 3, this is a NamedTuple where m[1] == m.name and m[2] == m.ispkg
+    modules = (m[1] for m in pkgutil.iter_modules(config_path, prefix=config_mod_prefix) if not m[2])
+    modules = (importlib.import_module(m) for m in modules)
+
+    for module in modules:
+        objects = (getattr(module, obj_name) for obj_name in dir(module))
+        config_classes = (
+            obj for obj in objects if
+                isinstance(obj, type) and
+                issubclass(obj, Config) and
+                obj is not Config
+        )
+        configs.update(config_classes)
+    # END POC Code
+
+    return list(configs)
+
+
+def parse_repo_config_files():
+    repo_config = {}
+    for config in all_repository_config_schemas():
+        section_name = config.section
+
+        if section_name not in repo_config:
+            repo_config.update(config.to_dict())
+        else:
+            if '{0}_description__'.format(config_item.name) in repo_config[config.section]:
+                raise Exception("Error: Two configuration items are declared with the same name Section: {0}, Key: {1}".format(config.section, config.name))
+
+            repo_config[config.section].update(config.to_dict()[config.section])
+
+    return repo_config
+
+
+def parse_config_files(config_dir):
+    """
+    Parse all configuration and return a dict with those values.
+    """
+    config = parse_repo_config_files()
+    system_config = parse_system_config_files(config_dir)
+
+    for section, config_items in system_config.items():
+        if section not in config:
+            print('WARNING: config file contains an unused section: Section: {0}'.format(section))
+            config.update[section] = config_items
+        else:
+            for key, value in config_items:
+                if '{0}_description__'.format(key) not in config[section]:
+                    print('WARNING: config file contains an unused config entry: Section: {0}, Key{1}'.format(section, key))
+
+                config[section][key] = value
+
+    return config
+
+
+def format_config():
+    """
+    Read the configuration definitions from all of the known repositories and return a string that
+    can be used as an example config file.
+
+    Example config file:
+        transaction:
+            to_install_description__: |
+                List of packages to be added to the upgrade transaction.
+                Signed packages which are already installed will be skipped.
+            to_remove_description__: |
+                List of packages to be removed from the upgrade transaction
+                initial-setup should be removed to avoid it asking for EULA acceptance during upgrade
+            to_remove:
+                - initial-setup
+            to_keep_description__: |
+                List of packages to be kept in the upgrade transaction
+            to_keep:
+                - leapp
+                - python2-leapp
+                - python3-leapp
+                - leapp-repository
+                - snactor
+    """
+    return SafeDumper(yaml.dump(parse_config_files(), dumper=SafeDumper))

repos/system_upgrade/common/configs/rhui.py

@@ -0,0 +1,76 @@
+"""
+Configuration keys for RHUI.
+
+In case of RHUI in private regions it usual that publicly known RHUI data
+is not valid. In such cases it's possible to provide the correct expected
+RHUI data to correct the in-place upgrade process.
+"""
+
+from leapp.models import fields
+from leapp.models.configs import Config
+
+
+class RhuiSrcPkg(Config):
+    section = "rhui"
+    name = "src_pkg"
+    type_ = fields.String(default="rhui")
+    description = """
+        The name of the source RHUI client RPM (installed on the system).
+        Default: rhui.
+    """
+
+
+class RhuiTargetPkg(Config):
+    section = "rhui"
+    name = "target_pkg"
+    type_ = fields.String(default="rhui")
+    description = """
+        The name of the target RHUI client RPM (to be installed on the system).
+        Default: rhui
+    """
+
+
+class RhuiLeappRhuiPkg(Config):
+    section = "rhui"
+    name = "leapp_rhui_pkg"
+    type_ = fields.String(default="leapp-rhui")
+    description = """
+    The name of the leapp-rhui RPM.  Default: leapp-rhui
+    """
+
+
+class RhuiLeappRhuiPkgRepo(Config):
+    section = "rhui"
+    name = "leapp_rhui_pkg_repo"
+    type_ = fields.String(default="rhel-base")
+    description = """
+        The repository ID containing the specified leapp-rhui RPM.
+        Default: rhel-base
+    """
+
+all_rhui_cfg = (RhuiSrcPkg, RhuiTargetPkg, RhuiLeappRhuiPkg, RhuiLeappRhuiPkg)
+# Usage: from configs import rhui
+#        class MyActor:
+#            [...]
+#            configs = all_rhui_cfg + (MyConfig,)
+
+### We need to implement fields.Map before this can be enabled
+'''
+class RhuiFileMap(Config):
+    section = "rhui"
+    name = "file_map"
+    type_ = fields.Map(fields.String())
+    description = """
+        Define directories to which paritcular files provided by the leapp-rhui
+        RPM should be installed. The files in 'files_map' are provided by
+        special Leapp rpms (per cloud) and are supposed to be delivered into the
+        repos/system_upgrade/common/files/rhui/<PROVIDER> directory.
+
+        These files are usually needed to get access to the target system repositories
+        using RHUI. Typically these are certificates, keys, and repofiles with the
+        target RHUI repositories.
+
+        The key is the name of the file, the value is the expected directory
+        where the file should be installed on the upgraded system.
+    """
+'''

repos/system_upgrade/common/configs/rpm.py

@@ -0,0 +1,51 @@
+"""
+Configuration keys for dnf transactions.
+"""
+
+from leapp.models import fields
+from leapp.models.configs import Config
+
+
+### Nested containers?
+### Duplication of default value in type_ and Config.  If we eliminate that, we need to extract default from the type_ for the documentation.
+### We probably want to allow dicts in Config.  But IIRC, dicts were specifically excluded for model fields.  Do we need something that restricts where fields are valid?
+### Test that type validation is strict.  For instance, giving an integer like 644 to a field.String() is an error.
+class Transaction_ToInstall(Config):
+    section = "transaction"
+    name = "to_install"
+    type_ = fields.List(fields.String(), default=[])
+    description = """
+        List of packages to be added to the upgrade transaction.
+        Signed packages which are already installed will be skipped.
+    """
+
+
+class Transaction_ToKeep(Config):
+    section = "transaction"
+    name = "to_keep"
+    type_ = fields.List(fields.String(), default=[
+        "leapp",
+        "python2-leapp",
+        "python3-leapp",
+        "leapp-repository",
+        "snactor",
+    ])
+    description = """
+        List of packages to be kept in the upgrade transaction. The default is
+        leapp, python2-leapp, python3-leapp, leapp-repository, snactor. If you
+        override this, remember to include the default values if applicable.
+    """
+
+
+class Transaction_ToRemove(Config):
+    section = "transaction"
+    name = "to_remove"
+    type_ = fields.List(fields.String(), default=[
+        "initial-setup",
+    ])
+    description = """
+        List of packages to be removed from the upgrade transaction. The default
+        is initial-setup which should be removed to avoid it asking for EULA
+        acceptance during upgrade. If you override this, remember to include the
+        default values if applicable.
+    """