distributionsignedrpmscanner: refactoring + gpg-pubkey fix

We have decided to refactor the code in the actor (coming history
time ago) to make it more readable.

Also it's fixing an old issue with gpg-pubkey detection as unsigned
rpm. gpg-pubkey is not a real package and it's just an entry in RPM DB
about the imported RPM GPG keys. Originally it has been checked whether
the packager is vendor/authority of the installed distribution and if
not, such a package (key) has been mared as unsigned.

This led to false positive reports, that we do not know what will
happen with gpg-pubkey packages (reported even multiple times..)
and that they might be removed or do another problems with the upgrade
transaction - which has not been true.

So I dropped the check for the packager and mark gpg-pubkey always
as signed. It's a question whether we should not ignore this package
always and do not put it to any signed/unsigned list. Handling it
in this way for now.

repos/system_upgrade/common/actors/distributionsignedrpmscanner/actor.py

@@ -1,22 +1,28 @@
-import json
-import os
-
 from leapp.actors import Actor
-from leapp.exceptions import StopActorExecutionError
-from leapp.libraries.common import rhui
-from leapp.libraries.common.config import get_env
-from leapp.libraries.stdlib import api
+from leapp.libraries.actor import distributionsignedrpmscanner
 from leapp.models import DistributionSignedRPM, InstalledRedHatSignedRPM, InstalledRPM, InstalledUnsignedRPM
 from leapp.tags import FactsPhaseTag, IPUWorkflowTag
 from leapp.utils.deprecation import suppress_deprecation
 
 
 @suppress_deprecation(InstalledRedHatSignedRPM)
 class DistributionSignedRpmScanner(Actor):
-    """Provide data about installed RPM Packages signed by the distribution.
+    """
+    Provide data about distribution signed & unsigned RPM packages.
+
+    For various checks and actions done during the upgrade it's important to
+    know what packages are signed by GPG keys of the installed linux system
+    distribution. RPMs that are not provided in the distribution could have
+    different versions, different behaviour, and also it could be completely
+    different application just with the same RPM name.
+
+    For that reasons, various actors rely on the DistributionSignedRPM message
+    to check whether particular package is installed, to be sure it provides
+    valid data. Fingerprints of distribution GPG keys are stored under
+      common/files/distro/<distro>/gpg_signatures.json
+    where <distro> is distribution ID of the installed system (e.g. centos, rhel).
 
-    After filtering the list of installed RPM packages by signature, a message
-    with relevant data will be produced.
+    If the file for the installed distribution is not find, end with error.
     """
 
     name = 'distribution_signed_rpm_scanner'
@@ -25,70 +31,4 @@ class DistributionSignedRpmScanner(Actor):
     tags = (IPUWorkflowTag, FactsPhaseTag)
 
     def process(self):
-        # TODO(pstodulk): refactor this function
-        # - move it to the private library
-        # - split it into several functions (so the main function stays small)
-        # FIXME(pstodulk): gpg-pubkey is handled wrong; it's not a real package
-        # and create FP report about unsigned RPMs. Keeping the fix for later.
-        distribution = self.configuration.os_release.release_id
-        distributions_path = api.get_common_folder_path('distro')
-
-        distribution_config = os.path.join(distributions_path, distribution, 'gpg-signatures.json')
-        if os.path.exists(distribution_config):
-            with open(distribution_config) as distro_config_file:
-                distro_config_json = json.load(distro_config_file)
-                distribution_keys = distro_config_json.get('keys', [])
-                distribution_packager = distro_config_json.get('packager', 'not-available')
-        else:
-            raise StopActorExecutionError(
-                'Cannot find distribution signature configuration.',
-                details={'Problem': 'Distribution {} was not found in {}.'.format(distribution, distributions_path)})
-
-        signed_pkgs = DistributionSignedRPM()
-        rh_signed_pkgs = InstalledRedHatSignedRPM()
-        unsigned_pkgs = InstalledUnsignedRPM()
-
-        all_signed = get_env('LEAPP_DEVEL_RPMS_ALL_SIGNED', '0') == '1'
-
-        def has_distributionsig(pkg):
-            return any(key in pkg.pgpsig for key in distribution_keys)
-
-        def is_gpg_pubkey(pkg):
-            """
-            Check if gpg-pubkey pkg exists or LEAPP_DEVEL_RPMS_ALL_SIGNED=1
-
-            gpg-pubkey is not signed as it would require another package
-            to verify its signature
-            """
-            return (    # pylint: disable-msg=consider-using-ternary
-                    pkg.name == 'gpg-pubkey'
-                    and pkg.packager.startswith(distribution_packager)
-                    or all_signed
-            )
-
-        def has_katello_prefix(pkg):
-            """Whitelist the katello package."""
-            return pkg.name.startswith('katello-ca-consumer')
-
-        whitelisted_cloud_pkgs = rhui.get_all_known_rhui_pkgs_for_current_upg()
-
-        for rpm_pkgs in self.consume(InstalledRPM):
-            for pkg in rpm_pkgs.items:
-                if any(
-                    [
-                        has_distributionsig(pkg),
-                        is_gpg_pubkey(pkg),
-                        has_katello_prefix(pkg),
-                        pkg.name in whitelisted_cloud_pkgs,
-                    ]
-                ):
-                    signed_pkgs.items.append(pkg)
-                    if distribution == 'rhel':
-                        rh_signed_pkgs.items.append(pkg)
-                    continue
-
-                unsigned_pkgs.items.append(pkg)
-
-        self.produce(signed_pkgs)
-        self.produce(rh_signed_pkgs)
-        self.produce(unsigned_pkgs)
+        distributionsignedrpmscanner.process()

repos/system_upgrade/common/actors/distributionsignedrpmscanner/libraries/distributionsignedrpmscanner.py

@@ -0,0 +1,72 @@
+import json
+import os
+
+from leapp.exceptions import StopActorExecutionError
+from leapp.libraries.common import rhui
+from leapp.libraries.common.config import get_env
+from leapp.libraries.stdlib import api
+from leapp.models import DistributionSignedRPM, InstalledRedHatSignedRPM, InstalledRPM, InstalledUnsignedRPM
+
+
+def get_distribution_data(distribution):
+    distributions_path = api.get_common_folder_path('distro')
+
+    distribution_config = os.path.join(distributions_path, distribution, 'gpg-signatures.json')
+    if os.path.exists(distribution_config):
+        with open(distribution_config) as distro_config_file:
+            distro_config_json = json.load(distro_config_file)
+            distro_keys = distro_config_json.get('keys', [])
+            # distro_packager = distro_config_json.get('packager', 'not-available')
+    else:
+        raise StopActorExecutionError(
+            'Cannot find distribution signature configuration.',
+            details={'Problem': 'Distribution {} was not found in {}.'.format(distribution, distributions_path)})
+    return distro_keys
+
+
+def is_distro_signed(pkg, distro_keys):
+    return any(key in pkg.pgpsig for key in distro_keys)
+
+
+def is_exceptional(pkg, allowlist):
+    """
+    Some packages should be marked always as signed
+
+    tl;dr; gpg-pubkey, katello packages, and rhui packages
+
+    gpg-pubkey is not real RPM. It's just an entry representing
+    gpg key imported inside the RPM DB. For that same reason, it cannot be
+    signed. Note that it cannot affect the upgrade transaction, so ignore
+    who vendored the key. Total majority of all machines have imported third
+    party gpg keys.
+
+    Katello packages have various names and are created on a Satellite server.
+
+    The allowlist is now used for any other package names that should be marked
+    always as signed for the particular upgrade.
+    """
+    return pkg.name == 'gpg-pubkey' or pkg.name.startswith('katello-ca-consumer') or pkg.name in allowlist
+
+
+def process():
+    distribution = api.current_actor().configuration.os_release.release_id
+    distro_keys = get_distribution_data(distribution)
+    all_signed = get_env('LEAPP_DEVEL_RPMS_ALL_SIGNED', '0') == '1'
+    rhui_pkgs = rhui.get_all_known_rhui_pkgs_for_current_upg()
+
+    signed_pkgs = DistributionSignedRPM()
+    rh_signed_pkgs = InstalledRedHatSignedRPM()
+    unsigned_pkgs = InstalledUnsignedRPM()
+
+    for rpm_pkgs in api.consume(InstalledRPM):
+        for pkg in rpm_pkgs.items:
+            if all_signed or is_distro_signed(pkg, distro_keys) or is_exceptional(pkg, rhui_pkgs):
+                signed_pkgs.items.append(pkg)
+                if distribution == 'rhel':
+                    rh_signed_pkgs.items.append(pkg)
+                continue
+            unsigned_pkgs.items.append(pkg)
+
+    api.produce(signed_pkgs)
+    api.produce(rh_signed_pkgs)
+    api.produce(unsigned_pkgs)

repos/system_upgrade/common/actors/distributionsignedrpmscanner/tests/test_distributionsignedrpmscanner.py

@@ -180,11 +180,11 @@ def test_gpg_pubkey_pkg(current_actor_context):
     current_actor_context.feed(InstalledRPM(items=installed_rpm))
     current_actor_context.run(config_model=mock_configs.CONFIG)
     assert current_actor_context.consume(DistributionSignedRPM)
-    assert len(current_actor_context.consume(DistributionSignedRPM)[0].items) == 1
+    assert len(current_actor_context.consume(DistributionSignedRPM)[0].items) == 2
     assert current_actor_context.consume(InstalledRedHatSignedRPM)
-    assert len(current_actor_context.consume(InstalledRedHatSignedRPM)[0].items) == 1
+    assert len(current_actor_context.consume(InstalledRedHatSignedRPM)[0].items) == 2
     assert current_actor_context.consume(InstalledUnsignedRPM)
-    assert len(current_actor_context.consume(InstalledUnsignedRPM)[0].items) == 1
+    assert not current_actor_context.consume(InstalledUnsignedRPM)[0].items
 
 
 def test_create_lookup():