Add MCO and DR HUB operator upgrades class

Introduce markers for mco, drhub and acm upgrades

Add tests for mco and drhub upgrades

Signed-off-by: Shylesh Kumar Mohan <[email protected]>

ocs_ci/framework/pytest_customization/marks.py

@@ -13,6 +13,9 @@
     ORDER_BEFORE_OCP_UPGRADE,
     ORDER_BEFORE_UPGRADE,
     ORDER_OCP_UPGRADE,
+    ORDER_MCO_UPGRADE,
+    ORDER_DR_HUB_UPGRADE,
+    ORDER_ACM_UPGRADE,
     ORDER_OCS_UPGRADE,
     ORDER_AFTER_OCP_UPGRADE,
     ORDER_AFTER_OCS_UPGRADE,
@@ -102,12 +105,23 @@
 order_pre_ocp_upgrade = pytest.mark.run(order=ORDER_BEFORE_OCP_UPGRADE)
 order_pre_ocs_upgrade = pytest.mark.run(order=ORDER_BEFORE_OCS_UPGRADE)
 order_ocp_upgrade = pytest.mark.run(order=ORDER_OCP_UPGRADE)
+order_mco_upgrade = pytest.mark.run(order=ORDER_MCO_UPGRADE)
+order_dr_hub_upgrade = pytest.mark.run(order=ORDER_DR_HUB_UPGRADE)
+order_acm_upgrade = pytest.mark.run(order=ORDER_ACM_UPGRADE)
 order_ocs_upgrade = pytest.mark.run(order=ORDER_OCS_UPGRADE)
 order_post_upgrade = pytest.mark.run(order=ORDER_AFTER_UPGRADE)
 order_post_ocp_upgrade = pytest.mark.run(order=ORDER_AFTER_OCP_UPGRADE)
 order_post_ocs_upgrade = pytest.mark.run(order=ORDER_AFTER_OCS_UPGRADE)
+# *_upgrade markers
 ocp_upgrade = compose(order_ocp_upgrade, pytest.mark.ocp_upgrade)
+# multicluster orchestrator
+mco_upgrade = compose(order_mco_upgrade, pytest.mark.mco_upgrade)
+# dr hub operator
+dr_hub_upgrade = compose(order_dr_hub_upgrade, pytest.mark.dr_hub_upgrade)
+# acm operator
+acm_upgrade = compose(order_acm_upgrade, pytest.mark.acm_upgrade)
 ocs_upgrade = compose(order_ocs_upgrade, pytest.mark.ocs_upgrade)
+# pre_*_upgrade markers
 pre_upgrade = compose(order_pre_upgrade, pytest.mark.pre_upgrade)
 pre_ocp_upgrade = compose(
     order_pre_ocp_upgrade,
@@ -117,12 +131,16 @@
     order_pre_ocs_upgrade,
     pytest.mark.pre_ocs_upgrade,
 )
+# post_*_upgrade markers
 post_upgrade = compose(order_post_upgrade, pytest.mark.post_upgrade)
 post_ocp_upgrade = compose(order_post_ocp_upgrade, pytest.mark.post_ocp_upgrade)
 post_ocs_upgrade = compose(order_post_ocs_upgrade, pytest.mark.post_ocs_upgrade)
 
 upgrade_marks = [
     ocp_upgrade,
+    mco_upgrade,
+    dr_hub_upgrade,
+    acm_upgrade,
     ocs_upgrade,
     pre_upgrade,
     pre_ocp_upgrade,

ocs_ci/ocs/constants.py

@@ -1123,6 +1123,12 @@
 ORDER_BEFORE_OCP_UPGRADE = 20
 ORDER_OCP_UPGRADE = 30
 ORDER_AFTER_OCP_UPGRADE = 40
+# Multicluster orchestrator
+ORDER_MCO_UPGRADE = 42
+# DR Hub operator
+ORDER_DR_HUB_UPGRADE = 44
+# ACM Operator
+ORDER_ACM_UPGRADE = 46
 ORDER_BEFORE_OCS_UPGRADE = 50
 ORDER_OCS_UPGRADE = 60
 ORDER_AFTER_OCS_UPGRADE = 70

ocs_ci/ocs/defaults.py

@@ -49,6 +49,8 @@
 FUSION_CATALOG_NAME = "isf-data-foundation-catalog"
 LIVE_CONTENT_SOURCE = "redhat-operators"
 OCS_CLIENT_OPERATOR_NAME = "ocs-client-operator"
+MCO_OPERATOR_NAME = "odf-multicluster-orchestrator"
+DR_HUB_OPERATOR_NAME = "odr-hub-operator"
 
 # Noobaa S3 bucket website configurations
 website_config = {

ocs_ci/ocs/dr_upgrade.py

@@ -0,0 +1,134 @@
+"""
+All DR operators upgrades implemented here ex: MulticlusterOrchestrator, Openshift DR operator
+
+"""
+
+import logging
+
+from ocs_ci.framework import config
+from ocs_ci.ocs.cluster import CephCluster, CephClusterExternal, CephHealthMonitor
+from ocs_ci.ocs.exceptions import TimeoutException
+from ocs_ci.ocs.ocs_upgrade import OCSUpgrade, verify_image_versions
+from ocs_ci.ocs import constants
+from ocs_ci.ocs import defaults
+from ocs_ci.deployment.helpers.external_cluster_helpers import (
+    ExternalCluster,
+    get_external_cluster_client,
+)
+from ocs_ci.ocs.resources.install_plan import wait_for_install_plan_and_approve
+from ocs_ci.utility.utils import TimeoutSampler
+
+
+log = logging.getLogger(__name__)
+
+DR_TO_CEPH_CLUSTER_MAP = {"regional-dr": CephCluster, "metro-dr": CephClusterExternal}
+
+
+class DRUpgrade(OCSUpgrade):
+    """
+    Base class for all DR operator upgrades
+
+    """
+
+    def __init__(
+        self,
+        namespace=constants.OPENSHIFT_OPERATORS,
+        version_before_upgrade=config.ENV_DATA.get("ocs_version"),
+        ocs_registry_image=config.UPGRADE.get("upgrade_ocs_registry_image"),
+        upgrade_in_current_source=config.UPGRADE.get(
+            "upgrade_in_current_source", False
+        ),
+    ):
+        self.namespace = (
+            namespace if namespace else config.ENV_DATA["cluster_namespace"]
+        )
+        self.version_before_upgrade = version_before_upgrade
+        self.ocs_registry_image = ocs_registry_image
+        self.upgrade_in_current_source = upgrade_in_current_source
+        self.ceph_cluster = DR_TO_CEPH_CLUSTER_MAP[
+            config.MULTICLUSTER["multicluster_mode"]
+        ]
+        self.external_cluster = None
+        self.operator_name = None
+
+        super.__init__(
+            self.namespace,
+            self.version_before_upgrade,
+            self.ocs_registry_image,
+            self.upgrade_in_current_source,
+        )
+
+    def run_upgrade(self):
+        self.upgrade_version = self.get_upgrade_version()
+        assert self.get_parsed_versions()[1] >= self.get_parsed_versions()[0], (
+            f"Version you would like to upgrade to: {self.upgrade_version} "
+            f"is not higher or equal to the version you currently running: "
+            f"{self.version_before_upgrade}"
+        )
+
+        # create external cluster object
+        if config.DEPLOYMENT["external_mode"]:
+            host, user, password, ssh_key = get_external_cluster_client()
+            self.external_cluster = ExternalCluster(host, user, password, ssh_key)
+        self.csv_name_pre_upgrade = self.get_csv_name_pre_upgrade()
+        self.pre_upgrade_images = self.get_pre_upgrade_image(self.csv_name_pre_upgrade)
+        self.load_version_config_file(self.upgrade_version)
+
+        with CephHealthMonitor(self.ceph_cluster):
+            self.channel = self.set_upgrade_channel(resource_name=self.operator_name)
+            self.set_upgrade_images()
+            # TODO: Overload this function
+            self.update_subscription(self.channel)
+            # In the case upgrade is not from 4.8 to 4.9 and we have manual approval strategy
+            # we need to wait and approve install plan, otherwise it's approved in the
+            # subscribe_ocs method.
+            subscription_plan_approval = config.DEPLOYMENT.get(
+                "subscription_plan_approval"
+            )
+            if subscription_plan_approval == "Manual":
+                wait_for_install_plan_and_approve(config.ENV_DATA["cluster_namespace"])
+
+            for sample in TimeoutSampler(
+                timeout=725,
+                sleep=5,
+                func=self.check_if_upgrade_completed,
+                channel=self.channel,
+                csv_name_pre_upgrade=self.csv_name_pre_upgrade,
+            ):
+                try:
+                    if sample:
+                        log.info("Upgrade success!")
+                        break
+                except TimeoutException:
+                    raise TimeoutException("No new CSV found after upgrade!")
+            old_image = self.get_images_post_upgrade(
+                self.channel, self.pre_upgrade_images, self.upgrade_version
+            )
+
+        verify_image_versions(
+            old_image,
+            self.get_parsed_versions()[1],
+            self.version_before_upgrade,
+        )
+
+
+class MultiClusterOrchestratorUpgrade(DRUpgrade):
+    """
+    A class to handle ODF MCO operator upgrades
+
+    """
+
+    def __init__(self):
+        super.__init__()
+        self.operator_name = defaults.MCO_OPERATOR_NAME
+
+
+class DRHubUpgrade(DRUpgrade):
+    """
+    A class to handle DR Hub operator upgrades
+
+    """
+
+    def __init__(self):
+        super.__init__()
+        self.operator_name = defaults.DR_HUB_OPERATOR_NAME

ocs_ci/ocs/ocs_upgrade.py

@@ -380,7 +380,7 @@ def get_pre_upgrade_image(self, csv_name_pre_upgrade):
         )
         return get_images(csv_pre_upgrade.get())
 
-    def set_upgrade_channel(self):
+    def set_upgrade_channel(self, resource_name=OCS_OPERATOR_NAME):
         """
         Wait for the new package manifest for upgrade.
 
@@ -390,7 +390,7 @@ def set_upgrade_channel(self):
         """
         operator_selector = get_selector_for_ocs_operator()
         package_manifest = PackageManifest(
-            resource_name=OCS_OPERATOR_NAME,
+            resource_name=resource_name,
             selector=operator_selector,
         )
         package_manifest.wait_for_resource()

ocs_ci/utility/multicluster.py

@@ -24,6 +24,9 @@ class MultiClusterUpgradeParametrize(object):
         "pre_ocp_upgrade",
         "ocp_upgrade",
         "post_ocp_upgrade",
+        "mco_upgrade",
+        "dr_hub_upgrade",
+        "acm_upgrade",
         "pre_ocs_upgrade",
         "ocs_upgrade",
         "post_ocs_upgrade",

tests/ecosystem/upgrade/test_upgrade.py

@@ -6,9 +6,13 @@
 from ocs_ci.framework.testlib import (
     ocs_upgrade,
     polarion_id,
+    mco_upgrade,
+    dr_hub_upgrade,
+    acm_upgrade,
 )
 from ocs_ci.ocs.disruptive_operations import worker_node_shutdown, osd_node_reboot
 from ocs_ci.ocs.ocs_upgrade import run_ocs_upgrade
+from ocs_ci.ocs.dr_upgrade import MultiClusterOrchestratorUpgrade, DRHubUpgrade
 from ocs_ci.utility.reporting import get_polarion_id
 
 log = logging.getLogger(__name__)
@@ -72,3 +76,25 @@ def test_upgrade():
     """
 
     run_ocs_upgrade()
+
+
+@mco_upgrade
+@multicluster_roles(["mdr_all_acm"])
+def test_mco_upgrade():
+    """
+    Test upgrade procedure for multicluster orchestrator operator
+
+    """
+    mco_upgrade_obj = MultiClusterOrchestratorUpgrade()
+    mco_upgrade_obj.run_upgrade()
+
+
+@dr_hub_upgrade
+@multicluster_roles(["mdr_all_acm"])
+def test_dr_hub_upgrade():
+    """
+    Test upgrade procedure for DR hub operator
+
+    """
+    dr_hub_upgrade_obj = DRHubUpgrade()
+    dr_hub_upgrade_obj.run_upgrade()