diff --git a/conf/README.md b/conf/README.md index 4f9e45f04ef..b4fdcad163c 100644 --- a/conf/README.md +++ b/conf/README.md @@ -212,6 +212,7 @@ higher priority). * `skip_ocp_deployment` - Skip the OCP deployment step or not (Default: false) * `skip_ocs_deployment` - Skip the OCS deployment step or not (Default: false) * `ocs_version` - Version of OCS that is being deployed +* `acm_version` - Version of acm to be used for this run (applicable mostly to DR scenarios) * `vm_template` - VMWare template to use for RHCOS images * `fio_storageutilization_min_mbps` - Minimal write speed of FIO used in workload_fio_storageutilization * `TF_LOG_LEVEL` - Terraform log level @@ -362,6 +363,9 @@ Upgrade related configuration data. * `ocp_arch` - Architecture type of the OCP image * `upgrade_logging_channel` - OCP logging channel to upgrade with * `upgrade_ui` - Perform upgrade via UI (Not all the versions are supported, please look at the code) +* `upgrade_acm_version` - ACM version to which we have to upgrade +* `upgrade_acm_registry_image` - ACM Image tag from brew which should be used to upgrade +example: /rh-osbs/iib:565330 #### AUTH diff --git a/ocs_ci/framework/__init__.py b/ocs_ci/framework/__init__.py index 7ecd7cedd7e..3930b52f877 100644 --- a/ocs_ci/framework/__init__.py +++ b/ocs_ci/framework/__init__.py @@ -37,6 +37,10 @@ class Config: COMPONENTS: dict = field(default_factory=dict) # Used for multicluster only MULTICLUSTER: dict = field(default_factory=dict) + # Use this variable to store any arbitrary key/values related + # to the upgrade context. Applicable only in the multicluster upgrade + # scenario + PREUPGRADE_CONFIG: dict = field(default_factory=dict) def __post_init__(self): self.reset() diff --git a/ocs_ci/framework/conf/default_config.yaml b/ocs_ci/framework/conf/default_config.yaml index 23f01d8feb8..4db39aa2863 100644 --- a/ocs_ci/framework/conf/default_config.yaml +++ b/ocs_ci/framework/conf/default_config.yaml @@ -362,3 +362,7 @@ MULTICLUSTER: acm_cluster: False primary_cluster: False active_acm_cluster: False + +PREUPGRADE_CONFIG: + AUTH: null + MULTICLUSTER: null diff --git a/ocs_ci/framework/pytest_customization/marks.py b/ocs_ci/framework/pytest_customization/marks.py index 7b3734bda08..7ca6a065332 100644 --- a/ocs_ci/framework/pytest_customization/marks.py +++ b/ocs_ci/framework/pytest_customization/marks.py @@ -14,6 +14,9 @@ ORDER_BEFORE_OCP_UPGRADE, ORDER_BEFORE_UPGRADE, ORDER_OCP_UPGRADE, + ORDER_MCO_UPGRADE, + ORDER_DR_HUB_UPGRADE, + ORDER_ACM_UPGRADE, ORDER_OCS_UPGRADE, ORDER_AFTER_OCP_UPGRADE, ORDER_AFTER_OCS_UPGRADE, @@ -117,12 +120,28 @@ order_pre_ocp_upgrade = pytest.mark.order(ORDER_BEFORE_OCP_UPGRADE) order_pre_ocs_upgrade = pytest.mark.order(ORDER_BEFORE_OCS_UPGRADE) order_ocp_upgrade = pytest.mark.order(ORDER_OCP_UPGRADE) +order_mco_upgrade = pytest.mark.order(ORDER_MCO_UPGRADE) +order_dr_hub_upgrade = pytest.mark.order(ORDER_DR_HUB_UPGRADE) +# dr cluster operator order is same as hub operator order except that +# it's applicable only on the managed clusters +order_dr_cluster_operator_upgrade = pytest.mark.order(ORDER_DR_HUB_UPGRADE) +order_acm_upgrade = pytest.mark.order(ORDER_ACM_UPGRADE) order_ocs_upgrade = pytest.mark.order(ORDER_OCS_UPGRADE) order_post_upgrade = pytest.mark.order(ORDER_AFTER_UPGRADE) order_post_ocp_upgrade = pytest.mark.order(ORDER_AFTER_OCP_UPGRADE) order_post_ocs_upgrade = pytest.mark.order(ORDER_AFTER_OCS_UPGRADE) ocp_upgrade = compose(order_ocp_upgrade, pytest.mark.ocp_upgrade) +# multicluster orchestrator +mco_upgrade = compose(order_mco_upgrade, pytest.mark.mco_upgrade) +# dr hub operator +dr_hub_upgrade = compose(order_dr_hub_upgrade, pytest.mark.dr_hub_upgrade) +dr_cluster_operator_upgrade = compose( + order_dr_cluster_operator_upgrade, pytest.mark.dr_cluster_operator_upgrade +) +# acm operator +acm_upgrade = compose(order_acm_upgrade, pytest.mark.acm_upgrade) ocs_upgrade = compose(order_ocs_upgrade, pytest.mark.ocs_upgrade) +# pre_*_upgrade markers pre_upgrade = compose(order_pre_upgrade, pytest.mark.pre_upgrade) pre_ocp_upgrade = compose( order_pre_ocp_upgrade, @@ -132,12 +151,16 @@ order_pre_ocs_upgrade, pytest.mark.pre_ocs_upgrade, ) +# post_*_upgrade markers post_upgrade = compose(order_post_upgrade, pytest.mark.post_upgrade) post_ocp_upgrade = compose(order_post_ocp_upgrade, pytest.mark.post_ocp_upgrade) post_ocs_upgrade = compose(order_post_ocs_upgrade, pytest.mark.post_ocs_upgrade) upgrade_marks = [ ocp_upgrade, + mco_upgrade, + dr_hub_upgrade, + acm_upgrade, ocs_upgrade, pre_upgrade, pre_ocp_upgrade, @@ -685,3 +708,12 @@ def get_current_test_marks(): config.DEPLOYMENT.get("kms_deployment") is True, reason="This test is not supported for KMS deployment.", ) + +# Mark the test with marker below to allow re-tries in ceph health fixture +# for known issues when waiting in re-balance and flip flop from health OK +# to 1-2 PGs waiting to be Clean +ceph_health_retry = pytest.mark.ceph_health_retry + +# Mark for Multicluster upgrade scenarios +config_index = pytest.mark.config_index +multicluster_roles = pytest.mark.multicluster_roles diff --git a/ocs_ci/framework/pytest_customization/ocscilib.py b/ocs_ci/framework/pytest_customization/ocscilib.py index 2a95c2eeac0..958ac3fd834 100644 --- a/ocs_ci/framework/pytest_customization/ocscilib.py +++ b/ocs_ci/framework/pytest_customization/ocscilib.py @@ -263,6 +263,16 @@ def pytest_addoption(parser): "(e.g. quay.io/rhceph-dev/ocs-olm-operator:latest-4.3)" ), ) + parser.addoption( + "--acm-version", + dest="acm_version", + help="acm version(e.g. 2.8) to be used for the current run", + ) + parser.addoption( + "--upgrade-acm-version", + dest="upgrade_acm_version", + help="acm version to upgrade(e.g. 2.8), use only with DR upgrade scenario", + ) parser.addoption( "--flexy-env-file", dest="flexy_env_file", help="Path to flexy environment file" ) @@ -567,6 +577,11 @@ def process_cluster_cli_params(config): upgrade_ocs_version = get_cli_param(config, "upgrade_ocs_version") if upgrade_ocs_version: ocsci_config.UPGRADE["upgrade_ocs_version"] = upgrade_ocs_version + # Storing previous version explicitly + # Useful in DR upgrade scenarios + ocsci_config.UPGRADE["pre_upgrade_ocs_version"] = ocsci_config.ENV_DATA[ + "ocs_version" + ] ocs_registry_image = get_cli_param(config, f"ocs_registry_image{suffix}") if ocs_registry_image: ocsci_config.DEPLOYMENT["ocs_registry_image"] = ocs_registry_image @@ -661,6 +676,12 @@ def process_cluster_cli_params(config): if custom_kubeconfig_location: os.environ["KUBECONFIG"] = custom_kubeconfig_location ocsci_config.RUN["kubeconfig"] = custom_kubeconfig_location + acm_version = get_cli_param(config, "--acm-version") + if acm_version: + ocsci_config.ENV_DATA["acm_version"] = acm_version + upgrade_acm_version = get_cli_param(config, "--upgrade-acm-version") + if upgrade_acm_version: + ocsci_config.UPGRADE["upgrade_acm_version"] = upgrade_acm_version def pytest_collection_modifyitems(session, config, items): diff --git a/ocs_ci/ocs/acm_upgrade.py b/ocs_ci/ocs/acm_upgrade.py new file mode 100644 index 00000000000..4b448891d1d --- /dev/null +++ b/ocs_ci/ocs/acm_upgrade.py @@ -0,0 +1,152 @@ +""" +ACM operator upgrade classes and utilities + +""" + +import logging +import tempfile +from pkg_resources import parse_version + +import requests + +from ocs_ci.ocs import constants +from ocs_ci.framework import config +from ocs_ci.ocs.ocp import OCP +from ocs_ci.utility import templating +from ocs_ci.utility.utils import get_ocp_version, get_running_acm_version, run_cmd + + +logger = logging.getLogger(__name__) + + +class ACMUpgrade(object): + def __init__(self): + self.namespace = constants.ACM_HUB_NAMESPACE + self.operator_name = constants.ACM_HUB_OPERATOR_NAME + # Since ACM upgrade happens followed by OCP upgrade in the sequence + # the config would have loaded upgrade parameters rather than pre-upgrade params + # Hence we can't rely on ENV_DATA['acm_version'] for the pre-upgrade version + # we need to dynamically find it + self.version_before_upgrade = self.get_acm_version_before_upgrade() + self.upgrade_version = config.UPGRADE["upgrade_acm_version"] + # In case if we are using registry image + self.acm_registry_image = config.UPGRADE.get("upgrade_acm_registry_image", "") + self.zstream_upgrade = False + + def get_acm_version_before_upgrade(self): + running_acm_version = get_running_acm_version() + return running_acm_version + + def get_parsed_versions(self): + parsed_version_before_upgrade = parse_version(self.version_before_upgrade) + parsed_upgrade_version = parse_version(self.upgrade_version) + + return parsed_version_before_upgrade, parsed_upgrade_version + + def run_upgrade(self): + self.version_change = ( + self.get_parsed_versions()[1] > self.get_parsed_versions()[0] + ) + if not self.version_change: + self.zstream_upgrade = True + # either this would be GA to Unreleased upgrade of same version OR + # GA to unreleased upgrade to higher version + if self.acm_registry_image and self.version_change: + self.upgrade_with_registry() + self.annotate_mch() + run_cmd(f"oc create -f {constants.ACM_BREW_ICSP_YAML}") + self.patch_channel() + else: + # GA to GA + self.upgrade_without_registry() + self.validate_upgrade() + + def upgrade_without_registry(self): + self.patch_channel() + + def patch_channel(self): + """ + GA to GA acm upgrade + + """ + patch = f'\'{{"spec": {{"channel": "release-{self.upgrade_version}"}}}}\'' + self.acm_patch_subscription(patch) + + def upgrade_with_registry(self): + """ + There are 2 scenarios with registry + 1. GA to unreleased same version (ex: 2.8.1 GA to 2.8.2 Unreleased) + 2. GA to unreleased higher version (ex: 2.8.9 GA to 2.9.1 Unreleased) + + """ + if self.acm_registry_image and (not self.version_change): + # This is GA to unreleased: same version + self.create_catalog_source() + else: + # This is GA to unreleased version: upgrade to next version + self.create_catalog_source() + patch = f'\'{{"spec":{{"source": "{constants.ACM_CATSRC_NAME}"}}}}\'' + self.acm_patch_subscription(patch) + + def annotate_mch(self): + annotation = f'\'{{"source": "{constants.ACM_CATSRC_NAME}"}}\'' + annotate_cmd = ( + f"oc -n {constants.ACM_HUB_NAMESPACE} annotate mch multiclusterhub " + f"installer.open-cluster-management.io/mce-subscription-spec={annotation}" + ) + run_cmd(annotate_cmd) + + def acm_patch_subscription(self, patch): + patch_cmd = ( + f"oc -n {constants.ACM_HUB_NAMESPACE} patch sub advanced-cluster-management " + f"-p {patch} --type merge" + ) + run_cmd(patch_cmd) + + def create_catalog_source(self): + logger.info("Creating ACM catalog source") + acm_catsrc = templating.load_yaml(constants.ACM_CATSRC) + if self.acm_registry_image: + acm_catsrc["spec"]["image"] = self.acm_registry_image + else: + # Update catalog source + resp = requests.get(constants.ACM_BREW_BUILD_URL, verify=False) + raw_msg = resp.json()["raw_messages"] + # TODO: Find way to get ocp version before upgrade + version_tag = raw_msg[0]["msg"]["pipeline"]["index_image"][ + f"v{get_ocp_version()}" + ].split(":")[1] + acm_catsrc["spec"]["image"] = ":".jon( + [constants.ACM_BREW_REPO, version_tag] + ) + acm_catsrc["metadata"]["name"] = constants.ACM_CATSRC_NAME + acm_catsrc["spec"]["publisher"] = "grpc" + acm_data_yaml = tempfile.NamedTemporaryFile( + mode="w+", prefix="acm_catsrc", delete=False + ) + templating.dump_data_to_temp_yaml(acm_catsrc, acm_data_yaml.name) + run_cmd(f"oc create -f {acm_data_yaml.name}", timeout=300) + + def validate_upgrade(self): + acm_sub = OCP( + namespace=self.namespace, + resource_name=self.operator_name, + kind="Subscription.operators.coreos.com", + ) + if not self.zstream_upgrade: + acm_prev_channel = f"release-{self.upgrade_version}" + else: + acm_prev_channel = config.ENV_DATA["acm_hub_channel"] + assert acm_sub.get().get("spec").get("channel") == acm_prev_channel + logger.info("Checking ACM status") + acm_mch = OCP( + kind=constants.ACM_MULTICLUSTER_HUB, + namespace=constants.ACM_HUB_NAMESPACE, + ) + acm_mch.wait_for_resource( + condition=constants.STATUS_RUNNING, + resource_name=constants.ACM_MULTICLUSTER_RESOURCE, + column="STATUS", + timeout=720, + sleep=5, + ) diff --git a/ocs_ci/ocs/cluster.py b/ocs_ci/ocs/cluster.py index d4f979a268f..7be4b3b13cf 100644 --- a/ocs_ci/ocs/cluster.py +++ b/ocs_ci/ocs/cluster.py @@ -69,6 +69,19 @@ logger = logging.getLogger(__name__) +class CephClusterMultiCluster(object): + """ + TODO: Implement this class later + This class will be used in case of multicluster scenario + and current cluster is ACM hence this cluster should point to + the ODF which is not in current context + + """ + + def __init__(self, cluster_conf=None): + pass + + class CephCluster(object): """ Handles all cluster related operations from ceph perspective @@ -84,11 +97,17 @@ class CephCluster(object): namespace (str): openshift Namespace where this cluster lives """ - def __init__(self): + def __init__(self, cluster_config=None): """ Cluster object initializer, this object needs to be initialized after cluster deployment. However its harmless to do anywhere. """ + if cluster_config: + logger.info( + "CephClusterMulticluster will be used to handle multicluster case" + ) + return CephClusterMultiCluster() + if config.ENV_DATA["mcg_only_deployment"] or ( config.ENV_DATA.get("platform") == constants.FUSIONAAS_PLATFORM and config.ENV_DATA["cluster_type"].lower() == "consumer" @@ -1035,6 +1054,18 @@ def delete_blockpool(self, pool_name): self.RBD.exec_oc_cmd(f"patch {patch}") +class MulticlusterCephHealthMonitor(object): + # TODO: This will be a placeholder for now + def __init__(self, ceph_cluster=None): + pass + + def __enter__(self): + pass + + def __exit__(self, exception_type, value, traceback): + pass + + class CephHealthMonitor(threading.Thread): """ Context manager class for monitoring ceph health status of CephCluster. @@ -1052,6 +1083,8 @@ def __init__(self, ceph_cluster, sleep=5): sleep (int): Number of seconds to sleep between health checks. """ + if isinstance(ceph_cluster, CephClusterMultiCluster): + return MulticlusterCephHealthMonitor() self.ceph_cluster = ceph_cluster self.sleep = sleep self.health_error_status = None diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 2c14f51b679..4b9413efc75 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -1673,6 +1673,12 @@ ORDER_BEFORE_OCP_UPGRADE = 20 ORDER_OCP_UPGRADE = 30 ORDER_AFTER_OCP_UPGRADE = 40 +# Multicluster orchestrator +ORDER_MCO_UPGRADE = 42 +# DR Hub operator +ORDER_DR_HUB_UPGRADE = 44 +# ACM Operator +ORDER_ACM_UPGRADE = 46 ORDER_BEFORE_OCS_UPGRADE = 50 ORDER_OCS_UPGRADE = 60 ORDER_AFTER_OCS_UPGRADE = 70 @@ -1715,6 +1721,14 @@ ARO_MASTER_SUBNET_ADDRESS_PREFIXES = "10.0.0.0/23" CLIENT_OPERATOR_CONFIGMAP = "ocs-client-operator-config" CLIENT_OPERATOR_CSI_IMAGES = "ocs-client-operator-csi-images" +MCO_SUBSCRIPTION = "odf-multicluster-orchestrator" +DR_HUB_OPERATOR_SUBSCRIPTION = ( + "odr-hub-operator-stable-PLACEHOLDER-redhat-operators-openshift-marketplace" +) +DR_HUB_OPERATOR_SUBSCRIPTION_LABEL = ( + "operators.coreos.com/odr-hub-operator.openshift-operators" +) +DR_CLUSTER_OPERATOR_SUBSCRIPTION = "ramen-dr-cluster-subscription" # UI Deployment constants HTPASSWD_SECRET_NAME = "htpass-secret" @@ -2699,14 +2713,22 @@ SUBMARINER_DOWNSTREAM_UNRELEASED = os.path.join( TEMPLATE_MULTICLUSTER_DIR, "submariner_downstream_unreleased_catsrc.yaml" ) +ACM_CATSRC = SUBMARINER_DOWNSTREAM_UNRELEASED +ACM_CATSRC_NAME = "acm-catalogsource" # We need to append version string at the end of this url SUBMARINER_DOWNSTREAM_UNRELEASED_BUILD_URL = ( "https://datagrepper.engineering.redhat.com/raw?topic=/topic/" "VirtualTopic.eng.ci.redhat-container-image.pipeline.complete" "&rows_per_page=25&delta=1296000&contains=submariner-operator-bundle-container-v" ) +ACM_BREW_BUILD_URL = ( + "https://datagrepper.engineering.redhat.com/raw?topic=/topic/" + "VirtualTopic.eng.ci.redhat-container-image.pipeline.complete" + "&rows_per_page=25&delta=1296000&contains=acm" +) SUBMARINER_BREW_REPO = "brew.registry.redhat.io/rh-osbs/iib" SUBCTL_DOWNSTREAM_URL = "registry.redhat.io/rhacm2/" +ACM_BREW_REPO = SUBMARINER_BREW_REPO # Multicluster related @@ -2745,6 +2767,7 @@ SUBMARINER_DOWNSTREAM_BREW_ICSP = os.path.join( TEMPLATE_DIR, "acm-deployment", "submariner_downstream_brew_icsp.yaml" ) +ACM_BREW_ICSP_YAML = os.path.join(TEMPLATE_DIR, "acm-deployment", "acm_brew_icsp.yaml") ACM_HUB_UNRELEASED_PULL_SECRET_TEMPLATE = "pull-secret.yaml.j2" ACM_ODF_MULTICLUSTER_ORCHESTRATOR_RESOURCE = "odf-multicluster-orchestrator" ACM_ODR_HUB_OPERATOR_RESOURCE = "odr-hub-operator" @@ -3090,3 +3113,5 @@ EDIT = "edit" DELETE = "delete" MACHINE_POOL_ACTIONS = [CREATE, EDIT, DELETE] +# MDR multicluster roles +MDR_ROLES = ["ActiveACM", "PassiveACM", "PrimaryODF", "SecondaryODF"] diff --git a/ocs_ci/ocs/defaults.py b/ocs_ci/ocs/defaults.py index 426e1da7201..437a4f350b0 100644 --- a/ocs_ci/ocs/defaults.py +++ b/ocs_ci/ocs/defaults.py @@ -58,6 +58,9 @@ OCS_CLIENT_OPERATOR_NAME = "ocs-client-operator" CEPHCSI_OPERATOR = "cephcsi-operator" ODF_DEPENDENCIES = "odf-dependencies" +MCO_OPERATOR_NAME = "odf-multicluster-orchestrator" +DR_HUB_OPERATOR_NAME = "odr-hub-operator" +DR_CLUSTER_OPERATOR_NAME = "odr-cluster-operator" # Noobaa S3 bucket website configurations website_config = { diff --git a/ocs_ci/ocs/dr_upgrade.py b/ocs_ci/ocs/dr_upgrade.py new file mode 100644 index 00000000000..1d31c3f5e7f --- /dev/null +++ b/ocs_ci/ocs/dr_upgrade.py @@ -0,0 +1,343 @@ +""" +All DR operators upgrades implemented here ex: MulticlusterOrchestrator, Openshift DR operator + +""" + +import logging +import time + +from ocs_ci.framework import config +from ocs_ci.ocs.exceptions import TimeoutException +from ocs_ci.ocs.ocp import OCP +from ocs_ci.ocs.ocs_upgrade import OCSUpgrade +from ocs_ci.ocs import constants +from ocs_ci.ocs import defaults +from ocs_ci.deployment.helpers.external_cluster_helpers import ( + ExternalCluster, + get_external_cluster_client, +) +from ocs_ci.ocs.resources import pod +from ocs_ci.ocs.resources.csv import CSV, check_all_csvs_are_succeeded +from ocs_ci.ocs.resources.install_plan import wait_for_install_plan_and_approve +from ocs_ci.ocs.resources.packagemanifest import PackageManifest +from ocs_ci.utility.utils import TimeoutSampler + + +log = logging.getLogger(__name__) + + +class DRUpgrade(OCSUpgrade): + """ + Base class for all DR operator upgrades + + """ + + def __init__( + self, + namespace=constants.OPENSHIFT_OPERATORS, + version_before_upgrade=None, + ocs_registry_image=None, + upgrade_in_current_source=config.UPGRADE.get( + "upgrade_in_current_source", False + ), + resource_name=None, + ): + if not version_before_upgrade: + if config.PREUPGRADE_CONFIG.get("ENV_DATA").get("ocs_version", ""): + version_before_upgrade = config.PREUPGRADE_CONFIG["ENV_DATA"].get( + "ocs_version" + ) + else: + version_before_upgrade = config.ENV_DATA.get("ocs_version") + if not ocs_registry_image: + ocs_registry_image = config.UPGRADE.get("upgrade_ocs_registry_image") + self.external_cluster = None + self.operator_name = None + self.subscription_name = None + self.pre_upgrade_data = dict() + self.post_upgrade_data = dict() + self.namespace = namespace + # Upgraded phases [pre_upgrade, post_upgrade] + self.upgrade_phase = "pre_upgrade" + if resource_name: + self.resource_name = resource_name + + super().__init__( + namespace, + version_before_upgrade, + ocs_registry_image, + upgrade_in_current_source, + ) + self.upgrade_version = self.get_upgrade_version() + + def run_upgrade(self): + assert self.get_parsed_versions()[1] >= self.get_parsed_versions()[0], ( + f"Version you would like to upgrade to: {self.upgrade_version} " + f"is not higher or equal to the version you currently running: " + f"{self.version_before_upgrade}" + ) + + # create external cluster object + if config.DEPLOYMENT["external_mode"]: + host, user, password, ssh_key = get_external_cluster_client() + self.external_cluster = ExternalCluster(host, user, password, ssh_key) + self.csv_name_pre_upgrade = self.get_csv_name_pre_upgrade( + resource_name=self.resource_name + ) + self.pre_upgrade_images = self.get_pre_upgrade_image(self.csv_name_pre_upgrade) + self.load_version_config_file(self.upgrade_version) + + self.channel = self.set_upgrade_channel(resource_name=self.operator_name) + self.set_upgrade_images() + # TODO: When we have to support colocated ACM on Managed cluster node + # we need to update subscriptions individually for DR operator as we don't want + # to upgrade ODF at the time of DR operator (MCO, DR Hub), ODF would follow the upgrade + # of DR operators + self.update_subscription(self.channel, self.subscription_name, self.namespace) + # In the case upgrade is not from 4.8 to 4.9 and we have manual approval strategy + # we need to wait and approve install plan, otherwise it's approved in the + # subscribe_ocs method. + subscription_plan_approval = config.DEPLOYMENT.get("subscription_plan_approval") + if subscription_plan_approval == "Manual": + wait_for_install_plan_and_approve(config.ENV_DATA["cluster_namespace"]) + + for sample in TimeoutSampler( + timeout=725, + sleep=5, + func=self.check_if_upgrade_completed, + channel=self.channel, + csv_name_pre_upgrade=self.csv_name_pre_upgrade, + ): + try: + if sample: + log.info("Upgrade success!") + break + except TimeoutException: + raise TimeoutException("No new CSV found after upgrade!") + + def update_subscription( + self, channel, subscription_name, namespace=constants.OPENSHIFT_OPERATORS + ): + subscription = OCP( + resource_name=subscription_name, + kind="subscription.operators.coreos.com", + # namespace could be different on managed clusters + # TODO: Handle different namespaces + namespace=namespace, + ) + current_source = subscription.data["spec"]["source"] + log.info(f"Current source: {current_source}") + mco_source = ( + current_source + if self.upgrade_in_current_source + else constants.OPERATOR_CATALOG_SOURCE_NAME + ) + patch_subscription_cmd = ( + f"patch subscription.operators.coreos.com {subscription_name} " + f'-n {self.namespace} --type merge -p \'{{"spec":{{"channel": ' + f'"{self.channel}", "source": "{mco_source}"}}}}\'' + ) + subscription.exec_oc_cmd(patch_subscription_cmd, out_yaml_format=False) + # Deliberately sleeping here as there are so many places down the line + # where ocs-ci will check CSV and it fails as changes take some time to reflect + time.sleep(60) + + def check_if_upgrade_completed(self, channel, csv_name_pre_upgrade): + """ + Check if DR operator finished it's upgrade + + Args: + channel: (str): DR operator subscription channel + csv_name_pre_upgrade: (str): DR operator name + + Returns: + bool: True if upgrade completed, False otherwise + + """ + if not check_all_csvs_are_succeeded(self.namespace): + log.warning("One of CSV is still not upgraded!") + return False + package_manifest = PackageManifest( + resource_name=self.operator_name, + subscription_plan_approval=self.subscription_plan_approval, + ) + csv_name_post_upgrade = package_manifest.get_current_csv(channel) + if csv_name_post_upgrade == csv_name_pre_upgrade: + log.info(f"CSV is still: {csv_name_post_upgrade}") + return False + else: + log.info(f"CSV now upgraded to: {csv_name_post_upgrade}") + return True + + def validate_upgrade(self): + # In case of both MCO and DRhub operator, validation steps are similar + # just the resource names changes + assert ( + self.post_upgrade_data.get("pod_status", "") == "Running" + ), f"Pod {self.pod_name_pattern} not in Running state post upgrade" + assert ( + self.post_upgrade_data.get("version", "") + != self.pre_upgrade_data["version"] + ), "CSV version not upgraded" + check_all_csvs_are_succeeded(namespace=self.namespace) + + def collect_data(self): + """ + Collect DR operator related pods and csv data + """ + pod_data = pod.get_all_pods(namespace=self.namespace) + for p in pod_data: + if self.pod_name_pattern in p.get()["metadata"]["name"]: + pod_obj = OCP( + namespace=self.namespace, + resource_name=p.get()["metadata"]["name"], + kind="Pod", + ) + if self.upgrade_phase == "pre_upgrade": + self.pre_upgrade_data["age"] = pod_obj.get_resource( + resource_name=p.get()["metadata"]["name"], column="AGE" + ) + self.pre_upgrade_data["pod_status"] = pod_obj.get_resource_status( + resource_name=p.get()["metadata"]["name"] + ) + if self.upgrade_phase == "post_upgrade": + self.post_upgrade_data["age"] = pod_obj.get_resource( + resource_name=p.get()["metadata"]["name"], column="AGE" + ) + self.post_upgrade_data["pod_status"] = pod_obj.get_resource_status( + resource_name=p.get()["metadata"]["name"] + ) + + # get pre-upgrade csv + csv_objs = CSV(namespace=self.namespace) + for csv in csv_objs.get()["items"]: + if self.operator_name in csv["metadata"]["name"]: + csv_obj = CSV( + namespace=self.namespace, resource_name=csv["metadata"]["name"] + ) + if self.upgrade_phase == "pre_upgrade": + self.pre_upgrade_data["version"] = csv_obj.get_resource( + resource_name=csv_obj.resource_name, column="VERSION" + ) + try: + self.pre_upgrade_data["version"] + except KeyError: + log.error( + f"Couldn't capture Pre-upgrade CSV version for {self.operator_name}" + ) + if self.upgrade_phase == "post_upgrade": + if self.upgrade_version in csv["metadata"]["name"]: + self.post_upgrade_data["version"] = csv_obj.get_resource( + resource_name=csv_obj.resource_name, column="VERSION" + ) + try: + self.post_upgrade_data["version"] + except KeyError: + log.error( + f"Couldn't capture Post upgrade CSV version for {self.operator_name}" + ) + # Make sure all csvs are in succeeded state + check_all_csvs_are_succeeded(namespace=self.namespace) + + +class MultiClusterOrchestratorUpgrade(DRUpgrade): + """ + A class to handle ODF MCO operator upgrades + + """ + + def __init__(self): + super().__init__(resource_name=defaults.MCO_OPERATOR_NAME) + self.operator_name = defaults.MCO_OPERATOR_NAME + self.subscription_name = constants.MCO_SUBSCRIPTION + self.pod_name_pattern = "odfmo-controller-manager" + + def run_upgrade(self): + # Collect some pre-upgrade data for comparision after the upgrade + self.collect_data() + assert ( + self.pre_upgrade_data.get("pod_status", "") == "Running" + ), "odfmo-controller pod is not in Running status" + super().run_upgrade() + self.upgrade_phase = "post_upgrade" + self.collect_data() + self.validate_upgrade() + + def validate_upgrade(self): + # validate csv VERSION, PHASE==Succeeded + # validate odfmo-controller-manager pods age + super().validate_upgrade() + + +class DRHubUpgrade(DRUpgrade): + """ + A class to handle DR Hub operator upgrades + + """ + + def __init__(self): + super().__init__(resource_name=defaults.DR_HUB_OPERATOR_NAME) + self.operator_name = defaults.DR_HUB_OPERATOR_NAME + for sample in TimeoutSampler( + 300, 10, OCP, kind=constants.SUBSCRIPTION_COREOS, namespace=self.namespace + ): + subscriptions = sample.get().get("items", []) + for subscription in subscriptions: + found_subscription_name = subscription.get("metadata", {}).get( + "name", "" + ) + if defaults.DR_HUB_OPERATOR_NAME in found_subscription_name: + log.info(f"Subscription found: {found_subscription_name}") + self.subscription_name = found_subscription_name + break + if not self.subscription_name: + log.error( + f"Couldn't find the subscription for {defaults.DR_HUB_OPERATOR_NAME}" + ) + self.pod_name_pattern = "ramen-hub-operator" + + def run_upgrade(self): + self.collect_data() + assert ( + self.pre_upgrade_data.get("pod_status", "") == "Running" + ), "ramen-hub-operator pod is not in Running status" + super().run_upgrade() + self.upgrade_phase = "post_upgrade" + self.collect_data() + self.validate_upgrade() + + def validate_upgrade(self): + # validate csv odr-hub-operator.v4.13.5-rhodf VERSION, PHASE + # validate pod/ramen-hub-operator- + super().validate_upgrade() + + +class DRClusterOperatorUpgrade(DRUpgrade): + """ + A class to handle DR Cluster operator upgrades + + """ + + def __init__(self): + super().__init__( + resource_name=defaults.DR_CLUSTER_OPERATOR_NAME, + namespace=constants.OPENSHIFT_DR_SYSTEM_NAMESPACE, + ) + self.operator_name = defaults.DR_CLUSTER_OPERATOR_NAME + self.subscription_name = constants.DR_CLUSTER_OPERATOR_SUBSCRIPTION + self.pod_name_pattern = "ramen-dr-cluster-operator" + + def run_upgrade(self): + self.collect_data() + assert ( + self.pre_upgrade_data.get("pod_status", "") == "Running" + ), "ramen-dr-operator pod is not in Running status" + super().run_upgrade() + self.upgrade_phase = "post_upgrade" + self.collect_data() + self.validate_upgrade() + + def validate_upgrade(self): + # validate csv odr-cluster-operator.v4.13.5-rhodf VERSION, PHASE + # validate pod/ramen-dr-cluster-operator- + return super().validate_upgrade() diff --git a/ocs_ci/ocs/ocs_upgrade.py b/ocs_ci/ocs/ocs_upgrade.py index 3e30b6ea063..8532293f76d 100644 --- a/ocs_ci/ocs/ocs_upgrade.py +++ b/ocs_ci/ocs/ocs_upgrade.py @@ -31,8 +31,12 @@ from ocs_ci.ocs.ocp import get_images, OCP from ocs_ci.ocs.node import get_nodes from ocs_ci.ocs.resources.catalog_source import CatalogSource, disable_specific_source -from ocs_ci.ocs.resources.csv import CSV, check_all_csvs_are_succeeded from ocs_ci.ocs.resources.daemonset import DaemonSet +from ocs_ci.ocs.resources.csv import ( + CSV, + check_all_csvs_are_succeeded, + get_csvs_start_with_prefix, +) from ocs_ci.ocs.resources.install_plan import wait_for_install_plan_and_approve from ocs_ci.ocs.resources.pod import get_noobaa_pods, verify_pods_upgraded from ocs_ci.ocs.resources.packagemanifest import ( @@ -62,6 +66,7 @@ from ocs_ci.ocs.exceptions import ( TimeoutException, ExternalClusterRGWAdminOpsUserException, + CSVNotFound, ) from ocs_ci.ocs.ui.base_ui import logger, login_ui from ocs_ci.ocs.ui.views import locators, ODF_OPERATOR @@ -365,23 +370,28 @@ def load_version_config_file(self, upgrade_version): config.REPORTING["ocs_must_gather_image"] = must_gather_image config.REPORTING["ocs_must_gather_latest_tag"] = must_gather_tag - def get_csv_name_pre_upgrade(self): + def get_csv_name_pre_upgrade(self, resource_name=OCS_OPERATOR_NAME): """ - Getting OCS operator name as displayed in CSV + Get pre-upgrade CSV name - Returns: - str: OCS operator name, as displayed in CSV + Ealier we used to depend on packagemanifest to find the pre-upgrade + csv name. Due to issues in catalogsource where csv names were not shown properly once + catalogsource for upgrade version has been created, we are taking new approach of + finding csv name from csv list and also look for pre-upgrade ocs version for finding out + the actual csv """ - operator_selector = get_selector_for_ocs_operator() - package_manifest = PackageManifest( - resource_name=OCS_OPERATOR_NAME, - selector=operator_selector, - subscription_plan_approval=self.subscription_plan_approval, - ) - channel = config.DEPLOYMENT.get("ocs_csv_channel") - return package_manifest.get_current_csv(channel) + csv_name = None + csv_list = get_csvs_start_with_prefix(resource_name, namespace=self.namespace) + for csv in csv_list: + if resource_name in csv.get("metadata").get("name"): + if config.PREUPGRADE_CONFIG.get("ENV_DATA").get( + "ocs_version" + ) in csv.get("metadata").get("name"): + csv_name = csv.get("metadata").get("name") + return csv_name + raise CSVNotFound(f"No preupgrade CSV found for {resource_name}") def get_pre_upgrade_image(self, csv_name_pre_upgrade): """ @@ -403,7 +413,7 @@ def get_pre_upgrade_image(self, csv_name_pre_upgrade): ) return get_images(csv_pre_upgrade.get()) - def set_upgrade_channel(self): + def set_upgrade_channel(self, resource_name=OCS_OPERATOR_NAME): """ Wait for the new package manifest for upgrade. @@ -413,7 +423,7 @@ def set_upgrade_channel(self): """ operator_selector = get_selector_for_ocs_operator() package_manifest = PackageManifest( - resource_name=OCS_OPERATOR_NAME, + resource_name=resource_name, selector=operator_selector, ) package_manifest.wait_for_resource() @@ -435,9 +445,14 @@ def update_subscription(self, channel): subscription_name = constants.ODF_SUBSCRIPTION else: subscription_name = constants.OCS_SUBSCRIPTION + kind_name = ( + "subscription.operators.coreos.com" + if config.multicluster + else "subscription" + ) subscription = OCP( resource_name=subscription_name, - kind="subscription", + kind=kind_name, namespace=config.ENV_DATA["cluster_namespace"], ) current_ocs_source = subscription.data["spec"]["source"] @@ -448,7 +463,7 @@ def update_subscription(self, channel): else constants.OPERATOR_CATALOG_SOURCE_NAME ) patch_subscription_cmd = ( - f"patch subscription {subscription_name} " + f"patch {kind_name} {subscription_name} " f'-n {self.namespace} --type merge -p \'{{"spec":{{"channel": ' f'"{channel}", "source": "{ocs_source}"}}}}\'' ) @@ -483,7 +498,13 @@ def check_if_upgrade_completed(self, channel, csv_name_pre_upgrade): log.info(f"CSV now upgraded to: {csv_name_post_upgrade}") return True - def get_images_post_upgrade(self, channel, pre_upgrade_images, upgrade_version): + def get_images_post_upgrade( + self, + channel, + pre_upgrade_images, + upgrade_version, + resource_name=OCS_OPERATOR_NAME, + ): """ Checks if all images of OCS cluster upgraded, and return list of all images if upgrade success @@ -499,7 +520,7 @@ def get_images_post_upgrade(self, channel, pre_upgrade_images, upgrade_version): """ operator_selector = get_selector_for_ocs_operator() package_manifest = PackageManifest( - resource_name=OCS_OPERATOR_NAME, + resource_name=resource_name, selector=operator_selector, subscription_plan_approval=self.subscription_plan_approval, ) diff --git a/ocs_ci/ocs/utils.py b/ocs_ci/ocs/utils.py index 494253dcbbc..f2dfbfdee96 100644 --- a/ocs_ci/ocs/utils.py +++ b/ocs_ci/ocs/utils.py @@ -1609,6 +1609,20 @@ def get_non_acm_cluster_config(): return non_acm_list +def get_non_acm_cluster_indexes(): + """ + Get config index of all non-acm clusters + + Returns: + list: of integer indexes of non-acm clusters + + """ + non_acm_indexes = list() + for cluster in get_non_acm_cluster_config(): + non_acm_indexes.append(cluster.MULTICLUSTER["multicluster_index"]) + return non_acm_indexes + + def get_all_acm_indexes(): """ Get indexes fro all ACM clusters @@ -1758,6 +1772,14 @@ def cluster_config_reindex(): ocsci_config.switch_acm_ctx() +def get_primary_cluster_index(): + """ + Get the index of primary cluster in case of multicluster scenario + """ + pcluster = get_primary_cluster_config() + return pcluster.MULTICLUSTER["multicluster_index"] + + def thread_init_class(class_init_operations, shutdown): if len(class_init_operations) > 0: executor = ThreadPoolExecutor(max_workers=len(class_init_operations)) diff --git a/ocs_ci/templates/acm-deployment/acm_brew_icsp.yaml b/ocs_ci/templates/acm-deployment/acm_brew_icsp.yaml new file mode 100644 index 00000000000..312328c85b1 --- /dev/null +++ b/ocs_ci/templates/acm-deployment/acm_brew_icsp.yaml @@ -0,0 +1,21 @@ +apiVersion: operator.openshift.io/v1alpha1 +kind: ImageContentSourcePolicy +metadata: + name: image-policy-brew +spec: + repositoryDigestMirrors: + - mirrors: + - brew.registry.redhat.io/rh-osbs/rhacm2 + source: registry.redhat.io/rhacm2 + - mirrors: + - brew.registry.redhat.io/rh-osbs + source: registry-proxy.engineering.redhat.com/rh-osbs + - mirrors: + - brew.registry.redhat.io/rh-osbs/multicluster-engine + source: registry.redhat.io/multicluster-engine + - mirrors: + - registry.redhat.io/rhacm2 + source: registry.stage.redhat.io/rhacm2 + - mirrors: + - registry.redhat.io/multicluster-engine + source: registry.stage.redhat.io/multicluster-engine diff --git a/ocs_ci/utility/multicluster.py b/ocs_ci/utility/multicluster.py new file mode 100644 index 00000000000..13153743572 --- /dev/null +++ b/ocs_ci/utility/multicluster.py @@ -0,0 +1,250 @@ +""" +All multicluster specific utility functions and classes can be here + +""" + +from abc import ABC, abstractmethod +import logging + +from ocs_ci.framework import config as ocsci_config +from ocs_ci.ocs.utils import ( + get_non_acm_cluster_indexes, + get_primary_cluster_index, + get_active_acm_index, + get_all_acm_indexes, +) +from ocs_ci.ocs.constants import MDR_ROLES + +log = logging.getLogger(__name__) + + +class MultiClusterUpgradeParametrize(ABC): + """ + This base class abstracts upgrade parametrization for multicluster scenarios: MDR, RDR and Managed service + + """ + + MULTICLUSTER_UPGRADE_MARKERS = [ + "pre_upgrade", + "pre_ocp_upgrade", + "ocp_upgrade", + "post_ocp_upgrade", + "mco_upgrade", + "dr_hub_upgrade", + "dr_cluster_operator_upgrade", + "acm_upgrade", + "pre_ocs_upgrade", + "ocs_upgrade", + "post_ocs_upgrade", + "post_upgrade", + ] + + def __init__(self): + self.roles = [] + # List of zones which are participating in this multicluster setup + self.zones = self.get_zone_info() + self.zones.sort() + self.zone_base_rank = 100 + # Each zone will be assigned with a rank + # This rank comes handy when we have to order the tests + self.zone_ranks = {} + + @abstractmethod + def get_roles(self, metafunc): + """ + should be overridden in the child class + Look for specific role markers based on multicluster scenario + + Args: + metafunc: Pytest metafunc fixture object + """ + pass + + def generate_zone_ranks(self): + """ + For each zone we would be generating the ranks, we will add the zone's respective indexes + to the base rank values which keeps the zone ranks apart and create spaces (for ranks) + in between to accomodate other tests + + """ + for i in range(len(self.zones)): + self.zone_ranks[f"{self.zones[i]}"] = ( + self.zone_base_rank + i * self.zone_base_rank + ) + log.info(f"zone ranks = {self.zone_ranks}") + + @abstractmethod + def generate_role_ranks(self): + """ + Based on the multicluster scenario, child class should generate the corresponding + role ranks. Roles are specific to multicluster scenarios + + """ + pass + + @abstractmethod + def generate_pytest_parameters(self, metafunc, roles): + """ + should be overridden in the child class. + This will be called for every testcase parametrization + + """ + pass + + def get_zone_info(self): + """ + Get the list of participating zones + + """ + zones = set() + for c in ocsci_config.clusters: + zones.add(c.ENV_DATA["zone"]) + return list(zones) + + +class MDRClusterUpgradeParametrize(MultiClusterUpgradeParametrize): + """ + This child class handles MDR upgrade scenario specific pytest parametrization + + """ + + def __init__(self): + super().__init__() + self.roles_to_param_tuples = dict() + self.roles_to_config_index_map = dict() + self.zone_role_map = dict() + self.all_mdr_roles = MDR_ROLES + + def config_init(self): + self.generate_zone_ranks() + self.generate_role_ranks() + self.generate_config_index_map() + # Reverse mapping of cluster's index to its role + self.index_to_role = { + index: role for role, index in self.roles_to_config_index_map.items() + } + self.generate_zone_role_map() + self.generate_role_to_param_tuple_map() + + def generate_config_index_map(self): + """ + Generate config indexes for all the MDRs cluster roles + ex: {"ActiveACM": 0, "PassiveACM": 2, "PrimaryODF": 1, "SecondaryODF": 3} + + """ + for cluster in ocsci_config.clusters: + cluster_index = cluster.MULTICLUSTER["multicluster_index"] + if cluster_index == get_active_acm_index(): + self.roles_to_config_index_map["ActiveACM"] = cluster_index + elif cluster_index == get_primary_cluster_index(): + self.roles_to_config_index_map["PrimaryODF"] = cluster_index + elif cluster_index in get_all_acm_indexes(): + # We would have already ruled out the ActiveACM in the first 'if' + self.roles_to_config_index_map["PassiveACM"] = cluster_index + else: + # Only option left is secondary odf + self.roles_to_config_index_map["SecondaryODF"] = cluster_index + + def generate_role_ranks(self): + """ + Based on current roles for MDR : ActiveACM:1, PassiveACM:1, PrimaryODF:2, SecondaryODF: 2 + + """ + # For now we will stick to this convention + self.role_ranks = { + "ActiveACM": 1, + "PassiveACM": 1, + "PrimaryODF": 2, + "SecondaryODF": 2, + } + + def generate_zone_role_map(self): + """ + Generate a map of Cluster's role vs zone in which clusters are located + ex: {"ActiveACM": 'a', "PassiveACM": 'b', "PrimaryODF": 'a'} + """ + for crole, cindex in self.roles_to_config_index_map.items(): + czone = ocsci_config.clusters[cindex].ENV_DATA.get("zone") + if czone: + self.zone_role_map[crole] = czone + + def generate_role_to_param_tuple_map(self): + """ + For each of the MDRs applicable roles store a tuple (zone_rank, role_rank, config_index) + ex: {"ActiveACM": (1, 1, 0), "PassiveACM": (2, 1, 2), "PrimaryODF": (1, 2, 1), "SecondarODF": (2, 2, 3)} + + """ + for role in self.all_mdr_roles: + self.roles_to_param_tuples[role] = ( + self.zone_ranks[self.zone_role_map[role]], + self.role_ranks[role], + self.roles_to_config_index_map[role], + ) + + def get_pytest_params_tuple(self, role): + """ + Get a tuple of parameters applicable to the given role + For ex: if role is 'ActiveACM', then get a tuple which is applicable to + that role. If the role is 'all' then we will get tuples of parameter + for all the roles applicable + Parmeter tuples looks like (zone_rank, role_rank, config_index) for a given role + + """ + param_list = None + if role.startswith("mdr-all"): + param_list = self.get_mdr_all_param_tuples(role) + else: + param_list = [self.roles_to_param_tuples[role]] + return param_list + + def get_mdr_all_param_tuples(self, role): + if "mdr-all-ocp" in role: + return self.get_all_roles_to_param_tuples() + elif "mdr-all-odf" in role: + return self.get_all_odf_roles_to_param_tuple() + elif "mdr-all-acm" in role: + return self.get_all_acm_roles_to_param_tuple() + + def get_all_acm_roles_to_param_tuple(self): + params_list = list() + for i in get_all_acm_indexes(): + params_list.append(self.roles_to_param_tuples[self.index_to_role[i]]) + return params_list + + def get_all_odf_roles_to_param_tuple(self): + params_list = list() + for i in get_non_acm_cluster_indexes(): + params_list.append(self.roles_to_param_tuples[self.index_to_role[i]]) + return params_list + + def get_all_roles_to_param_tuples(self): + param_list = list() + for t in self.roles_to_param_tuples.values(): + param_list.append(t) + return param_list + + def get_roles(self, metafunc): + # Return a list of roles applicable to the current test + for marker in metafunc.definition.iter_markers(): + if marker.name == "multicluster_roles": + return marker.args[0] + + def generate_pytest_parameters(self, metafunc, roles): + """ + We will have to parametrize the test based on the MDR roles to which the test is applicable to, + Parameters will be a tuple of (zone_rank, role_rank, config_index) + + """ + pytest_params = [] + for role in roles: + pytest_params.extend(self.get_pytest_params_tuple(role)) + return pytest_params + + +multicluster_upgrade_parametrizer = {"metro-dr": MDRClusterUpgradeParametrize} + + +def get_multicluster_upgrade_parametrizer(): + return multicluster_upgrade_parametrizer[ + ocsci_config.MULTICLUSTER["multicluster_mode"] + ]() diff --git a/ocs_ci/utility/utils.py b/ocs_ci/utility/utils.py index 71fa4db195e..b733ac463f9 100644 --- a/ocs_ci/utility/utils.py +++ b/ocs_ci/utility/utils.py @@ -5236,3 +5236,16 @@ def extract_image_urls(string_data): # Find all URLs that start with 'registry.redhat.io' image_urls = re.findall(r'registry\.redhat\.io[^\s"]+', string_data) return image_urls + + +def is_z_stream_upgrade(): + """ + Check whether this is a z-stream upgrade scenario + + Returns: + bool: True if its a z-stream upgrade else False + + """ + return config.UPGRADE.get("pre_upgrade_ocs_version", "") == config.UPGRADE.get( + "upgrade_ocs_version", "" + ) diff --git a/tests/conftest.py b/tests/conftest.py index 70993c5cb3a..fbcb163db76 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,7 @@ from math import floor from shutil import copyfile, rmtree from functools import partial +from copy import deepcopy import boto3 import yaml @@ -33,6 +34,7 @@ ignore_leftover_label, upgrade_marks, ignore_resource_not_found_error_label, + config_index, ) from ocs_ci.helpers.proxy import update_container_with_proxy_env @@ -150,6 +152,10 @@ from ocs_ci.utility.prometheus import PrometheusAPI from ocs_ci.utility.reporting import update_live_must_gather_image from ocs_ci.utility.retry import retry +from ocs_ci.utility.multicluster import ( + get_multicluster_upgrade_parametrizer, + MultiClusterUpgradeParametrize, +) from ocs_ci.utility.uninstall_openshift_logging import uninstall_cluster_logging from ocs_ci.utility.utils import ( ceph_health_check, @@ -340,6 +346,30 @@ def export_squad_marker_to_csv(items, filename=None): log.info("%s tests require action across %s files", num_tests, num_files) +def pytest_generate_tests(metafunc): + """ + This hook handles pytest dynamic pytest parametrization of tests related to + Multicluster scenarios + + Args: + metafunc (pytest fixture): metafunc pytest object to access info about + test function and its parameters + + """ + # For now we are only dealing with multicluster scenarios in this hook + if ocsci_config.multicluster: + upgrade_parametrizer = get_multicluster_upgrade_parametrizer() + # for various roles which are applicable to current test wrt multicluster, for ex: ACM, primary, secondary etc + roles = None + roles = upgrade_parametrizer.get_roles(metafunc) + if roles: + upgrade_parametrizer.config_init() + params = upgrade_parametrizer.generate_pytest_parameters(metafunc, roles) + for marker in metafunc.definition.iter_markers(): + if marker.name in upgrade_parametrizer.MULTICLUSTER_UPGRADE_MARKERS: + metafunc.parametrize("zone_rank, role_rank, config_index", params) + + def pytest_collection_modifyitems(session, config, items): """ A pytest hook to filter out skipped tests satisfying @@ -445,6 +475,63 @@ def pytest_collection_modifyitems(session, config, items): f" UI is not supported on {ocsci_config.ENV_DATA['platform'].lower()}" ) items.remove(item) + # If multicluster upgrade scenario + if ocsci_config.multicluster and ocsci_config.UPGRADE.get("upgrade", False): + for item in items: + if ( + list( + set([i.name for i in item.iter_markers()]).intersection( + (MultiClusterUpgradeParametrize.MULTICLUSTER_UPGRADE_MARKERS) + ) + ) + and ocsci_config.multicluster + ): + if getattr(item, "callspec", ""): + zone_rank = item.callspec.params["zone_rank"] + role_rank = item.callspec.params["role_rank"] + else: + continue + markers_update = [] + item_markers = copy.copy(item.own_markers) + if not item_markers: + # If testcase is in a class then own_markers will be empty + # hence we need to check item.instance.pytestmark + item_markers = copy.copy(item.instance.pytestmark) + for m in item_markers: + # fetch already marked 'order' value + if m.name == "order": + val = m.args[0] + # Sum of the base order value along with + # zone in which the cluster is and the cluster's role rank + # determines the order in which tests need to be executed + # Lower the sum, higher the rank hence it gets prioritized early + # in the test execution sequence + newval = val + zone_rank + role_rank + log.info(f"ORIGINAL = {val}, NEW={newval}") + markers_update.append((pytest.mark.order, newval)) + if item.own_markers: + item.own_markers.remove(m) + break + markers_update.append( + (config_index, item.callspec.params["config_index"]) + ) + # Apply all the markers now + for mark, param in markers_update: + if mark.name == "order": + item.add_marker(pytest.mark.order(param)) + else: + item.add_marker(mark(param)) + log.info(f"TEST={item.name}") + log.info( + f"MARKERS = {[(i.name, i.args, i.kwargs) for i in item.iter_markers()]}" + ) + # Update PREUPGRADE_CONFIG for each of the Config class + # so that in case of Y stream upgrade we will have preupgrade configurations for reference + # across the tests as Y stream upgrade will reload the config of target version + for cluster in ocsci_config.clusters: + for k in cluster.__dataclass_fields__.keys(): + if k != "PREUPGRADE_CONFIG": + cluster.PREUPGRADE_CONFIG[k] = deepcopy(getattr(cluster, k)) def pytest_collection_finish(session): @@ -459,6 +546,34 @@ def pytest_collection_finish(session): ocsci_config.RUN["number_of_tests"] = len(session.items) +def pytest_runtest_setup(item): + """ + Pytest hook where we want to switch context of the cluster + before any fixture runs + + """ + if ocsci_config.multicluster and ocsci_config.UPGRADE.get("upgrade", False): + for mark in item.iter_markers(): + if mark.name == "config_index": + log.info("Switching the test context to index: {mark.args[0]}") + ocsci_config.switch_ctx(mark.args[0]) + + +def pytest_fixture_setup(fixturedef, request): + """ + In case of multicluster upgrade scenarios, we want to make sure that before running + any fixture related to the testcase we need to switch the cluster context + + """ + # If this is the first fixture getting loaded then its the right time + # to switch context + if ocsci_config.multicluster and ocsci_config.UPGRADE.get("upgrade", ""): + if request.fixturenames.index(fixturedef.argname) == 0: + for mark in request.node.iter_markers(): + if mark.name == "config_index": + ocsci_config.switch_ctx(mark.args[0]) + + @pytest.fixture() def supported_configuration(): """ @@ -8747,6 +8862,16 @@ def virtctl_binary(): get_virtctl_tool() +@pytest.fixture() +def zone_rank(): + return None + + +@pytest.fixture() +def role_rank(): + return None + + @pytest.fixture() def nb_assign_user_role_fixture(request, mcg_obj_session): diff --git a/tests/functional/upgrade/test_upgrade.py b/tests/functional/upgrade/test_upgrade.py index ac836b07ed2..40f0fd25982 100644 --- a/tests/functional/upgrade/test_upgrade.py +++ b/tests/functional/upgrade/test_upgrade.py @@ -2,18 +2,40 @@ import pytest -from ocs_ci.framework.pytest_customization.marks import purple_squad +from ocs_ci.framework.pytest_customization.marks import ( + purple_squad, + multicluster_roles, +) from ocs_ci.framework.testlib import ( ocs_upgrade, polarion_id, + mco_upgrade, + dr_hub_upgrade, + dr_cluster_operator_upgrade, + acm_upgrade, ) +from ocs_ci.framework import config +from ocs_ci.ocs.acm_upgrade import ACMUpgrade from ocs_ci.ocs.disruptive_operations import worker_node_shutdown, osd_node_reboot from ocs_ci.ocs.ocs_upgrade import run_ocs_upgrade +from ocs_ci.ocs.dr_upgrade import ( + DRClusterOperatorUpgrade, + MultiClusterOrchestratorUpgrade, + DRHubUpgrade, +) from ocs_ci.utility.reporting import get_polarion_id +from ocs_ci.utility.utils import is_z_stream_upgrade log = logging.getLogger(__name__) +operator_map = { + "mco": MultiClusterOrchestratorUpgrade, + "drhub": DRHubUpgrade, + "drcluster": DRClusterOperatorUpgrade, +} + + @pytest.fixture() def teardown(request, nodes): def finalizer(): @@ -65,13 +87,84 @@ def test_osd_reboot(teardown, upgrade_stats): run_ocs_upgrade(operation=osd_node_reboot, upgrade_stats=upgrade_stats) +@pytest.fixture +def config_index(request): + return request.param if hasattr(request, "param") else None + + @purple_squad @ocs_upgrade @polarion_id(get_polarion_id(upgrade=True)) -def test_upgrade(upgrade_stats): +@multicluster_roles(["mdr-all-odf"]) +def test_upgrade(zone_rank, role_rank, config_index, upgrade_stats=None): """ Tests upgrade procedure of OCS cluster """ run_ocs_upgrade(upgrade_stats=upgrade_stats) + if config.multicluster and config.MULTICLUSTER["multicluster_mode"] == "metro-dr": + # Perform validation for MCO, dr hub operator and dr cluster operator here + # in case of z stream because we wouldn't call those tests in the case of + # z stream + if is_z_stream_upgrade(): + for operator, op_upgrade_cls in operator_map.items(): + temp = op_upgrade_cls() + log.info(f"Validating upgrade for {operator}") + temp.validate_upgrade() + + +@purple_squad +@mco_upgrade +@multicluster_roles(["mdr-all-acm"]) +def test_mco_upgrade(zone_rank, role_rank, config_index): + """ + Test upgrade procedure for multicluster orchestrator operator + + """ + mco_upgrade_obj = MultiClusterOrchestratorUpgrade() + mco_upgrade_obj.run_upgrade() + + +@purple_squad +@dr_hub_upgrade +@multicluster_roles(["mdr-all-acm"]) +def test_dr_hub_upgrade(zone_rank, role_rank, config_index): + """ + Test upgrade procedure for DR hub operator + + """ + if is_z_stream_upgrade(): + pytest.skip( + "This is z-stream upgrade and this component upgrade should have been taken care by ODF upgrade" + ) + dr_hub_upgrade_obj = DRHubUpgrade() + dr_hub_upgrade_obj.run_upgrade() + + +@purple_squad +@dr_cluster_operator_upgrade +@multicluster_roles(["mdr-all-odf"]) +def test_dr_cluster_upgrade(zone_rank, role_rank, config_index): + """ + Test upgrade procedure for DR cluster operator + + """ + if is_z_stream_upgrade(): + pytest.skip( + "This is z-stream upgrade and this component upgrade should have been taken care by ODF upgrade" + ) + dr_cluster_upgrade_obj = DRClusterOperatorUpgrade() + dr_cluster_upgrade_obj.run_upgrade() + + +@purple_squad +@acm_upgrade +@multicluster_roles(["mdr-all-acm"]) +def test_acm_upgrade(zone_rank, role_rank, config_index): + """ + Test upgrade procedure for ACM operator + + """ + acm_hub_upgrade_obj = ACMUpgrade() + acm_hub_upgrade_obj.run_upgrade() diff --git a/tests/functional/upgrade/test_upgrade_ocp.py b/tests/functional/upgrade/test_upgrade_ocp.py index a8495e89b03..4258cdfac31 100644 --- a/tests/functional/upgrade/test_upgrade_ocp.py +++ b/tests/functional/upgrade/test_upgrade_ocp.py @@ -20,17 +20,25 @@ load_config_file, ) from ocs_ci.framework.testlib import ManageTest, ocp_upgrade, ignore_leftovers -from ocs_ci.ocs.cluster import CephCluster, CephHealthMonitor +from ocs_ci.ocs.cluster import ( + CephCluster, + CephClusterMultiCluster, + CephHealthMonitor, + MulticlusterCephHealthMonitor, +) +from ocs_ci.ocs.utils import is_acm_cluster, get_non_acm_cluster_config from ocs_ci.utility.ocp_upgrade import ( pause_machinehealthcheck, resume_machinehealthcheck, ) +from ocs_ci.utility.multicluster import MDRClusterUpgradeParametrize from ocs_ci.utility.version import ( get_semantic_ocp_running_version, VERSION_4_8, ) from ocs_ci.framework.pytest_customization.marks import ( purple_squad, + multicluster_roles, ) logger = logging.getLogger(__name__) @@ -39,6 +47,7 @@ @ignore_leftovers @ocp_upgrade @purple_squad +@multicluster_roles(["mdr-all-ocp"]) class TestUpgradeOCP(ManageTest): """ 1. check cluster health @@ -78,7 +87,9 @@ def load_ocp_version_config_file(self, ocp_upgrade_version): f" {version_before_upgrade}, new config file will not be loaded" ) - def test_upgrade_ocp(self, reduce_and_resume_cluster_load): + def test_upgrade_ocp( + self, zone_rank, role_rank, config_index, reduce_and_resume_cluster_load + ): """ Tests OCS stability when upgrading OCP @@ -86,9 +97,25 @@ def test_upgrade_ocp(self, reduce_and_resume_cluster_load): cluster_ver = ocp.run_cmd("oc get clusterversions/version -o yaml") logger.debug(f"Cluster versions before upgrade:\n{cluster_ver}") - ceph_cluster = CephCluster() - with CephHealthMonitor(ceph_cluster): + if ( + config.multicluster + and config.MULTICLUSTER["multicluster_mode"] == "metro-dr" + and is_acm_cluster(config) + ): + # Find the ODF cluster in current zone + mdr_upgrade = MDRClusterUpgradeParametrize() + mdr_upgrade.config_init() + local_zone_odf = None + for cluster in get_non_acm_cluster_config(): + if config.ENV_DATA["zone"] == cluster.ENV_DATA["zone"]: + local_zone_odf = cluster + ceph_cluster = CephClusterMultiCluster(local_zone_odf) + health_monitor = MulticlusterCephHealthMonitor + else: + ceph_cluster = CephCluster() + health_monitor = CephHealthMonitor + with health_monitor(ceph_cluster): ocp_channel = config.UPGRADE.get( "ocp_channel", ocp.get_ocp_upgrade_channel() ) @@ -199,7 +226,7 @@ def test_upgrade_ocp(self, reduce_and_resume_cluster_load): # load new config file self.load_ocp_version_config_file(ocp_upgrade_version) - if not config.ENV_DATA["mcg_only_deployment"]: + if not config.ENV_DATA["mcg_only_deployment"] and not config.multicluster: new_ceph_cluster = CephCluster() # Increased timeout because of this bug: # https://bugzilla.redhat.com/show_bug.cgi?id=2038690