From 0c760fe8106293faf7709ee62b7d2ca506955a35 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Sat, 14 Oct 2023 13:38:05 +0100 Subject: [PATCH 01/32] MDR upgrade scenario implementation Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/deployment/deployment.py | 2 + .../framework/pytest_customization/marks.py | 8 + ocs_ci/utility/multicluster.py | 175 ++++++++++++++++++ tests/conftest.py | 62 +++++++ 4 files changed, 247 insertions(+) create mode 100644 ocs_ci/utility/multicluster.py diff --git a/ocs_ci/deployment/deployment.py b/ocs_ci/deployment/deployment.py index d9dd0ec7759..8be6c1bf384 100644 --- a/ocs_ci/deployment/deployment.py +++ b/ocs_ci/deployment/deployment.py @@ -197,6 +197,8 @@ class Deployment(object): is not needed), or point to a yaml file with custom storage class. """ + UPGRADE_MARKERS = [] + def __init__(self): self.platform = config.ENV_DATA["platform"] self.ocp_deployment_type = config.ENV_DATA["deployment_type"] diff --git a/ocs_ci/framework/pytest_customization/marks.py b/ocs_ci/framework/pytest_customization/marks.py index 7b3734bda08..f06df9a1113 100644 --- a/ocs_ci/framework/pytest_customization/marks.py +++ b/ocs_ci/framework/pytest_customization/marks.py @@ -685,3 +685,11 @@ def get_current_test_marks(): config.DEPLOYMENT.get("kms_deployment") is True, reason="This test is not supported for KMS deployment.", ) + +# Mark the test with marker below to allow re-tries in ceph health fixture +# for known issues when waiting in re-balance and flip flop from health OK +# to 1-2 PGs waiting to be Clean +ceph_health_retry = pytest.mark.ceph_health_retry + +# Mark for Multicluster upgrade scenarios +config_index = pytest.mark.config_index diff --git a/ocs_ci/utility/multicluster.py b/ocs_ci/utility/multicluster.py new file mode 100644 index 00000000000..95ea78a979e --- /dev/null +++ b/ocs_ci/utility/multicluster.py @@ -0,0 +1,175 @@ +""" +All multicluster specific utility functions and classes can be here + +""" + + +class MutliClusterUpgradeParametrize(object): + """ + This base class abstracts upgrade parametrization for multicluster scenarios: MDR, RDR and Managed service + + """ + + MULTICLUSTER_UPGRADE_MARKERS = [ + "pre_upgrade", + "pre_ocp_upgrade", + "ocp_upgrade", + "post_ocp_upgrade", + "pre_ocs_upgrade", + "ocs_upgrade", + "post_ocs_upgrade", + "post_upgrade" + ] + + def __init__(self): + self.roles = [] + # List of zones which are participating in this multicluster setup + self.zones = [] + self.zone_base_rank = 100 + # Each zone will be assigned with a rank + # This rank comes handy when we have to order the tests + self.zone_ranks = {} + + def get_roles(self, metafunc): + """ + should be overridden in the child class + Look for specific role markers based on multicluster scenario + + Args: + metafunc: Pytest metafunc fixture object + """ + pass + + def generate_zone_ranks(self): + """ + For each zone we would be generating the ranks, we will add the zone's respective indexes + to the base rank values which keeps the zone ranks apart and create spaces (for ranks) + in between to accomodate other tests + + """ + for i in range(len(self.zones)): + self.zone_ranks[f"{self.zones[i]}"] = self.zone_base_rank + i * self.zone_base_rank + + def generate_role_ranks(self): + """ + Based on the multicluster scenario, child class should generate the corresponding + role ranks. Roles are specific to multicluster scenarios + + """ + pass + + def generate_pytest_parameters(self, metafunc, roles): + """ + should be overridden in the child class. + This will be called for every testcase parametrization + + """ + pass + + def get_zone_info(self): + """ + Get the list of participating zones + + """ + #TODO: Implement this + return ['a', 'b', 'c'] + + +class MDRClusterUpgradeParametrize(MutliClusterUpgradeParametrize): + """ + This child class handles MDR upgrade scenario specific pytest parametrization + + """ + def __init__(self): + super().__init__() + self.zones = self.get_zone_info() + self.mdr_roles = self.get_mdr_roles() + self.generate_zone_ranks() + self.generate_role_ranks() + self.roles_to_param_tuples = {} + self.generate_role_to_param_tuple_map() + + # In ocs-ci we need to build this based on the config provided + def get_config_index_map(self): + # Convention: build a dictionary with role name as key and a tuple (zone_rank, config_index) as + #value. For ex: {'ActiveACM': (100, 0), 'PassiceACM: (200, 1), 'Primary': (100, 2) + # TO BE built before pytest_generate_test + # TODO: Implement this to fetch indexes at run time + return {'ActiveACM': 0, 'PassiveACM': 1, 'Primary_odf': 2, 'Secondary_odf': 3} + + def get_mdr_roles(self): + """ + All MDR applicable roles + """ + return ["ActiveACM", "PassiveACM", "Primary_odf", "Secondary_odf"] + + + def generate_role_ranks(self): + """ + Based on current roles for MDR : ActiveACM:1, PassiceACM:1, Primary:2, Secondary: 2 + + """ + # TODO: add some more dynamic stuff + self.role_ranks = {'ActiveACM': 1, 'PassiveACM': 1, 'Primary_odf': 2, 'Secondary_odf': 2} + + def get_zone_for_role(self, role): + """ + Get zone to cluster with a multicluster role mapping + + """ + # TODO: Fetch zone to cluster mapping dynamically + zone_role_map = {'ActiveACM': 'b', 'PassiveACM': 'c', 'Primary_odf': 'b', 'Secondary_odf': 'c'} + return zone_role_map[role] + + def generate_role_to_param_tuple_map(self): + """ + For each of the MDRs applicable roles store a tuple (zone_rank, role_rank, config_index) + + """ + for role in self.mdr_roles: + self.roles_to_param_tuples[role] = ( + self.zone_ranks[self.get_zone_for_role(role)], + self.role_ranks[role], + self.get_config_index_map()[role] + ) + + def get_pytest_params_tuple(self, role): + """ + Generate a tuple of parameters applicable to the given role + For ex: if role is 'ActiveACM', then generate a tuple which is applicable to + that role. If the role is 'all' then we will generate tuple of parameter + for each of the role applicable from MDRs perspective. + Parmeter tuples looks like (zone_rank, role_rank, config_index) + """ + param_list = list() + if role == 'all': + for t in self.roles_to_param_tuples.values(): + param_list.append(t) + param_list + else: + param_list.append(self.roles_to_param_tuples[role]) + return param_list + + def get_roles(self, metafunc): + # Return a list of roles applicable to the current test + for marker in metafunc.definition.iter_markers(): + if marker.name == 'mdr_roles': + return marker.args[0] + + def generate_pytest_parameters(self, metafunc, roles): + """ + We will have to parametrize the test based on the MDR roles to which the test is applicable to, + Parameters will be a tuple of (zone_rank, role_rank, config_index) + + """ + pytest_params = [] + for role in roles: + pytest_params.extend(self.get_pytest_params_tuple(role)) + return pytest_params + + +multicluster_upgrade_parametrizer = {"metro-dr": MDRClusterUpgradeParametrize} + + +def get_multicluster_upgrade_parametrizer(): + return multicluster_upgrade_parametrizer["metro-dr"]() \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 70993c5cb3a..f5f3f9c489b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,6 +33,7 @@ ignore_leftover_label, upgrade_marks, ignore_resource_not_found_error_label, + config_index, ) from ocs_ci.helpers.proxy import update_container_with_proxy_env @@ -150,6 +151,7 @@ from ocs_ci.utility.prometheus import PrometheusAPI from ocs_ci.utility.reporting import update_live_must_gather_image from ocs_ci.utility.retry import retry +from ocs_ci.utility.multicluster import get_multicluster_upgrade_parametrizer, MutliClusterUpgradeParametrize from ocs_ci.utility.uninstall_openshift_logging import uninstall_cluster_logging from ocs_ci.utility.utils import ( ceph_health_check, @@ -340,6 +342,30 @@ def export_squad_marker_to_csv(items, filename=None): log.info("%s tests require action across %s files", num_tests, num_files) +def pytest_generate_tests(metafunc): + """ + This hook handles pytest dynamic pytest parametrization of tests related to + Multicluster scenarios + + Args: + metafunc (pytest fixture): metafunc pytest object to access info about + test function and its parameters + + """ + # For now we are only dealing with multicluster scenarios in this hook + if ocsci_config.multicluster: + # for various roles which are applicable to current test wrt multicluster, for ex: ACM, primary, secondary etc + roles = None + upgrade_parametrizer = get_multicluster_upgrade_parametrizer() + roles = upgrade_parametrizer.get_roles(metafunc) + if roles: + params = upgrade_parametrizer.generate_pytest_parameters(metafunc, roles) + for marker in metafunc.definition.iter_markers(): + if marker.name in upgrade_parametrizer.MULTICLUSTER_UPGRADE_MARKERS: + metafunc.parametrize("zone_rank, role_rank, config_index", params) + + + def pytest_collection_modifyitems(session, config, items): """ A pytest hook to filter out skipped tests satisfying @@ -445,6 +471,27 @@ def pytest_collection_modifyitems(session, config, items): f" UI is not supported on {ocsci_config.ENV_DATA['platform'].lower()}" ) items.remove(item) + # If multicluster upgrade scenario + if ocsci_config.multicluster and ocsci_config['UPGRADE'].get('upgrade', ''): + for item in items: + if list(set([i.name for i in item.iter_markers()]).intersection((MutliClusterUpgradeParametrize.MULTICLUSTER_UPGRADE_MARKERS))) and ocsci_config.multicluster_scenario: + if getattr(item, 'callspec', ''): + zone_rank = item.callspec.params['zone_rank'] + role_rank = item.callspec.params['role_rank'] + else: + continue + markers_update=[] + for m in item.iter_markers(): + # fetch already marked 'order' value + if m.name == 'run': + val = m.kwargs.get('order') + newval = val + zone_rank + role_rank + markers_update.append((pytest.mark.order, newval)) + break + markers_update.append((config_index, item.callspec.params['config_index'])) + # Apply all the markers now + for mark, param in markers_update: + item.add_marker(mark(param)) def pytest_collection_finish(session): @@ -459,6 +506,21 @@ def pytest_collection_finish(session): ocsci_config.RUN["number_of_tests"] = len(session.items) +def pytest_fixture_setup(fixturedef, request): + """ + In case of multicluster upgrade scenarios, we want to make sure that before running + any fixture related to the testcase we need to switch the cluster context + + """ + # If this is the first fixture getting loaded then its the right time + # to switch context + if ocsci_config.multicluster and ocsci_config['UPGRADE'].get('upgrade', ''): + if request.fixturenames.index(fixturedef.argname) == 0: + for mark in request.node.iter_markers(): + if mark.name == 'config_index': + ocsci_config.switch_ctx(mark.args[0]) + + @pytest.fixture() def supported_configuration(): """ From 537e29d1f66ce6182e89d88b2f078d21eeb1294a Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Tue, 17 Oct 2023 15:53:34 +0100 Subject: [PATCH 02/32] MDR upgrade scenario 1. Pytest hooks for handling upgrade tests 2. Segregation of classes for handling different multicluster upgrades Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/deployment/deployment.py | 2 - ocs_ci/ocs/constants.py | 2 + ocs_ci/ocs/utils.py | 8 +++ ocs_ci/utility/multicluster.py | 115 +++++++++++++++++++++----------- tests/conftest.py | 39 +++++++---- 5 files changed, 111 insertions(+), 55 deletions(-) diff --git a/ocs_ci/deployment/deployment.py b/ocs_ci/deployment/deployment.py index 8be6c1bf384..d9dd0ec7759 100644 --- a/ocs_ci/deployment/deployment.py +++ b/ocs_ci/deployment/deployment.py @@ -197,8 +197,6 @@ class Deployment(object): is not needed), or point to a yaml file with custom storage class. """ - UPGRADE_MARKERS = [] - def __init__(self): self.platform = config.ENV_DATA["platform"] self.ocp_deployment_type = config.ENV_DATA["deployment_type"] diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 2c14f51b679..6e385c58041 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -3090,3 +3090,5 @@ EDIT = "edit" DELETE = "delete" MACHINE_POOL_ACTIONS = [CREATE, EDIT, DELETE] +# MDR multicluster roles +mdr_roles = ["ActiveACM", "PassiveACM", "Primary_odf", "Secondary_odf"] diff --git a/ocs_ci/ocs/utils.py b/ocs_ci/ocs/utils.py index 494253dcbbc..dcd190759c5 100644 --- a/ocs_ci/ocs/utils.py +++ b/ocs_ci/ocs/utils.py @@ -1758,6 +1758,14 @@ def cluster_config_reindex(): ocsci_config.switch_acm_ctx() +def get_primary_cluster_index(): + """ + Get the index of primary cluster in case of multicluster scenario + """ + pcluster = get_primary_cluster_config() + return pcluster.MULTICLUSTER["multicluster_index"] + + def thread_init_class(class_init_operations, shutdown): if len(class_init_operations) > 0: executor = ThreadPoolExecutor(max_workers=len(class_init_operations)) diff --git a/ocs_ci/utility/multicluster.py b/ocs_ci/utility/multicluster.py index 95ea78a979e..9c75acd39e4 100644 --- a/ocs_ci/utility/multicluster.py +++ b/ocs_ci/utility/multicluster.py @@ -3,6 +3,14 @@ """ +from ocs_ci.ocs.constants import mdr_roles +from ocs_ci.framework import config as ocsci_config +from ocs_ci.ocs.utils import ( + get_primary_cluster_index, + get_active_acm_index, + get_all_acm_indexes, +) + class MutliClusterUpgradeParametrize(object): """ @@ -11,14 +19,14 @@ class MutliClusterUpgradeParametrize(object): """ MULTICLUSTER_UPGRADE_MARKERS = [ - "pre_upgrade", - "pre_ocp_upgrade", - "ocp_upgrade", - "post_ocp_upgrade", - "pre_ocs_upgrade", - "ocs_upgrade", - "post_ocs_upgrade", - "post_upgrade" + "pre_upgrade", + "pre_ocp_upgrade", + "ocp_upgrade", + "post_ocp_upgrade", + "pre_ocs_upgrade", + "ocs_upgrade", + "post_ocs_upgrade", + "post_upgrade", ] def __init__(self): @@ -42,14 +50,16 @@ def get_roles(self, metafunc): def generate_zone_ranks(self): """ - For each zone we would be generating the ranks, we will add the zone's respective indexes + For each zone we would be generating the ranks, we will add the zone's respective indexes to the base rank values which keeps the zone ranks apart and create spaces (for ranks) in between to accomodate other tests """ for i in range(len(self.zones)): - self.zone_ranks[f"{self.zones[i]}"] = self.zone_base_rank + i * self.zone_base_rank - + self.zone_ranks[f"{self.zones[i]}"] = ( + self.zone_base_rank + i * self.zone_base_rank + ) + def generate_role_ranks(self): """ Based on the multicluster scenario, child class should generate the corresponding @@ -71,8 +81,10 @@ def get_zone_info(self): Get the list of participating zones """ - #TODO: Implement this - return ['a', 'b', 'c'] + zones = set() + for c in ocsci_config.clusters: + zones.add(c.ENV_DATA["zone"]) + return list(zones) class MDRClusterUpgradeParametrize(MutliClusterUpgradeParametrize): @@ -80,46 +92,71 @@ class MDRClusterUpgradeParametrize(MutliClusterUpgradeParametrize): This child class handles MDR upgrade scenario specific pytest parametrization """ + def __init__(self): super().__init__() + self.roles_to_param_tuples = dict() + self.roles_to_config_index_map = dict() + self.zone_role_map = dict() + self.zones = self.get_zone_info() self.mdr_roles = self.get_mdr_roles() self.generate_zone_ranks() self.generate_role_ranks() - self.roles_to_param_tuples = {} self.generate_role_to_param_tuple_map() + self.generate_config_index_map() + self.generate_zone_role_map() # In ocs-ci we need to build this based on the config provided def get_config_index_map(self): - # Convention: build a dictionary with role name as key and a tuple (zone_rank, config_index) as - #value. For ex: {'ActiveACM': (100, 0), 'PassiceACM: (200, 1), 'Primary': (100, 2) # TO BE built before pytest_generate_test - # TODO: Implement this to fetch indexes at run time - return {'ActiveACM': 0, 'PassiveACM': 1, 'Primary_odf': 2, 'Secondary_odf': 3} + if not self.roles_to_config_index_map: + self.generate_config_index_map() + return self.roles_to_config_index_map + + def generate_config_index_map(self): + for cluster in ocsci_config: + cluster_index = cluster.MULTICLUSTER["multicluster_index"] + if cluster_index == get_active_acm_index(): + self.roles_to_config_index_map["ActiveACM"] = cluster_index + elif cluster_index == get_primary_cluster_index(): + self.roles_to_config_index_map["Primary_odf"] = cluster_index + elif cluster_index in get_all_acm_indexes(): + # We would have already ruled out the ActiveACM in the first 'if' + self.roles_to_config_index_map["PassiveACM"] = cluster_index + else: + # Only option left is secondary odf + self.roles_to_config_index_map["Secondary_odf"] = cluster_index def get_mdr_roles(self): """ All MDR applicable roles - """ - return ["ActiveACM", "PassiveACM", "Primary_odf", "Secondary_odf"] + """ + return mdr_roles def generate_role_ranks(self): """ Based on current roles for MDR : ActiveACM:1, PassiceACM:1, Primary:2, Secondary: 2 """ - # TODO: add some more dynamic stuff - self.role_ranks = {'ActiveACM': 1, 'PassiveACM': 1, 'Primary_odf': 2, 'Secondary_odf': 2} + # For now we will stick to this convention + self.role_ranks = { + "ActiveACM": 1, + "PassiveACM": 1, + "Primary_odf": 2, + "Secondary_odf": 2, + } - def get_zone_for_role(self, role): + def generate_zone_role_map(self): """ - Get zone to cluster with a multicluster role mapping + Generate a map of Cluster's role vs zone in which clusters are located """ - # TODO: Fetch zone to cluster mapping dynamically - zone_role_map = {'ActiveACM': 'b', 'PassiveACM': 'c', 'Primary_odf': 'b', 'Secondary_odf': 'c'} - return zone_role_map[role] + for crole, cindex in self.roles_to_config_index_map.items(): + czone = ocsci_config.clusters[cindex].ENV_DATA.get("zone") + if czone: + self.zone_role_map[crole] = czone def generate_role_to_param_tuple_map(self): """ @@ -128,24 +165,24 @@ def generate_role_to_param_tuple_map(self): """ for role in self.mdr_roles: self.roles_to_param_tuples[role] = ( - self.zone_ranks[self.get_zone_for_role(role)], - self.role_ranks[role], - self.get_config_index_map()[role] + self.zone_ranks[self.zone_role_map[role]], + self.role_ranks[role], + self.get_config_index_map()[role], ) def get_pytest_params_tuple(self, role): """ Generate a tuple of parameters applicable to the given role - For ex: if role is 'ActiveACM', then generate a tuple which is applicable to - that role. If the role is 'all' then we will generate tuple of parameter - for each of the role applicable from MDRs perspective. - Parmeter tuples looks like (zone_rank, role_rank, config_index) + For ex: if role is 'ActiveACM', then generate a tuple which is applicable to + that role. If the role is 'all' then we will generate tuple of parameter + for each of the role applicable from MDRs perspective. + Parmeter tuples looks like (zone_rank, role_rank, config_index) """ - param_list = list() - if role == 'all': + param_list = list() + if role == "all": for t in self.roles_to_param_tuples.values(): param_list.append(t) - param_list + param_list else: param_list.append(self.roles_to_param_tuples[role]) return param_list @@ -153,7 +190,7 @@ def get_pytest_params_tuple(self, role): def get_roles(self, metafunc): # Return a list of roles applicable to the current test for marker in metafunc.definition.iter_markers(): - if marker.name == 'mdr_roles': + if marker.name == "mdr_roles": return marker.args[0] def generate_pytest_parameters(self, metafunc, roles): @@ -172,4 +209,4 @@ def generate_pytest_parameters(self, metafunc, roles): def get_multicluster_upgrade_parametrizer(): - return multicluster_upgrade_parametrizer["metro-dr"]() \ No newline at end of file + return multicluster_upgrade_parametrizer["metro-dr"]() diff --git a/tests/conftest.py b/tests/conftest.py index f5f3f9c489b..d0dcd3ea420 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -151,7 +151,10 @@ from ocs_ci.utility.prometheus import PrometheusAPI from ocs_ci.utility.reporting import update_live_must_gather_image from ocs_ci.utility.retry import retry -from ocs_ci.utility.multicluster import get_multicluster_upgrade_parametrizer, MutliClusterUpgradeParametrize +from ocs_ci.utility.multicluster import ( + get_multicluster_upgrade_parametrizer, + MutliClusterUpgradeParametrize, +) from ocs_ci.utility.uninstall_openshift_logging import uninstall_cluster_logging from ocs_ci.utility.utils import ( ceph_health_check, @@ -344,7 +347,7 @@ def export_squad_marker_to_csv(items, filename=None): def pytest_generate_tests(metafunc): """ - This hook handles pytest dynamic pytest parametrization of tests related to + This hook handles pytest dynamic pytest parametrization of tests related to Multicluster scenarios Args: @@ -365,7 +368,6 @@ def pytest_generate_tests(metafunc): metafunc.parametrize("zone_rank, role_rank, config_index", params) - def pytest_collection_modifyitems(session, config, items): """ A pytest hook to filter out skipped tests satisfying @@ -472,23 +474,32 @@ def pytest_collection_modifyitems(session, config, items): ) items.remove(item) # If multicluster upgrade scenario - if ocsci_config.multicluster and ocsci_config['UPGRADE'].get('upgrade', ''): + if ocsci_config.multicluster and ocsci_config["UPGRADE"].get("upgrade", ""): for item in items: - if list(set([i.name for i in item.iter_markers()]).intersection((MutliClusterUpgradeParametrize.MULTICLUSTER_UPGRADE_MARKERS))) and ocsci_config.multicluster_scenario: - if getattr(item, 'callspec', ''): - zone_rank = item.callspec.params['zone_rank'] - role_rank = item.callspec.params['role_rank'] + if ( + list( + set([i.name for i in item.iter_markers()]).intersection( + (MutliClusterUpgradeParametrize.MULTICLUSTER_UPGRADE_MARKERS) + ) + ) + and ocsci_config.multicluster_scenario + ): + if getattr(item, "callspec", ""): + zone_rank = item.callspec.params["zone_rank"] + role_rank = item.callspec.params["role_rank"] else: continue - markers_update=[] + markers_update = [] for m in item.iter_markers(): # fetch already marked 'order' value - if m.name == 'run': - val = m.kwargs.get('order') + if m.name == "run": + val = m.kwargs.get("order") newval = val + zone_rank + role_rank markers_update.append((pytest.mark.order, newval)) break - markers_update.append((config_index, item.callspec.params['config_index'])) + markers_update.append( + (config_index, item.callspec.params["config_index"]) + ) # Apply all the markers now for mark, param in markers_update: item.add_marker(mark(param)) @@ -514,10 +525,10 @@ def pytest_fixture_setup(fixturedef, request): """ # If this is the first fixture getting loaded then its the right time # to switch context - if ocsci_config.multicluster and ocsci_config['UPGRADE'].get('upgrade', ''): + if ocsci_config.multicluster and ocsci_config["UPGRADE"].get("upgrade", ""): if request.fixturenames.index(fixturedef.argname) == 0: for mark in request.node.iter_markers(): - if mark.name == 'config_index': + if mark.name == "config_index": ocsci_config.switch_ctx(mark.args[0]) From 1990c2168c1541a441c4cf830c94eaa85a1356a3 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Tue, 24 Oct 2023 20:51:19 +0100 Subject: [PATCH 03/32] address review comments from Petr decorate tests with role markers Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/framework/pytest_customization/marks.py | 1 + ocs_ci/utility/multicluster.py | 16 +++++----------- tests/conftest.py | 2 +- tests/functional/upgrade/test_upgrade.py | 3 ++- tests/functional/upgrade/test_upgrade_ocp.py | 2 ++ 5 files changed, 11 insertions(+), 13 deletions(-) diff --git a/ocs_ci/framework/pytest_customization/marks.py b/ocs_ci/framework/pytest_customization/marks.py index f06df9a1113..42c7a06ed94 100644 --- a/ocs_ci/framework/pytest_customization/marks.py +++ b/ocs_ci/framework/pytest_customization/marks.py @@ -693,3 +693,4 @@ def get_current_test_marks(): # Mark for Multicluster upgrade scenarios config_index = pytest.mark.config_index +multicluster_roles = pytest.mark.multicluster_roles diff --git a/ocs_ci/utility/multicluster.py b/ocs_ci/utility/multicluster.py index 9c75acd39e4..2cdc3668066 100644 --- a/ocs_ci/utility/multicluster.py +++ b/ocs_ci/utility/multicluster.py @@ -3,7 +3,6 @@ """ -from ocs_ci.ocs.constants import mdr_roles from ocs_ci.framework import config as ocsci_config from ocs_ci.ocs.utils import ( get_primary_cluster_index, @@ -100,7 +99,7 @@ def __init__(self): self.zone_role_map = dict() self.zones = self.get_zone_info() - self.mdr_roles = self.get_mdr_roles() + self.mdr_roles = self.get_roles() self.generate_zone_ranks() self.generate_role_ranks() self.generate_role_to_param_tuple_map() @@ -128,13 +127,6 @@ def generate_config_index_map(self): # Only option left is secondary odf self.roles_to_config_index_map["Secondary_odf"] = cluster_index - def get_mdr_roles(self): - """ - All MDR applicable roles - - """ - return mdr_roles - def generate_role_ranks(self): """ Based on current roles for MDR : ActiveACM:1, PassiceACM:1, Primary:2, Secondary: 2 @@ -190,7 +182,7 @@ def get_pytest_params_tuple(self, role): def get_roles(self, metafunc): # Return a list of roles applicable to the current test for marker in metafunc.definition.iter_markers(): - if marker.name == "mdr_roles": + if marker.name == "multicluster_roles": return marker.args[0] def generate_pytest_parameters(self, metafunc, roles): @@ -209,4 +201,6 @@ def generate_pytest_parameters(self, metafunc, roles): def get_multicluster_upgrade_parametrizer(): - return multicluster_upgrade_parametrizer["metro-dr"]() + return multicluster_upgrade_parametrizer[ + ocsci_config.MULTICLUSTER["multicluster_mode"] + ]() diff --git a/tests/conftest.py b/tests/conftest.py index d0dcd3ea420..09b59716d0e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -352,7 +352,7 @@ def pytest_generate_tests(metafunc): Args: metafunc (pytest fixture): metafunc pytest object to access info about - test function and its parameters + test function and its parameters """ # For now we are only dealing with multicluster scenarios in this hook diff --git a/tests/functional/upgrade/test_upgrade.py b/tests/functional/upgrade/test_upgrade.py index ac836b07ed2..340bba55b2d 100644 --- a/tests/functional/upgrade/test_upgrade.py +++ b/tests/functional/upgrade/test_upgrade.py @@ -2,7 +2,7 @@ import pytest -from ocs_ci.framework.pytest_customization.marks import purple_squad +from ocs_ci.framework.pytest_customization.marks import purple_squad, multicluster_roles from ocs_ci.framework.testlib import ( ocs_upgrade, polarion_id, @@ -68,6 +68,7 @@ def test_osd_reboot(teardown, upgrade_stats): @purple_squad @ocs_upgrade @polarion_id(get_polarion_id(upgrade=True)) +@multicluster_roles(["odf"]) def test_upgrade(upgrade_stats): """ Tests upgrade procedure of OCS cluster diff --git a/tests/functional/upgrade/test_upgrade_ocp.py b/tests/functional/upgrade/test_upgrade_ocp.py index a8495e89b03..764e67e17a2 100644 --- a/tests/functional/upgrade/test_upgrade_ocp.py +++ b/tests/functional/upgrade/test_upgrade_ocp.py @@ -31,6 +31,7 @@ ) from ocs_ci.framework.pytest_customization.marks import ( purple_squad, + multicluster_roles, ) logger = logging.getLogger(__name__) @@ -39,6 +40,7 @@ @ignore_leftovers @ocp_upgrade @purple_squad +@multicluster_roles(["ocp"]) class TestUpgradeOCP(ManageTest): """ 1. check cluster health From 342bfc0ebe1661225b18fee2ff10508bf9644e3a Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Thu, 26 Oct 2023 12:26:22 +0100 Subject: [PATCH 04/32] Update roles markers Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/constants.py | 2 +- ocs_ci/utility/multicluster.py | 38 ++++++++++---------- tests/functional/upgrade/test_upgrade_ocp.py | 3 +- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 6e385c58041..6f40e759c39 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -3091,4 +3091,4 @@ DELETE = "delete" MACHINE_POOL_ACTIONS = [CREATE, EDIT, DELETE] # MDR multicluster roles -mdr_roles = ["ActiveACM", "PassiveACM", "Primary_odf", "Secondary_odf"] +MDR_ROLES = ["ActiveACM", "PassiveACM", "Primary_odf", "Secondary_odf"] diff --git a/ocs_ci/utility/multicluster.py b/ocs_ci/utility/multicluster.py index 2cdc3668066..e2da836dbb6 100644 --- a/ocs_ci/utility/multicluster.py +++ b/ocs_ci/utility/multicluster.py @@ -9,6 +9,7 @@ get_active_acm_index, get_all_acm_indexes, ) +from ocs_ci.ocs.constants import MDR_ROLES class MutliClusterUpgradeParametrize(object): @@ -31,7 +32,7 @@ class MutliClusterUpgradeParametrize(object): def __init__(self): self.roles = [] # List of zones which are participating in this multicluster setup - self.zones = [] + self.zones = self.get_zone_info() self.zone_base_rank = 100 # Each zone will be assigned with a rank # This rank comes handy when we have to order the tests @@ -97,23 +98,20 @@ def __init__(self): self.roles_to_param_tuples = dict() self.roles_to_config_index_map = dict() self.zone_role_map = dict() + self.all_mdr_roles = MDR_ROLES - self.zones = self.get_zone_info() - self.mdr_roles = self.get_roles() self.generate_zone_ranks() self.generate_role_ranks() - self.generate_role_to_param_tuple_map() self.generate_config_index_map() + self.generate_role_to_param_tuple_map() self.generate_zone_role_map() - # In ocs-ci we need to build this based on the config provided - def get_config_index_map(self): - # TO BE built before pytest_generate_test - if not self.roles_to_config_index_map: - self.generate_config_index_map() - return self.roles_to_config_index_map - def generate_config_index_map(self): + """ + Generate config indexes for all the MDRs cluster roles + ex: {"ActiveACM": 0, "PassiceACM": 2, "PrimaryODF": 1, "SecondaryODF": 3} + + """ for cluster in ocsci_config: cluster_index = cluster.MULTICLUSTER["multicluster_index"] if cluster_index == get_active_acm_index(): @@ -143,7 +141,7 @@ def generate_role_ranks(self): def generate_zone_role_map(self): """ Generate a map of Cluster's role vs zone in which clusters are located - + ex: {"ActiveACM": 'a', "PassiveACM": 'b', "PrimaryODF": 'a'} """ for crole, cindex in self.roles_to_config_index_map.items(): czone = ocsci_config.clusters[cindex].ENV_DATA.get("zone") @@ -153,22 +151,24 @@ def generate_zone_role_map(self): def generate_role_to_param_tuple_map(self): """ For each of the MDRs applicable roles store a tuple (zone_rank, role_rank, config_index) + ex: {"ActiveACM": (1, 1, 0), "PassiceACM": (2, 1, 2), "PrimaryODF": (1, 2, 1), "SecondarODF": (2, 2, 3)} """ - for role in self.mdr_roles: + for role in self.all_mdr_roles: self.roles_to_param_tuples[role] = ( self.zone_ranks[self.zone_role_map[role]], self.role_ranks[role], - self.get_config_index_map()[role], + self.roles_to_config_index_map[role], ) def get_pytest_params_tuple(self, role): """ - Generate a tuple of parameters applicable to the given role - For ex: if role is 'ActiveACM', then generate a tuple which is applicable to - that role. If the role is 'all' then we will generate tuple of parameter - for each of the role applicable from MDRs perspective. - Parmeter tuples looks like (zone_rank, role_rank, config_index) + Get a tuple of parameters applicable to the given role + For ex: if role is 'ActiveACM', then get a tuple which is applicable to + that role. If the role is 'all' then we will get tuples of parameter + for all the roles applicable + Parmeter tuples looks like (zone_rank, role_rank, config_index) for a given role + """ param_list = list() if role == "all": diff --git a/tests/functional/upgrade/test_upgrade_ocp.py b/tests/functional/upgrade/test_upgrade_ocp.py index 764e67e17a2..eb8c360f7b6 100644 --- a/tests/functional/upgrade/test_upgrade_ocp.py +++ b/tests/functional/upgrade/test_upgrade_ocp.py @@ -40,7 +40,7 @@ @ignore_leftovers @ocp_upgrade @purple_squad -@multicluster_roles(["ocp"]) +@multicluster_roles(["mdr_all_ocp"]) class TestUpgradeOCP(ManageTest): """ 1. check cluster health @@ -90,7 +90,6 @@ def test_upgrade_ocp(self, reduce_and_resume_cluster_load): logger.debug(f"Cluster versions before upgrade:\n{cluster_ver}") ceph_cluster = CephCluster() with CephHealthMonitor(ceph_cluster): - ocp_channel = config.UPGRADE.get( "ocp_channel", ocp.get_ocp_upgrade_channel() ) From 8ac4eb30bcf85c96b9f44a0f79ed46ba7de3d0e7 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Fri, 27 Oct 2023 13:05:43 +0100 Subject: [PATCH 05/32] Add functions to choose the right parameter tuples based on roles Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/constants.py | 2 +- ocs_ci/ocs/utils.py | 14 ++++++ ocs_ci/utility/multicluster.py | 55 ++++++++++++++++++------ tests/conftest.py | 9 +++- tests/functional/upgrade/test_upgrade.py | 2 +- 5 files changed, 65 insertions(+), 17 deletions(-) diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 6f40e759c39..3dc4d3c69c8 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -3091,4 +3091,4 @@ DELETE = "delete" MACHINE_POOL_ACTIONS = [CREATE, EDIT, DELETE] # MDR multicluster roles -MDR_ROLES = ["ActiveACM", "PassiveACM", "Primary_odf", "Secondary_odf"] +MDR_ROLES = ["ActiveACM", "PassiveACM", "PrimaryODF", "SecondaryODF"] diff --git a/ocs_ci/ocs/utils.py b/ocs_ci/ocs/utils.py index dcd190759c5..f2dfbfdee96 100644 --- a/ocs_ci/ocs/utils.py +++ b/ocs_ci/ocs/utils.py @@ -1609,6 +1609,20 @@ def get_non_acm_cluster_config(): return non_acm_list +def get_non_acm_cluster_indexes(): + """ + Get config index of all non-acm clusters + + Returns: + list: of integer indexes of non-acm clusters + + """ + non_acm_indexes = list() + for cluster in get_non_acm_cluster_config(): + non_acm_indexes.append(cluster.MULTICLUSTER["multicluster_index"]) + return non_acm_indexes + + def get_all_acm_indexes(): """ Get indexes fro all ACM clusters diff --git a/ocs_ci/utility/multicluster.py b/ocs_ci/utility/multicluster.py index e2da836dbb6..e08abff2588 100644 --- a/ocs_ci/utility/multicluster.py +++ b/ocs_ci/utility/multicluster.py @@ -5,6 +5,7 @@ from ocs_ci.framework import config as ocsci_config from ocs_ci.ocs.utils import ( + get_non_acm_cluster_indexes, get_primary_cluster_index, get_active_acm_index, get_all_acm_indexes, @@ -12,7 +13,7 @@ from ocs_ci.ocs.constants import MDR_ROLES -class MutliClusterUpgradeParametrize(object): +class MultiClusterUpgradeParametrize(object): """ This base class abstracts upgrade parametrization for multicluster scenarios: MDR, RDR and Managed service @@ -87,7 +88,7 @@ def get_zone_info(self): return list(zones) -class MDRClusterUpgradeParametrize(MutliClusterUpgradeParametrize): +class MDRClusterUpgradeParametrize(MultiClusterUpgradeParametrize): """ This child class handles MDR upgrade scenario specific pytest parametrization @@ -103,6 +104,10 @@ def __init__(self): self.generate_zone_ranks() self.generate_role_ranks() self.generate_config_index_map() + # Reverse mapping of cluster's index to its role + self.index_to_role = { + index: role for role, index in self.roles_to_config_index_map.items() + } self.generate_role_to_param_tuple_map() self.generate_zone_role_map() @@ -117,25 +122,25 @@ def generate_config_index_map(self): if cluster_index == get_active_acm_index(): self.roles_to_config_index_map["ActiveACM"] = cluster_index elif cluster_index == get_primary_cluster_index(): - self.roles_to_config_index_map["Primary_odf"] = cluster_index + self.roles_to_config_index_map["PrimaryODF"] = cluster_index elif cluster_index in get_all_acm_indexes(): # We would have already ruled out the ActiveACM in the first 'if' self.roles_to_config_index_map["PassiveACM"] = cluster_index else: # Only option left is secondary odf - self.roles_to_config_index_map["Secondary_odf"] = cluster_index + self.roles_to_config_index_map["SecondaryODF"] = cluster_index def generate_role_ranks(self): """ - Based on current roles for MDR : ActiveACM:1, PassiceACM:1, Primary:2, Secondary: 2 + Based on current roles for MDR : ActiveACM:1, PassiceACM:1, PrimaryODF:2, SecondaryODF: 2 """ # For now we will stick to this convention self.role_ranks = { "ActiveACM": 1, "PassiveACM": 1, - "Primary_odf": 2, - "Secondary_odf": 2, + "PrimaryODF": 2, + "SecondaryODF": 2, } def generate_zone_role_map(self): @@ -170,13 +175,37 @@ def get_pytest_params_tuple(self, role): Parmeter tuples looks like (zone_rank, role_rank, config_index) for a given role """ - param_list = list() - if role == "all": - for t in self.roles_to_param_tuples.values(): - param_list.append(t) - param_list + param_list = None + if role.startswith("mdr-all"): + param_list = self.get_mdr_all_param_tuples(role) else: - param_list.append(self.roles_to_param_tuples[role]) + param_list = [self.roles_to_param_tuples[role]] + return param_list + + def get_mdr_all_param_tuples(self, role): + if "mdr-all-ocp" in role: + return self.get_all_roles_to_param_tuples() + elif "mdr-all-odf" in role: + return self.get_all_odf_roles_to_param_tuple() + elif "mdr-all-acm" in role: + return self.get_all_acm_roles_to_param_tuple() + + def get_all_acm_roles_to_param_tuple(self): + params_list = list() + for i in get_all_acm_indexes(): + params_list.append(self.roles_to_param_tuples[self.index_to_role[i]]) + return params_list + + def get_all_odf_roles_to_param_tuple(self): + params_list = list() + for i in get_non_acm_cluster_indexes(): + params_list.append(self.roles_to_param_tuples[self.index_to_role[i]]) + return params_list + + def get_all_roles_to_param_tuples(self): + param_list = list() + for t in self.roles_to_param_tuples.values(): + param_list.append(t) return param_list def get_roles(self, metafunc): diff --git a/tests/conftest.py b/tests/conftest.py index 09b59716d0e..ef560976085 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -153,7 +153,7 @@ from ocs_ci.utility.retry import retry from ocs_ci.utility.multicluster import ( get_multicluster_upgrade_parametrizer, - MutliClusterUpgradeParametrize, + MultiClusterUpgradeParametrize, ) from ocs_ci.utility.uninstall_openshift_logging import uninstall_cluster_logging from ocs_ci.utility.utils import ( @@ -479,7 +479,7 @@ def pytest_collection_modifyitems(session, config, items): if ( list( set([i.name for i in item.iter_markers()]).intersection( - (MutliClusterUpgradeParametrize.MULTICLUSTER_UPGRADE_MARKERS) + (MultiClusterUpgradeParametrize.MULTICLUSTER_UPGRADE_MARKERS) ) ) and ocsci_config.multicluster_scenario @@ -494,6 +494,11 @@ def pytest_collection_modifyitems(session, config, items): # fetch already marked 'order' value if m.name == "run": val = m.kwargs.get("order") + # Sum of the base order value along with + # zone in which the cluster is and the cluster's role rank + # determines the order in which tests need to be executed + # Lower the sum, higher the rank hence it gets prioritized early + # in the test execution sequence newval = val + zone_rank + role_rank markers_update.append((pytest.mark.order, newval)) break diff --git a/tests/functional/upgrade/test_upgrade.py b/tests/functional/upgrade/test_upgrade.py index 340bba55b2d..3569c27a335 100644 --- a/tests/functional/upgrade/test_upgrade.py +++ b/tests/functional/upgrade/test_upgrade.py @@ -68,7 +68,7 @@ def test_osd_reboot(teardown, upgrade_stats): @purple_squad @ocs_upgrade @polarion_id(get_polarion_id(upgrade=True)) -@multicluster_roles(["odf"]) +@multicluster_roles(["mdr_all_odf"]) def test_upgrade(upgrade_stats): """ Tests upgrade procedure of OCS cluster From c95f49f0a3aec5949ff5d2fbcb01b571f9bc86be Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Tue, 7 Nov 2023 21:09:23 +0000 Subject: [PATCH 06/32] Add MCO and DR HUB operator upgrades class Introduce markers for mco, drhub and acm upgrades Add tests for mco and drhub upgrades Signed-off-by: Shylesh Kumar Mohan --- .../framework/pytest_customization/marks.py | 14 ++ ocs_ci/ocs/constants.py | 6 + ocs_ci/ocs/defaults.py | 2 + ocs_ci/ocs/dr_upgrade.py | 134 ++++++++++++++++++ ocs_ci/ocs/ocs_upgrade.py | 4 +- ocs_ci/utility/multicluster.py | 3 + tests/functional/upgrade/test_upgrade.py | 27 ++++ 7 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 ocs_ci/ocs/dr_upgrade.py diff --git a/ocs_ci/framework/pytest_customization/marks.py b/ocs_ci/framework/pytest_customization/marks.py index 42c7a06ed94..f1e06f1c526 100644 --- a/ocs_ci/framework/pytest_customization/marks.py +++ b/ocs_ci/framework/pytest_customization/marks.py @@ -14,6 +14,9 @@ ORDER_BEFORE_OCP_UPGRADE, ORDER_BEFORE_UPGRADE, ORDER_OCP_UPGRADE, + ORDER_MCO_UPGRADE, + ORDER_DR_HUB_UPGRADE, + ORDER_ACM_UPGRADE, ORDER_OCS_UPGRADE, ORDER_AFTER_OCP_UPGRADE, ORDER_AFTER_OCS_UPGRADE, @@ -122,7 +125,14 @@ order_post_ocp_upgrade = pytest.mark.order(ORDER_AFTER_OCP_UPGRADE) order_post_ocs_upgrade = pytest.mark.order(ORDER_AFTER_OCS_UPGRADE) ocp_upgrade = compose(order_ocp_upgrade, pytest.mark.ocp_upgrade) +# multicluster orchestrator +mco_upgrade = compose(order_mco_upgrade, pytest.mark.mco_upgrade) +# dr hub operator +dr_hub_upgrade = compose(order_dr_hub_upgrade, pytest.mark.dr_hub_upgrade) +# acm operator +acm_upgrade = compose(order_acm_upgrade, pytest.mark.acm_upgrade) ocs_upgrade = compose(order_ocs_upgrade, pytest.mark.ocs_upgrade) +# pre_*_upgrade markers pre_upgrade = compose(order_pre_upgrade, pytest.mark.pre_upgrade) pre_ocp_upgrade = compose( order_pre_ocp_upgrade, @@ -132,12 +142,16 @@ order_pre_ocs_upgrade, pytest.mark.pre_ocs_upgrade, ) +# post_*_upgrade markers post_upgrade = compose(order_post_upgrade, pytest.mark.post_upgrade) post_ocp_upgrade = compose(order_post_ocp_upgrade, pytest.mark.post_ocp_upgrade) post_ocs_upgrade = compose(order_post_ocs_upgrade, pytest.mark.post_ocs_upgrade) upgrade_marks = [ ocp_upgrade, + mco_upgrade, + dr_hub_upgrade, + acm_upgrade, ocs_upgrade, pre_upgrade, pre_ocp_upgrade, diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 3dc4d3c69c8..e1fed7692d8 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -1673,6 +1673,12 @@ ORDER_BEFORE_OCP_UPGRADE = 20 ORDER_OCP_UPGRADE = 30 ORDER_AFTER_OCP_UPGRADE = 40 +# Multicluster orchestrator +ORDER_MCO_UPGRADE = 42 +# DR Hub operator +ORDER_DR_HUB_UPGRADE = 44 +# ACM Operator +ORDER_ACM_UPGRADE = 46 ORDER_BEFORE_OCS_UPGRADE = 50 ORDER_OCS_UPGRADE = 60 ORDER_AFTER_OCS_UPGRADE = 70 diff --git a/ocs_ci/ocs/defaults.py b/ocs_ci/ocs/defaults.py index 426e1da7201..a9cc4781447 100644 --- a/ocs_ci/ocs/defaults.py +++ b/ocs_ci/ocs/defaults.py @@ -58,6 +58,8 @@ OCS_CLIENT_OPERATOR_NAME = "ocs-client-operator" CEPHCSI_OPERATOR = "cephcsi-operator" ODF_DEPENDENCIES = "odf-dependencies" +MCO_OPERATOR_NAME = "odf-multicluster-orchestrator" +DR_HUB_OPERATOR_NAME = "odr-hub-operator" # Noobaa S3 bucket website configurations website_config = { diff --git a/ocs_ci/ocs/dr_upgrade.py b/ocs_ci/ocs/dr_upgrade.py new file mode 100644 index 00000000000..a94f27c5efe --- /dev/null +++ b/ocs_ci/ocs/dr_upgrade.py @@ -0,0 +1,134 @@ +""" +All DR operators upgrades implemented here ex: MulticlusterOrchestrator, Openshift DR operator + +""" + +import logging + +from ocs_ci.framework import config +from ocs_ci.ocs.cluster import CephCluster, CephClusterExternal, CephHealthMonitor +from ocs_ci.ocs.exceptions import TimeoutException +from ocs_ci.ocs.ocs_upgrade import OCSUpgrade, verify_image_versions +from ocs_ci.ocs import constants +from ocs_ci.ocs import defaults +from ocs_ci.deployment.helpers.external_cluster_helpers import ( + ExternalCluster, + get_external_cluster_client, +) +from ocs_ci.ocs.resources.install_plan import wait_for_install_plan_and_approve +from ocs_ci.utility.utils import TimeoutSampler + + +log = logging.getLogger(__name__) + +DR_TO_CEPH_CLUSTER_MAP = {"regional-dr": CephCluster, "metro-dr": CephClusterExternal} + + +class DRUpgrade(OCSUpgrade): + """ + Base class for all DR operator upgrades + + """ + + def __init__( + self, + namespace=constants.OPENSHIFT_OPERATORS, + version_before_upgrade=config.ENV_DATA.get("ocs_version"), + ocs_registry_image=config.UPGRADE.get("upgrade_ocs_registry_image"), + upgrade_in_current_source=config.UPGRADE.get( + "upgrade_in_current_source", False + ), + ): + self.namespace = ( + namespace if namespace else config.ENV_DATA["cluster_namespace"] + ) + self.version_before_upgrade = version_before_upgrade + self.ocs_registry_image = ocs_registry_image + self.upgrade_in_current_source = upgrade_in_current_source + self.ceph_cluster = DR_TO_CEPH_CLUSTER_MAP[ + config.MULTICLUSTER["multicluster_mode"] + ] + self.external_cluster = None + self.operator_name = None + + super.__init__( + self.namespace, + self.version_before_upgrade, + self.ocs_registry_image, + self.upgrade_in_current_source, + ) + + def run_upgrade(self): + self.upgrade_version = self.get_upgrade_version() + assert self.get_parsed_versions()[1] >= self.get_parsed_versions()[0], ( + f"Version you would like to upgrade to: {self.upgrade_version} " + f"is not higher or equal to the version you currently running: " + f"{self.version_before_upgrade}" + ) + + # create external cluster object + if config.DEPLOYMENT["external_mode"]: + host, user, password, ssh_key = get_external_cluster_client() + self.external_cluster = ExternalCluster(host, user, password, ssh_key) + self.csv_name_pre_upgrade = self.get_csv_name_pre_upgrade() + self.pre_upgrade_images = self.get_pre_upgrade_image(self.csv_name_pre_upgrade) + self.load_version_config_file(self.upgrade_version) + + with CephHealthMonitor(self.ceph_cluster): + self.channel = self.set_upgrade_channel(resource_name=self.operator_name) + self.set_upgrade_images() + # TODO: Overload this function + self.update_subscription(self.channel) + # In the case upgrade is not from 4.8 to 4.9 and we have manual approval strategy + # we need to wait and approve install plan, otherwise it's approved in the + # subscribe_ocs method. + subscription_plan_approval = config.DEPLOYMENT.get( + "subscription_plan_approval" + ) + if subscription_plan_approval == "Manual": + wait_for_install_plan_and_approve(config.ENV_DATA["cluster_namespace"]) + + for sample in TimeoutSampler( + timeout=725, + sleep=5, + func=self.check_if_upgrade_completed, + channel=self.channel, + csv_name_pre_upgrade=self.csv_name_pre_upgrade, + ): + try: + if sample: + log.info("Upgrade success!") + break + except TimeoutException: + raise TimeoutException("No new CSV found after upgrade!") + old_image = self.get_images_post_upgrade( + self.channel, self.pre_upgrade_images, self.upgrade_version + ) + + verify_image_versions( + old_image, + self.get_parsed_versions()[1], + self.version_before_upgrade, + ) + + +class MultiClusterOrchestratorUpgrade(DRUpgrade): + """ + A class to handle ODF MCO operator upgrades + + """ + + def __init__(self): + super.__init__() + self.operator_name = defaults.MCO_OPERATOR_NAME + + +class DRHubUpgrade(DRUpgrade): + """ + A class to handle DR Hub operator upgrades + + """ + + def __init__(self): + super.__init__() + self.operator_name = defaults.DR_HUB_OPERATOR_NAME diff --git a/ocs_ci/ocs/ocs_upgrade.py b/ocs_ci/ocs/ocs_upgrade.py index 3e30b6ea063..ed36b18c23e 100644 --- a/ocs_ci/ocs/ocs_upgrade.py +++ b/ocs_ci/ocs/ocs_upgrade.py @@ -403,7 +403,7 @@ def get_pre_upgrade_image(self, csv_name_pre_upgrade): ) return get_images(csv_pre_upgrade.get()) - def set_upgrade_channel(self): + def set_upgrade_channel(self, resource_name=OCS_OPERATOR_NAME): """ Wait for the new package manifest for upgrade. @@ -413,7 +413,7 @@ def set_upgrade_channel(self): """ operator_selector = get_selector_for_ocs_operator() package_manifest = PackageManifest( - resource_name=OCS_OPERATOR_NAME, + resource_name=resource_name, selector=operator_selector, ) package_manifest.wait_for_resource() diff --git a/ocs_ci/utility/multicluster.py b/ocs_ci/utility/multicluster.py index e08abff2588..942cfff5352 100644 --- a/ocs_ci/utility/multicluster.py +++ b/ocs_ci/utility/multicluster.py @@ -24,6 +24,9 @@ class MultiClusterUpgradeParametrize(object): "pre_ocp_upgrade", "ocp_upgrade", "post_ocp_upgrade", + "mco_upgrade", + "dr_hub_upgrade", + "acm_upgrade", "pre_ocs_upgrade", "ocs_upgrade", "post_ocs_upgrade", diff --git a/tests/functional/upgrade/test_upgrade.py b/tests/functional/upgrade/test_upgrade.py index 3569c27a335..3f558d8d054 100644 --- a/tests/functional/upgrade/test_upgrade.py +++ b/tests/functional/upgrade/test_upgrade.py @@ -6,9 +6,13 @@ from ocs_ci.framework.testlib import ( ocs_upgrade, polarion_id, + mco_upgrade, + dr_hub_upgrade, + acm_upgrade, ) from ocs_ci.ocs.disruptive_operations import worker_node_shutdown, osd_node_reboot from ocs_ci.ocs.ocs_upgrade import run_ocs_upgrade +from ocs_ci.ocs.dr_upgrade import MultiClusterOrchestratorUpgrade, DRHubUpgrade from ocs_ci.utility.reporting import get_polarion_id log = logging.getLogger(__name__) @@ -76,3 +80,26 @@ def test_upgrade(upgrade_stats): """ run_ocs_upgrade(upgrade_stats=upgrade_stats) + + +@mco_upgrade +@multicluster_roles(["mdr_all_acm"]) +def test_mco_upgrade(): + """ + Test upgrade procedure for multicluster orchestrator operator + + """ + mco_upgrade_obj = MultiClusterOrchestratorUpgrade() + mco_upgrade_obj.run_upgrade() + + +@dr_hub_upgrade +@multicluster_roles(["mdr_all_acm"]) +def test_dr_hub_upgrade(): + """ + Test upgrade procedure for DR hub operator + + """ + dr_hub_upgrade_obj = DRHubUpgrade() + dr_hub_upgrade_obj.run_upgrade() +>>>>>>> 4fbd44181 (Add MCO and DR HUB operator upgrades class) From 91e42dfdd2c5e357650935d4c6e8ed213bbb2d0d Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Sun, 12 Nov 2023 23:39:46 +0000 Subject: [PATCH 07/32] Add ACM upgrade class Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/acm_upgrade.py | 75 +++++++++++++++++++ ocs_ci/ocs/constants.py | 9 +++ .../acm-deployment/acm_brew_icsp.yaml | 21 ++++++ 3 files changed, 105 insertions(+) create mode 100644 ocs_ci/ocs/acm_upgrade.py create mode 100644 ocs_ci/templates/acm-deployment/acm_brew_icsp.yaml diff --git a/ocs_ci/ocs/acm_upgrade.py b/ocs_ci/ocs/acm_upgrade.py new file mode 100644 index 00000000000..6aa4443a9d6 --- /dev/null +++ b/ocs_ci/ocs/acm_upgrade.py @@ -0,0 +1,75 @@ +""" +ACM operator upgrade classes and utilities + +""" + +import logging +import tempfile + +import requests + +from ocs_ci.ocs import constants +from ocs_ci.utility import templating +from ocs_ci.utility.utils import get_ocp_version, get_running_acm_version, run_cmd +from ocs_ci.utility.version import get_semantic_version + +logger = logging.getLogger(__name__) + + +class ACMUpgrade(object): + def __init__(self): + self.namespace = constants.ACM_HUB_NAMESPACE + self.operator_name = constants.ACM_HUB_OPERATOR_NAME + # Since ACM upgrade happens followed by OCP upgrade in the sequence + # the config would have loaded upgrade parameters rather than pre-upgrade params + # Hence we can't rely on ENV_DATA['acm_version'] for the pre-upgrade version + # we need to dynamically find it + self.version_before_upgrade = self.get_acm_version_before_upgrade() + + def get_acm_version_before_upgrade(self): + running_acm_version = get_running_acm_version() + return get_semantic_version(running_acm_version) + + def run_upgrade(self): + self.create_catalog_source() + self.acm_patch_subscription() + self.annotate_mch() + run_cmd(f"oc create -f {constants.ACM_BREW_ICSP_YAML}") + + def annotate_mch(self): + annotation = f'\'{{"source": "{constants.ACM_CATSRC_NAME}"}}\'' + annotate_cmd = ( + f"oc -n {constants.ACM_HUB_NAMESPACE} annotate mch multiclusterhub " + f"installer.open-cluster-management.io/mce-subscription-spec={annotation}" + ) + run_cmd(annotate_cmd) + + def acm_patch_subscription(self): + patch = f'\'{{"spec": "{constants.ACM_CATSRC_NAME}"}}\'' + patch_cmd = ( + f"oc -n {constants.ACM_HUB_NAMESPACE} patch sub advanced-cluster-management " + f"-p {patch} --type merge" + ) + run_cmd(patch_cmd) + + def create_catalog_source(self): + logger.info("Creating ACM catalog source") + acm_catsrc = templating.load_yaml(constants.ACM_CATSRC) + # Update catalog source + resp = requests.get(constants.ACM_BREW_BUILD_URL, verify=False) + raw_msg = resp.json()["raw_messages"] + # TODO: Find way to get ocp version before upgrade + version_tag = raw_msg[0]["msg"]["pipeline"]["index_image"][ + f"v{get_ocp_version()}" + ].split(":")[1] + acm_catsrc["spec"]["image"] = ":".jon([constants.ACM_BREW_REPO, version_tag]) + acm_catsrc["metadata"]["name"] = constants.ACM_CATSRC_NAME + acm_data_yaml = tempfile.NamedTemporaryFile( + mode="w+", prefix="acm_catsrc", delete=False + ) + templating.dump_data_to_temp_yaml(acm_catsrc, acm_data_yaml.name) + run_cmd(f"oc create -f {acm_data_yaml.name}", timeout=300) + + def validate_upgrade(self): + # TODO: add validation of upgrade + pass diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index e1fed7692d8..989ba6d3195 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -2705,14 +2705,22 @@ SUBMARINER_DOWNSTREAM_UNRELEASED = os.path.join( TEMPLATE_MULTICLUSTER_DIR, "submariner_downstream_unreleased_catsrc.yaml" ) +ACM_CATSRC = SUBMARINER_DOWNSTREAM_UNRELEASED +ACM_CATSRC_NAME = "acm-catalogsource" # We need to append version string at the end of this url SUBMARINER_DOWNSTREAM_UNRELEASED_BUILD_URL = ( "https://datagrepper.engineering.redhat.com/raw?topic=/topic/" "VirtualTopic.eng.ci.redhat-container-image.pipeline.complete" "&rows_per_page=25&delta=1296000&contains=submariner-operator-bundle-container-v" ) +ACM_BREW_BUILD_URL = ( + "https://datagrepper.engineering.redhat.com/raw?topic=/topic/" + "VirtualTopic.eng.ci.redhat-container-image.pipeline.complete" + "&rows_per_page=25&delta=1296000&contains=acm" +) SUBMARINER_BREW_REPO = "brew.registry.redhat.io/rh-osbs/iib" SUBCTL_DOWNSTREAM_URL = "registry.redhat.io/rhacm2/" +ACM_BREW_REPO = SUBMARINER_BREW_REPO # Multicluster related @@ -2751,6 +2759,7 @@ SUBMARINER_DOWNSTREAM_BREW_ICSP = os.path.join( TEMPLATE_DIR, "acm-deployment", "submariner_downstream_brew_icsp.yaml" ) +ACM_BREW_ICSP_YAML = os.path.join(TEMPLATE_DIR, "acm-deployment", "acm_brew_icsp.yaml") ACM_HUB_UNRELEASED_PULL_SECRET_TEMPLATE = "pull-secret.yaml.j2" ACM_ODF_MULTICLUSTER_ORCHESTRATOR_RESOURCE = "odf-multicluster-orchestrator" ACM_ODR_HUB_OPERATOR_RESOURCE = "odr-hub-operator" diff --git a/ocs_ci/templates/acm-deployment/acm_brew_icsp.yaml b/ocs_ci/templates/acm-deployment/acm_brew_icsp.yaml new file mode 100644 index 00000000000..a22605f250b --- /dev/null +++ b/ocs_ci/templates/acm-deployment/acm_brew_icsp.yaml @@ -0,0 +1,21 @@ +apiVersion: operator.openshift.io/v1alpha1 +kind: ImageContentSourcePolicy +metadata: + name: image-policy-brew +spec: + repositoryDigestMirrors: + - mirrors: + - brew.registry.redhat.io/rh-osbs/rhacm2 + source: registry.redhat.io/rhacm2 + - mirrors: + - brew.registry.redhat.io/rh-osbs + source: registry-proxy.engineering.redhat.com/rh-osbs + - mirrors: + - brew.registry.redhat.io/rh-osbs/multicluster-engine + source: registry.redhat.io/multicluster-engine + - mirrors: + - registry.redhat.io/rhacm2 + source: registry.stage.redhat.io/rhacm2 + - mirrors: + - registry.redhat.io/multicluster-engine + source: registry.stage.redhat.io/multicluster-engine From 2022a229f5d1a576e67c3297b64280355653b59e Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Mon, 13 Nov 2023 22:33:19 +0000 Subject: [PATCH 08/32] Address review comments 1.Introduce abstract methods 2. Fix typos Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/dr_upgrade.py | 70 ++++++++++++++++------------------ ocs_ci/utility/multicluster.py | 13 +++++-- tests/conftest.py | 2 +- 3 files changed, 43 insertions(+), 42 deletions(-) diff --git a/ocs_ci/ocs/dr_upgrade.py b/ocs_ci/ocs/dr_upgrade.py index a94f27c5efe..0b70cf0cd71 100644 --- a/ocs_ci/ocs/dr_upgrade.py +++ b/ocs_ci/ocs/dr_upgrade.py @@ -6,7 +6,6 @@ import logging from ocs_ci.framework import config -from ocs_ci.ocs.cluster import CephCluster, CephClusterExternal, CephHealthMonitor from ocs_ci.ocs.exceptions import TimeoutException from ocs_ci.ocs.ocs_upgrade import OCSUpgrade, verify_image_versions from ocs_ci.ocs import constants @@ -21,8 +20,6 @@ log = logging.getLogger(__name__) -DR_TO_CEPH_CLUSTER_MAP = {"regional-dr": CephCluster, "metro-dr": CephClusterExternal} - class DRUpgrade(OCSUpgrade): """ @@ -45,9 +42,6 @@ def __init__( self.version_before_upgrade = version_before_upgrade self.ocs_registry_image = ocs_registry_image self.upgrade_in_current_source = upgrade_in_current_source - self.ceph_cluster = DR_TO_CEPH_CLUSTER_MAP[ - config.MULTICLUSTER["multicluster_mode"] - ] self.external_cluster = None self.operator_name = None @@ -74,36 +68,36 @@ def run_upgrade(self): self.pre_upgrade_images = self.get_pre_upgrade_image(self.csv_name_pre_upgrade) self.load_version_config_file(self.upgrade_version) - with CephHealthMonitor(self.ceph_cluster): - self.channel = self.set_upgrade_channel(resource_name=self.operator_name) - self.set_upgrade_images() - # TODO: Overload this function - self.update_subscription(self.channel) - # In the case upgrade is not from 4.8 to 4.9 and we have manual approval strategy - # we need to wait and approve install plan, otherwise it's approved in the - # subscribe_ocs method. - subscription_plan_approval = config.DEPLOYMENT.get( - "subscription_plan_approval" - ) - if subscription_plan_approval == "Manual": - wait_for_install_plan_and_approve(config.ENV_DATA["cluster_namespace"]) - - for sample in TimeoutSampler( - timeout=725, - sleep=5, - func=self.check_if_upgrade_completed, - channel=self.channel, - csv_name_pre_upgrade=self.csv_name_pre_upgrade, - ): - try: - if sample: - log.info("Upgrade success!") - break - except TimeoutException: - raise TimeoutException("No new CSV found after upgrade!") - old_image = self.get_images_post_upgrade( - self.channel, self.pre_upgrade_images, self.upgrade_version - ) + self.channel = self.set_upgrade_channel(resource_name=self.operator_name) + self.set_upgrade_images() + # TODO: When we have to support colocated ACM on Managed cluster node + # we need to update subscriptions individually for DR operator as we don't want + # to upgrade ODF at the time of DR operator (MCO, DR Hub), ODF would follow the upgrade + # of DR operators + self.update_subscription(self.channel) + # In the case upgrade is not from 4.8 to 4.9 and we have manual approval strategy + # we need to wait and approve install plan, otherwise it's approved in the + # subscribe_ocs method. + subscription_plan_approval = config.DEPLOYMENT.get("subscription_plan_approval") + if subscription_plan_approval == "Manual": + wait_for_install_plan_and_approve(config.ENV_DATA["cluster_namespace"]) + + for sample in TimeoutSampler( + timeout=725, + sleep=5, + func=self.check_if_upgrade_completed, + channel=self.channel, + csv_name_pre_upgrade=self.csv_name_pre_upgrade, + ): + try: + if sample: + log.info("Upgrade success!") + break + except TimeoutException: + raise TimeoutException("No new CSV found after upgrade!") + old_image = self.get_images_post_upgrade( + self.channel, self.pre_upgrade_images, self.upgrade_version + ) verify_image_versions( old_image, @@ -121,7 +115,9 @@ class MultiClusterOrchestratorUpgrade(DRUpgrade): def __init__(self): super.__init__() self.operator_name = defaults.MCO_OPERATOR_NAME - + + def run_upgrade(self): + return super().run_upgrade() class DRHubUpgrade(DRUpgrade): """ diff --git a/ocs_ci/utility/multicluster.py b/ocs_ci/utility/multicluster.py index 942cfff5352..7fa70f54b7a 100644 --- a/ocs_ci/utility/multicluster.py +++ b/ocs_ci/utility/multicluster.py @@ -3,6 +3,8 @@ """ +from abc import ABC, abstractmethod + from ocs_ci.framework import config as ocsci_config from ocs_ci.ocs.utils import ( get_non_acm_cluster_indexes, @@ -13,7 +15,7 @@ from ocs_ci.ocs.constants import MDR_ROLES -class MultiClusterUpgradeParametrize(object): +class MultiClusterUpgradeParametrize(ABC): """ This base class abstracts upgrade parametrization for multicluster scenarios: MDR, RDR and Managed service @@ -42,6 +44,7 @@ def __init__(self): # This rank comes handy when we have to order the tests self.zone_ranks = {} + @abstractmethod def get_roles(self, metafunc): """ should be overridden in the child class @@ -64,6 +67,7 @@ def generate_zone_ranks(self): self.zone_base_rank + i * self.zone_base_rank ) + @abstractmethod def generate_role_ranks(self): """ Based on the multicluster scenario, child class should generate the corresponding @@ -72,6 +76,7 @@ def generate_role_ranks(self): """ pass + @abstractmethod def generate_pytest_parameters(self, metafunc, roles): """ should be overridden in the child class. @@ -117,7 +122,7 @@ def __init__(self): def generate_config_index_map(self): """ Generate config indexes for all the MDRs cluster roles - ex: {"ActiveACM": 0, "PassiceACM": 2, "PrimaryODF": 1, "SecondaryODF": 3} + ex: {"ActiveACM": 0, "PassiveACM": 2, "PrimaryODF": 1, "SecondaryODF": 3} """ for cluster in ocsci_config: @@ -135,7 +140,7 @@ def generate_config_index_map(self): def generate_role_ranks(self): """ - Based on current roles for MDR : ActiveACM:1, PassiceACM:1, PrimaryODF:2, SecondaryODF: 2 + Based on current roles for MDR : ActiveACM:1, PassiveACM:1, PrimaryODF:2, SecondaryODF: 2 """ # For now we will stick to this convention @@ -159,7 +164,7 @@ def generate_zone_role_map(self): def generate_role_to_param_tuple_map(self): """ For each of the MDRs applicable roles store a tuple (zone_rank, role_rank, config_index) - ex: {"ActiveACM": (1, 1, 0), "PassiceACM": (2, 1, 2), "PrimaryODF": (1, 2, 1), "SecondarODF": (2, 2, 3)} + ex: {"ActiveACM": (1, 1, 0), "PassiveACM": (2, 1, 2), "PrimaryODF": (1, 2, 1), "SecondarODF": (2, 2, 3)} """ for role in self.all_mdr_roles: diff --git a/tests/conftest.py b/tests/conftest.py index ef560976085..440ae727523 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -482,7 +482,7 @@ def pytest_collection_modifyitems(session, config, items): (MultiClusterUpgradeParametrize.MULTICLUSTER_UPGRADE_MARKERS) ) ) - and ocsci_config.multicluster_scenario + and ocsci_config.multicluster ): if getattr(item, "callspec", ""): zone_rank = item.callspec.params["zone_rank"] From 080e440aad353528f46da921121042e9773fb493 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Thu, 30 Nov 2023 00:10:17 +0000 Subject: [PATCH 09/32] Add validations for MCO, DR and ACM upgrades Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/acm_upgrade.py | 26 ++++- ocs_ci/ocs/constants.py | 4 + ocs_ci/ocs/dr_upgrade.py | 121 ++++++++++++++++++++++- tests/functional/upgrade/test_upgrade.py | 16 ++- 4 files changed, 160 insertions(+), 7 deletions(-) diff --git a/ocs_ci/ocs/acm_upgrade.py b/ocs_ci/ocs/acm_upgrade.py index 6aa4443a9d6..aca0ba62556 100644 --- a/ocs_ci/ocs/acm_upgrade.py +++ b/ocs_ci/ocs/acm_upgrade.py @@ -9,6 +9,8 @@ import requests from ocs_ci.ocs import constants +from ocs_ci.framework import config +from ocs_ci.ocs.ocp import OCP from ocs_ci.utility import templating from ocs_ci.utility.utils import get_ocp_version, get_running_acm_version, run_cmd from ocs_ci.utility.version import get_semantic_version @@ -35,6 +37,7 @@ def run_upgrade(self): self.acm_patch_subscription() self.annotate_mch() run_cmd(f"oc create -f {constants.ACM_BREW_ICSP_YAML}") + self.validate_upgrade() def annotate_mch(self): annotation = f'\'{{"source": "{constants.ACM_CATSRC_NAME}"}}\'' @@ -71,5 +74,24 @@ def create_catalog_source(self): run_cmd(f"oc create -f {acm_data_yaml.name}", timeout=300) def validate_upgrade(self): - # TODO: add validation of upgrade - pass + acm_sub = OCP( + namespace=self.namespace, + resource_name=self.operator_name, + kind="Subscription", + ) + assert ( + acm_sub.get()["items"][0]["spec"]["channel"] + == config.ENV_DATA["acm_hub_channel"] + ) + logger.info("Checking ACM status") + acm_mch = OCP( + kind=constants.ACM_MULTICLUSTER_HUB, + namespace=constants.ACM_HUB_NAMESPACE, + ) + acm_mch.wait_for_resource( + condition=constants.STATUS_RUNNING, + resource_name=constants.ACM_MULTICLUSTER_RESOURCE, + column="STATUS", + timeout=720, + sleep=5, + ) diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 989ba6d3195..4505fae3dbf 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -1721,6 +1721,10 @@ ARO_MASTER_SUBNET_ADDRESS_PREFIXES = "10.0.0.0/23" CLIENT_OPERATOR_CONFIGMAP = "ocs-client-operator-config" CLIENT_OPERATOR_CSI_IMAGES = "ocs-client-operator-csi-images" +MCO_SUBSCRIPTION = "odf-multicluster-orchestrator" +DR_HUB_OPERATOR_SUBSCRIPTION = ( + "odr-hub-operator-stable-PLACEHOLDER-redhat-operators-openshift-marketplace" +) # UI Deployment constants HTPASSWD_SECRET_NAME = "htpass-secret" diff --git a/ocs_ci/ocs/dr_upgrade.py b/ocs_ci/ocs/dr_upgrade.py index 0b70cf0cd71..0ed586e4c90 100644 --- a/ocs_ci/ocs/dr_upgrade.py +++ b/ocs_ci/ocs/dr_upgrade.py @@ -7,6 +7,7 @@ from ocs_ci.framework import config from ocs_ci.ocs.exceptions import TimeoutException +from ocs_ci.ocs.ocp import OCP from ocs_ci.ocs.ocs_upgrade import OCSUpgrade, verify_image_versions from ocs_ci.ocs import constants from ocs_ci.ocs import defaults @@ -14,6 +15,8 @@ ExternalCluster, get_external_cluster_client, ) +from ocs_ci.ocs.resources import pod +from ocs_ci.ocs.resources.csv import CSV, check_all_csvs_are_succeeded from ocs_ci.ocs.resources.install_plan import wait_for_install_plan_and_approve from ocs_ci.utility.utils import TimeoutSampler @@ -44,6 +47,11 @@ def __init__( self.upgrade_in_current_source = upgrade_in_current_source self.external_cluster = None self.operator_name = None + self.subscription_name = None + self.pre_upgrade_data = None + self.post_upgrade_data = None + # Upgraded phases [pre_upgrade, post_upgrade] + self.upgrade_phase = "pre_upgrade" super.__init__( self.namespace, @@ -74,7 +82,7 @@ def run_upgrade(self): # we need to update subscriptions individually for DR operator as we don't want # to upgrade ODF at the time of DR operator (MCO, DR Hub), ODF would follow the upgrade # of DR operators - self.update_subscription(self.channel) + self.update_subscription(self.channel, self.subscription_name) # In the case upgrade is not from 4.8 to 4.9 and we have manual approval strategy # we need to wait and approve install plan, otherwise it's approved in the # subscribe_ocs method. @@ -98,13 +106,85 @@ def run_upgrade(self): old_image = self.get_images_post_upgrade( self.channel, self.pre_upgrade_images, self.upgrade_version ) - verify_image_versions( old_image, self.get_parsed_versions()[1], self.version_before_upgrade, ) + def update_subscription(self, channel, subscription_name): + subscription = OCP( + resource_name=subscription_name, + kind="subscription", + # namespace could be different on managed clusters + # TODO: Handle different namespaces + namespace=constants.OPENSHIFT_OPERATORS, + ) + current_source = subscription.data["spec"]["source"] + log.info(f"Current MCO source: {current_source}") + mco_source = ( + current_source + if self.upgrade_in_current_source + else constants.OPERATOR_CATALOG_SOURCE_NAME + ) + patch_subscription_cmd = ( + f"patch subscription {subscription_name} " + f'-n {self.namespace} --type merge -p \'{{"spec":{{"channel": ' + f'"{self.channel}", "source": "{mco_source}"}}}}\'' + ) + subscription.exec_oc_cmd(patch_subscription_cmd, out_yaml_format=False) + + def validate_upgrade(self): + # In case of both MCO and DRhub operator, validation steps are similar + # just the resource names changes + assert ( + self.post_upgrade_data["pod_status"] == "Running" + ), f"Pod {self.pod_name_pattern} not in Running state post upgrade" + assert ( + self.post_upgrade_data["age"] <= self.pre_upgrade_data["age"] + ), f"{self.pod_name_pattern} didn't restart after upgrade" + assert ( + self.post_upgrade_data["version"] != self.pre_upgrade_data["version"] + ), "CSV version not upgraded" + check_all_csvs_are_succeeded(namespace=self.namespace) + + def collect_data(self): + """ + Collect DR operator related pods and csv data + """ + pod_data = pod.get_all_pods(namespace=self.namespace) + for p in pod_data: + if self.pod_name_pattern in p["metadata"]["name"]: + pod_obj = OCP( + namespace=self.namespace, + resource_name=p["metadata"]["name"], + kind="Pod", + ) + if self.upgrade_phase == "pre_upgrade": + self.pre_upgrade_data["age"] = pod_obj.get_resource(column="AGE") + self.pre_upgrade_data["pod_status"] = pod_obj.get_resource_status() + if self.upgrade_phse == "post_upgrade": + self.post_upgrade_data["age"] = pod_obj.get_resource(column="AGE") + self.post_upgrade_data["pod_status"] = pod_obj.get_resource_status() + + # get pre-upgrade csv for MCO + csv_objs = CSV(namespace=self.namespace) + for csv in csv_objs: + if self.operator_name in csv["metadata"]["name"]: + csv_obj = CSV( + namespace=self.namespace, resource_name=csv["metadata"]["name"] + ) + if self.upgrade_phase == "pre_upgrade": + self.pre_upgrade_data["version"] = csv_obj.get_resource( + column="VERSION" + ) + if self.upgrade_phase == "post_upgrade": + self.post_upgrade_data["version"] = csv_obj.get_resource( + column="VERSION" + ) + # Make sure all csvs are in succeeded state + check_all_csvs_are_succeeded(namespace=self.namespace) + class MultiClusterOrchestratorUpgrade(DRUpgrade): """ @@ -115,9 +195,25 @@ class MultiClusterOrchestratorUpgrade(DRUpgrade): def __init__(self): super.__init__() self.operator_name = defaults.MCO_OPERATOR_NAME - + self.subscription_name = constants.MCO_SUBSCRIPTION + def run_upgrade(self): - return super().run_upgrade() + # Collect some pre-upgrade data for comparision after the upgrade + self.pod_name_pattern = "odfmo-contoller-manager" + self.collect_data() + assert ( + self.pre_upgrade_data["pod_status"] == "Running" + ), "odfmo-controller pod is not in Running status" + super().run_upgrade() + self.collect_data() + self.upgrade_phase = "post-upgrade" + self.validate_upgrade() + + def validate_upgrade(self): + # validate csv VERSION, PHASE==Succeeded + # validate odfmo-controller-manager pods age + super().validate_upgrade() + class DRHubUpgrade(DRUpgrade): """ @@ -128,3 +224,20 @@ class DRHubUpgrade(DRUpgrade): def __init__(self): super.__init__() self.operator_name = defaults.DR_HUB_OPERATOR_NAME + self.subscription_name = constants.DR_HUB_OPERATOR_SUBSCRIPTION + + def run_upgrade(self): + self.pod_name_pattern = "ramen-hub-operator" + self.collect_data() + assert ( + self.pre_upgrade_data["pod_status"] == "Running" + ), "ramen-hub-operator pod is not in Running status" + super().run_upgrade() + self.upgrade_phase = "post-upgrade" + self.collect_data() + self.validate_upgrade() + + def validate_upgrade(self): + # validate csv odr-hub-operator.v4.13.5-rhodf VERSION, PHASE + # validate pod/ramen-hub-operator- + super().validate_upgrade() diff --git a/tests/functional/upgrade/test_upgrade.py b/tests/functional/upgrade/test_upgrade.py index 3f558d8d054..7b46be32ee8 100644 --- a/tests/functional/upgrade/test_upgrade.py +++ b/tests/functional/upgrade/test_upgrade.py @@ -10,6 +10,7 @@ dr_hub_upgrade, acm_upgrade, ) +from ocs_ci.ocs.acm_upgrade import ACMUpgrade from ocs_ci.ocs.disruptive_operations import worker_node_shutdown, osd_node_reboot from ocs_ci.ocs.ocs_upgrade import run_ocs_upgrade from ocs_ci.ocs.dr_upgrade import MultiClusterOrchestratorUpgrade, DRHubUpgrade @@ -82,6 +83,7 @@ def test_upgrade(upgrade_stats): run_ocs_upgrade(upgrade_stats=upgrade_stats) +@purple_squad @mco_upgrade @multicluster_roles(["mdr_all_acm"]) def test_mco_upgrade(): @@ -93,6 +95,7 @@ def test_mco_upgrade(): mco_upgrade_obj.run_upgrade() +@purple_squad @dr_hub_upgrade @multicluster_roles(["mdr_all_acm"]) def test_dr_hub_upgrade(): @@ -102,4 +105,15 @@ def test_dr_hub_upgrade(): """ dr_hub_upgrade_obj = DRHubUpgrade() dr_hub_upgrade_obj.run_upgrade() ->>>>>>> 4fbd44181 (Add MCO and DR HUB operator upgrades class) + + +@purple_squad +@acm_upgrade +@multicluster_roles(["mdr_all_acm"]) +def test_acm_upgrade(): + """ + Test upgrade procedure for ACM operator + + """ + acm_hub_upgrade_obj = ACMUpgrade() + acm_hub_upgrade_obj.run_upgrade() From fb77a9387884f542820c3b7395aa846bd83213ef Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Mon, 4 Dec 2023 11:26:37 +0000 Subject: [PATCH 10/32] Add facility for handling acm upgrade versions Signed-off-by: Shylesh Kumar Mohan --- .../pytest_customization/ocscilib.py | 8 ++ ocs_ci/ocs/acm_upgrade.py | 74 +++++++++++++++---- 2 files changed, 68 insertions(+), 14 deletions(-) diff --git a/ocs_ci/framework/pytest_customization/ocscilib.py b/ocs_ci/framework/pytest_customization/ocscilib.py index 2a95c2eeac0..72d9ae46364 100644 --- a/ocs_ci/framework/pytest_customization/ocscilib.py +++ b/ocs_ci/framework/pytest_customization/ocscilib.py @@ -263,6 +263,11 @@ def pytest_addoption(parser): "(e.g. quay.io/rhceph-dev/ocs-olm-operator:latest-4.3)" ), ) + parser.addoption( + "--upgrade-acm-version", + dest="upgrade_acm_version", + help="acm version to upgrade(e.g. 2.8), use only with DR upgrade scenario", + ) parser.addoption( "--flexy-env-file", dest="flexy_env_file", help="Path to flexy environment file" ) @@ -661,6 +666,9 @@ def process_cluster_cli_params(config): if custom_kubeconfig_location: os.environ["KUBECONFIG"] = custom_kubeconfig_location ocsci_config.RUN["kubeconfig"] = custom_kubeconfig_location + upgrade_acm_version = get_cli_param(config, "--upgrade-acm-version") + if upgrade_acm_version: + ocsci_config.UPGRADE["upgrade_acm_version"] = upgrade_acm_version def pytest_collection_modifyitems(session, config, items): diff --git a/ocs_ci/ocs/acm_upgrade.py b/ocs_ci/ocs/acm_upgrade.py index aca0ba62556..40829b15af4 100644 --- a/ocs_ci/ocs/acm_upgrade.py +++ b/ocs_ci/ocs/acm_upgrade.py @@ -5,6 +5,7 @@ import logging import tempfile +from pkg_resources import parse_version import requests @@ -27,18 +28,59 @@ def __init__(self): # Hence we can't rely on ENV_DATA['acm_version'] for the pre-upgrade version # we need to dynamically find it self.version_before_upgrade = self.get_acm_version_before_upgrade() + self.upgrade_version = config.UPGRADE["upgrade_acm_version"] + # In case if we are using registry image + self.acm_registry_image = config.UPGRADE.get("upgrade_acm_registry_image", "") def get_acm_version_before_upgrade(self): running_acm_version = get_running_acm_version() return get_semantic_version(running_acm_version) + def get_parsed_versions(self): + parsed_version_before_upgrade = parse_version(self.version_before_upgrade) + parsed_upgrade_version = parse_version(self.upgrade_version) + + return parsed_version_before_upgrade, parsed_upgrade_version + def run_upgrade(self): - self.create_catalog_source() - self.acm_patch_subscription() - self.annotate_mch() - run_cmd(f"oc create -f {constants.ACM_BREW_ICSP_YAML}") + self.version_change = ( + self.get_parsed_versions()[1] > self.get_parsed_versions()[0] + ) + # either this would be GA to Unreleased upgrade of same version OR + # GA to unreleased upgrade to higher version + if self.acm_registry_image and self.version_change: + self.upgrade_with_registry() + self.annotate_mch() + run_cmd(f"oc create -f {constants.ACM_BREW_ICSP_YAML}") + else: + # GA to GA + self.upgrade_without_registry() self.validate_upgrade() + def upgrade_without_registry(self): + """ + GA to GA acm upgrade + + """ + patch = f'\'{{"spec": {{"channel": "release-{self.upgrade_version}"}}}}\'' + self.acm_patch_subscription(patch) + + def upgrade_with_registry(self): + """ + There are 2 scenarios with registry + 1. GA to unreleased same version (ex: 2.8.1 GA to 2.8.2 Unreleased) + 2. GA to unreleased higher version (ex: 2.8.9 GA to 2.9.1 Unreleased) + + """ + if self.acm_registry_image and (not self.version_change): + # This is GA to unreleased: same version + self.create_catalog_source() + else: + # This is GA to unreleased version: upgrade to next version + self.create_catalog_source() + patch = f'\'{{"spec": "{constants.ACM_CATSRC_NAME}"}}\'' + self.acm_patch_subscription(patch) + def annotate_mch(self): annotation = f'\'{{"source": "{constants.ACM_CATSRC_NAME}"}}\'' annotate_cmd = ( @@ -47,8 +89,7 @@ def annotate_mch(self): ) run_cmd(annotate_cmd) - def acm_patch_subscription(self): - patch = f'\'{{"spec": "{constants.ACM_CATSRC_NAME}"}}\'' + def acm_patch_subscription(self, patch): patch_cmd = ( f"oc -n {constants.ACM_HUB_NAMESPACE} patch sub advanced-cluster-management " f"-p {patch} --type merge" @@ -58,14 +99,19 @@ def acm_patch_subscription(self): def create_catalog_source(self): logger.info("Creating ACM catalog source") acm_catsrc = templating.load_yaml(constants.ACM_CATSRC) - # Update catalog source - resp = requests.get(constants.ACM_BREW_BUILD_URL, verify=False) - raw_msg = resp.json()["raw_messages"] - # TODO: Find way to get ocp version before upgrade - version_tag = raw_msg[0]["msg"]["pipeline"]["index_image"][ - f"v{get_ocp_version()}" - ].split(":")[1] - acm_catsrc["spec"]["image"] = ":".jon([constants.ACM_BREW_REPO, version_tag]) + if self.acm_registry_image: + acm_catsrc["spec"]["image"] = self.acm_registry_image + else: + # Update catalog source + resp = requests.get(constants.ACM_BREW_BUILD_URL, verify=False) + raw_msg = resp.json()["raw_messages"] + # TODO: Find way to get ocp version before upgrade + version_tag = raw_msg[0]["msg"]["pipeline"]["index_image"][ + f"v{get_ocp_version()}" + ].split(":")[1] + acm_catsrc["spec"]["image"] = ":".jon( + [constants.ACM_BREW_REPO, version_tag] + ) acm_catsrc["metadata"]["name"] = constants.ACM_CATSRC_NAME acm_data_yaml = tempfile.NamedTemporaryFile( mode="w+", prefix="acm_catsrc", delete=False From c2d23576eebdb783b1198b35910ac9236a75a52d Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Fri, 15 Dec 2023 17:00:43 +0000 Subject: [PATCH 11/32] Add skipif_z_stream markers to DR hub and DR cluster operator upgrade tests Signed-off-by: Shylesh Kumar Mohan --- .../framework/pytest_customization/marks.py | 12 +++++- .../pytest_customization/ocscilib.py | 5 +++ ocs_ci/ocs/constants.py | 3 ++ ocs_ci/ocs/defaults.py | 1 + ocs_ci/ocs/dr_upgrade.py | 28 ++++++++++++ ocs_ci/utility/multicluster.py | 5 ++- ocs_ci/utility/utils.py | 13 ++++++ tests/conftest.py | 3 +- tests/functional/upgrade/test_upgrade.py | 43 ++++++++++++++----- tests/functional/upgrade/test_upgrade_ocp.py | 4 +- 10 files changed, 101 insertions(+), 16 deletions(-) diff --git a/ocs_ci/framework/pytest_customization/marks.py b/ocs_ci/framework/pytest_customization/marks.py index f1e06f1c526..043657a1824 100644 --- a/ocs_ci/framework/pytest_customization/marks.py +++ b/ocs_ci/framework/pytest_customization/marks.py @@ -40,7 +40,7 @@ ) from ocs_ci.utility import version from ocs_ci.utility.aws import update_config_from_s3 -from ocs_ci.utility.utils import load_auth_config +from ocs_ci.utility.utils import is_z_stream_upgrade, load_auth_config # tier marks @@ -129,6 +129,9 @@ mco_upgrade = compose(order_mco_upgrade, pytest.mark.mco_upgrade) # dr hub operator dr_hub_upgrade = compose(order_dr_hub_upgrade, pytest.mark.dr_hub_upgrade) +dr_cluster_operator_upgrade = compose( + order_dr_cluster_operator_upgrade, pytest.mark.dr_cluster_operator_upgrade +) # acm operator acm_upgrade = compose(order_acm_upgrade, pytest.mark.acm_upgrade) ocs_upgrade = compose(order_ocs_upgrade, pytest.mark.ocs_upgrade) @@ -188,6 +191,13 @@ reason="This test cannot run on setup having more than three worker nodes", ) +# Skip if we are running z-stream upgrades +# Useful marker for DR upgrade cases +skipif_z_stream_upgrade = pytest.mark.skipif( + is_z_stream_upgrade(), + reason="This is z-stream upgrade and this component upgrade should have been taken care by other components", +) + # Skipif marks skipif_aws_creds_are_missing = pytest.mark.skipif( ( diff --git a/ocs_ci/framework/pytest_customization/ocscilib.py b/ocs_ci/framework/pytest_customization/ocscilib.py index 72d9ae46364..7a56621d003 100644 --- a/ocs_ci/framework/pytest_customization/ocscilib.py +++ b/ocs_ci/framework/pytest_customization/ocscilib.py @@ -572,6 +572,11 @@ def process_cluster_cli_params(config): upgrade_ocs_version = get_cli_param(config, "upgrade_ocs_version") if upgrade_ocs_version: ocsci_config.UPGRADE["upgrade_ocs_version"] = upgrade_ocs_version + # Storing previous version explicitly + # Useful in DR upgrade scenarios + ocsci_config.UPGRADE["pre_upgrade_ocs_version"] = ocsci_config.ENV_DATA[ + "ocs_version" + ] ocs_registry_image = get_cli_param(config, f"ocs_registry_image{suffix}") if ocs_registry_image: ocsci_config.DEPLOYMENT["ocs_registry_image"] = ocs_registry_image diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 4505fae3dbf..1d5daa2f13c 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -1725,6 +1725,9 @@ DR_HUB_OPERATOR_SUBSCRIPTION = ( "odr-hub-operator-stable-PLACEHOLDER-redhat-operators-openshift-marketplace" ) +DR_CLUSTER_OPERATOR_SUBSCRIPTION = ( + "odr-cluster-operator-stable-PLACEHOLDER-redhat-operators-openshift-marketplace" +) # UI Deployment constants HTPASSWD_SECRET_NAME = "htpass-secret" diff --git a/ocs_ci/ocs/defaults.py b/ocs_ci/ocs/defaults.py index a9cc4781447..437a4f350b0 100644 --- a/ocs_ci/ocs/defaults.py +++ b/ocs_ci/ocs/defaults.py @@ -60,6 +60,7 @@ ODF_DEPENDENCIES = "odf-dependencies" MCO_OPERATOR_NAME = "odf-multicluster-orchestrator" DR_HUB_OPERATOR_NAME = "odr-hub-operator" +DR_CLUSTER_OPERATOR_NAME = "odr-cluster-operator" # Noobaa S3 bucket website configurations website_config = { diff --git a/ocs_ci/ocs/dr_upgrade.py b/ocs_ci/ocs/dr_upgrade.py index 0ed586e4c90..8856aed91ef 100644 --- a/ocs_ci/ocs/dr_upgrade.py +++ b/ocs_ci/ocs/dr_upgrade.py @@ -241,3 +241,31 @@ def validate_upgrade(self): # validate csv odr-hub-operator.v4.13.5-rhodf VERSION, PHASE # validate pod/ramen-hub-operator- super().validate_upgrade() + + +class DRClusterOperatorUpgrade(DRUpgrade): + """ + A class to handle DR Cluster operator upgrades + + """ + + def __init__(self): + super.__init__() + self.operator_name = defaults.DR_CLUSTER_OPERATOR_NAME + self.subscription_name = constants.DR_CLUSTER_OPERATOR_SUBSCRIPTION + + def run_upgrade(self): + self.pod_name_pattern = "ramen-dr-cluster-operator" + self.collect_data() + assert ( + self.pre_upgrade_data["pod_status"] == "Running" + ), "ramen-dr-operator pod is not in Running status" + super().run_upgrade() + self.upgrade_phase = "post-upgrade" + self.collect_data() + self.validate_upgrade() + + def validate_upgrade(self): + # validate csv odr-cluster-operator.v4.13.5-rhodf VERSION, PHASE + # validate pod/ramen-dr-cluster-operator- + return super().validate_upgrade() diff --git a/ocs_ci/utility/multicluster.py b/ocs_ci/utility/multicluster.py index 7fa70f54b7a..019ceeb653b 100644 --- a/ocs_ci/utility/multicluster.py +++ b/ocs_ci/utility/multicluster.py @@ -109,6 +109,7 @@ def __init__(self): self.zone_role_map = dict() self.all_mdr_roles = MDR_ROLES + def config_init(self): self.generate_zone_ranks() self.generate_role_ranks() self.generate_config_index_map() @@ -116,8 +117,8 @@ def __init__(self): self.index_to_role = { index: role for role, index in self.roles_to_config_index_map.items() } - self.generate_role_to_param_tuple_map() self.generate_zone_role_map() + self.generate_role_to_param_tuple_map() def generate_config_index_map(self): """ @@ -125,7 +126,7 @@ def generate_config_index_map(self): ex: {"ActiveACM": 0, "PassiveACM": 2, "PrimaryODF": 1, "SecondaryODF": 3} """ - for cluster in ocsci_config: + for cluster in ocsci_config.clusters: cluster_index = cluster.MULTICLUSTER["multicluster_index"] if cluster_index == get_active_acm_index(): self.roles_to_config_index_map["ActiveACM"] = cluster_index diff --git a/ocs_ci/utility/utils.py b/ocs_ci/utility/utils.py index 71fa4db195e..b733ac463f9 100644 --- a/ocs_ci/utility/utils.py +++ b/ocs_ci/utility/utils.py @@ -5236,3 +5236,16 @@ def extract_image_urls(string_data): # Find all URLs that start with 'registry.redhat.io' image_urls = re.findall(r'registry\.redhat\.io[^\s"]+', string_data) return image_urls + + +def is_z_stream_upgrade(): + """ + Check whether this is a z-stream upgrade scenario + + Returns: + bool: True if its a z-stream upgrade else False + + """ + return config.UPGRADE.get("pre_upgrade_ocs_version", "") == config.UPGRADE.get( + "upgrade_ocs_version", "" + ) diff --git a/tests/conftest.py b/tests/conftest.py index 440ae727523..2bbf988f01a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -362,6 +362,7 @@ def pytest_generate_tests(metafunc): upgrade_parametrizer = get_multicluster_upgrade_parametrizer() roles = upgrade_parametrizer.get_roles(metafunc) if roles: + upgrade_parametrizer.config_init() params = upgrade_parametrizer.generate_pytest_parameters(metafunc, roles) for marker in metafunc.definition.iter_markers(): if marker.name in upgrade_parametrizer.MULTICLUSTER_UPGRADE_MARKERS: @@ -474,7 +475,7 @@ def pytest_collection_modifyitems(session, config, items): ) items.remove(item) # If multicluster upgrade scenario - if ocsci_config.multicluster and ocsci_config["UPGRADE"].get("upgrade", ""): + if ocsci_config.multicluster and ocsci_config.UPGRADE.get("upgrade", True): for item in items: if ( list( diff --git a/tests/functional/upgrade/test_upgrade.py b/tests/functional/upgrade/test_upgrade.py index 7b46be32ee8..494b7eb95b5 100644 --- a/tests/functional/upgrade/test_upgrade.py +++ b/tests/functional/upgrade/test_upgrade.py @@ -2,18 +2,27 @@ import pytest -from ocs_ci.framework.pytest_customization.marks import purple_squad, multicluster_roles +from ocs_ci.framework.pytest_customization.marks import ( + purple_squad, + multicluster_roles, + skipif_z_stream_upgrade, +) from ocs_ci.framework.testlib import ( ocs_upgrade, polarion_id, mco_upgrade, dr_hub_upgrade, + dr_cluster_operator_upgrade, acm_upgrade, ) from ocs_ci.ocs.acm_upgrade import ACMUpgrade from ocs_ci.ocs.disruptive_operations import worker_node_shutdown, osd_node_reboot from ocs_ci.ocs.ocs_upgrade import run_ocs_upgrade -from ocs_ci.ocs.dr_upgrade import MultiClusterOrchestratorUpgrade, DRHubUpgrade +from ocs_ci.ocs.dr_upgrade import ( + DRClusterOperatorUpgrade, + MultiClusterOrchestratorUpgrade, + DRHubUpgrade, +) from ocs_ci.utility.reporting import get_polarion_id log = logging.getLogger(__name__) @@ -73,8 +82,8 @@ def test_osd_reboot(teardown, upgrade_stats): @purple_squad @ocs_upgrade @polarion_id(get_polarion_id(upgrade=True)) -@multicluster_roles(["mdr_all_odf"]) -def test_upgrade(upgrade_stats): +@multicluster_roles(["mdr-all-odf"]) +def test_upgrade(zone_rank, role_rank, config_index, upgrade_stats): """ Tests upgrade procedure of OCS cluster @@ -85,8 +94,8 @@ def test_upgrade(upgrade_stats): @purple_squad @mco_upgrade -@multicluster_roles(["mdr_all_acm"]) -def test_mco_upgrade(): +@multicluster_roles(["mdr-all-acm"]) +def test_mco_upgrade(zone_rank, role_rank, config_index): """ Test upgrade procedure for multicluster orchestrator operator @@ -97,8 +106,9 @@ def test_mco_upgrade(): @purple_squad @dr_hub_upgrade -@multicluster_roles(["mdr_all_acm"]) -def test_dr_hub_upgrade(): +@skipif_z_stream_upgrade +@multicluster_roles(["mdr-all-acm"]) +def test_dr_hub_upgrade(zone_rank, role_rank, config_index): """ Test upgrade procedure for DR hub operator @@ -107,10 +117,23 @@ def test_dr_hub_upgrade(): dr_hub_upgrade_obj.run_upgrade() +@purple_squad +@dr_cluster_operator_upgrade +@skipif_z_stream_upgrade +@multicluster_roles(["mdr-all-odf"]) +def test_dr_cluster_upgrade(zone_rank, role_rank, config_index): + """ + Test upgrade procedure for DR cluster operator + + """ + dr_cluster_upgrade_obj = DRClusterOperatorUpgrade() + dr_cluster_upgrade_obj.run_upgrade() + + @purple_squad @acm_upgrade -@multicluster_roles(["mdr_all_acm"]) -def test_acm_upgrade(): +@multicluster_roles(["mdr-all-acm"]) +def test_acm_upgrade(zone_rank, role_rank, config_index): """ Test upgrade procedure for ACM operator diff --git a/tests/functional/upgrade/test_upgrade_ocp.py b/tests/functional/upgrade/test_upgrade_ocp.py index eb8c360f7b6..7a2dee61349 100644 --- a/tests/functional/upgrade/test_upgrade_ocp.py +++ b/tests/functional/upgrade/test_upgrade_ocp.py @@ -40,7 +40,7 @@ @ignore_leftovers @ocp_upgrade @purple_squad -@multicluster_roles(["mdr_all_ocp"]) +@multicluster_roles(["mdr-all-ocp"]) class TestUpgradeOCP(ManageTest): """ 1. check cluster health @@ -80,7 +80,7 @@ def load_ocp_version_config_file(self, ocp_upgrade_version): f" {version_before_upgrade}, new config file will not be loaded" ) - def test_upgrade_ocp(self, reduce_and_resume_cluster_load): + def test_upgrade_ocp(self, zone_rank, role_rank, config_index, reduce_and_resume_cluster_load): """ Tests OCS stability when upgrading OCP From b5071def1961fcbe344f39d0a736d3466cd3b28b Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Sat, 20 Jan 2024 22:36:56 +0000 Subject: [PATCH 12/32] Fix test ordering issues Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/utility/multicluster.py | 6 ++++++ tests/conftest.py | 12 +++++++++++- tests/functional/upgrade/test_upgrade_ocp.py | 4 +++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/ocs_ci/utility/multicluster.py b/ocs_ci/utility/multicluster.py index 019ceeb653b..13153743572 100644 --- a/ocs_ci/utility/multicluster.py +++ b/ocs_ci/utility/multicluster.py @@ -4,6 +4,7 @@ """ from abc import ABC, abstractmethod +import logging from ocs_ci.framework import config as ocsci_config from ocs_ci.ocs.utils import ( @@ -14,6 +15,8 @@ ) from ocs_ci.ocs.constants import MDR_ROLES +log = logging.getLogger(__name__) + class MultiClusterUpgradeParametrize(ABC): """ @@ -28,6 +31,7 @@ class MultiClusterUpgradeParametrize(ABC): "post_ocp_upgrade", "mco_upgrade", "dr_hub_upgrade", + "dr_cluster_operator_upgrade", "acm_upgrade", "pre_ocs_upgrade", "ocs_upgrade", @@ -39,6 +43,7 @@ def __init__(self): self.roles = [] # List of zones which are participating in this multicluster setup self.zones = self.get_zone_info() + self.zones.sort() self.zone_base_rank = 100 # Each zone will be assigned with a rank # This rank comes handy when we have to order the tests @@ -66,6 +71,7 @@ def generate_zone_ranks(self): self.zone_ranks[f"{self.zones[i]}"] = ( self.zone_base_rank + i * self.zone_base_rank ) + log.info(f"zone ranks = {self.zone_ranks}") @abstractmethod def generate_role_ranks(self): diff --git a/tests/conftest.py b/tests/conftest.py index 2bbf988f01a..e580aabb374 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -357,9 +357,9 @@ def pytest_generate_tests(metafunc): """ # For now we are only dealing with multicluster scenarios in this hook if ocsci_config.multicluster: + upgrade_parametrizer = get_multicluster_upgrade_parametrizer() # for various roles which are applicable to current test wrt multicluster, for ex: ACM, primary, secondary etc roles = None - upgrade_parametrizer = get_multicluster_upgrade_parametrizer() roles = upgrade_parametrizer.get_roles(metafunc) if roles: upgrade_parametrizer.config_init() @@ -501,6 +501,7 @@ def pytest_collection_modifyitems(session, config, items): # Lower the sum, higher the rank hence it gets prioritized early # in the test execution sequence newval = val + zone_rank + role_rank + log.info(f"ORIGINAL = {val}, NEW={newval}") markers_update.append((pytest.mark.order, newval)) break markers_update.append( @@ -508,7 +509,16 @@ def pytest_collection_modifyitems(session, config, items): ) # Apply all the markers now for mark, param in markers_update: + # if mark.name == 'run': + # for m in item.iter_markers(): + # if m.name == 'run': + # m.kwargs['order'] = param + # else: item.add_marker(mark(param)) + log.info(f"TEST={item.name}") + log.info( + f"MARKERS = {[(i.name, i.args, i.kwargs) for i in item.iter_markers()]}" + ) def pytest_collection_finish(session): diff --git a/tests/functional/upgrade/test_upgrade_ocp.py b/tests/functional/upgrade/test_upgrade_ocp.py index 7a2dee61349..01f4ed6a844 100644 --- a/tests/functional/upgrade/test_upgrade_ocp.py +++ b/tests/functional/upgrade/test_upgrade_ocp.py @@ -80,7 +80,9 @@ def load_ocp_version_config_file(self, ocp_upgrade_version): f" {version_before_upgrade}, new config file will not be loaded" ) - def test_upgrade_ocp(self, zone_rank, role_rank, config_index, reduce_and_resume_cluster_load): + def test_upgrade_ocp( + self, zone_rank, role_rank, config_index, reduce_and_resume_cluster_load + ): """ Tests OCS stability when upgrading OCP From 4d3d5a69c11232271b234a7305d37a80030cb1f9 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Thu, 1 Feb 2024 20:06:07 +0000 Subject: [PATCH 13/32] Handle CephCluster object in case of ACM cluster context during upgrade Signed-off-by: Shylesh Kumar Mohan --- conf/README.md | 3 ++ ocs_ci/ocs/cluster.py | 40 +++++++++++++------- ocs_ci/ocs/utils.py | 9 +++++ tests/conftest.py | 2 +- tests/functional/upgrade/test_upgrade_ocp.py | 14 ++++++- 5 files changed, 53 insertions(+), 15 deletions(-) diff --git a/conf/README.md b/conf/README.md index 4f9e45f04ef..ac2eeb68844 100644 --- a/conf/README.md +++ b/conf/README.md @@ -362,6 +362,9 @@ Upgrade related configuration data. * `ocp_arch` - Architecture type of the OCP image * `upgrade_logging_channel` - OCP logging channel to upgrade with * `upgrade_ui` - Perform upgrade via UI (Not all the versions are supported, please look at the code) +* `upgrade_acm_version` - ACM version to which we have to upgrade +* `upgrade_acm_registry_image` - ACM Image tag from brew which should be used to upgrade +example: /rh-osbs/iib:565330 #### AUTH diff --git a/ocs_ci/ocs/cluster.py b/ocs_ci/ocs/cluster.py index d4f979a268f..68f1fb5559d 100644 --- a/ocs_ci/ocs/cluster.py +++ b/ocs_ci/ocs/cluster.py @@ -68,6 +68,15 @@ logger = logging.getLogger(__name__) +class CephClusterMultiCluster(object): + """ + TODO: Implement this class later + This class will be used in case of multicluster scenario + and current cluster is ACM hence this cluster should point to + the ODF which is not in current context + + """ + class CephCluster(object): """ @@ -84,30 +93,35 @@ class CephCluster(object): namespace (str): openshift Namespace where this cluster lives """ - def __init__(self): + def __init__(self, cluster_config): """ Cluster object initializer, this object needs to be initialized after cluster deployment. However its harmless to do anywhere. """ - if config.ENV_DATA["mcg_only_deployment"] or ( - config.ENV_DATA.get("platform") == constants.FUSIONAAS_PLATFORM - and config.ENV_DATA["cluster_type"].lower() == "consumer" + if cluster_config: + logger.info("INITIALIZING") + self.config = cluster_config + else: + self.config = config + if self.config.ENV_DATA["mcg_only_deployment"] or ( + self.config.ENV_DATA.get("platform") == constants.FUSIONAAS_PLATFORM + and self.config.ENV_DATA["cluster_type"].lower() == "consumer" ): return # cluster_name is name of cluster in rook of type CephCluster - self.POD = ocp.OCP(kind="Pod", namespace=config.ENV_DATA["cluster_namespace"]) + self.POD = ocp.OCP(kind="Pod", namespace=self.config.ENV_DATA["cluster_namespace"]) self.CEPHCLUSTER = ocp.OCP( - kind="CephCluster", namespace=config.ENV_DATA["cluster_namespace"] + kind="CephCluster", namespace=self.config.ENV_DATA["cluster_namespace"] ) self.CEPHFS = ocp.OCP( - kind="CephFilesystem", namespace=config.ENV_DATA["cluster_namespace"] + kind="CephFilesystem", namespace=self.config.ENV_DATA["cluster_namespace"] ) self.RBD = ocp.OCP( - kind="CephBlockPool", namespace=config.ENV_DATA["cluster_namespace"] + kind="CephBlockPool", namespace=self.config.ENV_DATA["cluster_namespace"] ) self.DEP = ocp.OCP( - kind="Deployment", namespace=config.ENV_DATA["cluster_namespace"] + kind="Deployment", namespace=self.config.ENV_DATA["cluster_namespace"] ) self.cluster_resource_config = self.CEPHCLUSTER.get().get("items")[0] @@ -289,9 +303,9 @@ def cluster_health_check(self, timeout=None): self.scan_cluster() - if config.ENV_DATA[ + if self.config.ENV_DATA[ "platform" - ] in constants.HCI_PC_OR_MS_PLATFORM and config.ENV_DATA["cluster_type"] in [ + ] in constants.HCI_PC_OR_MS_PLATFORM and self.config.ENV_DATA["cluster_type"] in [ constants.MS_CONSUMER_TYPE, constant.HCI_CLIENT, ]: @@ -341,8 +355,8 @@ def cluster_health_check(self, timeout=None): # Check Noobaa health if ( - config.ENV_DATA["platform"] not in constants.MANAGED_SERVICE_PLATFORMS - and not config.COMPONENTS["disable_noobaa"] + self.config.ENV_DATA["platform"] not in constants.MANAGED_SERVICE_PLATFORMS + and not self.config.COMPONENTS["disable_noobaa"] ): # skip noobaa healthcheck due to bug https://bugzilla.redhat.com/show_bug.cgi?id=2075422 ocp_version = version.get_semantic_ocp_version_from_config() diff --git a/ocs_ci/ocs/utils.py b/ocs_ci/ocs/utils.py index f2dfbfdee96..9a4c235a666 100644 --- a/ocs_ci/ocs/utils.py +++ b/ocs_ci/ocs/utils.py @@ -1638,6 +1638,15 @@ def get_all_acm_indexes(): acm_indexes.append(cluster.MULTICLUSTER["multicluster_index"]) return acm_indexes +def is_acm_cluster(): + """ + Check whether the current cluster in context is an ACM cluster + + Returns: + bool: True if its an ACM cluster else False + """ + return ocsci_config.MULTICLUSTER["multicluster_index"] in get_all_acm_indexes() + def is_acm_cluster(cluster): """ diff --git a/tests/conftest.py b/tests/conftest.py index e580aabb374..40f17c6ea73 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -541,7 +541,7 @@ def pytest_fixture_setup(fixturedef, request): """ # If this is the first fixture getting loaded then its the right time # to switch context - if ocsci_config.multicluster and ocsci_config["UPGRADE"].get("upgrade", ""): + if ocsci_config.multicluster and ocsci_config.UPGRADE.get("upgrade", ""): if request.fixturenames.index(fixturedef.argname) == 0: for mark in request.node.iter_markers(): if mark.name == "config_index": diff --git a/tests/functional/upgrade/test_upgrade_ocp.py b/tests/functional/upgrade/test_upgrade_ocp.py index 01f4ed6a844..10487422e1e 100644 --- a/tests/functional/upgrade/test_upgrade_ocp.py +++ b/tests/functional/upgrade/test_upgrade_ocp.py @@ -21,10 +21,12 @@ ) from ocs_ci.framework.testlib import ManageTest, ocp_upgrade, ignore_leftovers from ocs_ci.ocs.cluster import CephCluster, CephHealthMonitor +from ocs_ci.ocs.utils import is_acm_cluster, get_non_acm_cluster_config from ocs_ci.utility.ocp_upgrade import ( pause_machinehealthcheck, resume_machinehealthcheck, ) +from ocs_ci.utility.multicluster import MDRClusterUpgradeParametrize from ocs_ci.utility.version import ( get_semantic_ocp_running_version, VERSION_4_8, @@ -90,7 +92,17 @@ def test_upgrade_ocp( cluster_ver = ocp.run_cmd("oc get clusterversions/version -o yaml") logger.debug(f"Cluster versions before upgrade:\n{cluster_ver}") - ceph_cluster = CephCluster() + if config.multicluster and config.MULTICLUSTER['multicluster_mode'] == 'metro-dr' and is_acm_cluster(): + # Find the ODF cluster in current zone + mdr_upgrade = MDRClusterUpgradeParametrize() + mdr_upgrade.config_init() + local_zone_odf = None + for cluster in get_non_acm_cluster_config(): + if config.ENV_DATA['zone'] == cluster.ENV_DATA['zone']: + local_zone_odf = cluster + ceph_cluster = CephCluster(local_zone_odf) + else: + ceph_cluster = CephCluster() with CephHealthMonitor(ceph_cluster): ocp_channel = config.UPGRADE.get( "ocp_channel", ocp.get_ocp_upgrade_channel() From f04e7c5c08e9b68b60cbad516d352c01c1b1e427 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Wed, 7 Feb 2024 09:38:52 +0000 Subject: [PATCH 14/32] Add dummy cephcluster classes for healthmonitor Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/cluster.py | 51 ++++++++++++++------ ocs_ci/ocs/utils.py | 1 + tests/functional/upgrade/test_upgrade_ocp.py | 8 ++- 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/ocs_ci/ocs/cluster.py b/ocs_ci/ocs/cluster.py index 68f1fb5559d..224054ebb5f 100644 --- a/ocs_ci/ocs/cluster.py +++ b/ocs_ci/ocs/cluster.py @@ -68,6 +68,7 @@ logger = logging.getLogger(__name__) + class CephClusterMultiCluster(object): """ TODO: Implement this class later @@ -77,6 +78,9 @@ class CephClusterMultiCluster(object): """ + def __init__(self): + pass + class CephCluster(object): """ @@ -99,29 +103,30 @@ def __init__(self, cluster_config): after cluster deployment. However its harmless to do anywhere. """ if cluster_config: - logger.info("INITIALIZING") - self.config = cluster_config - else: - self.config = config - if self.config.ENV_DATA["mcg_only_deployment"] or ( - self.config.ENV_DATA.get("platform") == constants.FUSIONAAS_PLATFORM - and self.config.ENV_DATA["cluster_type"].lower() == "consumer" + logger.info( + "CephClusterMulticluster will be used to handle multicluster case" + ) + return CephClusterMultiCluster() + + if config.ENV_DATA["mcg_only_deployment"] or ( + config.ENV_DATA.get("platform") == constants.FUSIONAAS_PLATFORM + and config.ENV_DATA["cluster_type"].lower() == "consumer" ): return # cluster_name is name of cluster in rook of type CephCluster - self.POD = ocp.OCP(kind="Pod", namespace=self.config.ENV_DATA["cluster_namespace"]) + self.POD = ocp.OCP(kind="Pod", namespace=config.ENV_DATA["cluster_namespace"]) self.CEPHCLUSTER = ocp.OCP( - kind="CephCluster", namespace=self.config.ENV_DATA["cluster_namespace"] + kind="CephCluster", namespace=config.ENV_DATA["cluster_namespace"] ) self.CEPHFS = ocp.OCP( - kind="CephFilesystem", namespace=self.config.ENV_DATA["cluster_namespace"] + kind="CephFilesystem", namespace=config.ENV_DATA["cluster_namespace"] ) self.RBD = ocp.OCP( - kind="CephBlockPool", namespace=self.config.ENV_DATA["cluster_namespace"] + kind="CephBlockPool", namespace=config.ENV_DATA["cluster_namespace"] ) self.DEP = ocp.OCP( - kind="Deployment", namespace=self.config.ENV_DATA["cluster_namespace"] + kind="Deployment", namespace=config.ENV_DATA["cluster_namespace"] ) self.cluster_resource_config = self.CEPHCLUSTER.get().get("items")[0] @@ -303,9 +308,9 @@ def cluster_health_check(self, timeout=None): self.scan_cluster() - if self.config.ENV_DATA[ + if config.ENV_DATA[ "platform" - ] in constants.HCI_PC_OR_MS_PLATFORM and self.config.ENV_DATA["cluster_type"] in [ + ] in constants.HCI_PC_OR_MS_PLATFORM and config.ENV_DATA["cluster_type"] in [ constants.MS_CONSUMER_TYPE, constant.HCI_CLIENT, ]: @@ -355,8 +360,8 @@ def cluster_health_check(self, timeout=None): # Check Noobaa health if ( - self.config.ENV_DATA["platform"] not in constants.MANAGED_SERVICE_PLATFORMS - and not self.config.COMPONENTS["disable_noobaa"] + config.ENV_DATA["platform"] not in constants.MANAGED_SERVICE_PLATFORMS + and not config.COMPONENTS["disable_noobaa"] ): # skip noobaa healthcheck due to bug https://bugzilla.redhat.com/show_bug.cgi?id=2075422 ocp_version = version.get_semantic_ocp_version_from_config() @@ -1049,6 +1054,18 @@ def delete_blockpool(self, pool_name): self.RBD.exec_oc_cmd(f"patch {patch}") +class MulticlusterCephHealthMonitor(object): + # TODO: This will be a placeholder for now + def __init__(self): + pass + + def __entry__(self): + pass + + def __exit__(self): + pass + + class CephHealthMonitor(threading.Thread): """ Context manager class for monitoring ceph health status of CephCluster. @@ -1066,6 +1083,8 @@ def __init__(self, ceph_cluster, sleep=5): sleep (int): Number of seconds to sleep between health checks. """ + if isinstance(ceph_cluster, CephClusterMultiCluster): + return MulticlusterCephHealthMonitor() self.ceph_cluster = ceph_cluster self.sleep = sleep self.health_error_status = None diff --git a/ocs_ci/ocs/utils.py b/ocs_ci/ocs/utils.py index 9a4c235a666..7deec50a3c8 100644 --- a/ocs_ci/ocs/utils.py +++ b/ocs_ci/ocs/utils.py @@ -1638,6 +1638,7 @@ def get_all_acm_indexes(): acm_indexes.append(cluster.MULTICLUSTER["multicluster_index"]) return acm_indexes + def is_acm_cluster(): """ Check whether the current cluster in context is an ACM cluster diff --git a/tests/functional/upgrade/test_upgrade_ocp.py b/tests/functional/upgrade/test_upgrade_ocp.py index 10487422e1e..f138ad36b74 100644 --- a/tests/functional/upgrade/test_upgrade_ocp.py +++ b/tests/functional/upgrade/test_upgrade_ocp.py @@ -92,13 +92,17 @@ def test_upgrade_ocp( cluster_ver = ocp.run_cmd("oc get clusterversions/version -o yaml") logger.debug(f"Cluster versions before upgrade:\n{cluster_ver}") - if config.multicluster and config.MULTICLUSTER['multicluster_mode'] == 'metro-dr' and is_acm_cluster(): + if ( + config.multicluster + and config.MULTICLUSTER["multicluster_mode"] == "metro-dr" + and is_acm_cluster() + ): # Find the ODF cluster in current zone mdr_upgrade = MDRClusterUpgradeParametrize() mdr_upgrade.config_init() local_zone_odf = None for cluster in get_non_acm_cluster_config(): - if config.ENV_DATA['zone'] == cluster.ENV_DATA['zone']: + if config.ENV_DATA["zone"] == cluster.ENV_DATA["zone"]: local_zone_odf = cluster ceph_cluster = CephCluster(local_zone_odf) else: From 0553332841acc454bd9798afcc30990dbcc4d579 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Mon, 19 Feb 2024 11:45:46 +0000 Subject: [PATCH 15/32] Fix minor zstream upgrade issues Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/cluster.py | 6 +- ocs_ci/ocs/dr_upgrade.py | 97 +++++++++++--------- ocs_ci/ocs/ocs_upgrade.py | 17 +++- tests/conftest.py | 13 +-- tests/functional/upgrade/test_upgrade_ocp.py | 14 ++- 5 files changed, 90 insertions(+), 57 deletions(-) diff --git a/ocs_ci/ocs/cluster.py b/ocs_ci/ocs/cluster.py index 224054ebb5f..25154a5ca11 100644 --- a/ocs_ci/ocs/cluster.py +++ b/ocs_ci/ocs/cluster.py @@ -78,7 +78,7 @@ class CephClusterMultiCluster(object): """ - def __init__(self): + def __init__(self, cluster_conf=None): pass @@ -1056,10 +1056,10 @@ def delete_blockpool(self, pool_name): class MulticlusterCephHealthMonitor(object): # TODO: This will be a placeholder for now - def __init__(self): + def __init__(self, ceph_cluster=None): pass - def __entry__(self): + def __enter__(self): pass def __exit__(self): diff --git a/ocs_ci/ocs/dr_upgrade.py b/ocs_ci/ocs/dr_upgrade.py index 8856aed91ef..453961b1774 100644 --- a/ocs_ci/ocs/dr_upgrade.py +++ b/ocs_ci/ocs/dr_upgrade.py @@ -33,31 +33,32 @@ class DRUpgrade(OCSUpgrade): def __init__( self, namespace=constants.OPENSHIFT_OPERATORS, - version_before_upgrade=config.ENV_DATA.get("ocs_version"), - ocs_registry_image=config.UPGRADE.get("upgrade_ocs_registry_image"), + version_before_upgrade=None, + ocs_registry_image=None, upgrade_in_current_source=config.UPGRADE.get( "upgrade_in_current_source", False ), + resource_name=None, ): - self.namespace = ( - namespace if namespace else config.ENV_DATA["cluster_namespace"] - ) - self.version_before_upgrade = version_before_upgrade - self.ocs_registry_image = ocs_registry_image - self.upgrade_in_current_source = upgrade_in_current_source + if not version_before_upgrade: + version_before_upgrade = config.ENV_DATA.get("ocs_version") + if not ocs_registry_image: + ocs_registry_image = config.UPGRADE.get("upgrade_ocs_registry_image") self.external_cluster = None self.operator_name = None self.subscription_name = None - self.pre_upgrade_data = None - self.post_upgrade_data = None + self.pre_upgrade_data = dict() + self.post_upgrade_data = dict() # Upgraded phases [pre_upgrade, post_upgrade] self.upgrade_phase = "pre_upgrade" - - super.__init__( - self.namespace, - self.version_before_upgrade, - self.ocs_registry_image, - self.upgrade_in_current_source, + if resource_name: + self.resource_name = resource_name + + super().__init__( + namespace, + version_before_upgrade, + ocs_registry_image, + upgrade_in_current_source, ) def run_upgrade(self): @@ -72,7 +73,9 @@ def run_upgrade(self): if config.DEPLOYMENT["external_mode"]: host, user, password, ssh_key = get_external_cluster_client() self.external_cluster = ExternalCluster(host, user, password, ssh_key) - self.csv_name_pre_upgrade = self.get_csv_name_pre_upgrade() + self.csv_name_pre_upgrade = self.get_csv_name_pre_upgrade( + resource_name=self.resource_name + ) self.pre_upgrade_images = self.get_pre_upgrade_image(self.csv_name_pre_upgrade) self.load_version_config_file(self.upgrade_version) @@ -104,7 +107,10 @@ def run_upgrade(self): except TimeoutException: raise TimeoutException("No new CSV found after upgrade!") old_image = self.get_images_post_upgrade( - self.channel, self.pre_upgrade_images, self.upgrade_version + self.channel, + self.pre_upgrade_images, + self.upgrade_version, + self.resource_name, ) verify_image_versions( old_image, @@ -115,7 +121,7 @@ def run_upgrade(self): def update_subscription(self, channel, subscription_name): subscription = OCP( resource_name=subscription_name, - kind="subscription", + kind="subscription.operators.coreos.com", # namespace could be different on managed clusters # TODO: Handle different namespaces namespace=constants.OPENSHIFT_OPERATORS, @@ -128,7 +134,7 @@ def update_subscription(self, channel, subscription_name): else constants.OPERATOR_CATALOG_SOURCE_NAME ) patch_subscription_cmd = ( - f"patch subscription {subscription_name} " + f"patch subscription.operators.coreos.com {subscription_name} " f'-n {self.namespace} --type merge -p \'{{"spec":{{"channel": ' f'"{self.channel}", "source": "{mco_source}"}}}}\'' ) @@ -138,13 +144,14 @@ def validate_upgrade(self): # In case of both MCO and DRhub operator, validation steps are similar # just the resource names changes assert ( - self.post_upgrade_data["pod_status"] == "Running" + self.post_upgrade_data.get("pod_status", "") == "Running" ), f"Pod {self.pod_name_pattern} not in Running state post upgrade" assert ( - self.post_upgrade_data["age"] <= self.pre_upgrade_data["age"] + self.post_upgrade_data.get("age", "") <= self.pre_upgrade_data["age"] ), f"{self.pod_name_pattern} didn't restart after upgrade" assert ( - self.post_upgrade_data["version"] != self.pre_upgrade_data["version"] + self.post_upgrade_data.get("version", "") + != self.pre_upgrade_data["version"] ), "CSV version not upgraded" check_all_csvs_are_succeeded(namespace=self.namespace) @@ -154,33 +161,41 @@ def collect_data(self): """ pod_data = pod.get_all_pods(namespace=self.namespace) for p in pod_data: - if self.pod_name_pattern in p["metadata"]["name"]: + if self.pod_name_pattern in p.get()["metadata"]["name"]: pod_obj = OCP( namespace=self.namespace, - resource_name=p["metadata"]["name"], + resource_name=p.get()["metadata"]["name"], kind="Pod", ) if self.upgrade_phase == "pre_upgrade": - self.pre_upgrade_data["age"] = pod_obj.get_resource(column="AGE") - self.pre_upgrade_data["pod_status"] = pod_obj.get_resource_status() - if self.upgrade_phse == "post_upgrade": - self.post_upgrade_data["age"] = pod_obj.get_resource(column="AGE") - self.post_upgrade_data["pod_status"] = pod_obj.get_resource_status() + self.pre_upgrade_data["age"] = pod_obj.get_resource( + resource_name=p.get()["metadata"]["name"], column="AGE" + ) + self.pre_upgrade_data["pod_status"] = pod_obj.get_resource_status( + resource_name=p.get()["metadata"]["name"] + ) + if self.upgrade_phase == "post_upgrade": + self.post_upgrade_data["age"] = pod_obj.get_resource( + resource_name=p.get()["metadata"]["name"], column="AGE" + ) + self.post_upgrade_data["pod_status"] = pod_obj.get_resource_status( + resource_name=p.get()["metadata"]["name"] + ) # get pre-upgrade csv for MCO csv_objs = CSV(namespace=self.namespace) - for csv in csv_objs: + for csv in csv_objs.get()["items"]: if self.operator_name in csv["metadata"]["name"]: csv_obj = CSV( namespace=self.namespace, resource_name=csv["metadata"]["name"] ) if self.upgrade_phase == "pre_upgrade": self.pre_upgrade_data["version"] = csv_obj.get_resource( - column="VERSION" + resource_name=csv_obj.resource_name, column="VERSION" ) if self.upgrade_phase == "post_upgrade": self.post_upgrade_data["version"] = csv_obj.get_resource( - column="VERSION" + resource_name=csv_obj.resource_name, column="VERSION" ) # Make sure all csvs are in succeeded state check_all_csvs_are_succeeded(namespace=self.namespace) @@ -193,20 +208,20 @@ class MultiClusterOrchestratorUpgrade(DRUpgrade): """ def __init__(self): - super.__init__() + super().__init__(resource_name=defaults.MCO_OPERATOR_NAME) self.operator_name = defaults.MCO_OPERATOR_NAME self.subscription_name = constants.MCO_SUBSCRIPTION def run_upgrade(self): # Collect some pre-upgrade data for comparision after the upgrade - self.pod_name_pattern = "odfmo-contoller-manager" + self.pod_name_pattern = "odfmo-controller-manager" self.collect_data() assert ( - self.pre_upgrade_data["pod_status"] == "Running" + self.pre_upgrade_data.get("pod_status", "") == "Running" ), "odfmo-controller pod is not in Running status" super().run_upgrade() + self.upgrade_phase = "post_upgrade" self.collect_data() - self.upgrade_phase = "post-upgrade" self.validate_upgrade() def validate_upgrade(self): @@ -222,7 +237,7 @@ class DRHubUpgrade(DRUpgrade): """ def __init__(self): - super.__init__() + super().__init__(resource_name=defaults.DR_HUB_OPERATOR_NAME) self.operator_name = defaults.DR_HUB_OPERATOR_NAME self.subscription_name = constants.DR_HUB_OPERATOR_SUBSCRIPTION @@ -233,7 +248,7 @@ def run_upgrade(self): self.pre_upgrade_data["pod_status"] == "Running" ), "ramen-hub-operator pod is not in Running status" super().run_upgrade() - self.upgrade_phase = "post-upgrade" + self.upgrade_phase = "post_upgrade" self.collect_data() self.validate_upgrade() @@ -250,7 +265,7 @@ class DRClusterOperatorUpgrade(DRUpgrade): """ def __init__(self): - super.__init__() + super().__init__(resource_name=defaults.DR_CLUSTER_OPERATOR_NAME) self.operator_name = defaults.DR_CLUSTER_OPERATOR_NAME self.subscription_name = constants.DR_CLUSTER_OPERATOR_SUBSCRIPTION @@ -261,7 +276,7 @@ def run_upgrade(self): self.pre_upgrade_data["pod_status"] == "Running" ), "ramen-dr-operator pod is not in Running status" super().run_upgrade() - self.upgrade_phase = "post-upgrade" + self.upgrade_phase = "post_upgrade" self.collect_data() self.validate_upgrade() diff --git a/ocs_ci/ocs/ocs_upgrade.py b/ocs_ci/ocs/ocs_upgrade.py index ed36b18c23e..1a49a9be260 100644 --- a/ocs_ci/ocs/ocs_upgrade.py +++ b/ocs_ci/ocs/ocs_upgrade.py @@ -365,7 +365,7 @@ def load_version_config_file(self, upgrade_version): config.REPORTING["ocs_must_gather_image"] = must_gather_image config.REPORTING["ocs_must_gather_latest_tag"] = must_gather_tag - def get_csv_name_pre_upgrade(self): + def get_csv_name_pre_upgrade(self, resource_name=OCS_OPERATOR_NAME): """ Getting OCS operator name as displayed in CSV @@ -373,9 +373,12 @@ def get_csv_name_pre_upgrade(self): str: OCS operator name, as displayed in CSV """ + import ipdb + + ipdb.set_trace() operator_selector = get_selector_for_ocs_operator() package_manifest = PackageManifest( - resource_name=OCS_OPERATOR_NAME, + resource_name=resource_name, selector=operator_selector, subscription_plan_approval=self.subscription_plan_approval, ) @@ -483,7 +486,13 @@ def check_if_upgrade_completed(self, channel, csv_name_pre_upgrade): log.info(f"CSV now upgraded to: {csv_name_post_upgrade}") return True - def get_images_post_upgrade(self, channel, pre_upgrade_images, upgrade_version): + def get_images_post_upgrade( + self, + channel, + pre_upgrade_images, + upgrade_version, + resource_name=OCS_OPERATOR_NAME, + ): """ Checks if all images of OCS cluster upgraded, and return list of all images if upgrade success @@ -499,7 +508,7 @@ def get_images_post_upgrade(self, channel, pre_upgrade_images, upgrade_version): """ operator_selector = get_selector_for_ocs_operator() package_manifest = PackageManifest( - resource_name=OCS_OPERATOR_NAME, + resource_name=resource_name, selector=operator_selector, subscription_plan_approval=self.subscription_plan_approval, ) diff --git a/tests/conftest.py b/tests/conftest.py index 40f17c6ea73..a810e646f70 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -509,12 +509,13 @@ def pytest_collection_modifyitems(session, config, items): ) # Apply all the markers now for mark, param in markers_update: - # if mark.name == 'run': - # for m in item.iter_markers(): - # if m.name == 'run': - # m.kwargs['order'] = param - # else: - item.add_marker(mark(param)) + if mark.name == "run": + for m in item.iter_markers(): + if m.name == "run": + # m.kwargs['order'] = param + item.add_marker(pytest.mark.run(order=param)) + else: + item.add_marker(mark(param)) log.info(f"TEST={item.name}") log.info( f"MARKERS = {[(i.name, i.args, i.kwargs) for i in item.iter_markers()]}" diff --git a/tests/functional/upgrade/test_upgrade_ocp.py b/tests/functional/upgrade/test_upgrade_ocp.py index f138ad36b74..e10e287ca86 100644 --- a/tests/functional/upgrade/test_upgrade_ocp.py +++ b/tests/functional/upgrade/test_upgrade_ocp.py @@ -20,7 +20,12 @@ load_config_file, ) from ocs_ci.framework.testlib import ManageTest, ocp_upgrade, ignore_leftovers -from ocs_ci.ocs.cluster import CephCluster, CephHealthMonitor +from ocs_ci.ocs.cluster import ( + CephCluster, + CephClusterMultiCluster, + CephHealthMonitor, + MulticlusterCephHealthMonitor, +) from ocs_ci.ocs.utils import is_acm_cluster, get_non_acm_cluster_config from ocs_ci.utility.ocp_upgrade import ( pause_machinehealthcheck, @@ -104,10 +109,13 @@ def test_upgrade_ocp( for cluster in get_non_acm_cluster_config(): if config.ENV_DATA["zone"] == cluster.ENV_DATA["zone"]: local_zone_odf = cluster - ceph_cluster = CephCluster(local_zone_odf) + ceph_cluster = CephClusterMultiCluster(local_zone_odf) + health_monitor = MulticlusterCephHealthMonitor else: ceph_cluster = CephCluster() - with CephHealthMonitor(ceph_cluster): + health_monitor = CephHealthMonitor + + with health_monitor(ceph_cluster): ocp_channel = config.UPGRADE.get( "ocp_channel", ocp.get_ocp_upgrade_channel() ) From b5d0a305bc1341434be11c1bfec933f518777adb Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Thu, 22 Feb 2024 17:28:22 +0000 Subject: [PATCH 16/32] Make adjustments for the new order marker Signed-off-by: Shylesh Kumar Mohan --- .../framework/pytest_customization/marks.py | 6 ++++++ tests/conftest.py | 20 +++++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/ocs_ci/framework/pytest_customization/marks.py b/ocs_ci/framework/pytest_customization/marks.py index 043657a1824..a55cd2ce6d6 100644 --- a/ocs_ci/framework/pytest_customization/marks.py +++ b/ocs_ci/framework/pytest_customization/marks.py @@ -120,6 +120,12 @@ order_pre_ocp_upgrade = pytest.mark.order(ORDER_BEFORE_OCP_UPGRADE) order_pre_ocs_upgrade = pytest.mark.order(ORDER_BEFORE_OCS_UPGRADE) order_ocp_upgrade = pytest.mark.order(ORDER_OCP_UPGRADE) +order_mco_upgrade = pytest.mark.order(ORDER_MCO_UPGRADE) +order_dr_hub_upgrade = pytest.mark.order(ORDER_DR_HUB_UPGRADE) +# dr cluster operator order is same as hub operator order except that +# it's applicable only on the managed clusters +order_dr_cluster_operator_upgrade = pytest.mark.order(ORDER_DR_HUB_UPGRADE) +order_acm_upgrade = pytest.mark.order(ORDER_ACM_UPGRADE) order_ocs_upgrade = pytest.mark.order(ORDER_OCS_UPGRADE) order_post_upgrade = pytest.mark.order(ORDER_AFTER_UPGRADE) order_post_ocp_upgrade = pytest.mark.order(ORDER_AFTER_OCP_UPGRADE) diff --git a/tests/conftest.py b/tests/conftest.py index a810e646f70..f861aedcb7a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -491,10 +491,15 @@ def pytest_collection_modifyitems(session, config, items): else: continue markers_update = [] - for m in item.iter_markers(): + item_markers = copy(item.own_markers) + if not item_markers: + # If testcase is in a class then own_markers will be empty + # hence we need to check item.instance.pytestmark + item_markers = copy(item.instance.pytestmark) + for m in item_markers: # fetch already marked 'order' value - if m.name == "run": - val = m.kwargs.get("order") + if m.name == "order": + val = m.args[0] # Sum of the base order value along with # zone in which the cluster is and the cluster's role rank # determines the order in which tests need to be executed @@ -503,17 +508,16 @@ def pytest_collection_modifyitems(session, config, items): newval = val + zone_rank + role_rank log.info(f"ORIGINAL = {val}, NEW={newval}") markers_update.append((pytest.mark.order, newval)) + if item.own_markers: + item.own_markers.remove(m) break markers_update.append( (config_index, item.callspec.params["config_index"]) ) # Apply all the markers now for mark, param in markers_update: - if mark.name == "run": - for m in item.iter_markers(): - if m.name == "run": - # m.kwargs['order'] = param - item.add_marker(pytest.mark.run(order=param)) + if mark.name == "order": + item.add_marker(pytest.mark.order(param)) else: item.add_marker(mark(param)) log.info(f"TEST={item.name}") From 3971abdc1b53af5df14da737efc21fb3e4572ff0 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Tue, 12 Mar 2024 10:27:55 +0000 Subject: [PATCH 17/32] Minor fixes for test ordering Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/cluster.py | 4 ++-- ocs_ci/ocs/ocs_upgrade.py | 3 --- tests/conftest.py | 19 ++++++++++++++++--- tests/functional/upgrade/test_upgrade_ocp.py | 2 +- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/ocs_ci/ocs/cluster.py b/ocs_ci/ocs/cluster.py index 25154a5ca11..7be4b3b13cf 100644 --- a/ocs_ci/ocs/cluster.py +++ b/ocs_ci/ocs/cluster.py @@ -97,7 +97,7 @@ class CephCluster(object): namespace (str): openshift Namespace where this cluster lives """ - def __init__(self, cluster_config): + def __init__(self, cluster_config=None): """ Cluster object initializer, this object needs to be initialized after cluster deployment. However its harmless to do anywhere. @@ -1062,7 +1062,7 @@ def __init__(self, ceph_cluster=None): def __enter__(self): pass - def __exit__(self): + def __exit__(self, exception_type, value, traceback): pass diff --git a/ocs_ci/ocs/ocs_upgrade.py b/ocs_ci/ocs/ocs_upgrade.py index 1a49a9be260..98c4b15f49f 100644 --- a/ocs_ci/ocs/ocs_upgrade.py +++ b/ocs_ci/ocs/ocs_upgrade.py @@ -373,9 +373,6 @@ def get_csv_name_pre_upgrade(self, resource_name=OCS_OPERATOR_NAME): str: OCS operator name, as displayed in CSV """ - import ipdb - - ipdb.set_trace() operator_selector = get_selector_for_ocs_operator() package_manifest = PackageManifest( resource_name=resource_name, diff --git a/tests/conftest.py b/tests/conftest.py index f861aedcb7a..dae2d6357ec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -475,7 +475,7 @@ def pytest_collection_modifyitems(session, config, items): ) items.remove(item) # If multicluster upgrade scenario - if ocsci_config.multicluster and ocsci_config.UPGRADE.get("upgrade", True): + if ocsci_config.multicluster and ocsci_config.UPGRADE.get("upgrade", False): for item in items: if ( list( @@ -491,11 +491,11 @@ def pytest_collection_modifyitems(session, config, items): else: continue markers_update = [] - item_markers = copy(item.own_markers) + item_markers = copy.copy(item.own_markers) if not item_markers: # If testcase is in a class then own_markers will be empty # hence we need to check item.instance.pytestmark - item_markers = copy(item.instance.pytestmark) + item_markers = copy.copy(item.instance.pytestmark) for m in item_markers: # fetch already marked 'order' value if m.name == "order": @@ -538,6 +538,19 @@ def pytest_collection_finish(session): ocsci_config.RUN["number_of_tests"] = len(session.items) +def pytest_runtest_setup(item): + """ + Pytest hook where we want to switch context of the cluster + before any fixture runs + + """ + if ocsci_config.multicluster and ocsci_config.UPGRADE.get("upgrade", False): + for mark in item.iter_markers(): + if mark.name == "config_index": + log.info("Switching the test context to index: {mark.args[0]}") + ocsci_config.switch_ctx(mark.args[0]) + + def pytest_fixture_setup(fixturedef, request): """ In case of multicluster upgrade scenarios, we want to make sure that before running diff --git a/tests/functional/upgrade/test_upgrade_ocp.py b/tests/functional/upgrade/test_upgrade_ocp.py index e10e287ca86..4de5f4cb5a4 100644 --- a/tests/functional/upgrade/test_upgrade_ocp.py +++ b/tests/functional/upgrade/test_upgrade_ocp.py @@ -226,7 +226,7 @@ def test_upgrade_ocp( # load new config file self.load_ocp_version_config_file(ocp_upgrade_version) - if not config.ENV_DATA["mcg_only_deployment"]: + if not config.ENV_DATA["mcg_only_deployment"] and not config.multicluster: new_ceph_cluster = CephCluster() # Increased timeout because of this bug: # https://bugzilla.redhat.com/show_bug.cgi?id=2038690 From 41f2eca89d0356c61eebc4edda8e83abca6b7daa Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Tue, 30 Apr 2024 10:28:39 +0100 Subject: [PATCH 18/32] Fix few issues in upgrades Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/framework/pytest_customization/marks.py | 9 +-------- ocs_ci/ocs/acm_upgrade.py | 18 +++++++++++------- ocs_ci/ocs/dr_upgrade.py | 17 ++++++++++++----- ocs_ci/ocs/ocs_upgrade.py | 9 +++++++-- tests/functional/upgrade/test_upgrade.py | 12 +++++++++--- 5 files changed, 40 insertions(+), 25 deletions(-) diff --git a/ocs_ci/framework/pytest_customization/marks.py b/ocs_ci/framework/pytest_customization/marks.py index a55cd2ce6d6..7ca6a065332 100644 --- a/ocs_ci/framework/pytest_customization/marks.py +++ b/ocs_ci/framework/pytest_customization/marks.py @@ -40,7 +40,7 @@ ) from ocs_ci.utility import version from ocs_ci.utility.aws import update_config_from_s3 -from ocs_ci.utility.utils import is_z_stream_upgrade, load_auth_config +from ocs_ci.utility.utils import load_auth_config # tier marks @@ -197,13 +197,6 @@ reason="This test cannot run on setup having more than three worker nodes", ) -# Skip if we are running z-stream upgrades -# Useful marker for DR upgrade cases -skipif_z_stream_upgrade = pytest.mark.skipif( - is_z_stream_upgrade(), - reason="This is z-stream upgrade and this component upgrade should have been taken care by other components", -) - # Skipif marks skipif_aws_creds_are_missing = pytest.mark.skipif( ( diff --git a/ocs_ci/ocs/acm_upgrade.py b/ocs_ci/ocs/acm_upgrade.py index 40829b15af4..853b5caaa8c 100644 --- a/ocs_ci/ocs/acm_upgrade.py +++ b/ocs_ci/ocs/acm_upgrade.py @@ -14,7 +14,7 @@ from ocs_ci.ocs.ocp import OCP from ocs_ci.utility import templating from ocs_ci.utility.utils import get_ocp_version, get_running_acm_version, run_cmd -from ocs_ci.utility.version import get_semantic_version + logger = logging.getLogger(__name__) @@ -31,10 +31,11 @@ def __init__(self): self.upgrade_version = config.UPGRADE["upgrade_acm_version"] # In case if we are using registry image self.acm_registry_image = config.UPGRADE.get("upgrade_acm_registry_image", "") + self.zstream_upgrade = False def get_acm_version_before_upgrade(self): running_acm_version = get_running_acm_version() - return get_semantic_version(running_acm_version) + return running_acm_version def get_parsed_versions(self): parsed_version_before_upgrade = parse_version(self.version_before_upgrade) @@ -46,6 +47,8 @@ def run_upgrade(self): self.version_change = ( self.get_parsed_versions()[1] > self.get_parsed_versions()[0] ) + if not self.version_change: + self.zstream_upgrade = True # either this would be GA to Unreleased upgrade of same version OR # GA to unreleased upgrade to higher version if self.acm_registry_image and self.version_change: @@ -123,12 +126,13 @@ def validate_upgrade(self): acm_sub = OCP( namespace=self.namespace, resource_name=self.operator_name, - kind="Subscription", - ) - assert ( - acm_sub.get()["items"][0]["spec"]["channel"] - == config.ENV_DATA["acm_hub_channel"] + kind="Subscription.operators.coreos.com", ) + if not self.zstream_upgrade: + acm_prev_channel = f"release-{self.upgrade_version}" + else: + acm_prev_channel = config.ENV_DATA["acm_hub_channel"] + assert acm_sub.get().get("spec").get("channel") == acm_prev_channel logger.info("Checking ACM status") acm_mch = OCP( kind=constants.ACM_MULTICLUSTER_HUB, diff --git a/ocs_ci/ocs/dr_upgrade.py b/ocs_ci/ocs/dr_upgrade.py index 453961b1774..e8e8199c416 100644 --- a/ocs_ci/ocs/dr_upgrade.py +++ b/ocs_ci/ocs/dr_upgrade.py @@ -239,13 +239,15 @@ class DRHubUpgrade(DRUpgrade): def __init__(self): super().__init__(resource_name=defaults.DR_HUB_OPERATOR_NAME) self.operator_name = defaults.DR_HUB_OPERATOR_NAME - self.subscription_name = constants.DR_HUB_OPERATOR_SUBSCRIPTION + self.subscription_name = constants.DR_HUB_OPERATOR_SUBSCRIPTION.replace( + "PLACEHOLDER", config.ENV_DATA["ocs_version"] + ) def run_upgrade(self): self.pod_name_pattern = "ramen-hub-operator" self.collect_data() assert ( - self.pre_upgrade_data["pod_status"] == "Running" + self.pre_upgrade_data.get("pod_status", "") == "Running" ), "ramen-hub-operator pod is not in Running status" super().run_upgrade() self.upgrade_phase = "post_upgrade" @@ -265,15 +267,20 @@ class DRClusterOperatorUpgrade(DRUpgrade): """ def __init__(self): - super().__init__(resource_name=defaults.DR_CLUSTER_OPERATOR_NAME) + super().__init__( + resource_name=defaults.DR_CLUSTER_OPERATOR_NAME, + namespace=constants.OPENSHIFT_DR_SYSTEM_NAMESPACE, + ) self.operator_name = defaults.DR_CLUSTER_OPERATOR_NAME - self.subscription_name = constants.DR_CLUSTER_OPERATOR_SUBSCRIPTION + self.subscription_name = constants.DR_CLUSTER_OPERATOR_SUBSCRIPTION.replace( + "PLACEHOLDER", config.ENV_DATA["ocs_version"] + ) def run_upgrade(self): self.pod_name_pattern = "ramen-dr-cluster-operator" self.collect_data() assert ( - self.pre_upgrade_data["pod_status"] == "Running" + self.pre_upgrade_data.get("pod_status", "") == "Running" ), "ramen-dr-operator pod is not in Running status" super().run_upgrade() self.upgrade_phase = "post_upgrade" diff --git a/ocs_ci/ocs/ocs_upgrade.py b/ocs_ci/ocs/ocs_upgrade.py index 98c4b15f49f..a1cc7a479eb 100644 --- a/ocs_ci/ocs/ocs_upgrade.py +++ b/ocs_ci/ocs/ocs_upgrade.py @@ -435,9 +435,14 @@ def update_subscription(self, channel): subscription_name = constants.ODF_SUBSCRIPTION else: subscription_name = constants.OCS_SUBSCRIPTION + kind_name = ( + "subscription.operators.coreos.com" + if config.multicluster + else "subscription" + ) subscription = OCP( resource_name=subscription_name, - kind="subscription", + kind=kind_name, namespace=config.ENV_DATA["cluster_namespace"], ) current_ocs_source = subscription.data["spec"]["source"] @@ -448,7 +453,7 @@ def update_subscription(self, channel): else constants.OPERATOR_CATALOG_SOURCE_NAME ) patch_subscription_cmd = ( - f"patch subscription {subscription_name} " + f"patch {kind_name} {subscription_name} " f'-n {self.namespace} --type merge -p \'{{"spec":{{"channel": ' f'"{channel}", "source": "{ocs_source}"}}}}\'' ) diff --git a/tests/functional/upgrade/test_upgrade.py b/tests/functional/upgrade/test_upgrade.py index 494b7eb95b5..1c0ad28ba7d 100644 --- a/tests/functional/upgrade/test_upgrade.py +++ b/tests/functional/upgrade/test_upgrade.py @@ -5,7 +5,6 @@ from ocs_ci.framework.pytest_customization.marks import ( purple_squad, multicluster_roles, - skipif_z_stream_upgrade, ) from ocs_ci.framework.testlib import ( ocs_upgrade, @@ -24,6 +23,7 @@ DRHubUpgrade, ) from ocs_ci.utility.reporting import get_polarion_id +from ocs_ci.utility.utils import is_z_stream_upgrade log = logging.getLogger(__name__) @@ -106,26 +106,32 @@ def test_mco_upgrade(zone_rank, role_rank, config_index): @purple_squad @dr_hub_upgrade -@skipif_z_stream_upgrade @multicluster_roles(["mdr-all-acm"]) def test_dr_hub_upgrade(zone_rank, role_rank, config_index): """ Test upgrade procedure for DR hub operator """ + if is_z_stream_upgrade(): + pytest.skip( + "This is z-stream upgrade and this component upgrade should have been taken care by ODF upgrade" + ) dr_hub_upgrade_obj = DRHubUpgrade() dr_hub_upgrade_obj.run_upgrade() @purple_squad @dr_cluster_operator_upgrade -@skipif_z_stream_upgrade @multicluster_roles(["mdr-all-odf"]) def test_dr_cluster_upgrade(zone_rank, role_rank, config_index): """ Test upgrade procedure for DR cluster operator """ + if is_z_stream_upgrade(): + pytest.skip( + "This is z-stream upgrade and this component upgrade should have been taken care by ODF upgrade" + ) dr_cluster_upgrade_obj = DRClusterOperatorUpgrade() dr_cluster_upgrade_obj.run_upgrade() From aa4df5a33def43e2b57370d33b50cd9d34222141 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Tue, 4 Jun 2024 23:01:08 +0100 Subject: [PATCH 19/32] Add validation for dr operators as part of ocs upgrade test in the case of zstream upgrade Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/dr_upgrade.py | 6 +++--- tests/functional/upgrade/test_upgrade.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/ocs_ci/ocs/dr_upgrade.py b/ocs_ci/ocs/dr_upgrade.py index e8e8199c416..fee43dd075a 100644 --- a/ocs_ci/ocs/dr_upgrade.py +++ b/ocs_ci/ocs/dr_upgrade.py @@ -211,10 +211,10 @@ def __init__(self): super().__init__(resource_name=defaults.MCO_OPERATOR_NAME) self.operator_name = defaults.MCO_OPERATOR_NAME self.subscription_name = constants.MCO_SUBSCRIPTION + self.pod_name_pattern = "odfmo-controller-manager" def run_upgrade(self): # Collect some pre-upgrade data for comparision after the upgrade - self.pod_name_pattern = "odfmo-controller-manager" self.collect_data() assert ( self.pre_upgrade_data.get("pod_status", "") == "Running" @@ -242,9 +242,9 @@ def __init__(self): self.subscription_name = constants.DR_HUB_OPERATOR_SUBSCRIPTION.replace( "PLACEHOLDER", config.ENV_DATA["ocs_version"] ) + self.pod_name_pattern = "ramen-hub-operator" def run_upgrade(self): - self.pod_name_pattern = "ramen-hub-operator" self.collect_data() assert ( self.pre_upgrade_data.get("pod_status", "") == "Running" @@ -275,9 +275,9 @@ def __init__(self): self.subscription_name = constants.DR_CLUSTER_OPERATOR_SUBSCRIPTION.replace( "PLACEHOLDER", config.ENV_DATA["ocs_version"] ) + self.pod_name_pattern = "ramen-dr-cluster-operator" def run_upgrade(self): - self.pod_name_pattern = "ramen-dr-cluster-operator" self.collect_data() assert ( self.pre_upgrade_data.get("pod_status", "") == "Running" diff --git a/tests/functional/upgrade/test_upgrade.py b/tests/functional/upgrade/test_upgrade.py index 1c0ad28ba7d..07d9158e518 100644 --- a/tests/functional/upgrade/test_upgrade.py +++ b/tests/functional/upgrade/test_upgrade.py @@ -14,6 +14,7 @@ dr_cluster_operator_upgrade, acm_upgrade, ) +from ocs_ci.framework import config from ocs_ci.ocs.acm_upgrade import ACMUpgrade from ocs_ci.ocs.disruptive_operations import worker_node_shutdown, osd_node_reboot from ocs_ci.ocs.ocs_upgrade import run_ocs_upgrade @@ -28,6 +29,13 @@ log = logging.getLogger(__name__) +operator_map = { + "mco": MultiClusterOrchestratorUpgrade, + "drhub": DRHubUpgrade, + "drcluster": DRClusterOperatorUpgrade, +} + + @pytest.fixture() def teardown(request, nodes): def finalizer(): @@ -90,6 +98,15 @@ def test_upgrade(zone_rank, role_rank, config_index, upgrade_stats): """ run_ocs_upgrade(upgrade_stats=upgrade_stats) + if config.multicluster and config.MULTICLUSTER["multicluster_mode"] == "metro-dr": + # Perform validation for MCO, dr hub operator and dr cluster operator here + # in case of z stream because we wouldn't call those tests in the case of + # z stream + if is_z_stream_upgrade(): + for operator, op_upgrade_cls in operator_map.items(): + temp = op_upgrade_cls() + log.info(f"Validating upgrade for {operator}") + temp.validate_upgrade() @purple_squad From 4f37772713703613f4ab914a376b6d2a89ef1415 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Tue, 25 Jun 2024 23:49:25 +0100 Subject: [PATCH 20/32] Add --acm-version CLI option Signed-off-by: Shylesh Kumar Mohan --- conf/README.md | 1 + ocs_ci/framework/pytest_customization/ocscilib.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/conf/README.md b/conf/README.md index ac2eeb68844..b4fdcad163c 100644 --- a/conf/README.md +++ b/conf/README.md @@ -212,6 +212,7 @@ higher priority). * `skip_ocp_deployment` - Skip the OCP deployment step or not (Default: false) * `skip_ocs_deployment` - Skip the OCS deployment step or not (Default: false) * `ocs_version` - Version of OCS that is being deployed +* `acm_version` - Version of acm to be used for this run (applicable mostly to DR scenarios) * `vm_template` - VMWare template to use for RHCOS images * `fio_storageutilization_min_mbps` - Minimal write speed of FIO used in workload_fio_storageutilization * `TF_LOG_LEVEL` - Terraform log level diff --git a/ocs_ci/framework/pytest_customization/ocscilib.py b/ocs_ci/framework/pytest_customization/ocscilib.py index 7a56621d003..958ac3fd834 100644 --- a/ocs_ci/framework/pytest_customization/ocscilib.py +++ b/ocs_ci/framework/pytest_customization/ocscilib.py @@ -263,6 +263,11 @@ def pytest_addoption(parser): "(e.g. quay.io/rhceph-dev/ocs-olm-operator:latest-4.3)" ), ) + parser.addoption( + "--acm-version", + dest="acm_version", + help="acm version(e.g. 2.8) to be used for the current run", + ) parser.addoption( "--upgrade-acm-version", dest="upgrade_acm_version", @@ -671,6 +676,9 @@ def process_cluster_cli_params(config): if custom_kubeconfig_location: os.environ["KUBECONFIG"] = custom_kubeconfig_location ocsci_config.RUN["kubeconfig"] = custom_kubeconfig_location + acm_version = get_cli_param(config, "--acm-version") + if acm_version: + ocsci_config.ENV_DATA["acm_version"] = acm_version upgrade_acm_version = get_cli_param(config, "--upgrade-acm-version") if upgrade_acm_version: ocsci_config.UPGRADE["upgrade_acm_version"] = upgrade_acm_version From 3e59c3b85701a207cd5119f09ee8a5df07b51897 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Wed, 31 Jul 2024 22:16:05 +0100 Subject: [PATCH 21/32] Handle reload of the configs during upgrades Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/framework/__init__.py | 4 ++++ ocs_ci/ocs/ocs_upgrade.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/ocs_ci/framework/__init__.py b/ocs_ci/framework/__init__.py index 7ecd7cedd7e..12b22119092 100644 --- a/ocs_ci/framework/__init__.py +++ b/ocs_ci/framework/__init__.py @@ -37,6 +37,10 @@ class Config: COMPONENTS: dict = field(default_factory=dict) # Used for multicluster only MULTICLUSTER: dict = field(default_factory=dict) + # Use this variable to store any arbitrary key/values related + # to the upgrade context. Applicable only in the multicluster upgrade + # scenario + PREUPGRADE_CONFIG = field(default_factory=dict) def __post_init__(self): self.reset() diff --git a/ocs_ci/ocs/ocs_upgrade.py b/ocs_ci/ocs/ocs_upgrade.py index a1cc7a479eb..52ddc013e98 100644 --- a/ocs_ci/ocs/ocs_upgrade.py +++ b/ocs_ci/ocs/ocs_upgrade.py @@ -4,6 +4,7 @@ from pkg_resources import parse_version from tempfile import NamedTemporaryFile import time +import yaml from selenium.webdriver.common.by import By from ocs_ci.framework import config @@ -312,6 +313,18 @@ def get_parsed_versions(self): return parsed_version_before_upgrade, parsed_upgrade_version + def store_pre_upgrade_ctx(self): + """ + Store pre upgrade ctx values in config.upgrade_ctx + + """ + version_config_file = os.path.join( + constants.OCS_VERSION_CONF_DIR, f"ocs-{self.version_before_upgrade}.yaml" + ) + buf = yaml.safe_load(version_config_file) + + + def load_version_config_file(self, upgrade_version): """ Loads config file to the ocs-ci config with upgrade version @@ -333,6 +346,15 @@ def load_version_config_file(self, upgrade_version): version_config_file = os.path.join( constants.OCS_VERSION_CONF_DIR, f"ocs-{upgrade_version}.yaml" ) + if config.multicluster: + # In case of multicluster upgrade + # we need to preserve the old config values so that + # any following tests will not read overwritten values + # we want to do it only once and the first test which hits this function + # will have the responsibility to store the previous config values + # TODO: CHANGE THIS TO STORE THE CONTEXT IN THE RESPECTIVE + # CONFIG CLASS DURING INITIAL STAGES OF THE UPGRADE RUN + self.save_pre_upgrade_ctx(version_config_file) log.info(f"Reloading config file for OCS/ODF version: {upgrade_version}.") load_config_file(version_config_file) else: From bd364f30ae287dc99a70605b98436e0f28eef933 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Wed, 11 Sep 2024 13:42:26 +0530 Subject: [PATCH 22/32] Save Preupgrade conf values in PREUPGRADE_CONFIG attr of the Config class Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/framework/__init__.py | 2 +- ocs_ci/framework/conf/default_config.yaml | 4 +++ ocs_ci/ocs/dr_upgrade.py | 5 +++- ocs_ci/ocs/ocs_upgrade.py | 27 +++----------------- ocs_ci/ocs/utils.py | 10 -------- tests/conftest.py | 8 ++++++ tests/functional/upgrade/test_upgrade_ocp.py | 2 +- 7 files changed, 22 insertions(+), 36 deletions(-) diff --git a/ocs_ci/framework/__init__.py b/ocs_ci/framework/__init__.py index 12b22119092..3930b52f877 100644 --- a/ocs_ci/framework/__init__.py +++ b/ocs_ci/framework/__init__.py @@ -40,7 +40,7 @@ class Config: # Use this variable to store any arbitrary key/values related # to the upgrade context. Applicable only in the multicluster upgrade # scenario - PREUPGRADE_CONFIG = field(default_factory=dict) + PREUPGRADE_CONFIG: dict = field(default_factory=dict) def __post_init__(self): self.reset() diff --git a/ocs_ci/framework/conf/default_config.yaml b/ocs_ci/framework/conf/default_config.yaml index 23f01d8feb8..56637adf82d 100644 --- a/ocs_ci/framework/conf/default_config.yaml +++ b/ocs_ci/framework/conf/default_config.yaml @@ -362,3 +362,7 @@ MULTICLUSTER: acm_cluster: False primary_cluster: False active_acm_cluster: False + +PREUPGRADE_CONFIG: + AUTH: null + MULTICLUSTER: null \ No newline at end of file diff --git a/ocs_ci/ocs/dr_upgrade.py b/ocs_ci/ocs/dr_upgrade.py index fee43dd075a..da436369d08 100644 --- a/ocs_ci/ocs/dr_upgrade.py +++ b/ocs_ci/ocs/dr_upgrade.py @@ -41,7 +41,10 @@ def __init__( resource_name=None, ): if not version_before_upgrade: - version_before_upgrade = config.ENV_DATA.get("ocs_version") + if config.PREUPGRADE_CONFIG.get('ENV_DATA').get("ocs_version", ''): + version_before_upgrade = config.PREUPGRADE_CONFIG['ENV_DATA'].get("ocs_version") + else: + version_before_upgrade = config.ENV_DATA.get("ocs_version") if not ocs_registry_image: ocs_registry_image = config.UPGRADE.get("upgrade_ocs_registry_image") self.external_cluster = None diff --git a/ocs_ci/ocs/ocs_upgrade.py b/ocs_ci/ocs/ocs_upgrade.py index 52ddc013e98..795f1d29a10 100644 --- a/ocs_ci/ocs/ocs_upgrade.py +++ b/ocs_ci/ocs/ocs_upgrade.py @@ -4,7 +4,6 @@ from pkg_resources import parse_version from tempfile import NamedTemporaryFile import time -import yaml from selenium.webdriver.common.by import By from ocs_ci.framework import config @@ -313,18 +312,6 @@ def get_parsed_versions(self): return parsed_version_before_upgrade, parsed_upgrade_version - def store_pre_upgrade_ctx(self): - """ - Store pre upgrade ctx values in config.upgrade_ctx - - """ - version_config_file = os.path.join( - constants.OCS_VERSION_CONF_DIR, f"ocs-{self.version_before_upgrade}.yaml" - ) - buf = yaml.safe_load(version_config_file) - - - def load_version_config_file(self, upgrade_version): """ Loads config file to the ocs-ci config with upgrade version @@ -346,15 +333,6 @@ def load_version_config_file(self, upgrade_version): version_config_file = os.path.join( constants.OCS_VERSION_CONF_DIR, f"ocs-{upgrade_version}.yaml" ) - if config.multicluster: - # In case of multicluster upgrade - # we need to preserve the old config values so that - # any following tests will not read overwritten values - # we want to do it only once and the first test which hits this function - # will have the responsibility to store the previous config values - # TODO: CHANGE THIS TO STORE THE CONTEXT IN THE RESPECTIVE - # CONFIG CLASS DURING INITIAL STAGES OF THE UPGRADE RUN - self.save_pre_upgrade_ctx(version_config_file) log.info(f"Reloading config file for OCS/ODF version: {upgrade_version}.") load_config_file(version_config_file) else: @@ -401,7 +379,10 @@ def get_csv_name_pre_upgrade(self, resource_name=OCS_OPERATOR_NAME): selector=operator_selector, subscription_plan_approval=self.subscription_plan_approval, ) - channel = config.DEPLOYMENT.get("ocs_csv_channel") + if config.PREUPGRADE_CONFIG.get("DEPLOYMENT").get("ocs_csv_channel"): + channel = config.PREUPGRADE_CONFIG.get("DEPLOYMENT").get("ocs_csv_channel") + else: + channel = config.DEPLOYMENT.get("ocs_csv_channel") return package_manifest.get_current_csv(channel) diff --git a/ocs_ci/ocs/utils.py b/ocs_ci/ocs/utils.py index 7deec50a3c8..f2dfbfdee96 100644 --- a/ocs_ci/ocs/utils.py +++ b/ocs_ci/ocs/utils.py @@ -1639,16 +1639,6 @@ def get_all_acm_indexes(): return acm_indexes -def is_acm_cluster(): - """ - Check whether the current cluster in context is an ACM cluster - - Returns: - bool: True if its an ACM cluster else False - """ - return ocsci_config.MULTICLUSTER["multicluster_index"] in get_all_acm_indexes() - - def is_acm_cluster(cluster): """ Checks given cluster is acm cluster diff --git a/tests/conftest.py b/tests/conftest.py index dae2d6357ec..556cdbbf4d6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,7 @@ from math import floor from shutil import copyfile, rmtree from functools import partial +from copy import deepcopy import boto3 import yaml @@ -524,6 +525,13 @@ def pytest_collection_modifyitems(session, config, items): log.info( f"MARKERS = {[(i.name, i.args, i.kwargs) for i in item.iter_markers()]}" ) + # Update PREUPGRADE_CONFIG for each of the Config class + # so that in case of Y stream upgrade we will have preupgrade configurations for reference + # across the tests as Y stream upgrade will reload the config of target version + for cluster in ocsci_config.clusters: + for k in cluster.__dataclass_fields__.keys(): + if k != 'PREUPGRADE_CONFIG': + cluster.PREUPGRADE_CONFIG[k] = deepcopy(getattr(cluster, k)) def pytest_collection_finish(session): diff --git a/tests/functional/upgrade/test_upgrade_ocp.py b/tests/functional/upgrade/test_upgrade_ocp.py index 4de5f4cb5a4..4258cdfac31 100644 --- a/tests/functional/upgrade/test_upgrade_ocp.py +++ b/tests/functional/upgrade/test_upgrade_ocp.py @@ -100,7 +100,7 @@ def test_upgrade_ocp( if ( config.multicluster and config.MULTICLUSTER["multicluster_mode"] == "metro-dr" - and is_acm_cluster() + and is_acm_cluster(config) ): # Find the ODF cluster in current zone mdr_upgrade = MDRClusterUpgradeParametrize() From 24e558b215d64ac23fda48fb1a5ae81e6f0cfea5 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Tue, 1 Oct 2024 23:08:32 +0530 Subject: [PATCH 23/32] Fix subscription naming issue Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/constants.py | 8 +++-- ocs_ci/ocs/dr_upgrade.py | 42 +++++++++++++++--------- tests/functional/upgrade/test_upgrade.py | 1 + 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 1d5daa2f13c..06953aabb28 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -1725,9 +1725,11 @@ DR_HUB_OPERATOR_SUBSCRIPTION = ( "odr-hub-operator-stable-PLACEHOLDER-redhat-operators-openshift-marketplace" ) -DR_CLUSTER_OPERATOR_SUBSCRIPTION = ( - "odr-cluster-operator-stable-PLACEHOLDER-redhat-operators-openshift-marketplace" -) +DR_HUB_OPERATOR_SUBSCRIPTION_LABEL = "operators.coreos.com/odr-hub-operator.openshift-operators" +#DR_CLUSTER_OPERATOR_SUBSCRIPTION = ( +# "odr-cluster-operator-stable-PLACEHOLDER-redhat-operators-openshift-marketplace" +#) +DR_CLUSTER_OPERATOR_SUBSCRIPTION = "ramen-dr-cluster-subscription" # UI Deployment constants HTPASSWD_SECRET_NAME = "htpass-secret" diff --git a/ocs_ci/ocs/dr_upgrade.py b/ocs_ci/ocs/dr_upgrade.py index da436369d08..7e2a8c75ec9 100644 --- a/ocs_ci/ocs/dr_upgrade.py +++ b/ocs_ci/ocs/dr_upgrade.py @@ -52,6 +52,7 @@ def __init__( self.subscription_name = None self.pre_upgrade_data = dict() self.post_upgrade_data = dict() + self.namespace = namespace # Upgraded phases [pre_upgrade, post_upgrade] self.upgrade_phase = "pre_upgrade" if resource_name: @@ -88,7 +89,7 @@ def run_upgrade(self): # we need to update subscriptions individually for DR operator as we don't want # to upgrade ODF at the time of DR operator (MCO, DR Hub), ODF would follow the upgrade # of DR operators - self.update_subscription(self.channel, self.subscription_name) + self.update_subscription(self.channel, self.subscription_name, self.namespace ) # In the case upgrade is not from 4.8 to 4.9 and we have manual approval strategy # we need to wait and approve install plan, otherwise it's approved in the # subscribe_ocs method. @@ -115,22 +116,17 @@ def run_upgrade(self): self.upgrade_version, self.resource_name, ) - verify_image_versions( - old_image, - self.get_parsed_versions()[1], - self.version_before_upgrade, - ) - def update_subscription(self, channel, subscription_name): + def update_subscription(self, channel, subscription_name, namespace=constants.OPENSHIFT_OPERATORS): subscription = OCP( resource_name=subscription_name, kind="subscription.operators.coreos.com", # namespace could be different on managed clusters # TODO: Handle different namespaces - namespace=constants.OPENSHIFT_OPERATORS, + namespace=namespace, ) current_source = subscription.data["spec"]["source"] - log.info(f"Current MCO source: {current_source}") + log.info(f"Current source: {current_source}") mco_source = ( current_source if self.upgrade_in_current_source @@ -242,9 +238,24 @@ class DRHubUpgrade(DRUpgrade): def __init__(self): super().__init__(resource_name=defaults.DR_HUB_OPERATOR_NAME) self.operator_name = defaults.DR_HUB_OPERATOR_NAME - self.subscription_name = constants.DR_HUB_OPERATOR_SUBSCRIPTION.replace( - "PLACEHOLDER", config.ENV_DATA["ocs_version"] - ) + #self.subscription_name = constants.DR_HUB_OPERATOR_SUBSCRIPTION.replace( + # "PLACEHOLDER", config.ENV_DATA["ocs_version"] + #) + for sample in TimeoutSampler( + 300, 10, OCP, kind=constants.SUBSCRIPTION_COREOS, namespace=self.namespace + ): + subscriptions = sample.get().get("items", []) + for subscription in subscriptions: + found_subscription_name = subscription.get("metadata", {}).get( + "name", "" + ) + if defaults.DR_HUB_OPERATOR_NAME in found_subscription_name: + log.info(f"Subscription found: {found_subscription_name}") + self.subscription_name = found_subscription_name + break + if not self.subscription_name: + log.error(f"Couldn't find the subscription for {defaults.DR_HUB_OPERATOR_NAME}") + self.pod_name_pattern = "ramen-hub-operator" def run_upgrade(self): @@ -275,9 +286,10 @@ def __init__(self): namespace=constants.OPENSHIFT_DR_SYSTEM_NAMESPACE, ) self.operator_name = defaults.DR_CLUSTER_OPERATOR_NAME - self.subscription_name = constants.DR_CLUSTER_OPERATOR_SUBSCRIPTION.replace( - "PLACEHOLDER", config.ENV_DATA["ocs_version"] - ) + #self.subscription_name = constants.DR_CLUSTER_OPERATOR_SUBSCRIPTION.replace( + # "PLACEHOLDER", config.ENV_DATA["ocs_version"] + #) + self.subscription_name = constants.DR_CLUSTER_OPERATOR_SUBSCRIPTION self.pod_name_pattern = "ramen-dr-cluster-operator" def run_upgrade(self): diff --git a/tests/functional/upgrade/test_upgrade.py b/tests/functional/upgrade/test_upgrade.py index 07d9158e518..bddef77011a 100644 --- a/tests/functional/upgrade/test_upgrade.py +++ b/tests/functional/upgrade/test_upgrade.py @@ -23,6 +23,7 @@ MultiClusterOrchestratorUpgrade, DRHubUpgrade, ) +from ocs_ci.ocs import constants from ocs_ci.utility.reporting import get_polarion_id from ocs_ci.utility.utils import is_z_stream_upgrade From 25d4dbb1fd98a53901dd7895686d0ce99792c580 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Fri, 11 Oct 2024 17:19:33 +0530 Subject: [PATCH 24/32] Rely on CSV list to find pre-upgrade csvs rather than PackageManifest Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/constants.py | 3 -- ocs_ci/ocs/dr_upgrade.py | 22 ++------------ ocs_ci/ocs/ocs_upgrade.py | 38 ++++++++++++++---------- tests/functional/upgrade/test_upgrade.py | 1 - 4 files changed, 26 insertions(+), 38 deletions(-) diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 06953aabb28..c05b8425bf2 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -1726,9 +1726,6 @@ "odr-hub-operator-stable-PLACEHOLDER-redhat-operators-openshift-marketplace" ) DR_HUB_OPERATOR_SUBSCRIPTION_LABEL = "operators.coreos.com/odr-hub-operator.openshift-operators" -#DR_CLUSTER_OPERATOR_SUBSCRIPTION = ( -# "odr-cluster-operator-stable-PLACEHOLDER-redhat-operators-openshift-marketplace" -#) DR_CLUSTER_OPERATOR_SUBSCRIPTION = "ramen-dr-cluster-subscription" # UI Deployment constants diff --git a/ocs_ci/ocs/dr_upgrade.py b/ocs_ci/ocs/dr_upgrade.py index 7e2a8c75ec9..8368ab12ad7 100644 --- a/ocs_ci/ocs/dr_upgrade.py +++ b/ocs_ci/ocs/dr_upgrade.py @@ -8,7 +8,7 @@ from ocs_ci.framework import config from ocs_ci.ocs.exceptions import TimeoutException from ocs_ci.ocs.ocp import OCP -from ocs_ci.ocs.ocs_upgrade import OCSUpgrade, verify_image_versions +from ocs_ci.ocs.ocs_upgrade import OCSUpgrade from ocs_ci.ocs import constants from ocs_ci.ocs import defaults from ocs_ci.deployment.helpers.external_cluster_helpers import ( @@ -89,7 +89,7 @@ def run_upgrade(self): # we need to update subscriptions individually for DR operator as we don't want # to upgrade ODF at the time of DR operator (MCO, DR Hub), ODF would follow the upgrade # of DR operators - self.update_subscription(self.channel, self.subscription_name, self.namespace ) + self.update_subscription(self.channel, self.subscription_name, self.namespace) # In the case upgrade is not from 4.8 to 4.9 and we have manual approval strategy # we need to wait and approve install plan, otherwise it's approved in the # subscribe_ocs method. @@ -110,12 +110,6 @@ def run_upgrade(self): break except TimeoutException: raise TimeoutException("No new CSV found after upgrade!") - old_image = self.get_images_post_upgrade( - self.channel, - self.pre_upgrade_images, - self.upgrade_version, - self.resource_name, - ) def update_subscription(self, channel, subscription_name, namespace=constants.OPENSHIFT_OPERATORS): subscription = OCP( @@ -145,9 +139,6 @@ def validate_upgrade(self): assert ( self.post_upgrade_data.get("pod_status", "") == "Running" ), f"Pod {self.pod_name_pattern} not in Running state post upgrade" - assert ( - self.post_upgrade_data.get("age", "") <= self.pre_upgrade_data["age"] - ), f"{self.pod_name_pattern} didn't restart after upgrade" assert ( self.post_upgrade_data.get("version", "") != self.pre_upgrade_data["version"] @@ -238,9 +229,6 @@ class DRHubUpgrade(DRUpgrade): def __init__(self): super().__init__(resource_name=defaults.DR_HUB_OPERATOR_NAME) self.operator_name = defaults.DR_HUB_OPERATOR_NAME - #self.subscription_name = constants.DR_HUB_OPERATOR_SUBSCRIPTION.replace( - # "PLACEHOLDER", config.ENV_DATA["ocs_version"] - #) for sample in TimeoutSampler( 300, 10, OCP, kind=constants.SUBSCRIPTION_COREOS, namespace=self.namespace ): @@ -255,7 +243,6 @@ def __init__(self): break if not self.subscription_name: log.error(f"Couldn't find the subscription for {defaults.DR_HUB_OPERATOR_NAME}") - self.pod_name_pattern = "ramen-hub-operator" def run_upgrade(self): @@ -286,10 +273,7 @@ def __init__(self): namespace=constants.OPENSHIFT_DR_SYSTEM_NAMESPACE, ) self.operator_name = defaults.DR_CLUSTER_OPERATOR_NAME - #self.subscription_name = constants.DR_CLUSTER_OPERATOR_SUBSCRIPTION.replace( - # "PLACEHOLDER", config.ENV_DATA["ocs_version"] - #) - self.subscription_name = constants.DR_CLUSTER_OPERATOR_SUBSCRIPTION + self.subscription_name = constants.DR_CLUSTER_OPERATOR_SUBSCRIPTION self.pod_name_pattern = "ramen-dr-cluster-operator" def run_upgrade(self): diff --git a/ocs_ci/ocs/ocs_upgrade.py b/ocs_ci/ocs/ocs_upgrade.py index 795f1d29a10..c4b2cabe9de 100644 --- a/ocs_ci/ocs/ocs_upgrade.py +++ b/ocs_ci/ocs/ocs_upgrade.py @@ -31,8 +31,12 @@ from ocs_ci.ocs.ocp import get_images, OCP from ocs_ci.ocs.node import get_nodes from ocs_ci.ocs.resources.catalog_source import CatalogSource, disable_specific_source -from ocs_ci.ocs.resources.csv import CSV, check_all_csvs_are_succeeded from ocs_ci.ocs.resources.daemonset import DaemonSet +from ocs_ci.ocs.resources.csv import ( + CSV, + check_all_csvs_are_succeeded, + get_csvs_start_with_prefix, +) from ocs_ci.ocs.resources.install_plan import wait_for_install_plan_and_approve from ocs_ci.ocs.resources.pod import get_noobaa_pods, verify_pods_upgraded from ocs_ci.ocs.resources.packagemanifest import ( @@ -62,6 +66,7 @@ from ocs_ci.ocs.exceptions import ( TimeoutException, ExternalClusterRGWAdminOpsUserException, + CSVNotFound, ) from ocs_ci.ocs.ui.base_ui import logger, login_ui from ocs_ci.ocs.ui.views import locators, ODF_OPERATOR @@ -367,24 +372,27 @@ def load_version_config_file(self, upgrade_version): def get_csv_name_pre_upgrade(self, resource_name=OCS_OPERATOR_NAME): """ - Getting OCS operator name as displayed in CSV + Get pre-upgrade CSV name - Returns: - str: OCS operator name, as displayed in CSV + Ealier we used to depend on packagemanifest to find the pre-upgrade + csv name. Due to issues in catalogsource where csv names were not shown properly once + catalogsource for upgrade version has been created, we are taking new approach of + finding csv name from csv list and also look for pre-upgrade ocs version for finding out + the actual csv """ - operator_selector = get_selector_for_ocs_operator() - package_manifest = PackageManifest( - resource_name=resource_name, - selector=operator_selector, - subscription_plan_approval=self.subscription_plan_approval, - ) - if config.PREUPGRADE_CONFIG.get("DEPLOYMENT").get("ocs_csv_channel"): - channel = config.PREUPGRADE_CONFIG.get("DEPLOYMENT").get("ocs_csv_channel") - else: - channel = config.DEPLOYMENT.get("ocs_csv_channel") - return package_manifest.get_current_csv(channel) + csv_name = None + csv_list = get_csvs_start_with_prefix(resource_name, namespace=self.namespace) + for csv in csv_list: + if resource_name in csv.get("metadata").get("name"): + if config.PREUPGRADE_CONFIG.get("ENV_DATA").get( + "ocs_version" + ) in csv.get("metadata").get("name"): + csv_name = csv.get("metadata").get("name") + if not csv_name: + raise CSVNotFound(f"No preupgrade CSV found for {resource_name}") + return csv_name def get_pre_upgrade_image(self, csv_name_pre_upgrade): """ diff --git a/tests/functional/upgrade/test_upgrade.py b/tests/functional/upgrade/test_upgrade.py index bddef77011a..07d9158e518 100644 --- a/tests/functional/upgrade/test_upgrade.py +++ b/tests/functional/upgrade/test_upgrade.py @@ -23,7 +23,6 @@ MultiClusterOrchestratorUpgrade, DRHubUpgrade, ) -from ocs_ci.ocs import constants from ocs_ci.utility.reporting import get_polarion_id from ocs_ci.utility.utils import is_z_stream_upgrade From dc866bfa4c22badf6ce20508b3f3dd093c9137e6 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Fri, 18 Oct 2024 19:05:16 +0530 Subject: [PATCH 25/32] Fix acm issues: 1. Catsrc name patching issue 2. Format icsp yaml 3. patch channel name Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/acm_upgrade.py | 7 +++- .../acm-deployment/acm_brew_icsp.yaml | 32 +++++++++---------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/ocs_ci/ocs/acm_upgrade.py b/ocs_ci/ocs/acm_upgrade.py index 853b5caaa8c..4b448891d1d 100644 --- a/ocs_ci/ocs/acm_upgrade.py +++ b/ocs_ci/ocs/acm_upgrade.py @@ -55,12 +55,16 @@ def run_upgrade(self): self.upgrade_with_registry() self.annotate_mch() run_cmd(f"oc create -f {constants.ACM_BREW_ICSP_YAML}") + self.patch_channel() else: # GA to GA self.upgrade_without_registry() self.validate_upgrade() def upgrade_without_registry(self): + self.patch_channel() + + def patch_channel(self): """ GA to GA acm upgrade @@ -81,7 +85,7 @@ def upgrade_with_registry(self): else: # This is GA to unreleased version: upgrade to next version self.create_catalog_source() - patch = f'\'{{"spec": "{constants.ACM_CATSRC_NAME}"}}\'' + patch = f'\'{{"spec":{{"source": "{constants.ACM_CATSRC_NAME}"}}}}\'' self.acm_patch_subscription(patch) def annotate_mch(self): @@ -116,6 +120,7 @@ def create_catalog_source(self): [constants.ACM_BREW_REPO, version_tag] ) acm_catsrc["metadata"]["name"] = constants.ACM_CATSRC_NAME + acm_catsrc["spec"]["publisher"] = "grpc" acm_data_yaml = tempfile.NamedTemporaryFile( mode="w+", prefix="acm_catsrc", delete=False ) diff --git a/ocs_ci/templates/acm-deployment/acm_brew_icsp.yaml b/ocs_ci/templates/acm-deployment/acm_brew_icsp.yaml index a22605f250b..312328c85b1 100644 --- a/ocs_ci/templates/acm-deployment/acm_brew_icsp.yaml +++ b/ocs_ci/templates/acm-deployment/acm_brew_icsp.yaml @@ -3,19 +3,19 @@ kind: ImageContentSourcePolicy metadata: name: image-policy-brew spec: - repositoryDigestMirrors: - - mirrors: - - brew.registry.redhat.io/rh-osbs/rhacm2 - source: registry.redhat.io/rhacm2 - - mirrors: - - brew.registry.redhat.io/rh-osbs - source: registry-proxy.engineering.redhat.com/rh-osbs - - mirrors: - - brew.registry.redhat.io/rh-osbs/multicluster-engine - source: registry.redhat.io/multicluster-engine - - mirrors: - - registry.redhat.io/rhacm2 - source: registry.stage.redhat.io/rhacm2 - - mirrors: - - registry.redhat.io/multicluster-engine - source: registry.stage.redhat.io/multicluster-engine + repositoryDigestMirrors: + - mirrors: + - brew.registry.redhat.io/rh-osbs/rhacm2 + source: registry.redhat.io/rhacm2 + - mirrors: + - brew.registry.redhat.io/rh-osbs + source: registry-proxy.engineering.redhat.com/rh-osbs + - mirrors: + - brew.registry.redhat.io/rh-osbs/multicluster-engine + source: registry.redhat.io/multicluster-engine + - mirrors: + - registry.redhat.io/rhacm2 + source: registry.stage.redhat.io/rhacm2 + - mirrors: + - registry.redhat.io/multicluster-engine + source: registry.stage.redhat.io/multicluster-engine From 222e30b7b817c7eb76610b22cabb9dec4dacbd45 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Mon, 4 Nov 2024 13:46:49 +0530 Subject: [PATCH 26/32] Sleep for 10 seconds after updating the subscription Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/dr_upgrade.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ocs_ci/ocs/dr_upgrade.py b/ocs_ci/ocs/dr_upgrade.py index 8368ab12ad7..a526770156e 100644 --- a/ocs_ci/ocs/dr_upgrade.py +++ b/ocs_ci/ocs/dr_upgrade.py @@ -4,6 +4,7 @@ """ import logging +import time from ocs_ci.framework import config from ocs_ci.ocs.exceptions import TimeoutException @@ -132,6 +133,7 @@ def update_subscription(self, channel, subscription_name, namespace=constants.OP f'"{self.channel}", "source": "{mco_source}"}}}}\'' ) subscription.exec_oc_cmd(patch_subscription_cmd, out_yaml_format=False) + time.sleep(10) def validate_upgrade(self): # In case of both MCO and DRhub operator, validation steps are similar @@ -175,7 +177,8 @@ def collect_data(self): # get pre-upgrade csv for MCO csv_objs = CSV(namespace=self.namespace) for csv in csv_objs.get()["items"]: - if self.operator_name in csv["metadata"]["name"]: + if (self.operator_name in csv["metadata"]["name"] + and self.upgrade_version in csv["metadata"]["name"]): csv_obj = CSV( namespace=self.namespace, resource_name=csv["metadata"]["name"] ) From 52d8c73aae92f8dcb9998742fd9e84130fc0aadb Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Tue, 19 Nov 2024 22:40:45 +0530 Subject: [PATCH 27/32] Collect upgrade version in the init Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/dr_upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocs_ci/ocs/dr_upgrade.py b/ocs_ci/ocs/dr_upgrade.py index a526770156e..e0fc20e5205 100644 --- a/ocs_ci/ocs/dr_upgrade.py +++ b/ocs_ci/ocs/dr_upgrade.py @@ -65,9 +65,9 @@ def __init__( ocs_registry_image, upgrade_in_current_source, ) + self.upgrade_version = self.get_upgrade_version() def run_upgrade(self): - self.upgrade_version = self.get_upgrade_version() assert self.get_parsed_versions()[1] >= self.get_parsed_versions()[0], ( f"Version you would like to upgrade to: {self.upgrade_version} " f"is not higher or equal to the version you currently running: " From a03d448af63e879df36efaa1135fea5af1c7fbc7 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Tue, 26 Nov 2024 21:39:10 +0530 Subject: [PATCH 28/32] Add pre and post upgrade CSV version key check Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/dr_upgrade.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/ocs_ci/ocs/dr_upgrade.py b/ocs_ci/ocs/dr_upgrade.py index e0fc20e5205..4fbaf4d38f5 100644 --- a/ocs_ci/ocs/dr_upgrade.py +++ b/ocs_ci/ocs/dr_upgrade.py @@ -133,7 +133,9 @@ def update_subscription(self, channel, subscription_name, namespace=constants.OP f'"{self.channel}", "source": "{mco_source}"}}}}\'' ) subscription.exec_oc_cmd(patch_subscription_cmd, out_yaml_format=False) - time.sleep(10) + # Deliberately sleeping here as there are so many places down the line + # where ocs-ci will check CSV and it fails as changes take some time to reflect + time.sleep(20) def validate_upgrade(self): # In case of both MCO and DRhub operator, validation steps are similar @@ -174,11 +176,10 @@ def collect_data(self): resource_name=p.get()["metadata"]["name"] ) - # get pre-upgrade csv for MCO + # get pre-upgrade csv csv_objs = CSV(namespace=self.namespace) for csv in csv_objs.get()["items"]: - if (self.operator_name in csv["metadata"]["name"] - and self.upgrade_version in csv["metadata"]["name"]): + if self.operator_name in csv["metadata"]["name"]: csv_obj = CSV( namespace=self.namespace, resource_name=csv["metadata"]["name"] ) @@ -186,10 +187,19 @@ def collect_data(self): self.pre_upgrade_data["version"] = csv_obj.get_resource( resource_name=csv_obj.resource_name, column="VERSION" ) + try: + self.pre_upgrade_data["version"] + except KeyError: + log.error(f"Couldn't capture Pre-upgrade CSV version for {self.operator_name}") if self.upgrade_phase == "post_upgrade": - self.post_upgrade_data["version"] = csv_obj.get_resource( - resource_name=csv_obj.resource_name, column="VERSION" - ) + if self.upgrade_version in csv["metadata"]["name"]: + self.post_upgrade_data["version"] = csv_obj.get_resource( + resource_name=csv_obj.resource_name, column="VERSION" + ) + try: + self.post_upgrade_data["version"] + except KeyError: + log.error(f"Couldn't capture Post upgrade CSV version for {self.operator_name}") # Make sure all csvs are in succeeded state check_all_csvs_are_succeeded(namespace=self.namespace) From 360a18dfcf741e0dca9d34cc089e41e2b3fcc722 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Tue, 3 Dec 2024 22:34:36 +0530 Subject: [PATCH 29/32] implement check_if_upgrade_completed() function for DR class Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/dr_upgrade.py | 54 ++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/ocs_ci/ocs/dr_upgrade.py b/ocs_ci/ocs/dr_upgrade.py index 4fbaf4d38f5..1d31c3f5e7f 100644 --- a/ocs_ci/ocs/dr_upgrade.py +++ b/ocs_ci/ocs/dr_upgrade.py @@ -19,6 +19,7 @@ from ocs_ci.ocs.resources import pod from ocs_ci.ocs.resources.csv import CSV, check_all_csvs_are_succeeded from ocs_ci.ocs.resources.install_plan import wait_for_install_plan_and_approve +from ocs_ci.ocs.resources.packagemanifest import PackageManifest from ocs_ci.utility.utils import TimeoutSampler @@ -42,8 +43,10 @@ def __init__( resource_name=None, ): if not version_before_upgrade: - if config.PREUPGRADE_CONFIG.get('ENV_DATA').get("ocs_version", ''): - version_before_upgrade = config.PREUPGRADE_CONFIG['ENV_DATA'].get("ocs_version") + if config.PREUPGRADE_CONFIG.get("ENV_DATA").get("ocs_version", ""): + version_before_upgrade = config.PREUPGRADE_CONFIG["ENV_DATA"].get( + "ocs_version" + ) else: version_before_upgrade = config.ENV_DATA.get("ocs_version") if not ocs_registry_image: @@ -112,7 +115,9 @@ def run_upgrade(self): except TimeoutException: raise TimeoutException("No new CSV found after upgrade!") - def update_subscription(self, channel, subscription_name, namespace=constants.OPENSHIFT_OPERATORS): + def update_subscription( + self, channel, subscription_name, namespace=constants.OPENSHIFT_OPERATORS + ): subscription = OCP( resource_name=subscription_name, kind="subscription.operators.coreos.com", @@ -133,9 +138,36 @@ def update_subscription(self, channel, subscription_name, namespace=constants.OP f'"{self.channel}", "source": "{mco_source}"}}}}\'' ) subscription.exec_oc_cmd(patch_subscription_cmd, out_yaml_format=False) - # Deliberately sleeping here as there are so many places down the line + # Deliberately sleeping here as there are so many places down the line # where ocs-ci will check CSV and it fails as changes take some time to reflect - time.sleep(20) + time.sleep(60) + + def check_if_upgrade_completed(self, channel, csv_name_pre_upgrade): + """ + Check if DR operator finished it's upgrade + + Args: + channel: (str): DR operator subscription channel + csv_name_pre_upgrade: (str): DR operator name + + Returns: + bool: True if upgrade completed, False otherwise + + """ + if not check_all_csvs_are_succeeded(self.namespace): + log.warning("One of CSV is still not upgraded!") + return False + package_manifest = PackageManifest( + resource_name=self.operator_name, + subscription_plan_approval=self.subscription_plan_approval, + ) + csv_name_post_upgrade = package_manifest.get_current_csv(channel) + if csv_name_post_upgrade == csv_name_pre_upgrade: + log.info(f"CSV is still: {csv_name_post_upgrade}") + return False + else: + log.info(f"CSV now upgraded to: {csv_name_post_upgrade}") + return True def validate_upgrade(self): # In case of both MCO and DRhub operator, validation steps are similar @@ -190,7 +222,9 @@ def collect_data(self): try: self.pre_upgrade_data["version"] except KeyError: - log.error(f"Couldn't capture Pre-upgrade CSV version for {self.operator_name}") + log.error( + f"Couldn't capture Pre-upgrade CSV version for {self.operator_name}" + ) if self.upgrade_phase == "post_upgrade": if self.upgrade_version in csv["metadata"]["name"]: self.post_upgrade_data["version"] = csv_obj.get_resource( @@ -199,7 +233,9 @@ def collect_data(self): try: self.post_upgrade_data["version"] except KeyError: - log.error(f"Couldn't capture Post upgrade CSV version for {self.operator_name}") + log.error( + f"Couldn't capture Post upgrade CSV version for {self.operator_name}" + ) # Make sure all csvs are in succeeded state check_all_csvs_are_succeeded(namespace=self.namespace) @@ -255,7 +291,9 @@ def __init__(self): self.subscription_name = found_subscription_name break if not self.subscription_name: - log.error(f"Couldn't find the subscription for {defaults.DR_HUB_OPERATOR_NAME}") + log.error( + f"Couldn't find the subscription for {defaults.DR_HUB_OPERATOR_NAME}" + ) self.pod_name_pattern = "ramen-hub-operator" def run_upgrade(self): From ba2330ccac185dbb75f762c00aa284aed108ea03 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Mon, 9 Dec 2024 18:37:20 +0530 Subject: [PATCH 30/32] Fix tox issues for black Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/constants.py | 4 +++- tests/conftest.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index c05b8425bf2..4b9413efc75 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -1725,7 +1725,9 @@ DR_HUB_OPERATOR_SUBSCRIPTION = ( "odr-hub-operator-stable-PLACEHOLDER-redhat-operators-openshift-marketplace" ) -DR_HUB_OPERATOR_SUBSCRIPTION_LABEL = "operators.coreos.com/odr-hub-operator.openshift-operators" +DR_HUB_OPERATOR_SUBSCRIPTION_LABEL = ( + "operators.coreos.com/odr-hub-operator.openshift-operators" +) DR_CLUSTER_OPERATOR_SUBSCRIPTION = "ramen-dr-cluster-subscription" # UI Deployment constants diff --git a/tests/conftest.py b/tests/conftest.py index 556cdbbf4d6..28ead9fa528 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -530,7 +530,7 @@ def pytest_collection_modifyitems(session, config, items): # across the tests as Y stream upgrade will reload the config of target version for cluster in ocsci_config.clusters: for k in cluster.__dataclass_fields__.keys(): - if k != 'PREUPGRADE_CONFIG': + if k != "PREUPGRADE_CONFIG": cluster.PREUPGRADE_CONFIG[k] = deepcopy(getattr(cluster, k)) From 02962bb8bca082329e4f44ed9b58b32e4b905c1b Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Fri, 13 Dec 2024 17:24:22 +0530 Subject: [PATCH 31/32] Address review comments from Petr Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/framework/conf/default_config.yaml | 4 ++-- ocs_ci/ocs/ocs_upgrade.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ocs_ci/framework/conf/default_config.yaml b/ocs_ci/framework/conf/default_config.yaml index 56637adf82d..4db39aa2863 100644 --- a/ocs_ci/framework/conf/default_config.yaml +++ b/ocs_ci/framework/conf/default_config.yaml @@ -364,5 +364,5 @@ MULTICLUSTER: active_acm_cluster: False PREUPGRADE_CONFIG: - AUTH: null - MULTICLUSTER: null \ No newline at end of file + AUTH: null + MULTICLUSTER: null diff --git a/ocs_ci/ocs/ocs_upgrade.py b/ocs_ci/ocs/ocs_upgrade.py index c4b2cabe9de..8532293f76d 100644 --- a/ocs_ci/ocs/ocs_upgrade.py +++ b/ocs_ci/ocs/ocs_upgrade.py @@ -390,9 +390,8 @@ def get_csv_name_pre_upgrade(self, resource_name=OCS_OPERATOR_NAME): "ocs_version" ) in csv.get("metadata").get("name"): csv_name = csv.get("metadata").get("name") - if not csv_name: - raise CSVNotFound(f"No preupgrade CSV found for {resource_name}") - return csv_name + return csv_name + raise CSVNotFound(f"No preupgrade CSV found for {resource_name}") def get_pre_upgrade_image(self, csv_name_pre_upgrade): """ From d7e5d857efdebdfaf3ced93aa5e4e0e620d26dfc Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Wed, 18 Dec 2024 22:08:53 +0530 Subject: [PATCH 32/32] Handle default values for DR params in test function for non-DR cases Signed-off-by: Shylesh Kumar Mohan --- tests/conftest.py | 24 +++++++++++++++++------- tests/functional/upgrade/test_upgrade.py | 7 ++++++- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 28ead9fa528..fbcb163db76 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -525,13 +525,13 @@ def pytest_collection_modifyitems(session, config, items): log.info( f"MARKERS = {[(i.name, i.args, i.kwargs) for i in item.iter_markers()]}" ) - # Update PREUPGRADE_CONFIG for each of the Config class - # so that in case of Y stream upgrade we will have preupgrade configurations for reference - # across the tests as Y stream upgrade will reload the config of target version - for cluster in ocsci_config.clusters: - for k in cluster.__dataclass_fields__.keys(): - if k != "PREUPGRADE_CONFIG": - cluster.PREUPGRADE_CONFIG[k] = deepcopy(getattr(cluster, k)) + # Update PREUPGRADE_CONFIG for each of the Config class + # so that in case of Y stream upgrade we will have preupgrade configurations for reference + # across the tests as Y stream upgrade will reload the config of target version + for cluster in ocsci_config.clusters: + for k in cluster.__dataclass_fields__.keys(): + if k != "PREUPGRADE_CONFIG": + cluster.PREUPGRADE_CONFIG[k] = deepcopy(getattr(cluster, k)) def pytest_collection_finish(session): @@ -8862,6 +8862,16 @@ def virtctl_binary(): get_virtctl_tool() +@pytest.fixture() +def zone_rank(): + return None + + +@pytest.fixture() +def role_rank(): + return None + + @pytest.fixture() def nb_assign_user_role_fixture(request, mcg_obj_session): diff --git a/tests/functional/upgrade/test_upgrade.py b/tests/functional/upgrade/test_upgrade.py index 07d9158e518..40f0fd25982 100644 --- a/tests/functional/upgrade/test_upgrade.py +++ b/tests/functional/upgrade/test_upgrade.py @@ -87,11 +87,16 @@ def test_osd_reboot(teardown, upgrade_stats): run_ocs_upgrade(operation=osd_node_reboot, upgrade_stats=upgrade_stats) +@pytest.fixture +def config_index(request): + return request.param if hasattr(request, "param") else None + + @purple_squad @ocs_upgrade @polarion_id(get_polarion_id(upgrade=True)) @multicluster_roles(["mdr-all-odf"]) -def test_upgrade(zone_rank, role_rank, config_index, upgrade_stats): +def test_upgrade(zone_rank, role_rank, config_index, upgrade_stats=None): """ Tests upgrade procedure of OCS cluster