From 6cd45258b4fa9fbf695690fa853c5fdb9789e58d Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Thu, 19 Oct 2023 20:46:38 +0100 Subject: [PATCH 01/12] Provision for installing submariner downstream unreleased build Signed-off-by: Shylesh Kumar Mohan --- .../submariner_downstream_unreleased.yaml | 4 ++ ocs_ci/deployment/acm.py | 21 +++++++++ ocs_ci/ocs/acm/acm.py | 45 +++++++++++++++++++ ocs_ci/ocs/constants.py | 13 ++++++ .../acm_downstream_brew_icsp.yaml | 18 ++++++++ ...bmariner_downstream_unreleased_catsrc.yaml | 16 +++++++ ocs_ci/utility/utils.py | 27 +++++++++++ 7 files changed, 144 insertions(+) create mode 100644 conf/ocsci/submariner_downstream_unreleased.yaml create mode 100644 ocs_ci/templates/acm-deployment/acm_downstream_brew_icsp.yaml create mode 100644 ocs_ci/templates/ocs-deployment/multicluster/submariner_downstream_unreleased_catsrc.yaml diff --git a/conf/ocsci/submariner_downstream_unreleased.yaml b/conf/ocsci/submariner_downstream_unreleased.yaml new file mode 100644 index 00000000000..a26c53b153f --- /dev/null +++ b/conf/ocsci/submariner_downstream_unreleased.yaml @@ -0,0 +1,4 @@ +ENV_DATA: + submariner_source: "downstream" + submariner_release_type: "unreleased" + submariner_version: "0.16.0" diff --git a/ocs_ci/deployment/acm.py b/ocs_ci/deployment/acm.py index 3268842de61..b02b49aa0b7 100644 --- a/ocs_ci/deployment/acm.py +++ b/ocs_ci/deployment/acm.py @@ -16,6 +16,7 @@ CommandFailed, DRPrimaryNotFoundException, ) +from ocs_ci.utility import templating from ocs_ci.ocs.utils import get_non_acm_cluster_config from ocs_ci.utility.utils import run_cmd, run_cmd_interactive from ocs_ci.ocs.node import get_typed_worker_nodes, label_nodes @@ -62,6 +63,8 @@ class Submariner(object): def __init__(self): # whether upstream OR downstream self.source = config.ENV_DATA["submariner_source"] + # released/unreleased + self.submariner_release_type = config.ENV_DATA.get("submariner_release_type") # Deployment type: self.deployment_type = config.ENV_DATA.get("submariner_deployment") # Designated broker cluster index where broker will be deployed @@ -94,9 +97,27 @@ def deploy_downstream(self): login_to_acm() acm_obj = AcmAddClusters() + if self.submariner_release_type == "unreleased": + old_ctx = config.cur_index + for cluster in get_non_acm_cluster_config(): + config.switch_ctx(cluster.MULTICLUSTER["multicluster_index"]) + self.create_acm_brew_icsp() + config.switch_ctx(old_ctx) acm_obj.install_submariner_ui() acm_obj.submariner_validation_ui() + def create_acm_brew_icsp(self): + """ + This is a prereq for downstream unreleased submariner + + """ + icsp_data = templating.load_yaml(constants.ACM_DOWNSTREAM_BREW_ICSP) + icsp_data_yaml = tempfile.NamedTemporaryFile( + mode="w+", prefix="acm_icsp", delete=False + ) + templating.dump_data_to_temp_yaml(icsp_data, icsp_data_yaml.name) + run_cmd(f"oc create -f {icsp_data_yaml.name}", timeout=300) + def download_binary(self): if self.source == "upstream": # This script puts the platform specific binary in ~/.local/bin diff --git a/ocs_ci/ocs/acm/acm.py b/ocs_ci/ocs/acm/acm.py index e46a6b22c25..a7f907b2a9d 100644 --- a/ocs_ci/ocs/acm/acm.py +++ b/ocs_ci/ocs/acm/acm.py @@ -1,6 +1,7 @@ import logging import time import os +import tempfile from selenium.webdriver.support.wait import WebDriverWait from selenium.webdriver.support import expected_conditions as ec @@ -24,6 +25,8 @@ TimeoutSampler, get_running_acm_version, string_chunkify, + chained_subprocess_pipes, + run_cmd, ) from ocs_ci.ocs.ui.acm_ui import AcmPageNavigator from ocs_ci.ocs.ui.base_ui import login_ui, SeleniumDriver @@ -152,6 +155,46 @@ def install_submariner_ui(self, globalnet=True): for s in get_non_acm_cluster_config() if s.MULTICLUSTER["multicluster_index"] != primary_index ][0] + # submariner catalogsource creation + if config.ENV_DATA["submariner_release_type"] == "unreleased": + submariner_downstream_unreleased = templating.load_yaml( + constants.SUBMARINER_DOWNSTREAM_UNRELEASED + ) + # Update catalog source + submariner_full_url = "".join( + [ + constants.SUBMARINER_DOWNSTREAM_UNRELEASED_BUILD_URL, + config.ENV_DATA["submariner_version"], + ] + ) + curl_cmd = f"curl --retry 3 --retry-delay 5 -Ls {submariner_full_url}" + jq_cmd1 = ( + 'jq -r \'[.raw_messages[].msg | select(.pipeline.status=="complete") |' + "{{nvr: .artifact.nvr, index_image: .pipeline.index_image}}] | .[0]'" + ) + + jq_cmd2 = "jq -r '.index_image.\"v4.14\"'" + cut_cmd1 = "cut -d'/' -f3-" + cut_cmd2 = "cut -d':' -f2-" + cmd_exec = chained_subprocess_pipes( + [curl_cmd, jq_cmd1, jq_cmd2, cut_cmd1, cut_cmd2] + ) + version_tag = cmd_exec.communicate()[0].decode() + + image_url = submariner_downstream_unreleased["spec"]["image"] + image_url = image_url.replace("PLACE_HOLDER", version_tag) + submariner_downstream_unreleased["spec"]["image"] = image_url + submariner_data_yaml = tempfile.NamedTemporaryFile( + mode="w+", prefix="submariner_downstream_unreleased", delete=False + ) + templating.dump_data_to_temp_yaml( + submariner_downstream_unreleased, submariner_data_yaml + ) + old_ctx = config.cur_index + for cluster in get_non_acm_cluster_config(): + config.switch_ctx(cluster.MULTICLUSTER["multicluster_index"]) + run_cmd(f"oc create -f {submariner_data_yaml.name}", timeout=300) + config.switch_ctx(old_ctx) cluster_name_a = cluster_env.get(f"cluster_name_{primary_index}") cluster_name_b = cluster_env.get(f"cluster_name_{secondary_index}") @@ -239,6 +282,8 @@ def install_submariner_ui(self, globalnet=True): check_globalnet == constants.GLOBALNET_STATUS ), "Globalnet was not enabled" log.info("Globalnet is enabled") + # TODO: Use custom submariner sucscription from UI in case of downstream unreleased submariner + self.take_screenshot() log.info("Click on 'Install'") self.do_click(self.page_nav["install-btn"]) diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index fb0e87df186..22c1e05cac5 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -849,6 +849,7 @@ ) MDR_BACKUP_SCHEDULE_RESOURCE = "schedule-acm" + # DR constants SUBMARINER_DOWNLOAD_URL = "https://get.submariner.io" DR_DEFAULT_NAMESPACE = "openshift-dr-systems" @@ -2069,6 +2070,15 @@ SUBMARINER_GATEWAY_ACTIVE_LABEL = "gateway.submariner.io/status=active" SUBMARINER_GATEWAY_NODE_LABEL = "submariner.io/gateway=true" GLOBALNET_STATUS = "True" +SUBMARINER_DOWNSTREAM_UNRELEASED = os.path.join( + TEMPLATE_MULTICLUSTER_DIR, "submariner_downstream_unreleased_catsrc.yaml" +) +# We need to append version string at the end of this url +SUBMARINER_DOWNSTREAM_UNRELEASED_BUILD_URL = ( + "https://datagrepper.engineering.redhat.com/raw?topic=/topic/" + "VirtualTopic.eng.ci.redhat-container-image.pipeline.complete" + "&rows_per_page=25&delta=1296000&contains=submariner-operator-bundle-container-v" +) # Multicluster related @@ -2102,6 +2112,9 @@ ACM_HUB_UNRELEASED_ICSP_YAML = os.path.join( TEMPLATE_DIR, "acm-deployment", "imagecontentsourcepolicy.yaml" ) +ACM_DOWNSTREAM_BREW_ICSP = os.path.join( + TEMPLATE_DIR, "acm-deployment", "acm_downstream_brew_icsp.yaml" +) ACM_HUB_UNRELEASED_PULL_SECRET_TEMPLATE = "pull-secret.yaml.j2" ACM_ODF_MULTICLUSTER_ORCHESTRATOR_RESOURCE = "odf-multicluster-orchestrator" ACM_ODR_HUB_OPERATOR_RESOURCE = "odr-hub-operator" diff --git a/ocs_ci/templates/acm-deployment/acm_downstream_brew_icsp.yaml b/ocs_ci/templates/acm-deployment/acm_downstream_brew_icsp.yaml new file mode 100644 index 00000000000..4a4c76e8aa0 --- /dev/null +++ b/ocs_ci/templates/acm-deployment/acm_downstream_brew_icsp.yaml @@ -0,0 +1,18 @@ +apiVersion: operator.openshift.io/v1alpha1 +kind: ImageContentSourcePolicy +metadata: + name: brew-registry +spec: + repositoryDigestMirrors: + - mirrors: + - brew.registry.redhat.io + source: registry.redhat.io + - mirrors: + - brew.registry.redhat.io/rhacm2 + source: registry.redhat.io/rhacm2 + - mirrors: + - brew.registry.redhat.io + source: registry.stage.redhat.io + - mirrors: + - brew.registry.redhat.io + source: registry-proxy.engineering.redhat.com diff --git a/ocs_ci/templates/ocs-deployment/multicluster/submariner_downstream_unreleased_catsrc.yaml b/ocs_ci/templates/ocs-deployment/multicluster/submariner_downstream_unreleased_catsrc.yaml new file mode 100644 index 00000000000..bab28efe1df --- /dev/null +++ b/ocs_ci/templates/ocs-deployment/multicluster/submariner_downstream_unreleased_catsrc.yaml @@ -0,0 +1,16 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + name: submariner-catalogsource + namespace: openshift-marketplace +spec: + icon: + base64data: "" + mediatype: "" + image: brew.registry.redhat.io/rh-osbs/iib:PLACE_HOLDER + publisher: Red Hat + sourceType: grpc + priority: 100 + updateStrategy: + registryPoll: + interval: 15m diff --git a/ocs_ci/utility/utils.py b/ocs_ci/utility/utils.py index a85aead0163..94cd8b3de5b 100644 --- a/ocs_ci/utility/utils.py +++ b/ocs_ci/utility/utils.py @@ -4553,3 +4553,30 @@ def is_cluster_y_version_upgraded(): ) > version_module.get_semantic_version(prev_version_num, only_major_minor=True): is_upgraded = True return is_upgraded + + +def chained_subprocess_pipes(cmd_list): + """ + We can use this function wherever we have chain of commands which + need to be piped. Direct shell piping has some parsing issues hence implementing + this using subprocess output redirections which mimics shell pipes. + Commands will be run as per the command ordering inside the list and output of + each command will be fed as input to the next command in the list + + + Args: + cmd_list (list): list of commands to be run whose output need to be piped + + Returns: + Popen object of the last command executed + + """ + pipe_buf = None + for cmd in cmd_list: + if not pipe_buf: + pipe_buf = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + else: + pipe_buf = subprocess.Popen( + shlex.split(cmd), stdin=pipe_buf.stdout, stdout=subprocess.PIPE + ) + return pipe_buf From caa7b810c617336bd4b2dcd7029ced85991fd44b Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Fri, 20 Oct 2023 21:57:34 +0100 Subject: [PATCH 02/12] Handle UI part of unreleased submariner info Signed-off-by: Shylesh Kumar Mohan --- .../ocsci/submariner_downstream_unreleased.yaml | 1 + ocs_ci/ocs/acm/acm.py | 17 +++++++++++++++-- ocs_ci/ocs/ui/views.py | 8 ++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/conf/ocsci/submariner_downstream_unreleased.yaml b/conf/ocsci/submariner_downstream_unreleased.yaml index a26c53b153f..677219eebbb 100644 --- a/conf/ocsci/submariner_downstream_unreleased.yaml +++ b/conf/ocsci/submariner_downstream_unreleased.yaml @@ -2,3 +2,4 @@ ENV_DATA: submariner_source: "downstream" submariner_release_type: "unreleased" submariner_version: "0.16.0" + submariner_unreleased_channel: "stable-0.16" diff --git a/ocs_ci/ocs/acm/acm.py b/ocs_ci/ocs/acm/acm.py index a7f907b2a9d..69555d1e3bd 100644 --- a/ocs_ci/ocs/acm/acm.py +++ b/ocs_ci/ocs/acm/acm.py @@ -265,6 +265,8 @@ def install_submariner_ui(self, globalnet=True): ) self.do_click(self.page_nav["gateway-count-btn"]) self.do_click(self.page_nav["gateway-count-btn"]) + if config.ENV_DATA.get("submariner_release_type") == "unreleased": + self.submariner_unreleased_downstream_info() log.info("Click on Next button") self.do_click(self.page_nav["next-btn"]) log.info("Click on 'Enable NAT-T' to uncheck it [2]") @@ -274,6 +276,8 @@ def install_submariner_ui(self, globalnet=True): ) self.do_click(self.page_nav["gateway-count-btn"]) self.do_click(self.page_nav["gateway-count-btn"]) + if config.ENV_DATA.get("submariner_release_type") == "unreleased": + self.submariner_unreleased_downstream_info() log.info("Click on Next button [2]") self.do_click(self.page_nav["next-btn"]) if ocs_version >= version.VERSION_4_13 and globalnet: @@ -282,12 +286,21 @@ def install_submariner_ui(self, globalnet=True): check_globalnet == constants.GLOBALNET_STATUS ), "Globalnet was not enabled" log.info("Globalnet is enabled") - # TODO: Use custom submariner sucscription from UI in case of downstream unreleased submariner - self.take_screenshot() log.info("Click on 'Install'") self.do_click(self.page_nav["install-btn"]) + def submariner_unreleased_downstream_info(self): + self.do_click(self.page_nav["submariner-custom-subscription"]) + self.do_clear(self.page_nav["submariner-custom-source"]) + self.do_send_keys( + self.page_nav["submariner-custom-source"], "submariner-catalogsource" + ) + self.do_send_keys( + self.page_nav["submariner-custom-channel"], + config.ENV_DATA["submariner_unreleased_channel"], + ) + def submariner_validation_ui(self): """ This function validates submariner status on ACM console which connects 2 managed OCP clusters. diff --git a/ocs_ci/ocs/ui/views.py b/ocs_ci/ocs/ui/views.py index 71791c1fa77..fcd3b03e23d 100644 --- a/ocs_ci/ocs/ui/views.py +++ b/ocs_ci/ocs/ui/views.py @@ -879,6 +879,13 @@ **acm_configuration_4_12, } +acm_configuration_4_14 = { + **acm_configuration_4_13, + "submariner-custom-subscription": ("isCustomSubscription", By.ID), + "submariner-custom-source": ("source", By.ID), + "submariner-custom-channel": ("channel", By.ID), +} + add_capacity = { "ocs_operator": ( 'a[data-test-operator-row="OpenShift Container Storage"]', @@ -1710,6 +1717,7 @@ **acm_configuration_4_11, **acm_configuration_4_12, **acm_configuration_4_13, + **acm_configuration_4_14, }, "validation": { **validation, From 689db13fd153884d124b9cf203726b1df7b2cd1e Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Mon, 23 Oct 2023 11:39:33 +0100 Subject: [PATCH 03/12] Add globalnet flag with default enabled Signed-off-by: Shylesh Kumar Mohan --- conf/ocsci/enable_globalnet.yaml | 2 ++ conf/ocsci/submariner_downstream_unreleased.yaml | 1 + 2 files changed, 3 insertions(+) create mode 100644 conf/ocsci/enable_globalnet.yaml diff --git a/conf/ocsci/enable_globalnet.yaml b/conf/ocsci/enable_globalnet.yaml new file mode 100644 index 00000000000..722d3b4a375 --- /dev/null +++ b/conf/ocsci/enable_globalnet.yaml @@ -0,0 +1,2 @@ +ENV_DATA: + enable_globalnet: true diff --git a/conf/ocsci/submariner_downstream_unreleased.yaml b/conf/ocsci/submariner_downstream_unreleased.yaml index 677219eebbb..c47f03ae421 100644 --- a/conf/ocsci/submariner_downstream_unreleased.yaml +++ b/conf/ocsci/submariner_downstream_unreleased.yaml @@ -3,3 +3,4 @@ ENV_DATA: submariner_release_type: "unreleased" submariner_version: "0.16.0" submariner_unreleased_channel: "stable-0.16" + enable_globalnet: true From bd7f8833f77118fc0f288d200883a0e139654cee Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Tue, 24 Oct 2023 13:39:18 +0100 Subject: [PATCH 04/12] Update StorageCluster based on globalnet flag Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/deployment/deployment.py | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/ocs_ci/deployment/deployment.py b/ocs_ci/deployment/deployment.py index 16a408cc341..39a138787a9 100644 --- a/ocs_ci/deployment/deployment.py +++ b/ocs_ci/deployment/deployment.py @@ -83,6 +83,8 @@ ocs_install_verification, setup_ceph_debug, get_osd_count, + StorageCluster, + verify_storage_cluster, ) from ocs_ci.ocs.uninstall import uninstall_ocs from ocs_ci.ocs.utils import ( @@ -351,6 +353,41 @@ def do_deploy_ocs(self): "ocs_registry_image", None ) ocs_install_verification(ocs_registry_image=ocs_registry_image) + # if we have Globalnet enabled in case of submariner with RDR + # we need to add a flag to storagecluster + if config.ENV_DATA.get("enable_globalnet", None): + for cluster in get_non_acm_cluster_config(): + config.switch_ctx(cluster.MULTICLUSTER["multicluster_index"]) + storage_cluster_name = config.ENV_DATA["storage_cluster_name"] + logger.info( + "Updating the StorageCluster resource for globalnet" + ) + storage_cluster = StorageCluster( + resource_name=storage_cluster_name, + namespace=config.ENV_DATA["cluster_namespace"], + ) + storage_cluster.reload_data() + storage_cluster.wait_for_phase(phase="Ready", timeout=1000) + multicluster_service = { + "network": { + "multiClusterService": { + "clusterID": config.ENV_DATA["cluster_name"], + "enabled": "true", + } + } + } + storage_cluster.data.get("spec").merge_dict( + multicluster_service + ) + # todo: oc apply updated resource + storage_cluster_yaml = tempfile.NamedTemporaryFile( + mode="w+", prefix="storageclusterupdate", delete=False + ) + templating.dump_data_to_temp_yaml( + storage_cluster.data, storage_cluster_yaml.name + ) + run_cmd(f"oc apply -f {storage_cluster.name}", timeout=300) + verify_storage_cluster() config.reset_ctx() else: logger.warning("OCS deployment will be skipped") From e2e87ce6374c3b7f0327c3dd0f3ec3d6c21b95f0 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Wed, 25 Oct 2023 23:15:19 +0100 Subject: [PATCH 05/12] Address few more review comments by Petr Simplify build tag extraction for submariner Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/deployment/acm.py | 7 ++++- ocs_ci/deployment/deployment.py | 5 ++-- ocs_ci/ocs/acm/acm.py | 27 +++++++------------ ocs_ci/ocs/constants.py | 1 + ...bmariner_downstream_unreleased_catsrc.yaml | 2 +- ocs_ci/utility/utils.py | 27 ------------------- 6 files changed, 20 insertions(+), 49 deletions(-) diff --git a/ocs_ci/deployment/acm.py b/ocs_ci/deployment/acm.py index b02b49aa0b7..04b46c78d8f 100644 --- a/ocs_ci/deployment/acm.py +++ b/ocs_ci/deployment/acm.py @@ -18,7 +18,11 @@ ) from ocs_ci.utility import templating from ocs_ci.ocs.utils import get_non_acm_cluster_config -from ocs_ci.utility.utils import run_cmd, run_cmd_interactive +from ocs_ci.utility.utils import ( + run_cmd, + run_cmd_interactive, + wait_for_machineconfigpool_status, +) from ocs_ci.ocs.node import get_typed_worker_nodes, label_nodes logger = logging.getLogger(__name__) @@ -117,6 +121,7 @@ def create_acm_brew_icsp(self): ) templating.dump_data_to_temp_yaml(icsp_data, icsp_data_yaml.name) run_cmd(f"oc create -f {icsp_data_yaml.name}", timeout=300) + wait_for_machineconfigpool_status(node_type="all") def download_binary(self): if self.source == "upstream": diff --git a/ocs_ci/deployment/deployment.py b/ocs_ci/deployment/deployment.py index 39a138787a9..57d72b48a39 100644 --- a/ocs_ci/deployment/deployment.py +++ b/ocs_ci/deployment/deployment.py @@ -376,10 +376,9 @@ def do_deploy_ocs(self): } } } - storage_cluster.data.get("spec").merge_dict( - multicluster_service + merge_dict( + storage_cluster.data.get("spec"), multicluster_service ) - # todo: oc apply updated resource storage_cluster_yaml = tempfile.NamedTemporaryFile( mode="w+", prefix="storageclusterupdate", delete=False ) diff --git a/ocs_ci/ocs/acm/acm.py b/ocs_ci/ocs/acm/acm.py index 69555d1e3bd..0fb71fe5cbb 100644 --- a/ocs_ci/ocs/acm/acm.py +++ b/ocs_ci/ocs/acm/acm.py @@ -2,6 +2,7 @@ import time import os import tempfile +import requests from selenium.webdriver.support.wait import WebDriverWait from selenium.webdriver.support import expected_conditions as ec @@ -23,9 +24,9 @@ from ocs_ci.ocs.utils import get_non_acm_cluster_config, get_primary_cluster_config from ocs_ci.utility.utils import ( TimeoutSampler, + get_ocp_version, get_running_acm_version, string_chunkify, - chained_subprocess_pipes, run_cmd, ) from ocs_ci.ocs.ui.acm_ui import AcmPageNavigator @@ -167,28 +168,20 @@ def install_submariner_ui(self, globalnet=True): config.ENV_DATA["submariner_version"], ] ) - curl_cmd = f"curl --retry 3 --retry-delay 5 -Ls {submariner_full_url}" - jq_cmd1 = ( - 'jq -r \'[.raw_messages[].msg | select(.pipeline.status=="complete") |' - "{{nvr: .artifact.nvr, index_image: .pipeline.index_image}}] | .[0]'" - ) - jq_cmd2 = "jq -r '.index_image.\"v4.14\"'" - cut_cmd1 = "cut -d'/' -f3-" - cut_cmd2 = "cut -d':' -f2-" - cmd_exec = chained_subprocess_pipes( - [curl_cmd, jq_cmd1, jq_cmd2, cut_cmd1, cut_cmd2] + resp = requests.get(submariner_full_url, verify=False) + raw_msg = resp.json()["raw_messages"] + version_tag = raw_msg[0]["msg"]["pipeline"]["index_image"][ + f"v{get_ocp_version()}" + ].split(":")[1] + submariner_downstream_unreleased["spec"]["image"] = ":".join( + [constants.SUBMARINER_BREW_REPO, version_tag] ) - version_tag = cmd_exec.communicate()[0].decode() - - image_url = submariner_downstream_unreleased["spec"]["image"] - image_url = image_url.replace("PLACE_HOLDER", version_tag) - submariner_downstream_unreleased["spec"]["image"] = image_url submariner_data_yaml = tempfile.NamedTemporaryFile( mode="w+", prefix="submariner_downstream_unreleased", delete=False ) templating.dump_data_to_temp_yaml( - submariner_downstream_unreleased, submariner_data_yaml + submariner_downstream_unreleased, submariner_data_yaml.name ) old_ctx = config.cur_index for cluster in get_non_acm_cluster_config(): diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 22c1e05cac5..f4c6313e7dd 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -2079,6 +2079,7 @@ "VirtualTopic.eng.ci.redhat-container-image.pipeline.complete" "&rows_per_page=25&delta=1296000&contains=submariner-operator-bundle-container-v" ) +SUBMARINER_BREW_REPO = "brew.registry.redhat.io/rh-osbs/iib" # Multicluster related diff --git a/ocs_ci/templates/ocs-deployment/multicluster/submariner_downstream_unreleased_catsrc.yaml b/ocs_ci/templates/ocs-deployment/multicluster/submariner_downstream_unreleased_catsrc.yaml index bab28efe1df..b931afe2ce8 100644 --- a/ocs_ci/templates/ocs-deployment/multicluster/submariner_downstream_unreleased_catsrc.yaml +++ b/ocs_ci/templates/ocs-deployment/multicluster/submariner_downstream_unreleased_catsrc.yaml @@ -7,7 +7,7 @@ spec: icon: base64data: "" mediatype: "" - image: brew.registry.redhat.io/rh-osbs/iib:PLACE_HOLDER + image: PLACE_HOLDER publisher: Red Hat sourceType: grpc priority: 100 diff --git a/ocs_ci/utility/utils.py b/ocs_ci/utility/utils.py index 94cd8b3de5b..a85aead0163 100644 --- a/ocs_ci/utility/utils.py +++ b/ocs_ci/utility/utils.py @@ -4553,30 +4553,3 @@ def is_cluster_y_version_upgraded(): ) > version_module.get_semantic_version(prev_version_num, only_major_minor=True): is_upgraded = True return is_upgraded - - -def chained_subprocess_pipes(cmd_list): - """ - We can use this function wherever we have chain of commands which - need to be piped. Direct shell piping has some parsing issues hence implementing - this using subprocess output redirections which mimics shell pipes. - Commands will be run as per the command ordering inside the list and output of - each command will be fed as input to the next command in the list - - - Args: - cmd_list (list): list of commands to be run whose output need to be piped - - Returns: - Popen object of the last command executed - - """ - pipe_buf = None - for cmd in cmd_list: - if not pipe_buf: - pipe_buf = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) - else: - pipe_buf = subprocess.Popen( - shlex.split(cmd), stdin=pipe_buf.stdout, stdout=subprocess.PIPE - ) - return pipe_buf From c9ed0a1fd26db2f6e9d69dd8b6b5f5ca65c1796b Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Thu, 26 Oct 2023 17:31:14 +0100 Subject: [PATCH 06/12] replace oc apply with oc patch for global net updates Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/deployment/deployment.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/ocs_ci/deployment/deployment.py b/ocs_ci/deployment/deployment.py index 57d72b48a39..3c4898afc02 100644 --- a/ocs_ci/deployment/deployment.py +++ b/ocs_ci/deployment/deployment.py @@ -368,25 +368,23 @@ def do_deploy_ocs(self): ) storage_cluster.reload_data() storage_cluster.wait_for_phase(phase="Ready", timeout=1000) - multicluster_service = { - "network": { - "multiClusterService": { - "clusterID": config.ENV_DATA["cluster_name"], - "enabled": "true", - } - } - } - merge_dict( - storage_cluster.data.get("spec"), multicluster_service + ptch = ( + f'\'{{"spec": {{"network": {{"multiClusterService": ' + f"{{\"clusterID\": \"{config.ENV_DATA['cluster_name']}\", \"enabled\": true}}}}}}}}'" ) - storage_cluster_yaml = tempfile.NamedTemporaryFile( - mode="w+", prefix="storageclusterupdate", delete=False + ptch_cmd = ( + f"oc patch storagecluster/{storage_cluster.data.get('metadata').get('name')} " + f"-n openshift-storage --type merge --patch {ptch}" ) - templating.dump_data_to_temp_yaml( - storage_cluster.data, storage_cluster_yaml.name - ) - run_cmd(f"oc apply -f {storage_cluster.name}", timeout=300) + run_cmd(ptch_cmd) verify_storage_cluster() + storage_cluster.reload_data() + assert ( + storage_cluster.data.get("spec") + .get("network") + .get("multiClusterService") + .get("enabled") + ), "Failed to update StorageCluster globalnet" config.reset_ctx() else: logger.warning("OCS deployment will be skipped") From 2ddc236efbb9614b98344f2410ad5a765bfc37b2 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Fri, 27 Oct 2023 11:04:32 +0100 Subject: [PATCH 07/12] Add ocs install verification after patching storagecluster Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/deployment/deployment.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ocs_ci/deployment/deployment.py b/ocs_ci/deployment/deployment.py index 3c4898afc02..34cd8988234 100644 --- a/ocs_ci/deployment/deployment.py +++ b/ocs_ci/deployment/deployment.py @@ -377,7 +377,9 @@ def do_deploy_ocs(self): f"-n openshift-storage --type merge --patch {ptch}" ) run_cmd(ptch_cmd) - verify_storage_cluster() + ocs_registry_image = config.DEPLOYMENT.get( + "ocs_registry_image", None + ) storage_cluster.reload_data() assert ( storage_cluster.data.get("spec") @@ -385,6 +387,9 @@ def do_deploy_ocs(self): .get("multiClusterService") .get("enabled") ), "Failed to update StorageCluster globalnet" + ocs_install_verification( + timeout=2000, ocs_registry_image=ocs_registry_image + ) config.reset_ctx() else: logger.warning("OCS deployment will be skipped") From 80e4feca75762c1e3d0eb6a7bfb5e2b073fd23a4 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Fri, 27 Oct 2023 13:06:45 +0100 Subject: [PATCH 08/12] Fix flake8 failures Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/deployment/deployment.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ocs_ci/deployment/deployment.py b/ocs_ci/deployment/deployment.py index 34cd8988234..72d88031ae4 100644 --- a/ocs_ci/deployment/deployment.py +++ b/ocs_ci/deployment/deployment.py @@ -84,7 +84,6 @@ setup_ceph_debug, get_osd_count, StorageCluster, - verify_storage_cluster, ) from ocs_ci.ocs.uninstall import uninstall_ocs from ocs_ci.ocs.utils import ( From a890fa41fd7dd521a5c7129251359d0787f3ff21 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Fri, 27 Oct 2023 13:50:39 +0100 Subject: [PATCH 09/12] Add the submariner config parameters in README file Signed-off-by: Shylesh Kumar Mohan --- conf/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conf/README.md b/conf/README.md index 3400b288456..eac213b5653 100644 --- a/conf/README.md +++ b/conf/README.md @@ -256,6 +256,10 @@ higher priority). * `nb_nfs_mount` - NFS mount point used specifically for testing noobaa db NFS mount test * `custom_default_storageclass_names` - Set to true if custom storageclass names use instead of default one. * `storageclassnames` - Under this key, custom storage class names for `cephFilesystems`, `cephObjectStores`, `cephBlockPools`, `cephNonResilientPools`, `nfs` and for `encryption` are defined. +* `submariner_source` - Source from which we take submariner build, ex: upstream, downstream, downstream_unreleased +* `submariner_release_type` - Released OR Unreleased submariner build +* `enable_globalnet` - enable or disable globalnet for submariner +* `submariner_unreleased_channel` - submariner channel for unreleased downstream build #### UPGRADE From 2bc663a000d8a5021a05ae9395c4f0a899101ece Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Tue, 31 Oct 2023 17:08:13 +0000 Subject: [PATCH 10/12] Update enable_globalnet flag to be defaut false Signed-off-by: Shylesh Kumar Mohan --- conf/README.md | 2 +- conf/ocsci/disable_globalnet.yaml | 2 ++ conf/ocsci/enable_globalnet.yaml | 2 -- ocs_ci/deployment/acm.py | 2 +- ocs_ci/ocs/constants.py | 4 ++-- ...am_brew_icsp.yaml => submariner_downstream_brew_icsp.yaml} | 0 6 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 conf/ocsci/disable_globalnet.yaml delete mode 100644 conf/ocsci/enable_globalnet.yaml rename ocs_ci/templates/acm-deployment/{acm_downstream_brew_icsp.yaml => submariner_downstream_brew_icsp.yaml} (100%) diff --git a/conf/README.md b/conf/README.md index eac213b5653..97afeb9efc1 100644 --- a/conf/README.md +++ b/conf/README.md @@ -258,7 +258,7 @@ higher priority). * `storageclassnames` - Under this key, custom storage class names for `cephFilesystems`, `cephObjectStores`, `cephBlockPools`, `cephNonResilientPools`, `nfs` and for `encryption` are defined. * `submariner_source` - Source from which we take submariner build, ex: upstream, downstream, downstream_unreleased * `submariner_release_type` - Released OR Unreleased submariner build -* `enable_globalnet` - enable or disable globalnet for submariner +* `enable_globalnet` - enable or disable globalnet for submariner, default: true * `submariner_unreleased_channel` - submariner channel for unreleased downstream build #### UPGRADE diff --git a/conf/ocsci/disable_globalnet.yaml b/conf/ocsci/disable_globalnet.yaml new file mode 100644 index 00000000000..e5041e99c12 --- /dev/null +++ b/conf/ocsci/disable_globalnet.yaml @@ -0,0 +1,2 @@ +ENV_DATA: + enable_globalnet: false diff --git a/conf/ocsci/enable_globalnet.yaml b/conf/ocsci/enable_globalnet.yaml deleted file mode 100644 index 722d3b4a375..00000000000 --- a/conf/ocsci/enable_globalnet.yaml +++ /dev/null @@ -1,2 +0,0 @@ -ENV_DATA: - enable_globalnet: true diff --git a/ocs_ci/deployment/acm.py b/ocs_ci/deployment/acm.py index 04b46c78d8f..ca6936b7787 100644 --- a/ocs_ci/deployment/acm.py +++ b/ocs_ci/deployment/acm.py @@ -115,7 +115,7 @@ def create_acm_brew_icsp(self): This is a prereq for downstream unreleased submariner """ - icsp_data = templating.load_yaml(constants.ACM_DOWNSTREAM_BREW_ICSP) + icsp_data = templating.load_yaml(constants.SUBMARINER_DOWNSTREAM_BREW_ICSP) icsp_data_yaml = tempfile.NamedTemporaryFile( mode="w+", prefix="acm_icsp", delete=False ) diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index f4c6313e7dd..19c66ff9cb1 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -2113,8 +2113,8 @@ ACM_HUB_UNRELEASED_ICSP_YAML = os.path.join( TEMPLATE_DIR, "acm-deployment", "imagecontentsourcepolicy.yaml" ) -ACM_DOWNSTREAM_BREW_ICSP = os.path.join( - TEMPLATE_DIR, "acm-deployment", "acm_downstream_brew_icsp.yaml" +SUBMARINER_DOWNSTREAM_BREW_ICSP = os.path.join( + TEMPLATE_DIR, "acm-deployment", "submariner_downstream_brew_icsp.yaml" ) ACM_HUB_UNRELEASED_PULL_SECRET_TEMPLATE = "pull-secret.yaml.j2" ACM_ODF_MULTICLUSTER_ORCHESTRATOR_RESOURCE = "odf-multicluster-orchestrator" diff --git a/ocs_ci/templates/acm-deployment/acm_downstream_brew_icsp.yaml b/ocs_ci/templates/acm-deployment/submariner_downstream_brew_icsp.yaml similarity index 100% rename from ocs_ci/templates/acm-deployment/acm_downstream_brew_icsp.yaml rename to ocs_ci/templates/acm-deployment/submariner_downstream_brew_icsp.yaml From c5f35dcca9433fcaafb9efec8976775745558b5e Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Thu, 2 Nov 2023 23:43:19 +0000 Subject: [PATCH 11/12] Move submariner_version parameter to ocp version specific files Validation for serviceexports Addressed few review comments Signed-off-by: Shylesh Kumar Mohan --- conf/README.md | 2 +- .../submariner_downstream_unreleased.yaml | 3 +- ocs_ci/deployment/deployment.py | 5 ++- .../conf/ocp_version/ocp-4.13-config.yaml | 1 + .../conf/ocp_version/ocp-4.13-ga-config.yaml | 1 + .../conf/ocp_version/ocp-4.14-config.yaml | 1 + ocs_ci/ocs/acm/acm.py | 8 ++++- ocs_ci/ocs/resources/storage_cluster.py | 32 +++++++++++++++++++ 8 files changed, 48 insertions(+), 5 deletions(-) diff --git a/conf/README.md b/conf/README.md index 97afeb9efc1..d77df36290e 100644 --- a/conf/README.md +++ b/conf/README.md @@ -256,7 +256,7 @@ higher priority). * `nb_nfs_mount` - NFS mount point used specifically for testing noobaa db NFS mount test * `custom_default_storageclass_names` - Set to true if custom storageclass names use instead of default one. * `storageclassnames` - Under this key, custom storage class names for `cephFilesystems`, `cephObjectStores`, `cephBlockPools`, `cephNonResilientPools`, `nfs` and for `encryption` are defined. -* `submariner_source` - Source from which we take submariner build, ex: upstream, downstream, downstream_unreleased +* `submariner_source` - Source from which we take submariner build, ex: upstream, downstream * `submariner_release_type` - Released OR Unreleased submariner build * `enable_globalnet` - enable or disable globalnet for submariner, default: true * `submariner_unreleased_channel` - submariner channel for unreleased downstream build diff --git a/conf/ocsci/submariner_downstream_unreleased.yaml b/conf/ocsci/submariner_downstream_unreleased.yaml index c47f03ae421..d0cf85c6beb 100644 --- a/conf/ocsci/submariner_downstream_unreleased.yaml +++ b/conf/ocsci/submariner_downstream_unreleased.yaml @@ -1,6 +1,5 @@ ENV_DATA: submariner_source: "downstream" submariner_release_type: "unreleased" - submariner_version: "0.16.0" - submariner_unreleased_channel: "stable-0.16" + submariner_unreleased_channel: "" enable_globalnet: true diff --git a/ocs_ci/deployment/deployment.py b/ocs_ci/deployment/deployment.py index 72d88031ae4..cab8b6fbb52 100644 --- a/ocs_ci/deployment/deployment.py +++ b/ocs_ci/deployment/deployment.py @@ -354,7 +354,10 @@ def do_deploy_ocs(self): ocs_install_verification(ocs_registry_image=ocs_registry_image) # if we have Globalnet enabled in case of submariner with RDR # we need to add a flag to storagecluster - if config.ENV_DATA.get("enable_globalnet", None): + if ( + config.ENV_DATA.get("enable_globalnet", True) + and config.MULTICLUSTER["multicluster_mode"] == "regional-dr" + ): for cluster in get_non_acm_cluster_config(): config.switch_ctx(cluster.MULTICLUSTER["multicluster_index"]) storage_cluster_name = config.ENV_DATA["storage_cluster_name"] diff --git a/ocs_ci/framework/conf/ocp_version/ocp-4.13-config.yaml b/ocs_ci/framework/conf/ocp_version/ocp-4.13-config.yaml index 898476c09d1..d1e525d252c 100644 --- a/ocs_ci/framework/conf/ocp_version/ocp-4.13-config.yaml +++ b/ocs_ci/framework/conf/ocp_version/ocp-4.13-config.yaml @@ -12,3 +12,4 @@ ENV_DATA: vm_template: "rhcos-413.92.202305021736-0-vmware.x86_64" acm_hub_channel: release-2.8 acm_version: "2.8" + submariner_version: "0.15.0" diff --git a/ocs_ci/framework/conf/ocp_version/ocp-4.13-ga-config.yaml b/ocs_ci/framework/conf/ocp_version/ocp-4.13-ga-config.yaml index 3130a97e8b6..f0f69b75f64 100644 --- a/ocs_ci/framework/conf/ocp_version/ocp-4.13-ga-config.yaml +++ b/ocs_ci/framework/conf/ocp_version/ocp-4.13-ga-config.yaml @@ -18,3 +18,4 @@ ENV_DATA: vm_template: "rhcos-4.13.0-x86_64-vmware.x86_64" acm_hub_channel: release-2.8 acm_version: "2.8" + submariner_version: "0.15.0"" diff --git a/ocs_ci/framework/conf/ocp_version/ocp-4.14-config.yaml b/ocs_ci/framework/conf/ocp_version/ocp-4.14-config.yaml index 1dbf6132a9d..3a2b1de39f4 100644 --- a/ocs_ci/framework/conf/ocp_version/ocp-4.14-config.yaml +++ b/ocs_ci/framework/conf/ocp_version/ocp-4.14-config.yaml @@ -12,3 +12,4 @@ ENV_DATA: vm_template: "rhcos-414.92.202303281555-0-vmware.x86_64" acm_hub_channel: release-2.9 acm_version: "2.9" + submariner_version: "0.16.0" diff --git a/ocs_ci/ocs/acm/acm.py b/ocs_ci/ocs/acm/acm.py index 0fb71fe5cbb..c628d662fd8 100644 --- a/ocs_ci/ocs/acm/acm.py +++ b/ocs_ci/ocs/acm/acm.py @@ -289,9 +289,15 @@ def submariner_unreleased_downstream_info(self): self.do_send_keys( self.page_nav["submariner-custom-source"], "submariner-catalogsource" ) + submariner_unreleased_channel = ( + config.ENV_DATA["submariner_unreleased_channel"] + if config.ENV_DATA["submariner_unreleased_channel"] + else config.ENV_DATA["submariner_version"].rpartition(".")[0] + ) + channel_name = "stable-" + submariner_unreleased_channel self.do_send_keys( self.page_nav["submariner-custom-channel"], - config.ENV_DATA["submariner_unreleased_channel"], + channel_name, ) def submariner_validation_ui(self): diff --git a/ocs_ci/ocs/resources/storage_cluster.py b/ocs_ci/ocs/resources/storage_cluster.py index 63672632efc..440380f87b4 100644 --- a/ocs_ci/ocs/resources/storage_cluster.py +++ b/ocs_ci/ocs/resources/storage_cluster.py @@ -756,6 +756,13 @@ def ocs_install_verification( verify_storage_device_class(device_class) verify_device_class_in_osd_tree(ct_pod, device_class) + # RDR with globalnet submariner + if ( + config.ENV_DATA.get("enable_globalnet", True) + and config.MULTICLUSTER["multicluster_mode"] == "regional-dr" + ): + validate_serviceexport() + def mcg_only_install_verification(ocs_registry_image=None): """ @@ -2535,3 +2542,28 @@ def patch_storage_cluster_for_custom_storage_class( else: log.error(f"Invalid action: '{action}'") return False + + +def validate_serviceexport(): + """ + validate the serviceexport resource + Number of osds and mons should match + + """ + serviceexport = OCP( + kind="ServiceExport", namespace=constants.OPENSHIFT_STORAGE_NAMESPACE + ) + osd_count = 0 + mon_count = 0 + for ent in serviceexport.get().get("items"): + if "osd" in ent["metadata"]["name"]: + osd_count += 1 + elif "mon" in ent["metadata"]["name"]: + mon_count += 1 + assert ( + osd_count == get_osd_count() + ), f"osd serviceexport count mismatch {osd_count} != {get_osd_count()} " + + assert mon_count == len( + get_mon_pods() + ), f"Mon serviceexport count mismatch {mon_count} != {len(get_mon_pods())}" From 8a5356f176991b27cb05f3ddefe11273f0998f34 Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Mohan Date: Sun, 5 Nov 2023 11:11:53 +0000 Subject: [PATCH 12/12] Add retries to service export validation Signed-off-by: Shylesh Kumar Mohan --- ocs_ci/ocs/resources/storage_cluster.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ocs_ci/ocs/resources/storage_cluster.py b/ocs_ci/ocs/resources/storage_cluster.py index 440380f87b4..afd22162bcd 100644 --- a/ocs_ci/ocs/resources/storage_cluster.py +++ b/ocs_ci/ocs/resources/storage_cluster.py @@ -2544,6 +2544,7 @@ def patch_storage_cluster_for_custom_storage_class( return False +@retry(AssertionError, 50, 10, 1) def validate_serviceexport(): """ validate the serviceexport resource