diff --git a/conf/README.md b/conf/README.md index 3400b288456..d77df36290e 100644 --- a/conf/README.md +++ b/conf/README.md @@ -256,6 +256,10 @@ higher priority). * `nb_nfs_mount` - NFS mount point used specifically for testing noobaa db NFS mount test * `custom_default_storageclass_names` - Set to true if custom storageclass names use instead of default one. * `storageclassnames` - Under this key, custom storage class names for `cephFilesystems`, `cephObjectStores`, `cephBlockPools`, `cephNonResilientPools`, `nfs` and for `encryption` are defined. +* `submariner_source` - Source from which we take submariner build, ex: upstream, downstream +* `submariner_release_type` - Released OR Unreleased submariner build +* `enable_globalnet` - enable or disable globalnet for submariner, default: true +* `submariner_unreleased_channel` - submariner channel for unreleased downstream build #### UPGRADE diff --git a/conf/ocsci/disable_globalnet.yaml b/conf/ocsci/disable_globalnet.yaml new file mode 100644 index 00000000000..e5041e99c12 --- /dev/null +++ b/conf/ocsci/disable_globalnet.yaml @@ -0,0 +1,2 @@ +ENV_DATA: + enable_globalnet: false diff --git a/conf/ocsci/submariner_downstream_unreleased.yaml b/conf/ocsci/submariner_downstream_unreleased.yaml new file mode 100644 index 00000000000..d0cf85c6beb --- /dev/null +++ b/conf/ocsci/submariner_downstream_unreleased.yaml @@ -0,0 +1,5 @@ +ENV_DATA: + submariner_source: "downstream" + submariner_release_type: "unreleased" + submariner_unreleased_channel: "" + enable_globalnet: true diff --git a/ocs_ci/deployment/acm.py b/ocs_ci/deployment/acm.py index 3268842de61..ca6936b7787 100644 --- a/ocs_ci/deployment/acm.py +++ b/ocs_ci/deployment/acm.py @@ -16,8 +16,13 @@ CommandFailed, DRPrimaryNotFoundException, ) +from ocs_ci.utility import templating from ocs_ci.ocs.utils import get_non_acm_cluster_config -from ocs_ci.utility.utils import run_cmd, run_cmd_interactive +from ocs_ci.utility.utils import ( + run_cmd, + run_cmd_interactive, + wait_for_machineconfigpool_status, +) from ocs_ci.ocs.node import get_typed_worker_nodes, label_nodes logger = logging.getLogger(__name__) @@ -62,6 +67,8 @@ class Submariner(object): def __init__(self): # whether upstream OR downstream self.source = config.ENV_DATA["submariner_source"] + # released/unreleased + self.submariner_release_type = config.ENV_DATA.get("submariner_release_type") # Deployment type: self.deployment_type = config.ENV_DATA.get("submariner_deployment") # Designated broker cluster index where broker will be deployed @@ -94,9 +101,28 @@ def deploy_downstream(self): login_to_acm() acm_obj = AcmAddClusters() + if self.submariner_release_type == "unreleased": + old_ctx = config.cur_index + for cluster in get_non_acm_cluster_config(): + config.switch_ctx(cluster.MULTICLUSTER["multicluster_index"]) + self.create_acm_brew_icsp() + config.switch_ctx(old_ctx) acm_obj.install_submariner_ui() acm_obj.submariner_validation_ui() + def create_acm_brew_icsp(self): + """ + This is a prereq for downstream unreleased submariner + + """ + icsp_data = templating.load_yaml(constants.SUBMARINER_DOWNSTREAM_BREW_ICSP) + icsp_data_yaml = tempfile.NamedTemporaryFile( + mode="w+", prefix="acm_icsp", delete=False + ) + templating.dump_data_to_temp_yaml(icsp_data, icsp_data_yaml.name) + run_cmd(f"oc create -f {icsp_data_yaml.name}", timeout=300) + wait_for_machineconfigpool_status(node_type="all") + def download_binary(self): if self.source == "upstream": # This script puts the platform specific binary in ~/.local/bin diff --git a/ocs_ci/deployment/deployment.py b/ocs_ci/deployment/deployment.py index 16a408cc341..cab8b6fbb52 100644 --- a/ocs_ci/deployment/deployment.py +++ b/ocs_ci/deployment/deployment.py @@ -83,6 +83,7 @@ ocs_install_verification, setup_ceph_debug, get_osd_count, + StorageCluster, ) from ocs_ci.ocs.uninstall import uninstall_ocs from ocs_ci.ocs.utils import ( @@ -351,6 +352,46 @@ def do_deploy_ocs(self): "ocs_registry_image", None ) ocs_install_verification(ocs_registry_image=ocs_registry_image) + # if we have Globalnet enabled in case of submariner with RDR + # we need to add a flag to storagecluster + if ( + config.ENV_DATA.get("enable_globalnet", True) + and config.MULTICLUSTER["multicluster_mode"] == "regional-dr" + ): + for cluster in get_non_acm_cluster_config(): + config.switch_ctx(cluster.MULTICLUSTER["multicluster_index"]) + storage_cluster_name = config.ENV_DATA["storage_cluster_name"] + logger.info( + "Updating the StorageCluster resource for globalnet" + ) + storage_cluster = StorageCluster( + resource_name=storage_cluster_name, + namespace=config.ENV_DATA["cluster_namespace"], + ) + storage_cluster.reload_data() + storage_cluster.wait_for_phase(phase="Ready", timeout=1000) + ptch = ( + f'\'{{"spec": {{"network": {{"multiClusterService": ' + f"{{\"clusterID\": \"{config.ENV_DATA['cluster_name']}\", \"enabled\": true}}}}}}}}'" + ) + ptch_cmd = ( + f"oc patch storagecluster/{storage_cluster.data.get('metadata').get('name')} " + f"-n openshift-storage --type merge --patch {ptch}" + ) + run_cmd(ptch_cmd) + ocs_registry_image = config.DEPLOYMENT.get( + "ocs_registry_image", None + ) + storage_cluster.reload_data() + assert ( + storage_cluster.data.get("spec") + .get("network") + .get("multiClusterService") + .get("enabled") + ), "Failed to update StorageCluster globalnet" + ocs_install_verification( + timeout=2000, ocs_registry_image=ocs_registry_image + ) config.reset_ctx() else: logger.warning("OCS deployment will be skipped") diff --git a/ocs_ci/framework/conf/ocp_version/ocp-4.13-config.yaml b/ocs_ci/framework/conf/ocp_version/ocp-4.13-config.yaml index 898476c09d1..d1e525d252c 100644 --- a/ocs_ci/framework/conf/ocp_version/ocp-4.13-config.yaml +++ b/ocs_ci/framework/conf/ocp_version/ocp-4.13-config.yaml @@ -12,3 +12,4 @@ ENV_DATA: vm_template: "rhcos-413.92.202305021736-0-vmware.x86_64" acm_hub_channel: release-2.8 acm_version: "2.8" + submariner_version: "0.15.0" diff --git a/ocs_ci/framework/conf/ocp_version/ocp-4.13-ga-config.yaml b/ocs_ci/framework/conf/ocp_version/ocp-4.13-ga-config.yaml index 3130a97e8b6..f0f69b75f64 100644 --- a/ocs_ci/framework/conf/ocp_version/ocp-4.13-ga-config.yaml +++ b/ocs_ci/framework/conf/ocp_version/ocp-4.13-ga-config.yaml @@ -18,3 +18,4 @@ ENV_DATA: vm_template: "rhcos-4.13.0-x86_64-vmware.x86_64" acm_hub_channel: release-2.8 acm_version: "2.8" + submariner_version: "0.15.0"" diff --git a/ocs_ci/framework/conf/ocp_version/ocp-4.14-config.yaml b/ocs_ci/framework/conf/ocp_version/ocp-4.14-config.yaml index 1dbf6132a9d..3a2b1de39f4 100644 --- a/ocs_ci/framework/conf/ocp_version/ocp-4.14-config.yaml +++ b/ocs_ci/framework/conf/ocp_version/ocp-4.14-config.yaml @@ -12,3 +12,4 @@ ENV_DATA: vm_template: "rhcos-414.92.202303281555-0-vmware.x86_64" acm_hub_channel: release-2.9 acm_version: "2.9" + submariner_version: "0.16.0" diff --git a/ocs_ci/ocs/acm/acm.py b/ocs_ci/ocs/acm/acm.py index e46a6b22c25..c628d662fd8 100644 --- a/ocs_ci/ocs/acm/acm.py +++ b/ocs_ci/ocs/acm/acm.py @@ -1,6 +1,8 @@ import logging import time import os +import tempfile +import requests from selenium.webdriver.support.wait import WebDriverWait from selenium.webdriver.support import expected_conditions as ec @@ -22,8 +24,10 @@ from ocs_ci.ocs.utils import get_non_acm_cluster_config, get_primary_cluster_config from ocs_ci.utility.utils import ( TimeoutSampler, + get_ocp_version, get_running_acm_version, string_chunkify, + run_cmd, ) from ocs_ci.ocs.ui.acm_ui import AcmPageNavigator from ocs_ci.ocs.ui.base_ui import login_ui, SeleniumDriver @@ -152,6 +156,38 @@ def install_submariner_ui(self, globalnet=True): for s in get_non_acm_cluster_config() if s.MULTICLUSTER["multicluster_index"] != primary_index ][0] + # submariner catalogsource creation + if config.ENV_DATA["submariner_release_type"] == "unreleased": + submariner_downstream_unreleased = templating.load_yaml( + constants.SUBMARINER_DOWNSTREAM_UNRELEASED + ) + # Update catalog source + submariner_full_url = "".join( + [ + constants.SUBMARINER_DOWNSTREAM_UNRELEASED_BUILD_URL, + config.ENV_DATA["submariner_version"], + ] + ) + + resp = requests.get(submariner_full_url, verify=False) + raw_msg = resp.json()["raw_messages"] + version_tag = raw_msg[0]["msg"]["pipeline"]["index_image"][ + f"v{get_ocp_version()}" + ].split(":")[1] + submariner_downstream_unreleased["spec"]["image"] = ":".join( + [constants.SUBMARINER_BREW_REPO, version_tag] + ) + submariner_data_yaml = tempfile.NamedTemporaryFile( + mode="w+", prefix="submariner_downstream_unreleased", delete=False + ) + templating.dump_data_to_temp_yaml( + submariner_downstream_unreleased, submariner_data_yaml.name + ) + old_ctx = config.cur_index + for cluster in get_non_acm_cluster_config(): + config.switch_ctx(cluster.MULTICLUSTER["multicluster_index"]) + run_cmd(f"oc create -f {submariner_data_yaml.name}", timeout=300) + config.switch_ctx(old_ctx) cluster_name_a = cluster_env.get(f"cluster_name_{primary_index}") cluster_name_b = cluster_env.get(f"cluster_name_{secondary_index}") @@ -222,6 +258,8 @@ def install_submariner_ui(self, globalnet=True): ) self.do_click(self.page_nav["gateway-count-btn"]) self.do_click(self.page_nav["gateway-count-btn"]) + if config.ENV_DATA.get("submariner_release_type") == "unreleased": + self.submariner_unreleased_downstream_info() log.info("Click on Next button") self.do_click(self.page_nav["next-btn"]) log.info("Click on 'Enable NAT-T' to uncheck it [2]") @@ -231,6 +269,8 @@ def install_submariner_ui(self, globalnet=True): ) self.do_click(self.page_nav["gateway-count-btn"]) self.do_click(self.page_nav["gateway-count-btn"]) + if config.ENV_DATA.get("submariner_release_type") == "unreleased": + self.submariner_unreleased_downstream_info() log.info("Click on Next button [2]") self.do_click(self.page_nav["next-btn"]) if ocs_version >= version.VERSION_4_13 and globalnet: @@ -243,6 +283,23 @@ def install_submariner_ui(self, globalnet=True): log.info("Click on 'Install'") self.do_click(self.page_nav["install-btn"]) + def submariner_unreleased_downstream_info(self): + self.do_click(self.page_nav["submariner-custom-subscription"]) + self.do_clear(self.page_nav["submariner-custom-source"]) + self.do_send_keys( + self.page_nav["submariner-custom-source"], "submariner-catalogsource" + ) + submariner_unreleased_channel = ( + config.ENV_DATA["submariner_unreleased_channel"] + if config.ENV_DATA["submariner_unreleased_channel"] + else config.ENV_DATA["submariner_version"].rpartition(".")[0] + ) + channel_name = "stable-" + submariner_unreleased_channel + self.do_send_keys( + self.page_nav["submariner-custom-channel"], + channel_name, + ) + def submariner_validation_ui(self): """ This function validates submariner status on ACM console which connects 2 managed OCP clusters. diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index fb0e87df186..19c66ff9cb1 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -849,6 +849,7 @@ ) MDR_BACKUP_SCHEDULE_RESOURCE = "schedule-acm" + # DR constants SUBMARINER_DOWNLOAD_URL = "https://get.submariner.io" DR_DEFAULT_NAMESPACE = "openshift-dr-systems" @@ -2069,6 +2070,16 @@ SUBMARINER_GATEWAY_ACTIVE_LABEL = "gateway.submariner.io/status=active" SUBMARINER_GATEWAY_NODE_LABEL = "submariner.io/gateway=true" GLOBALNET_STATUS = "True" +SUBMARINER_DOWNSTREAM_UNRELEASED = os.path.join( + TEMPLATE_MULTICLUSTER_DIR, "submariner_downstream_unreleased_catsrc.yaml" +) +# We need to append version string at the end of this url +SUBMARINER_DOWNSTREAM_UNRELEASED_BUILD_URL = ( + "https://datagrepper.engineering.redhat.com/raw?topic=/topic/" + "VirtualTopic.eng.ci.redhat-container-image.pipeline.complete" + "&rows_per_page=25&delta=1296000&contains=submariner-operator-bundle-container-v" +) +SUBMARINER_BREW_REPO = "brew.registry.redhat.io/rh-osbs/iib" # Multicluster related @@ -2102,6 +2113,9 @@ ACM_HUB_UNRELEASED_ICSP_YAML = os.path.join( TEMPLATE_DIR, "acm-deployment", "imagecontentsourcepolicy.yaml" ) +SUBMARINER_DOWNSTREAM_BREW_ICSP = os.path.join( + TEMPLATE_DIR, "acm-deployment", "submariner_downstream_brew_icsp.yaml" +) ACM_HUB_UNRELEASED_PULL_SECRET_TEMPLATE = "pull-secret.yaml.j2" ACM_ODF_MULTICLUSTER_ORCHESTRATOR_RESOURCE = "odf-multicluster-orchestrator" ACM_ODR_HUB_OPERATOR_RESOURCE = "odr-hub-operator" diff --git a/ocs_ci/ocs/resources/storage_cluster.py b/ocs_ci/ocs/resources/storage_cluster.py index 63672632efc..afd22162bcd 100644 --- a/ocs_ci/ocs/resources/storage_cluster.py +++ b/ocs_ci/ocs/resources/storage_cluster.py @@ -756,6 +756,13 @@ def ocs_install_verification( verify_storage_device_class(device_class) verify_device_class_in_osd_tree(ct_pod, device_class) + # RDR with globalnet submariner + if ( + config.ENV_DATA.get("enable_globalnet", True) + and config.MULTICLUSTER["multicluster_mode"] == "regional-dr" + ): + validate_serviceexport() + def mcg_only_install_verification(ocs_registry_image=None): """ @@ -2535,3 +2542,29 @@ def patch_storage_cluster_for_custom_storage_class( else: log.error(f"Invalid action: '{action}'") return False + + +@retry(AssertionError, 50, 10, 1) +def validate_serviceexport(): + """ + validate the serviceexport resource + Number of osds and mons should match + + """ + serviceexport = OCP( + kind="ServiceExport", namespace=constants.OPENSHIFT_STORAGE_NAMESPACE + ) + osd_count = 0 + mon_count = 0 + for ent in serviceexport.get().get("items"): + if "osd" in ent["metadata"]["name"]: + osd_count += 1 + elif "mon" in ent["metadata"]["name"]: + mon_count += 1 + assert ( + osd_count == get_osd_count() + ), f"osd serviceexport count mismatch {osd_count} != {get_osd_count()} " + + assert mon_count == len( + get_mon_pods() + ), f"Mon serviceexport count mismatch {mon_count} != {len(get_mon_pods())}" diff --git a/ocs_ci/ocs/ui/views.py b/ocs_ci/ocs/ui/views.py index 71791c1fa77..fcd3b03e23d 100644 --- a/ocs_ci/ocs/ui/views.py +++ b/ocs_ci/ocs/ui/views.py @@ -879,6 +879,13 @@ **acm_configuration_4_12, } +acm_configuration_4_14 = { + **acm_configuration_4_13, + "submariner-custom-subscription": ("isCustomSubscription", By.ID), + "submariner-custom-source": ("source", By.ID), + "submariner-custom-channel": ("channel", By.ID), +} + add_capacity = { "ocs_operator": ( 'a[data-test-operator-row="OpenShift Container Storage"]', @@ -1710,6 +1717,7 @@ **acm_configuration_4_11, **acm_configuration_4_12, **acm_configuration_4_13, + **acm_configuration_4_14, }, "validation": { **validation, diff --git a/ocs_ci/templates/acm-deployment/submariner_downstream_brew_icsp.yaml b/ocs_ci/templates/acm-deployment/submariner_downstream_brew_icsp.yaml new file mode 100644 index 00000000000..4a4c76e8aa0 --- /dev/null +++ b/ocs_ci/templates/acm-deployment/submariner_downstream_brew_icsp.yaml @@ -0,0 +1,18 @@ +apiVersion: operator.openshift.io/v1alpha1 +kind: ImageContentSourcePolicy +metadata: + name: brew-registry +spec: + repositoryDigestMirrors: + - mirrors: + - brew.registry.redhat.io + source: registry.redhat.io + - mirrors: + - brew.registry.redhat.io/rhacm2 + source: registry.redhat.io/rhacm2 + - mirrors: + - brew.registry.redhat.io + source: registry.stage.redhat.io + - mirrors: + - brew.registry.redhat.io + source: registry-proxy.engineering.redhat.com diff --git a/ocs_ci/templates/ocs-deployment/multicluster/submariner_downstream_unreleased_catsrc.yaml b/ocs_ci/templates/ocs-deployment/multicluster/submariner_downstream_unreleased_catsrc.yaml new file mode 100644 index 00000000000..b931afe2ce8 --- /dev/null +++ b/ocs_ci/templates/ocs-deployment/multicluster/submariner_downstream_unreleased_catsrc.yaml @@ -0,0 +1,16 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + name: submariner-catalogsource + namespace: openshift-marketplace +spec: + icon: + base64data: "" + mediatype: "" + image: PLACE_HOLDER + publisher: Red Hat + sourceType: grpc + priority: 100 + updateStrategy: + registryPoll: + interval: 15m