Provision for installing submariner downstream unreleased build

conf/README.md

@@ -256,6 +256,10 @@ higher priority).
 * `nb_nfs_mount` - NFS mount point used specifically for testing noobaa db NFS mount test
 * `custom_default_storageclass_names` - Set to true if custom storageclass names use instead of default one.
 * `storageclassnames` - Under this key, custom storage class names for `cephFilesystems`, `cephObjectStores`, `cephBlockPools`, `cephNonResilientPools`, `nfs` and for `encryption` are defined.
+* `submariner_source` - Source from which we take submariner build, ex: upstream, downstream
+* `submariner_release_type` - Released OR Unreleased submariner build
+* `enable_globalnet` - enable or disable globalnet for submariner, default: true
+* `submariner_unreleased_channel` - submariner channel for unreleased downstream build
 
 #### UPGRADE
 

conf/ocsci/disable_globalnet.yaml

@@ -0,0 +1,2 @@
+ENV_DATA:
+  enable_globalnet: false 

conf/ocsci/submariner_downstream_unreleased.yaml

@@ -0,0 +1,5 @@
+ENV_DATA:
+  submariner_source: "downstream"
+  submariner_release_type: "unreleased"
+  submariner_unreleased_channel: ""
+  enable_globalnet: true

ocs_ci/deployment/acm.py

@@ -16,8 +16,13 @@
     CommandFailed,
     DRPrimaryNotFoundException,
 )
+from ocs_ci.utility import templating
 from ocs_ci.ocs.utils import get_non_acm_cluster_config
-from ocs_ci.utility.utils import run_cmd, run_cmd_interactive
+from ocs_ci.utility.utils import (
+    run_cmd,
+    run_cmd_interactive,
+    wait_for_machineconfigpool_status,
+)
 from ocs_ci.ocs.node import get_typed_worker_nodes, label_nodes
 
 logger = logging.getLogger(__name__)
@@ -62,6 +67,8 @@ class Submariner(object):
     def __init__(self):
         # whether upstream OR downstream
         self.source = config.ENV_DATA["submariner_source"]
+        # released/unreleased
+        self.submariner_release_type = config.ENV_DATA.get("submariner_release_type")
         # Deployment type:
         self.deployment_type = config.ENV_DATA.get("submariner_deployment")
         # Designated broker cluster index where broker will be deployed
@@ -94,9 +101,28 @@ def deploy_downstream(self):
 
         login_to_acm()
         acm_obj = AcmAddClusters()
+        if self.submariner_release_type == "unreleased":
+            old_ctx = config.cur_index
+            for cluster in get_non_acm_cluster_config():
+                config.switch_ctx(cluster.MULTICLUSTER["multicluster_index"])
+                self.create_acm_brew_icsp()
+            config.switch_ctx(old_ctx)
         acm_obj.install_submariner_ui()
         acm_obj.submariner_validation_ui()
 
+    def create_acm_brew_icsp(self):
+        """
+        This is a prereq for downstream unreleased submariner
+
+        """
+        icsp_data = templating.load_yaml(constants.SUBMARINER_DOWNSTREAM_BREW_ICSP)
+        icsp_data_yaml = tempfile.NamedTemporaryFile(
+            mode="w+", prefix="acm_icsp", delete=False
+        )
+        templating.dump_data_to_temp_yaml(icsp_data, icsp_data_yaml.name)
+        run_cmd(f"oc create -f {icsp_data_yaml.name}", timeout=300)
+        wait_for_machineconfigpool_status(node_type="all")
+
     def download_binary(self):
         if self.source == "upstream":
             # This script puts the platform specific binary in ~/.local/bin

ocs_ci/deployment/deployment.py

@@ -83,6 +83,7 @@
     ocs_install_verification,
     setup_ceph_debug,
     get_osd_count,
+    StorageCluster,
 )
 from ocs_ci.ocs.uninstall import uninstall_ocs
 from ocs_ci.ocs.utils import (
@@ -351,6 +352,46 @@ def do_deploy_ocs(self):
                             "ocs_registry_image", None
                         )
                         ocs_install_verification(ocs_registry_image=ocs_registry_image)
+                # if we have Globalnet enabled in case of submariner with RDR
+                # we need to add a flag to storagecluster
+                if (
+                    config.ENV_DATA.get("enable_globalnet", True)
+                    and config.MULTICLUSTER["multicluster_mode"] == "regional-dr"
+                ):
+                    for cluster in get_non_acm_cluster_config():
+                        config.switch_ctx(cluster.MULTICLUSTER["multicluster_index"])
+                        storage_cluster_name = config.ENV_DATA["storage_cluster_name"]
+                        logger.info(
+                            "Updating the StorageCluster resource for globalnet"
+                        )
+                        storage_cluster = StorageCluster(
+                            resource_name=storage_cluster_name,
+                            namespace=config.ENV_DATA["cluster_namespace"],
+                        )
+                        storage_cluster.reload_data()
+                        storage_cluster.wait_for_phase(phase="Ready", timeout=1000)
+                        ptch = (
+                            f'\'{{"spec": {{"network": {{"multiClusterService": '
+                            f"{{\"clusterID\": \"{config.ENV_DATA['cluster_name']}\", \"enabled\": true}}}}}}}}'"
+                        )
+                        ptch_cmd = (
+                            f"oc patch storagecluster/{storage_cluster.data.get('metadata').get('name')} "
+                            f"-n openshift-storage  --type merge --patch {ptch}"
+                        )
+                        run_cmd(ptch_cmd)
+                        ocs_registry_image = config.DEPLOYMENT.get(
+                            "ocs_registry_image", None
+                        )
+                        storage_cluster.reload_data()
+                        assert (
+                            storage_cluster.data.get("spec")
+                            .get("network")
+                            .get("multiClusterService")
+                            .get("enabled")
+                        ), "Failed to update StorageCluster globalnet"
+                        ocs_install_verification(
+                            timeout=2000, ocs_registry_image=ocs_registry_image
+                        )
                 config.reset_ctx()
         else:
             logger.warning("OCS deployment will be skipped")

ocs_ci/framework/conf/ocp_version/ocp-4.13-config.yaml

@@ -12,3 +12,4 @@ ENV_DATA:
   vm_template: "rhcos-413.92.202305021736-0-vmware.x86_64"
   acm_hub_channel: release-2.8
   acm_version: "2.8"
+  submariner_version: "0.15.0"

ocs_ci/framework/conf/ocp_version/ocp-4.13-ga-config.yaml

@@ -18,3 +18,4 @@ ENV_DATA:
   vm_template: "rhcos-4.13.0-x86_64-vmware.x86_64"
   acm_hub_channel: release-2.8
   acm_version: "2.8"
+  submariner_version: "0.15.0""

ocs_ci/framework/conf/ocp_version/ocp-4.14-config.yaml

@@ -12,3 +12,4 @@ ENV_DATA:
   vm_template: "rhcos-414.92.202303281555-0-vmware.x86_64"
   acm_hub_channel: release-2.9
   acm_version: "2.9"
+  submariner_version: "0.16.0"

ocs_ci/ocs/acm/acm.py

@@ -1,6 +1,8 @@
 import logging
 import time
 import os
+import tempfile
+import requests
 
 from selenium.webdriver.support.wait import WebDriverWait
 from selenium.webdriver.support import expected_conditions as ec
@@ -22,8 +24,10 @@
 from ocs_ci.ocs.utils import get_non_acm_cluster_config, get_primary_cluster_config
 from ocs_ci.utility.utils import (
     TimeoutSampler,
+    get_ocp_version,
     get_running_acm_version,
     string_chunkify,
+    run_cmd,
 )
 from ocs_ci.ocs.ui.acm_ui import AcmPageNavigator
 from ocs_ci.ocs.ui.base_ui import login_ui, SeleniumDriver
@@ -152,6 +156,38 @@ def install_submariner_ui(self, globalnet=True):
             for s in get_non_acm_cluster_config()
             if s.MULTICLUSTER["multicluster_index"] != primary_index
         ][0]
+        # submariner catalogsource creation
+        if config.ENV_DATA["submariner_release_type"] == "unreleased":
+            submariner_downstream_unreleased = templating.load_yaml(
+                constants.SUBMARINER_DOWNSTREAM_UNRELEASED
+            )
+            # Update catalog source
+            submariner_full_url = "".join(
+                [
+                    constants.SUBMARINER_DOWNSTREAM_UNRELEASED_BUILD_URL,
+                    config.ENV_DATA["submariner_version"],
+                ]
+            )
+
+            resp = requests.get(submariner_full_url, verify=False)
+            raw_msg = resp.json()["raw_messages"]
+            version_tag = raw_msg[0]["msg"]["pipeline"]["index_image"][
+                f"v{get_ocp_version()}"
+            ].split(":")[1]
+            submariner_downstream_unreleased["spec"]["image"] = ":".join(
+                [constants.SUBMARINER_BREW_REPO, version_tag]
+            )
+            submariner_data_yaml = tempfile.NamedTemporaryFile(
+                mode="w+", prefix="submariner_downstream_unreleased", delete=False
+            )
+            templating.dump_data_to_temp_yaml(
+                submariner_downstream_unreleased, submariner_data_yaml.name
+            )
+            old_ctx = config.cur_index
+            for cluster in get_non_acm_cluster_config():
+                config.switch_ctx(cluster.MULTICLUSTER["multicluster_index"])
+                run_cmd(f"oc create -f {submariner_data_yaml.name}", timeout=300)
+            config.switch_ctx(old_ctx)
 
         cluster_name_a = cluster_env.get(f"cluster_name_{primary_index}")
         cluster_name_b = cluster_env.get(f"cluster_name_{secondary_index}")
@@ -222,6 +258,8 @@ def install_submariner_ui(self, globalnet=True):
         )
         self.do_click(self.page_nav["gateway-count-btn"])
         self.do_click(self.page_nav["gateway-count-btn"])
+        if config.ENV_DATA.get("submariner_release_type") == "unreleased":
+            self.submariner_unreleased_downstream_info()
         log.info("Click on Next button")
         self.do_click(self.page_nav["next-btn"])
         log.info("Click on 'Enable NAT-T' to uncheck it [2]")
@@ -231,6 +269,8 @@ def install_submariner_ui(self, globalnet=True):
         )
         self.do_click(self.page_nav["gateway-count-btn"])
         self.do_click(self.page_nav["gateway-count-btn"])
+        if config.ENV_DATA.get("submariner_release_type") == "unreleased":
+            self.submariner_unreleased_downstream_info()
         log.info("Click on Next button [2]")
         self.do_click(self.page_nav["next-btn"])
         if ocs_version >= version.VERSION_4_13 and globalnet:
@@ -243,6 +283,23 @@ def install_submariner_ui(self, globalnet=True):
         log.info("Click on 'Install'")
         self.do_click(self.page_nav["install-btn"])
 
+    def submariner_unreleased_downstream_info(self):
+        self.do_click(self.page_nav["submariner-custom-subscription"])
+        self.do_clear(self.page_nav["submariner-custom-source"])
+        self.do_send_keys(
+            self.page_nav["submariner-custom-source"], "submariner-catalogsource"
+        )
+        submariner_unreleased_channel = (
+            config.ENV_DATA["submariner_unreleased_channel"]
+            if config.ENV_DATA["submariner_unreleased_channel"]
+            else config.ENV_DATA["submariner_version"].rpartition(".")[0]
+        )
+        channel_name = "stable-" + submariner_unreleased_channel
+        self.do_send_keys(
+            self.page_nav["submariner-custom-channel"],
+            channel_name,
+        )
+
     def submariner_validation_ui(self):
         """
         This function validates submariner status on ACM console which connects 2 managed OCP clusters.

ocs_ci/ocs/constants.py

@@ -849,6 +849,7 @@
 )
 MDR_BACKUP_SCHEDULE_RESOURCE = "schedule-acm"
 
+
 # DR constants
 SUBMARINER_DOWNLOAD_URL = "https://get.submariner.io"
 DR_DEFAULT_NAMESPACE = "openshift-dr-systems"
@@ -2069,6 +2070,16 @@
 SUBMARINER_GATEWAY_ACTIVE_LABEL = "gateway.submariner.io/status=active"
 SUBMARINER_GATEWAY_NODE_LABEL = "submariner.io/gateway=true"
 GLOBALNET_STATUS = "True"
+SUBMARINER_DOWNSTREAM_UNRELEASED = os.path.join(
+    TEMPLATE_MULTICLUSTER_DIR, "submariner_downstream_unreleased_catsrc.yaml"
+)
+# We need to append version string at the end of this url
+SUBMARINER_DOWNSTREAM_UNRELEASED_BUILD_URL = (
+    "https://datagrepper.engineering.redhat.com/raw?topic=/topic/"
+    "VirtualTopic.eng.ci.redhat-container-image.pipeline.complete"
+    "&rows_per_page=25&delta=1296000&contains=submariner-operator-bundle-container-v"
+)
+SUBMARINER_BREW_REPO = "brew.registry.redhat.io/rh-osbs/iib"
 
 # Multicluster related
 
@@ -2102,6 +2113,9 @@
 ACM_HUB_UNRELEASED_ICSP_YAML = os.path.join(
     TEMPLATE_DIR, "acm-deployment", "imagecontentsourcepolicy.yaml"
 )
+SUBMARINER_DOWNSTREAM_BREW_ICSP = os.path.join(
+    TEMPLATE_DIR, "acm-deployment", "submariner_downstream_brew_icsp.yaml"
+)
 ACM_HUB_UNRELEASED_PULL_SECRET_TEMPLATE = "pull-secret.yaml.j2"
 ACM_ODF_MULTICLUSTER_ORCHESTRATOR_RESOURCE = "odf-multicluster-orchestrator"
 ACM_ODR_HUB_OPERATOR_RESOURCE = "odr-hub-operator"

ocs_ci/ocs/resources/storage_cluster.py

@@ -756,6 +756,13 @@ def ocs_install_verification(
             verify_storage_device_class(device_class)
             verify_device_class_in_osd_tree(ct_pod, device_class)
 
+    # RDR with globalnet submariner
+    if (
+        config.ENV_DATA.get("enable_globalnet", True)
+        and config.MULTICLUSTER["multicluster_mode"] == "regional-dr"
+    ):
+        validate_serviceexport()
+
 
 def mcg_only_install_verification(ocs_registry_image=None):
     """
@@ -2535,3 +2542,29 @@ def patch_storage_cluster_for_custom_storage_class(
     else:
         log.error(f"Invalid action: '{action}'")
         return False
+
+
+@retry(AssertionError, 50, 10, 1)
+def validate_serviceexport():
+    """
+    validate the serviceexport resource
+    Number of osds and mons should match
+
+    """
+    serviceexport = OCP(
+        kind="ServiceExport", namespace=constants.OPENSHIFT_STORAGE_NAMESPACE
+    )
+    osd_count = 0
+    mon_count = 0
+    for ent in serviceexport.get().get("items"):
+        if "osd" in ent["metadata"]["name"]:
+            osd_count += 1
+        elif "mon" in ent["metadata"]["name"]:
+            mon_count += 1
+    assert (
+        osd_count == get_osd_count()
+    ), f"osd serviceexport count mismatch {osd_count} != {get_osd_count()} "
+
+    assert mon_count == len(
+        get_mon_pods()
+    ), f"Mon serviceexport count mismatch {mon_count} != {len(get_mon_pods())}"

ocs_ci/ocs/ui/views.py

@@ -879,6 +879,13 @@
     **acm_configuration_4_12,
 }
 
+acm_configuration_4_14 = {
+    **acm_configuration_4_13,
+    "submariner-custom-subscription": ("isCustomSubscription", By.ID),
+    "submariner-custom-source": ("source", By.ID),
+    "submariner-custom-channel": ("channel", By.ID),
+}
+
 add_capacity = {
     "ocs_operator": (
         'a[data-test-operator-row="OpenShift Container Storage"]',
@@ -1710,6 +1717,7 @@
             **acm_configuration_4_11,
             **acm_configuration_4_12,
             **acm_configuration_4_13,
+            **acm_configuration_4_14,
         },
         "validation": {
             **validation,

ocs_ci/templates/acm-deployment/submariner_downstream_brew_icsp.yaml

@@ -0,0 +1,18 @@
+apiVersion: operator.openshift.io/v1alpha1
+kind: ImageContentSourcePolicy
+metadata:
+  name: brew-registry
+spec:
+  repositoryDigestMirrors:
+  - mirrors:
+    - brew.registry.redhat.io
+    source: registry.redhat.io
+  - mirrors:
+    - brew.registry.redhat.io/rhacm2
+    source: registry.redhat.io/rhacm2
+  - mirrors:
+    - brew.registry.redhat.io
+    source: registry.stage.redhat.io
+  - mirrors:
+    - brew.registry.redhat.io
+    source: registry-proxy.engineering.redhat.com

ocs_ci/templates/ocs-deployment/multicluster/submariner_downstream_unreleased_catsrc.yaml

@@ -0,0 +1,16 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: CatalogSource
+metadata:
+  name: submariner-catalogsource
+  namespace: openshift-marketplace
+spec:
+  icon:
+    base64data: ""
+    mediatype: ""
+  image: PLACE_HOLDER
+  publisher: Red Hat
+  sourceType: grpc
+  priority: 100
+  updateStrategy:
+    registryPoll:
+      interval: 15m