Merge pull request #9640 from vavuthu/switch_ibm_region_dynamically_b…

…ased_on_lb_count

use region dynamically for IBM cloud based on load balancers

conf/ocsci/ibmcloud_region_dynamic_switching.yaml

@@ -0,0 +1,5 @@
+---
+# Use this config for dynamic switching of region in IBM cloud
+# This is mainly due to current limitation 50 load balancers per region
+ENV_DATA:
+  enable_region_dynamic_switching: true

ocs_ci/deployment/ibmcloud.py

@@ -12,6 +12,7 @@
 from ocs_ci.deployment.ocp import OCPDeployment as BaseOCPDeployment
 from ocs_ci.framework import config
 from ocs_ci.ocs import constants
+from ocs_ci.ocs.defaults import IBM_CLOUD_LOAD_BALANCER_QUOTA
 from ocs_ci.ocs.exceptions import (
     CommandFailed,
     UnsupportedPlatformVersionError,
@@ -157,11 +158,22 @@ def deploy_ocp(self, log_cli_level="DEBUG"):
             raise UnsupportedPlatformVersionError(
                 "IBM Cloud IPI deployments are only supported on OCP versions >= 4.10"
             )
+        # By default, IBM cloud has load balancer limit of 50 per region.
+        # switch to us-south, if current load balancers are more than 45.
+        # https://cloud.ibm.com/docs/vpc?topic=vpc-quotas
+        ibmcloud.login()
+        if config.ENV_DATA.get("enable_region_dynamic_switching"):
+            lb_count = self.get_load_balancers_count()
+            if lb_count > (IBM_CLOUD_LOAD_BALANCER_QUOTA - 5):
+                logger.info(
+                    "Switching region to us-south due to lack of load balancers"
+                )
+                config.ENV_DATA["region"] = "us-south"
+                ibmcloud.login()
         self.ocp_deployment = self.OCPDeployment()
         self.ocp_deployment.deploy_prereq()
 
         # IBM Cloud specific prereqs
-        ibmcloud.login()
         cco.configure_cloud_credential_operator()
         self.export_api_key()
         self.manually_create_iam_for_vpc()
@@ -445,3 +457,32 @@ def export_api_key():
         logger.info("Exporting IC_API_KEY environment variable")
         api_key = config.AUTH["ibmcloud"]["api_key"]
         os.environ["IC_API_KEY"] = api_key
+
+    def get_load_balancers(self):
+        """
+        Gets the load balancers
+
+        Returns:
+            json: load balancers in json format
+
+        """
+        cmd = "ibmcloud is lbs --output json"
+        out = exec_cmd(cmd)
+        load_balancers = json.loads(out.stdout)
+        logger.debug(f"load balancers: {load_balancers}")
+        return load_balancers
+
+    def get_load_balancers_count(self):
+        """
+        Gets the number of load balancers
+
+        Return:
+            int: number of load balancers
+
+        """
+        load_balancers_count = len(self.get_load_balancers())
+        region = config.ENV_DATA.get("region")
+        logger.info(
+            f"Current load balancers count in region {region} is {load_balancers_count}"
+        )
+        return load_balancers_count

ocs_ci/ocs/defaults.py

@@ -163,3 +163,7 @@
 
 # CrushDeviceClass
 CRUSH_DEVICE_CLASS = "ssd"
+
+
+# IBM Cloud
+IBM_CLOUD_LOAD_BALANCER_QUOTA = 50

ocs_ci/utility/ibmcloud.py

@@ -49,6 +49,33 @@ def login():
     config.RUN["ibmcloud_last_login"] = time.time()
 
 
+def set_region():
+    """
+    Sets the cluster region to ENV_DATA
+    """
+    region = get_region(config.ENV_DATA["cluster_path"])
+    logger.info(f"cluster region is {region}")
+    logger.info(f"updating region {region} to ENV_DATA ")
+    config.ENV_DATA["region"] = region
+
+
+def get_region(cluster_path):
+    """
+    Get region from metadata.json in given cluster_path
+
+    Args:
+        cluster_path: path to cluster install directory
+
+    Returns:
+        str: region where cluster is deployed
+
+    """
+    metadata_file = os.path.join(cluster_path, "metadata.json")
+    with open(metadata_file) as f:
+        metadata = json.load(f)
+    return metadata["ibmcloud"]["region"]
+
+
 def run_ibmcloud_cmd(cmd, secrets=None, timeout=600, ignore_error=False, **kwargs):
     """
     Wrapper function for `run_cmd` which if needed will perform IBM Cloud login

tests/conftest.py

@@ -1759,6 +1759,7 @@ def cluster_teardown_finalizer():
         deployer.deploy_cluster(log_cli_level)
     else:
         if ocsci_config.ENV_DATA["platform"] == constants.IBMCLOUD_PLATFORM:
+            ibmcloud.set_region()
             ibmcloud.login()
     if "cephcluster" not in ocsci_config.RUN.keys():
         check_clusters()