From 045f9bf928b102b99bf5e8f1f37fa79194df9036 Mon Sep 17 00:00:00 2001
From: prsurve <prsurve@redhat.com>
Date: Sun, 25 Feb 2024 21:50:20 +0530
Subject: [PATCH 01/11] increase timeout

Signed-off-by: prsurve <prsurve@redhat.com>
---
 tests/functional/z_cluster/nodes/test_nodes_maintenance.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/functional/z_cluster/nodes/test_nodes_maintenance.py b/tests/functional/z_cluster/nodes/test_nodes_maintenance.py
index df1e4e0eaa9..4752c9283a8 100644
--- a/tests/functional/z_cluster/nodes/test_nodes_maintenance.py
+++ b/tests/functional/z_cluster/nodes/test_nodes_maintenance.py
@@ -255,6 +255,8 @@ def test_node_maintenance_restart_activate(
         wait_for_nodes_status(
             node_names=[typed_node_name],
             status=constants.NODE_READY_SCHEDULING_DISABLED,
+            timeout=600,
+            sleep=20,
         )
 
         # Mark the node back to schedulable

From 86061bd755f4be33975b4ccb2cd861f0ec97ee7d Mon Sep 17 00:00:00 2001
From: prsurve <prsurve@redhat.com>
Date: Mon, 5 Feb 2024 17:45:28 +0530
Subject: [PATCH 02/11] Add Ability to perform Node operation on IBM Cloud IPI
 deployment

Signed-off-by: prsurve <prsurve@redhat.com>
---
 ocs_ci/ocs/constants.py                       |   2 +
 ocs_ci/ocs/platform_nodes.py                  | 113 ++++++++
 ocs_ci/utility/ibmcloud.py                    | 268 ++++++++++++++++++
 .../z_cluster/nodes/test_disk_failures.py     |   3 +-
 4 files changed, 385 insertions(+), 1 deletion(-)

diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py
index 9c9300a6bfa..07744b6f31f 100644
--- a/ocs_ci/ocs/constants.py
+++ b/ocs_ci/ocs/constants.py
@@ -130,6 +130,8 @@
 STATUS_WAITFORUSERTOCLEANUP = "WaitOnUserToCleanUp"
 STATUS_POWERON = "ON"
 STATUS_POWEROFF = "OFF"
+STATUS_STOPPED = "stopped"
+STATUS_STOPPING = "stopping"
 
 # NooBaa statuses
 BS_AUTH_FAILED = "AUTH_FAILED"
diff --git a/ocs_ci/ocs/platform_nodes.py b/ocs_ci/ocs/platform_nodes.py
index cbd6ff9fff5..cf1f8805cf9 100644
--- a/ocs_ci/ocs/platform_nodes.py
+++ b/ocs_ci/ocs/platform_nodes.py
@@ -37,6 +37,7 @@
     get_node_objs,
     get_typed_worker_nodes,
     get_nodes,
+    get_worker_nodes,
 )
 from ocs_ci.ocs.resources.pvc import get_deviceset_pvs
 from ocs_ci.ocs.resources import pod
@@ -99,6 +100,7 @@ def __init__(self):
             "fusion_aas": AWSNodes,
             "hci_baremetal": IBMCloudBMNodes,
             "kubevirt_vm": KubevirtVMNodes,
+            "ibm_cloud_ipi": IBMCloudIPI,
         }
 
     def get_nodes_platform(self):
@@ -117,6 +119,9 @@ def get_nodes_platform(self):
         ):
             platform = "kubevirt_vm"
 
+        if config.ENV_DATA["platform"] == constants.IBMCLOUD_PLATFORM:
+            if config.ENV_DATA["deployment_type"] == "ipi":
+                platform += "_ipi"
         return self.cls_map[platform]()
 
 
@@ -3345,3 +3350,111 @@ def terminate_nodes(self, nodes, wait=True):
 
         """
         raise NotImplementedError("Terminate nodes functionality not implemented")
+
+
+class IBMCloudIPI(object):
+    """
+    A base class for nodes related operations.
+    Should be inherited by specific platform classes
+    """
+
+    def __init__(self):
+        from ocs_ci.utility import ibmcloud
+
+        super(IBMCloudIPI, self).__init__()
+        self.ibmcloud_ipi = ibmcloud.IBMCloudIPI()
+
+    def get_data_volumes(self):
+        pvs = get_deviceset_pvs()
+        s = [
+            pv.get().get("spec").get("csi").get("volumeAttributes").get("volumeId")
+            for pv in pvs
+        ]
+        logger.info(s)
+        return s
+
+    def get_node_by_attached_volume(self, volume):
+        volume_kube_path = f"kubernetes.io/csi/vpc.block.csi.ibm.io^{volume}"
+        all_nodes = get_node_objs(get_worker_nodes())
+        for node in all_nodes:
+            for volume in node.data["status"]["volumesAttached"]:
+                if volume_kube_path in volume.values():
+                    return node
+
+    def stop_nodes(self, nodes, force=True):
+        self.ibmcloud_ipi.stop_nodes(nodes=nodes, force=force)
+
+    def start_nodes(self, nodes):
+        self.ibmcloud_ipi.start_nodes(nodes=nodes)
+
+    def restart_nodes(self, nodes, wait=True):
+        self.ibmcloud_ipi.restart_nodes(nodes=nodes, wait=wait)
+
+    def restart_nodes_by_stop_and_start(self, nodes, force=True):
+        self.ibmcloud_ipi.restart_nodes_by_stop_and_start(nodes=nodes, force=force)
+
+    def detach_volume(self, volume, node=None, delete_from_backend=True):
+        self.ibmcloud_ipi.detach_volume(volume=volume, node=node)
+
+    def attach_volume(self, volume, node):
+        self.ibmcloud_ipi.attach_volume(volume=volume, node=node)
+
+    def wait_for_volume_attach(self, volume):
+        self.ibmcloud_ipi.wait_for_volume_attach(volume=volume)
+
+    def restart_nodes_by_stop_and_start_teardown(self):
+        self.ibmcloud_ipi.restart_nodes_by_stop_and_start_force()
+
+    def create_and_attach_nodes_to_cluster(self, node_conf, node_type, num_nodes):
+        """
+        Create nodes and attach them to cluster
+        Use this function if you want to do both creation/attachment in
+        a single call
+        Args:
+            node_conf (dict): of node configuration
+            node_type (str): type of node to be created RHCOS/RHEL
+            num_nodes (int): Number of node instances to be created
+        """
+        node_list = self.create_nodes(node_conf, node_type, num_nodes)
+        self.attach_nodes_to_cluster(node_list)
+
+    def create_nodes(self, node_conf, node_type, num_nodes):
+        raise NotImplementedError("Create nodes functionality not implemented")
+
+    def attach_nodes_to_cluster(self, node_list):
+        raise NotImplementedError(
+            "attach nodes to cluster functionality is not implemented"
+        )
+
+    def read_default_config(self, default_config_path):
+        """
+        Commonly used function to read default config
+        Args:
+            default_config_path (str): Path to default config file
+        Returns:
+            dict: of default config loaded
+        """
+        assert os.path.exists(default_config_path), "Config file doesnt exists"
+
+        with open(default_config_path) as f:
+            default_config_dict = yaml.safe_load(f)
+
+        return default_config_dict
+
+    def terminate_nodes(self, nodes, wait=True):
+        self.ibmcloud_ipi.terminate_nodes(nodes=nodes, wait=wait)
+
+    def wait_for_nodes_to_stop(self, nodes):
+        raise NotImplementedError(
+            "wait for nodes to stop functionality is not implemented"
+        )
+
+    def wait_for_nodes_to_terminate(self, nodes):
+        raise NotImplementedError(
+            "wait for nodes to terminate functionality is not implemented"
+        )
+
+    def wait_for_nodes_to_stop_or_terminate(self, nodes):
+        raise NotImplementedError(
+            "wait for nodes to stop or terminate functionality is not implemented"
+        )
diff --git a/ocs_ci/utility/ibmcloud.py b/ocs_ci/utility/ibmcloud.py
index a9712227f4b..3236f2350c5 100644
--- a/ocs_ci/utility/ibmcloud.py
+++ b/ocs_ci/utility/ibmcloud.py
@@ -16,6 +16,7 @@
 from ocs_ci.framework import config
 from ocs_ci.ocs import constants
 from ocs_ci.ocs.defaults import IBM_CLOUD_REGIONS
+from ocs_ci.ocs.ocp import OCP
 from ocs_ci.ocs.exceptions import (
     APIRequestError,
     CommandFailed,
@@ -24,6 +25,7 @@
     NodeHasNoAttachedVolume,
     TimeoutExpiredError,
 )
+from ocs_ci.ocs.resources.ocs import OCS
 from ocs_ci.utility import version as util_version
 from ocs_ci.utility.utils import get_infra_id, get_ocp_version, run_cmd, TimeoutSampler
 from ocs_ci.ocs.node import get_nodes
@@ -574,6 +576,272 @@ def delete_volume_id(self, volume):
             logger.info("volume is not deleted")
 
 
+class IBMCloudIPI(object):
+    def restart_nodes(self, nodes, wait=True, timeout=900):
+        """
+        Reboot the nodes on IBM Cloud.
+        Args:
+            nodes (list): The worker node instance
+            wait (bool): Wait for the VMs to stop
+            timeout (int): Timeout for the command, defaults to 900 seconds.
+        """
+        logger.info("restarting nodes")
+
+        for node in nodes:
+            cmd = f"ibmcloud is instance-reboot {node.name} -f"
+            out = run_ibmcloud_cmd(cmd)
+            logger.info(f"Node restart command output: {out}")
+
+        if wait:
+            for node in nodes:
+                sample = TimeoutSampler(
+                    timeout=timeout,
+                    sleep=10,
+                    func=self.check_node_status,
+                    node_name=node.name,
+                )
+                sample.wait_for_func_status(result=constants.STATUS_RUNNING.lower())
+
+    def start_nodes(self, nodes):
+        """
+        Start the nodes on IBM Cloud
+        Args:
+            nodes (list): The OCS objects of the nodes
+        """
+        # logger.info(nodes)
+        for node in nodes:
+            # logger.info(node.get())
+            cmd = f"ibmcloud is instance-start {node.name}"
+            out = run_ibmcloud_cmd(cmd)
+            logger.info(f"Node start command output: {out}")
+
+    def stop_nodes(self, nodes, force=True, wait=True):
+        """
+        Stop the nodes on IBM Cloud
+        Args:
+            nodes (list): The OCS objects of the nodes
+            force (bool): True for VM ungraceful power off, False for
+                graceful VM shutdown
+            wait (bool): Wait for the VMs to stop
+        """
+        for node in nodes:
+            cmd = f"ibmcloud is instance-stop {node.name} --force={force}"
+            out = run_ibmcloud_cmd(cmd)
+            logger.info(f"Node Stop command output: {out}")
+
+        if wait:
+            for node in nodes:
+                sample = TimeoutSampler(
+                    timeout=300,
+                    sleep=10,
+                    func=self.check_node_status,
+                    node_name=node.name,
+                )
+                sample.wait_for_func_status(result=constants.STATUS_STOPPED)
+
+    def restart_nodes_by_stop_and_start(
+        self, nodes, wait=True, force=True, timeout=300
+    ):
+        """
+        Restart nodes by stopping and starting VM in IBM Cloud
+        Args:
+            nodes (list): The OCS objects of the nodes
+            wait (bool): True in case wait for status is needed,
+                False otherwise
+            force (bool): True for force instance stop, False otherwise
+            timeout (int): Timeout for the command, defaults to 300 seconds.
+        """
+        logger.info(f"Stopping instances {list(nodes)}")
+        self.stop_nodes(nodes=nodes, force=force)
+        if wait:
+            for node in nodes:
+                sample = TimeoutSampler(
+                    timeout=timeout,
+                    sleep=10,
+                    func=self.check_node_status,
+                    node_name=node.name,
+                )
+                sample.wait_for_func_status(result=constants.STATUS_STOPPED)
+        logger.info(f"Starting instances {list(nodes)}")
+
+        self.start_nodes(nodes=nodes)
+        if wait:
+            for node in nodes:
+                sample = TimeoutSampler(
+                    timeout=timeout,
+                    sleep=10,
+                    func=self.check_node_status,
+                    node_name=node.name,
+                )
+                sample.wait_for_func_status(result=constants.STATUS_RUNNING.lower())
+
+    def check_node_status(self, node_name):
+        """
+        Check the node status in IBM cloud
+        Args:
+            node_name (str): Node name
+        Returns:
+            str: Status of node
+        """
+        try:
+            cmd = f"ibmcloud is instance {node_name} --output json"
+
+            out = run_ibmcloud_cmd(cmd)
+            out = json.loads(out)
+            return out["status"]
+        except CommandFailed as cf:
+            if "Instance not found" in str(cf):
+                return True
+        return False
+
+    def restart_nodes_by_stop_and_start_force(self):
+        """
+        Make sure all nodes are up by the end of the test on IBM Cloud.
+        """
+        resource_name = None
+        stop_node_list = []
+        stopping_node_list = []
+        cmd = "ibmcloud is ins --all-resource-groups --output json"
+        out = run_ibmcloud_cmd(cmd)
+        all_resource_grp = json.loads(out)
+        cluster_name = config.ENV_DATA["cluster_name"]
+        for resource_name in all_resource_grp:
+            if cluster_name in resource_name["resource_group"]["name"]:
+                resource_name = resource_name["resource_group"]["name"]
+                break
+        assert resource_name, "Resource Not found"
+        cmd = f"ibmcloud is ins --resource-group-name {resource_name} --output json"
+        out = run_ibmcloud_cmd(cmd)
+        all_instance_output = json.loads(out)
+        for instance_name in all_instance_output:
+            if instance_name["status"] == constants.STATUS_STOPPED:
+                node_obj = OCP(kind="Node", resource_name=instance_name["name"]).get()
+                node_obj_ocs = OCS(**node_obj)
+                stop_node_list.append(node_obj_ocs)
+            if instance_name["status"] == constants.STATUS_STOPPED:
+                node_obj = OCP(kind="Node", resource_name=instance_name["name"]).get()
+                node_obj_ocs = OCS(**node_obj)
+                stopping_node_list.append(node_obj_ocs)
+                stop_node_list.append(node_obj_ocs)
+        logger.info("Force stopping node which are in stopping state")
+        self.stop_nodes(nodes=stop_node_list, force=True, wait=True)
+        logger.info("Starting Stopped Node")
+        self.start_nodes(nodes=stop_node_list)
+
+    def terminate_nodes(self, nodes, wait=True):
+        """
+        Terminate the Node in IBMCloud
+        Args:
+            nodes (list): The OCS objects of the nodes
+            wait (bool): True in case wait for status is needed,
+                False otherwise
+        """
+        for node in nodes:
+            cmd = f"ibmcloud is instance-delete {node.name} -f"
+            out = run_ibmcloud_cmd(cmd)
+            logger.info(f"Node deletion command output: {out}")
+            break
+
+        if wait:
+            for node in nodes:
+                sample = TimeoutSampler(
+                    timeout=300,
+                    sleep=10,
+                    func=self.check_node_status,
+                    node_name=node.name,
+                )
+                sample.wait_for_func_status(result=True)
+                break
+
+    def detach_volume(self, volume, node=None):
+        """
+        Detach volume from node on IBM Cloud.
+        Args:
+            volume (str): volume id.
+            node (OCS): worker node id to detach.
+        """
+
+        logger.info(f"volume is : {volume}")
+
+        cmd = f"ibmcloud is volume {volume} --output json"
+        out = run_ibmcloud_cmd(cmd)
+        out = json.loads(out)
+
+        if out["status"] == "available":
+            attachment_id = out["volume_attachments"][0]["id"]
+            cmd = (
+                f"ibmcloud is instance-volume-attachment-update {node.name} {attachment_id} "
+                f"--output json --auto-delete=false"
+            )
+            out = run_ibmcloud_cmd(cmd)
+            out = json.loads(out)
+            logger.info(f"Update command output: {out}")
+            cmd = (
+                f"ibmcloud is instance-volume-attachment-detach {node.name} {attachment_id} "
+                f"--output=json --force"
+            )
+            out = run_ibmcloud_cmd(cmd)
+            logger.info(f"detachment command output: {out}")
+
+    def attach_volume(self, volume, node):
+        """
+        Attach volume to node on IBM Cloud.
+        Args:
+            volume (str): volume id.
+            node (OCS): worker node id to attach.
+        """
+        logger.info(
+            f"attach_volumes:{node[0].get()['metadata']['labels']['failure-domain.beta.kubernetes.io/zone']}"
+        )
+
+        logger.info(f"volume is : {volume}")
+
+        cmd = f"ibmcloud is volume {volume} --output json"
+        out = run_ibmcloud_cmd(cmd)
+        out = json.loads(out)
+
+        if len(out["volume_attachments"]) == 0:
+
+            logger.info(f"attachment command output: {out}")
+            cmd = f"ibmcloud is instance-volume-attachment-add data-vol-name {node[0].name} {volume} --output json"
+            out = run_ibmcloud_cmd(cmd)
+            out = json.loads(out)
+            logger.info(f"attachment command output: {out}")
+        else:
+            logger.info(f"volume is already attached to node: {out}")
+
+    def is_volume_attached(self, volume):
+        """
+        Check if volume is attached to node or not.
+        Args:
+            volume (str): The volume to check for to attached
+        Returns:
+            bool: 'True' if volume is attached otherwise 'False'
+        """
+        logger.info("Checking volume attachment status")
+        cmd = f"ibmcloud is volume {volume} --output json"
+        out = run_ibmcloud_cmd(cmd)
+        out = json.loads(out)
+        return out["volume_attachments"]
+
+    def wait_for_volume_attach(self, volume):
+        """
+        Checks volume is attached to node or not
+        Args:
+            volume (str): The volume to wait for to be attached
+        Returns:
+            bool: True if the volume has been attached to the
+                instance, False otherwise
+        """
+        try:
+            for sample in TimeoutSampler(300, 3, self.is_volume_attached, volume):
+                if sample:
+                    return True
+        except TimeoutExpiredError:
+            logger.info("Volume is not attached to node")
+            return False
+
+
 def label_nodes_region():
     """
     Apply the region label to the worker nodes.
diff --git a/tests/functional/z_cluster/nodes/test_disk_failures.py b/tests/functional/z_cluster/nodes/test_disk_failures.py
index 9a55ec4b2d2..4cc751630e6 100644
--- a/tests/functional/z_cluster/nodes/test_disk_failures.py
+++ b/tests/functional/z_cluster/nodes/test_disk_failures.py
@@ -84,7 +84,7 @@ def detach_volume_and_wait_for_attach(self, nodes, data_volume, worker_node):
             else:
                 assert nodes.wait_for_volume_attach(data_volume), (
                     f"Volume {data_volume} failed to be re-attached to worker "
-                    f"node {worker_node}"
+                    f"node {worker_node.name}"
                 )
 
     @pytest.fixture(autouse=True)
@@ -141,6 +141,7 @@ def init_sanity(self):
         self.sanity_helpers = Sanity()
 
     @skipif_managed_service
+    @skipif_ibm_cloud
     @skipif_hci_provider_and_client
     @cloud_platform_required
     @pytest.mark.polarion_id("OCS-1085")

From 72ee91d61964863646ab79cc853215cd08eed8ad Mon Sep 17 00:00:00 2001
From: prsurve <prsurve@redhat.com>
Date: Mon, 4 Mar 2024 09:34:10 +0530
Subject: [PATCH 03/11] Fix Tox failure

Signed-off-by: prsurve <prsurve@redhat.com>
---
 ocs_ci/utility/ibmcloud.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/ocs_ci/utility/ibmcloud.py b/ocs_ci/utility/ibmcloud.py
index 3236f2350c5..b275946521d 100644
--- a/ocs_ci/utility/ibmcloud.py
+++ b/ocs_ci/utility/ibmcloud.py
@@ -646,8 +646,7 @@ def restart_nodes_by_stop_and_start(
         Restart nodes by stopping and starting VM in IBM Cloud
         Args:
             nodes (list): The OCS objects of the nodes
-            wait (bool): True in case wait for status is needed,
-                False otherwise
+            wait (bool): True in case wait for status is needed, False otherwise
             force (bool): True for force instance stop, False otherwise
             timeout (int): Timeout for the command, defaults to 300 seconds.
         """
@@ -733,8 +732,7 @@ def terminate_nodes(self, nodes, wait=True):
         Terminate the Node in IBMCloud
         Args:
             nodes (list): The OCS objects of the nodes
-            wait (bool): True in case wait for status is needed,
-                False otherwise
+            wait (bool): True in case wait for status is needed, False otherwise
         """
         for node in nodes:
             cmd = f"ibmcloud is instance-delete {node.name} -f"
@@ -830,8 +828,7 @@ def wait_for_volume_attach(self, volume):
         Args:
             volume (str): The volume to wait for to be attached
         Returns:
-            bool: True if the volume has been attached to the
-                instance, False otherwise
+            bool: True if the volume has been attached to the instance, False otherwise
         """
         try:
             for sample in TimeoutSampler(300, 3, self.is_volume_attached, volume):

From 84abdb4ca920b3ac0782524194f179904ec7a1e4 Mon Sep 17 00:00:00 2001
From: prsurve <prsurve@redhat.com>
Date: Mon, 4 Mar 2024 09:46:44 +0530
Subject: [PATCH 04/11] Fix Tox failure

Signed-off-by: prsurve <prsurve@redhat.com>
---
 ocs_ci/utility/ibmcloud.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ocs_ci/utility/ibmcloud.py b/ocs_ci/utility/ibmcloud.py
index b275946521d..a90a0303842 100644
--- a/ocs_ci/utility/ibmcloud.py
+++ b/ocs_ci/utility/ibmcloud.py
@@ -620,8 +620,7 @@ def stop_nodes(self, nodes, force=True, wait=True):
         Stop the nodes on IBM Cloud
         Args:
             nodes (list): The OCS objects of the nodes
-            force (bool): True for VM ungraceful power off, False for
-                graceful VM shutdown
+            force (bool): True for VM ungraceful power off, False for graceful VM shutdown
             wait (bool): Wait for the VMs to stop
         """
         for node in nodes:

From e16838b005aa97477ee0eccbc81c4b3cd456f7cd Mon Sep 17 00:00:00 2001
From: prsurve <prsurve@redhat.com>
Date: Wed, 27 Mar 2024 16:29:02 +0530
Subject: [PATCH 05/11] Resolve Comment

Signed-off-by: prsurve <prsurve@redhat.com>
---
 ocs_ci/utility/ibmcloud.py | 38 +++++++++++++++++++++++++-------------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/ocs_ci/utility/ibmcloud.py b/ocs_ci/utility/ibmcloud.py
index a90a0303842..eb5e06a5003 100644
--- a/ocs_ci/utility/ibmcloud.py
+++ b/ocs_ci/utility/ibmcloud.py
@@ -599,8 +599,9 @@ def restart_nodes(self, nodes, wait=True, timeout=900):
                     sleep=10,
                     func=self.check_node_status,
                     node_name=node.name,
+                    node_status=constants.STATUS_RUNNING.lower(),
                 )
-                sample.wait_for_func_status(result=constants.STATUS_RUNNING.lower())
+                sample.wait_for_func_status(result=True)
 
     def start_nodes(self, nodes):
         """
@@ -635,8 +636,9 @@ def stop_nodes(self, nodes, force=True, wait=True):
                     sleep=10,
                     func=self.check_node_status,
                     node_name=node.name,
+                    node_status=constants.STATUS_STOPPED,
                 )
-                sample.wait_for_func_status(result=constants.STATUS_STOPPED)
+                sample.wait_for_func_status(result=True)
 
     def restart_nodes_by_stop_and_start(
         self, nodes, wait=True, force=True, timeout=300
@@ -658,8 +660,9 @@ def restart_nodes_by_stop_and_start(
                     sleep=10,
                     func=self.check_node_status,
                     node_name=node.name,
+                    node_status=constants.STATUS_STOPPED,
                 )
-                sample.wait_for_func_status(result=constants.STATUS_STOPPED)
+                sample.wait_for_func_status(result=True)
         logger.info(f"Starting instances {list(nodes)}")
 
         self.start_nodes(nodes=nodes)
@@ -670,14 +673,17 @@ def restart_nodes_by_stop_and_start(
                     sleep=10,
                     func=self.check_node_status,
                     node_name=node.name,
+                    node_status=constants.STATUS_RUNNING.lower(),
                 )
-                sample.wait_for_func_status(result=constants.STATUS_RUNNING.lower())
+                sample.wait_for_func_status(result=True)
 
-    def check_node_status(self, node_name):
+    def check_node_status(self, node_name, node_status):
         """
         Check the node status in IBM cloud
+
         Args:
             node_name (str): Node name
+
         Returns:
             str: Status of node
         """
@@ -686,7 +692,11 @@ def check_node_status(self, node_name):
 
             out = run_ibmcloud_cmd(cmd)
             out = json.loads(out)
-            return out["status"]
+            # return out["status"]
+            if out["status"] == node_status:
+                return True
+            else:
+                return False
         except CommandFailed as cf:
             if "Instance not found" in str(cf):
                 return True
@@ -698,7 +708,6 @@ def restart_nodes_by_stop_and_start_force(self):
         """
         resource_name = None
         stop_node_list = []
-        stopping_node_list = []
         cmd = "ibmcloud is ins --all-resource-groups --output json"
         out = run_ibmcloud_cmd(cmd)
         all_resource_grp = json.loads(out)
@@ -719,7 +728,6 @@ def restart_nodes_by_stop_and_start_force(self):
             if instance_name["status"] == constants.STATUS_STOPPED:
                 node_obj = OCP(kind="Node", resource_name=instance_name["name"]).get()
                 node_obj_ocs = OCS(**node_obj)
-                stopping_node_list.append(node_obj_ocs)
                 stop_node_list.append(node_obj_ocs)
         logger.info("Force stopping node which are in stopping state")
         self.stop_nodes(nodes=stop_node_list, force=True, wait=True)
@@ -755,7 +763,7 @@ def detach_volume(self, volume, node=None):
         Detach volume from node on IBM Cloud.
         Args:
             volume (str): volume id.
-            node (OCS): worker node id to detach.
+            node (OCS): worker node object to detach.
         """
 
         logger.info(f"volume is : {volume}")
@@ -785,10 +793,10 @@ def attach_volume(self, volume, node):
         Attach volume to node on IBM Cloud.
         Args:
             volume (str): volume id.
-            node (OCS): worker node id to attach.
+            node (OCS): worker node object to attach.
         """
         logger.info(
-            f"attach_volumes:{node[0].get()['metadata']['labels']['failure-domain.beta.kubernetes.io/zone']}"
+            f"attach_volumes:{node.get()['metadata']['labels']['failure-domain.beta.kubernetes.io/zone']}"
         )
 
         logger.info(f"volume is : {volume}")
@@ -800,7 +808,7 @@ def attach_volume(self, volume, node):
         if len(out["volume_attachments"]) == 0:
 
             logger.info(f"attachment command output: {out}")
-            cmd = f"ibmcloud is instance-volume-attachment-add data-vol-name {node[0].name} {volume} --output json"
+            cmd = f"ibmcloud is instance-volume-attachment-add data-vol-name {node.name} {volume} --output json"
             out = run_ibmcloud_cmd(cmd)
             out = json.loads(out)
             logger.info(f"attachment command output: {out}")
@@ -810,8 +818,10 @@ def attach_volume(self, volume, node):
     def is_volume_attached(self, volume):
         """
         Check if volume is attached to node or not.
+
         Args:
             volume (str): The volume to check for to attached
+
         Returns:
             bool: 'True' if volume is attached otherwise 'False'
         """
@@ -819,7 +829,7 @@ def is_volume_attached(self, volume):
         cmd = f"ibmcloud is volume {volume} --output json"
         out = run_ibmcloud_cmd(cmd)
         out = json.loads(out)
-        return out["volume_attachments"]
+        return True if len(out["volume_attachments"]) > 0 else False
 
     def wait_for_volume_attach(self, volume):
         """
@@ -833,6 +843,8 @@ def wait_for_volume_attach(self, volume):
             for sample in TimeoutSampler(300, 3, self.is_volume_attached, volume):
                 if sample:
                     return True
+                else:
+                    return False
         except TimeoutExpiredError:
             logger.info("Volume is not attached to node")
             return False

From 17d4a3a84288593b660b47182a5d7a28557ebd69 Mon Sep 17 00:00:00 2001
From: prsurve <prsurve@redhat.com>
Date: Thu, 28 Mar 2024 14:16:36 +0530
Subject: [PATCH 06/11] Resolve Comment

Signed-off-by: prsurve <prsurve@redhat.com>
---
 ocs_ci/utility/ibmcloud.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ocs_ci/utility/ibmcloud.py b/ocs_ci/utility/ibmcloud.py
index eb5e06a5003..a55168be811 100644
--- a/ocs_ci/utility/ibmcloud.py
+++ b/ocs_ci/utility/ibmcloud.py
@@ -683,16 +683,16 @@ def check_node_status(self, node_name, node_status):
 
         Args:
             node_name (str): Node name
+            node_status (str): Status of Node Running or Stopped
 
         Returns:
-            str: Status of node
+            bool: True if status matches else False
         """
         try:
             cmd = f"ibmcloud is instance {node_name} --output json"
 
             out = run_ibmcloud_cmd(cmd)
             out = json.loads(out)
-            # return out["status"]
             if out["status"] == node_status:
                 return True
             else:

From 1bc0185e87374bc03889f845a545f8d645855a29 Mon Sep 17 00:00:00 2001
From: prsurve <prsurve@redhat.com>
Date: Fri, 12 Apr 2024 15:07:11 +0530
Subject: [PATCH 07/11] Remove skip if for ibm

Signed-off-by: prsurve <prsurve@redhat.com>
---
 tests/functional/z_cluster/nodes/test_disk_failures.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/functional/z_cluster/nodes/test_disk_failures.py b/tests/functional/z_cluster/nodes/test_disk_failures.py
index 4cc751630e6..d8f50889543 100644
--- a/tests/functional/z_cluster/nodes/test_disk_failures.py
+++ b/tests/functional/z_cluster/nodes/test_disk_failures.py
@@ -141,7 +141,6 @@ def init_sanity(self):
         self.sanity_helpers = Sanity()
 
     @skipif_managed_service
-    @skipif_ibm_cloud
     @skipif_hci_provider_and_client
     @cloud_platform_required
     @pytest.mark.polarion_id("OCS-1085")

From b6214e541d2545c0d4fdee41df02fae758c35f17 Mon Sep 17 00:00:00 2001
From: prsurve <prsurve@redhat.com>
Date: Tue, 23 Apr 2024 11:17:02 +0530
Subject: [PATCH 08/11] Add skip for disk failure test

Signed-off-by: prsurve <prsurve@redhat.com>
---
 tests/functional/z_cluster/nodes/test_disk_failures.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/functional/z_cluster/nodes/test_disk_failures.py b/tests/functional/z_cluster/nodes/test_disk_failures.py
index d8f50889543..4cc751630e6 100644
--- a/tests/functional/z_cluster/nodes/test_disk_failures.py
+++ b/tests/functional/z_cluster/nodes/test_disk_failures.py
@@ -141,6 +141,7 @@ def init_sanity(self):
         self.sanity_helpers = Sanity()
 
     @skipif_managed_service
+    @skipif_ibm_cloud
     @skipif_hci_provider_and_client
     @cloud_platform_required
     @pytest.mark.polarion_id("OCS-1085")

From a2cfcfcf908391edc0581e3ffcd5647983e0ea84 Mon Sep 17 00:00:00 2001
From: prsurve <prsurve@redhat.com>
Date: Mon, 13 May 2024 14:19:13 +0530
Subject: [PATCH 09/11] Print node names

Signed-off-by: prsurve <prsurve@redhat.com>
---
 ocs_ci/utility/ibmcloud.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ocs_ci/utility/ibmcloud.py b/ocs_ci/utility/ibmcloud.py
index a55168be811..9773c4e10a5 100644
--- a/ocs_ci/utility/ibmcloud.py
+++ b/ocs_ci/utility/ibmcloud.py
@@ -651,7 +651,7 @@ def restart_nodes_by_stop_and_start(
             force (bool): True for force instance stop, False otherwise
             timeout (int): Timeout for the command, defaults to 300 seconds.
         """
-        logger.info(f"Stopping instances {list(nodes)}")
+        logger.info(f"Stopping instances {list(node.name for node in nodes )}")
         self.stop_nodes(nodes=nodes, force=force)
         if wait:
             for node in nodes:
@@ -663,7 +663,7 @@ def restart_nodes_by_stop_and_start(
                     node_status=constants.STATUS_STOPPED,
                 )
                 sample.wait_for_func_status(result=True)
-        logger.info(f"Starting instances {list(nodes)}")
+        logger.info(f"Starting instances {list(node.name for node in nodes )}")
 
         self.start_nodes(nodes=nodes)
         if wait:

From f61804c1d09bb5ec3f75f601c7935e17cd4c648d Mon Sep 17 00:00:00 2001
From: prsurve <prsurve@redhat.com>
Date: Wed, 4 Sep 2024 15:07:26 +0530
Subject: [PATCH 10/11] fix comment

Signed-off-by: prsurve <prsurve@redhat.com>
---
 ocs_ci/ocs/platform_nodes.py | 2 +-
 ocs_ci/utility/ibmcloud.py   | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/ocs_ci/ocs/platform_nodes.py b/ocs_ci/ocs/platform_nodes.py
index cf1f8805cf9..515b49e5d8b 100644
--- a/ocs_ci/ocs/platform_nodes.py
+++ b/ocs_ci/ocs/platform_nodes.py
@@ -3354,7 +3354,7 @@ def terminate_nodes(self, nodes, wait=True):
 
 class IBMCloudIPI(object):
     """
-    A base class for nodes related operations.
+    An IBM Cloud IPI class for node related operations
     Should be inherited by specific platform classes
     """
 
diff --git a/ocs_ci/utility/ibmcloud.py b/ocs_ci/utility/ibmcloud.py
index 9773c4e10a5..d59782f0bd3 100644
--- a/ocs_ci/utility/ibmcloud.py
+++ b/ocs_ci/utility/ibmcloud.py
@@ -577,6 +577,10 @@ def delete_volume_id(self, volume):
 
 
 class IBMCloudIPI(object):
+    """
+    Wrapper for Ibm Cloud IPI
+
+    """
     def restart_nodes(self, nodes, wait=True, timeout=900):
         """
         Reboot the nodes on IBM Cloud.

From 4e9c46b930dd17e1a803da729f9bd99f92a0fd5e Mon Sep 17 00:00:00 2001
From: prsurve <prsurve@redhat.com>
Date: Mon, 16 Sep 2024 16:30:56 +0530
Subject: [PATCH 11/11] fix tox

Signed-off-by: prsurve <prsurve@redhat.com>
---
 ocs_ci/utility/ibmcloud.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ocs_ci/utility/ibmcloud.py b/ocs_ci/utility/ibmcloud.py
index d59782f0bd3..61d91daa521 100644
--- a/ocs_ci/utility/ibmcloud.py
+++ b/ocs_ci/utility/ibmcloud.py
@@ -581,6 +581,7 @@ class IBMCloudIPI(object):
     Wrapper for Ibm Cloud IPI
 
     """
+
     def restart_nodes(self, nodes, wait=True, timeout=900):
         """
         Reboot the nodes on IBM Cloud.