Merge pull request #10721 from dahorak/enhance-ai-ocp-deployment-rebo…

…ot-not-connected-server

reboot server if not properly discovered by Assisted Installer

ocs_ci/deployment/assisted_installer.py

@@ -319,6 +319,13 @@ def wait_for_discovered_nodes(self, expected_nodes):
                 )
                 break
 
+    def get_infra_env_hosts(self):
+        """
+        Return:
+            list: list of discovered hosts in the Infrastructure Environment
+        """
+        return self.api.get_infra_env_hosts(infra_env_id=self.infra_id)
+
     @retry(HostValidationFailed, tries=5, delay=60, backoff=1)
     def verify_validations_info_for_discovered_nodes(self):
         """

ocs_ci/deployment/baremetal.py

@@ -21,7 +21,7 @@
 from ocs_ci.deployment.ocp import OCPDeployment as BaseOCPDeployment
 from ocs_ci.deployment import assisted_installer
 from ocs_ci.ocs import constants, ocp, exceptions
-from ocs_ci.ocs.exceptions import CommandFailed, RhcosImageNotFound
+from ocs_ci.ocs.exceptions import CommandFailed, RhcosImageNotFound, TimeoutExpiredError
 from ocs_ci.ocs.node import get_nodes
 from ocs_ci.ocs.openshift_ops import OCP
 from ocs_ci.utility import ibmcloud_bm
@@ -1054,7 +1054,17 @@ def deploy(self, log_cli_level="DEBUG"):
             expected_node_num = (
                 config.ENV_DATA["master_replicas"] + config.ENV_DATA["worker_replicas"]
             )
-            self.ai_cluster.wait_for_discovered_nodes(expected_node_num)
+            try:
+                self.ai_cluster.wait_for_discovered_nodes(expected_node_num)
+            except TimeoutExpiredError:
+                discovered_hosts = [
+                    host["requested_hostname"]
+                    for host in self.ai_cluster.get_infra_env_hosts()
+                ]
+                for machine in master_nodes + worker_nodes:
+                    if machine not in discovered_hosts:
+                        self.set_pxe_boot_and_reboot(machine)
+                self.ai_cluster.wait_for_discovered_nodes(expected_node_num)
 
             # verify validations info
             self.ai_cluster.verify_validations_info_for_discovered_nodes()