Make waiting for scheduler Pod more robust (#793)

dask · Aug 1, 2023 · 579cd46 · 579cd46
1 parent 55e8bb1
commit 579cd46
Showing 1 changed file with 11 additions and 7 deletions.
diff --git a/dask_kubernetes/common/networking.py b/dask_kubernetes/common/networking.py
@@ -200,13 +200,17 @@ async def wait_for_scheduler(cluster_name, namespace, timeout=None):
     while True:
         async with kubernetes.client.api_client.ApiClient() as api_client:
             k8s_api = kubernetes.client.CoreV1Api(api_client)
-            pods = await k8s_api.list_namespaced_pod(
-                namespace=namespace,
-                label_selector=f"dask.org/component=scheduler,dask.org/cluster-name={cluster_name}",
-            )
-        pod = await Pod.objects(api, namespace=namespace).get_by_name(
-            pods.items[0].metadata.name
-        )
+            try:
+                [pod] = (
+                    await k8s_api.list_namespaced_pod(
+                        namespace=namespace,
+                        label_selector=f"dask.org/component=scheduler,dask.org/cluster-name={cluster_name}",
+                    )
+                ).items
+            except ValueError:
+                await asyncio.sleep(0.25)
+                continue
+        pod = await Pod.objects(api, namespace=namespace).get_by_name(pod.metadata.name)
         phase = pod.obj["status"]["phase"]
         if phase == "Running":
             if not pod_start_time: