From 3188e31ef46d45eed9cb808438cf1a18e90cddbb Mon Sep 17 00:00:00 2001
From: Oren Zigdon <oren.zigdon@gmail.com>
Date: Sun, 2 Jun 2024 10:58:34 +0300
Subject: [PATCH 01/12] Initialize class `kernel_launch_terminate_on_events` in
 init method.

---
 enterprise_gateway/services/processproxies/container.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/enterprise_gateway/services/processproxies/container.py b/enterprise_gateway/services/processproxies/container.py
index 6378b633..a566e783 100644
--- a/enterprise_gateway/services/processproxies/container.py
+++ b/enterprise_gateway/services/processproxies/container.py
@@ -8,6 +8,7 @@
 import abc
 import os
 import signal
+from collections import defaultdict
 from typing import Any
 
 import urllib3  # docker ends up using this and it causes lots of noise, so turn off warnings
@@ -46,6 +47,14 @@ def __init__(self, kernel_manager: RemoteKernelManager, proxy_config: dict):
         super().__init__(kernel_manager, proxy_config)
         self.container_name = ""
         self.assigned_node_ip = None
+        self._initialize_kernel_launch_terminate_on_events()
+
+    def _initialize_kernel_launch_terminate_on_events(self):
+        self.kernel_launch_terminate_on_events = defaultdict(dict)
+        for configuration in self.kernel_manager.parent.kernel_launch_terminate_on_events:
+            self.kernel_launch_terminate_on_events[
+                configuration["type"]
+            ][configuration["reason"]] = configuration["timeout_in_seconds"]
 
     def _determine_kernel_images(self, **kwargs: dict[str, Any] | None) -> None:
         """

From add94cbeebb2ea18b61c93504acbf25c35f9b989 Mon Sep 17 00:00:00 2001
From: Oren Zigdon <oren.zigdon@gmail.com>
Date: Sun, 2 Jun 2024 15:06:43 +0300
Subject: [PATCH 02/12] Add pending container handling in
 `confirm_remote_startup` method - comparing the container events to the
 configuration events.

---
 .../services/processproxies/container.py      | 42 ++++++++++++++++++-
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/enterprise_gateway/services/processproxies/container.py b/enterprise_gateway/services/processproxies/container.py
index a566e783..e144bcc2 100644
--- a/enterprise_gateway/services/processproxies/container.py
+++ b/enterprise_gateway/services/processproxies/container.py
@@ -47,6 +47,7 @@ def __init__(self, kernel_manager: RemoteKernelManager, proxy_config: dict):
         super().__init__(kernel_manager, proxy_config)
         self.container_name = ""
         self.assigned_node_ip = None
+        self.kernel_events_to_occurrence_time = {}
         self._initialize_kernel_launch_terminate_on_events()
 
     def _initialize_kernel_launch_terminate_on_events(self):
@@ -199,7 +200,15 @@ async def confirm_remote_startup(self) -> None:
         """Confirms the container has started and returned necessary connection information."""
         self.log.debug("Trying to confirm kernel container startup status")
         self.start_time = RemoteProcessProxy.get_current_time()
-        i = 0
+        self.kernel_events_to_occurrence_time = {}
+        i = 1
+        container_status = self.get_container_status(i)
+        while not container_status:
+            i += 1
+            self.detect_launch_failure()
+            await self.handle_timeout()
+            container_status self.get_container_status(i)
+
         ready_to_connect = False  # we're ready to connect when we have a connection file to use
         while not ready_to_connect:
             i += 1
@@ -212,6 +221,8 @@ async def confirm_remote_startup(self) -> None:
                         http_status_code=500,
                         reason=f"Error starting kernel container; status: '{container_status}'.",
                     )
+                elif container_status == "pending":
+                    self._handle_pending_kernel()
                 else:
                     if self.assigned_host:
                         ready_to_connect = await self.receive_connection_info()
@@ -220,7 +231,29 @@ async def confirm_remote_startup(self) -> None:
                         )
                         self.pgid = 0
             else:
-                self.detect_launch_failure()
+                self.log_and_raise(
+                    http_status_code=500,
+                    reason="Error starting kernel container; status was not available. Perhaps the kernel pod died unexpectedly"
+                )
+        self.kernel_events_to_occurrence_time = {}
+
+    def _handle_pending_kernel(self):
+        self.log.debug("Sampling kernel container events")
+        kernel_pod_events = self.get_container_events()
+        for event in kernel_pod_events:
+            if event.type in self.kernel_launch_terminate_on_events and event.reason in self.kernel_launch_terminate_on_events[event.type]:
+                hashed_event = hash(f"{event.type}{event.reason}")
+                if hashed_event not in self.kernel_events_to_occurrence_time:
+                    self.kernel_events_to_occurrence_time[hashed_event] = RemoteProcessProxy.get_current_time()
+                if RemoteProcessProxy.get_time_diff(
+                    RemoteProcessProxy.get_current_time(),
+                    self.kernel_events_to_occurrence_time[hashed_event]
+                ) >= self.kernel_launch_terminate_on_events[event.type][event.reason]:
+                    self.kill()
+                    self.log_and_raise(
+                        http_status_code=409,
+                        reason=f"Error starting kernel container; The container encountered an event which may cause a longer than usual startup: '{event.reason} - {event.message[:64]}'"
+                    )
 
     def get_process_info(self) -> dict[str, Any]:
         """Captures the base information necessary for kernel persistence relative to containers."""
@@ -252,6 +285,11 @@ def get_container_status(self, iteration: int | None) -> str:
         """Returns the current container state (in lowercase) or the empty string if not available."""
         raise NotImplementedError
 
+    @abc.abstractmethod
+    def get_container_events(self) -> list:
+        """Returns a list of container events, or empty list if the container has no events."""
+        raise NotImplementedError
+
     @abc.abstractmethod
     def terminate_container_resources(self):
         """Terminate any artifacts created on behalf of the container's lifetime."""

From dc2f587785e03aaf721e2601fe7bac99bc91d0bd Mon Sep 17 00:00:00 2001
From: Oren Zigdon <oren.zigdon@gmail.com>
Date: Sun, 2 Jun 2024 15:08:15 +0300
Subject: [PATCH 03/12] Implement `get_container_events` in k8s.

---
 enterprise_gateway/services/processproxies/k8s.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/enterprise_gateway/services/processproxies/k8s.py b/enterprise_gateway/services/processproxies/k8s.py
index 00e6bf17..a0065323 100644
--- a/enterprise_gateway/services/processproxies/k8s.py
+++ b/enterprise_gateway/services/processproxies/k8s.py
@@ -110,6 +110,18 @@ def get_container_status(self, iteration: int | None) -> str:
 
         return pod_status
 
+    def get_container_events(self) -> list:
+        """Return container events"""
+        pod_events = []
+        core_v1_api = client.CoreV1Api()
+        if self.container_name:
+            ret = core_v1_api.list_namespaced_event(
+                namespace=self.kernel_namespace, field_selector=f"involvedObject.name={self.container_name}"
+            )
+            if ret and ret.items:
+                pod_events = ret.items
+        return pod_events
+
     def delete_managed_object(self, termination_stati: list[str]) -> bool:
         """Deletes the object managed by this process-proxy
 

From e0244fb0ba99a6a6c28d7333d7d4f46c7c1979e3 Mon Sep 17 00:00:00 2001
From: Oren Zigdon <oren.zigdon@gmail.com>
Date: Sun, 2 Jun 2024 15:46:12 +0300
Subject: [PATCH 04/12] Add `kernel_launch_terminate_on_events` configuration
 to `RemoteMappingKernelManager`.

---
 .../services/kernels/remotemanager.py         | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/enterprise_gateway/services/kernels/remotemanager.py b/enterprise_gateway/services/kernels/remotemanager.py
index f906d68f..2c164959 100644
--- a/enterprise_gateway/services/kernels/remotemanager.py
+++ b/enterprise_gateway/services/kernels/remotemanager.py
@@ -11,14 +11,16 @@
 import signal
 import time
 import uuid
+import json
 from typing import Any, ClassVar
 
 from jupyter_client.ioloop.manager import AsyncIOLoopKernelManager
 from jupyter_client.kernelspec import KernelSpec
 from jupyter_server.services.kernels.kernelmanager import AsyncMappingKernelManager
 from tornado import web
-from traitlets import directional_link
+from traitlets import directional_link, default
 from traitlets import log as traitlets_log
+from traitlets import List as ListTrait
 from zmq import IO_THREADS, MAX_SOCKETS, Context
 
 from enterprise_gateway.mixins import EnterpriseGatewayConfigMixin
@@ -161,6 +163,21 @@ class RemoteMappingKernelManager(AsyncMappingKernelManager):
     Extends the AsyncMappingKernelManager with support for managing remote kernels via the process-proxy.
     """
 
+    kernel_launch_terminate_on_events_env = "EG_KERNEL_LAUNCH_TERMINATE_ON_EVENTS"
+    kernel_launch_terminate_on_events_default_value = []
+    kernel_launch_terminate_on_events = ListTrait(
+        default_value=kernel_launch_terminate_on_events_default_value,
+        config=True,
+        help="""Comma-separated list of dictionaries, each describing an event by `type`, `reason`,
+         and `timeout_in_seconds` (e.g. [{"type": "Warning", "reason": "FailedMount", "timeout_in_seconds": 0}]).
+         Kernel pod events will be sampled during startup, and if an event described in this list is detected,
+         the kernel launch will be terminated after the set timeout. Only available for container kernels. """
+    )
+
+    @default("kernel_launch_terminate_on_events")
+    def _kernel_launch_terminate_on_events_default(self) -> list:
+        return json.loads(os.getenv(self.kernel_launch_terminate_on_events_env, "[]"))
+
     def _context_default(self) -> Context:
         """
         We override the _context_default method in

From eff70e1fe267874f21fd868c1c6ea9475b004b9a Mon Sep 17 00:00:00 2001
From: Oren Zigdon <oren.zigdon@gmail.com>
Date: Sun, 2 Jun 2024 16:10:02 +0300
Subject: [PATCH 05/12] Beautify code and add comments.

---
 .../services/processproxies/container.py      | 28 ++++++++-----------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/enterprise_gateway/services/processproxies/container.py b/enterprise_gateway/services/processproxies/container.py
index e144bcc2..c8b4cc08 100644
--- a/enterprise_gateway/services/processproxies/container.py
+++ b/enterprise_gateway/services/processproxies/container.py
@@ -51,6 +51,12 @@ def __init__(self, kernel_manager: RemoteKernelManager, proxy_config: dict):
         self._initialize_kernel_launch_terminate_on_events()
 
     def _initialize_kernel_launch_terminate_on_events(self):
+        """
+        Parse the `kernel_launch_terminate_on_events` configuration, for easier access during startup.
+        [{"type": "Warning", "reason": "FailedMount", "timeout_in_seconds": 0},
+        {"type": Warning", "reason": "Unschedulable", "timeout_in_seconds": 30}] ->
+        {"Warning": {"FailedMount": 0, "Unschedulable": 30}}
+        """
         self.kernel_launch_terminate_on_events = defaultdict(dict)
         for configuration in self.kernel_manager.parent.kernel_launch_terminate_on_events:
             self.kernel_launch_terminate_on_events[
@@ -201,14 +207,7 @@ async def confirm_remote_startup(self) -> None:
         self.log.debug("Trying to confirm kernel container startup status")
         self.start_time = RemoteProcessProxy.get_current_time()
         self.kernel_events_to_occurrence_time = {}
-        i = 1
-        container_status = self.get_container_status(i)
-        while not container_status:
-            i += 1
-            self.detect_launch_failure()
-            await self.handle_timeout()
-            container_status self.get_container_status(i)
-
+        i = 0
         ready_to_connect = False  # we're ready to connect when we have a connection file to use
         while not ready_to_connect:
             i += 1
@@ -231,10 +230,7 @@ async def confirm_remote_startup(self) -> None:
                         )
                         self.pgid = 0
             else:
-                self.log_and_raise(
-                    http_status_code=500,
-                    reason="Error starting kernel container; status was not available. Perhaps the kernel pod died unexpectedly"
-                )
+                self.detect_launch_failure()
         self.kernel_events_to_occurrence_time = {}
 
     def _handle_pending_kernel(self):
@@ -242,12 +238,12 @@ def _handle_pending_kernel(self):
         kernel_pod_events = self.get_container_events()
         for event in kernel_pod_events:
             if event.type in self.kernel_launch_terminate_on_events and event.reason in self.kernel_launch_terminate_on_events[event.type]:
-                hashed_event = hash(f"{event.type}{event.reason}")
-                if hashed_event not in self.kernel_events_to_occurrence_time:
-                    self.kernel_events_to_occurrence_time[hashed_event] = RemoteProcessProxy.get_current_time()
+                event_key = f"{event.type}{event.reason}"
+                if event_key not in self.kernel_events_to_occurrence_time:
+                    self.kernel_events_to_occurrence_time[event_key] = RemoteProcessProxy.get_current_time()
                 if RemoteProcessProxy.get_time_diff(
                     RemoteProcessProxy.get_current_time(),
-                    self.kernel_events_to_occurrence_time[hashed_event]
+                    self.kernel_events_to_occurrence_time[event_key]
                 ) >= self.kernel_launch_terminate_on_events[event.type][event.reason]:
                     self.kill()
                     self.log_and_raise(

From 69826378a98dc533028219ee248fd46af015a3db Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sun, 2 Jun 2024 13:59:07 +0000
Subject: [PATCH 06/12] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .../services/kernels/remotemanager.py         |  8 +++---
 .../services/processproxies/container.py      | 28 ++++++++++++-------
 .../services/processproxies/k8s.py            |  3 +-
 3 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/enterprise_gateway/services/kernels/remotemanager.py b/enterprise_gateway/services/kernels/remotemanager.py
index 2c164959..cf4ac3a7 100644
--- a/enterprise_gateway/services/kernels/remotemanager.py
+++ b/enterprise_gateway/services/kernels/remotemanager.py
@@ -6,21 +6,21 @@
 from __future__ import annotations
 
 import asyncio
+import json
 import os
 import re
 import signal
 import time
 import uuid
-import json
 from typing import Any, ClassVar
 
 from jupyter_client.ioloop.manager import AsyncIOLoopKernelManager
 from jupyter_client.kernelspec import KernelSpec
 from jupyter_server.services.kernels.kernelmanager import AsyncMappingKernelManager
 from tornado import web
-from traitlets import directional_link, default
-from traitlets import log as traitlets_log
 from traitlets import List as ListTrait
+from traitlets import default, directional_link
+from traitlets import log as traitlets_log
 from zmq import IO_THREADS, MAX_SOCKETS, Context
 
 from enterprise_gateway.mixins import EnterpriseGatewayConfigMixin
@@ -171,7 +171,7 @@ class RemoteMappingKernelManager(AsyncMappingKernelManager):
         help="""Comma-separated list of dictionaries, each describing an event by `type`, `reason`,
          and `timeout_in_seconds` (e.g. [{"type": "Warning", "reason": "FailedMount", "timeout_in_seconds": 0}]).
          Kernel pod events will be sampled during startup, and if an event described in this list is detected,
-         the kernel launch will be terminated after the set timeout. Only available for container kernels. """
+         the kernel launch will be terminated after the set timeout. Only available for container kernels. """,
     )
 
     @default("kernel_launch_terminate_on_events")
diff --git a/enterprise_gateway/services/processproxies/container.py b/enterprise_gateway/services/processproxies/container.py
index c8b4cc08..413e4375 100644
--- a/enterprise_gateway/services/processproxies/container.py
+++ b/enterprise_gateway/services/processproxies/container.py
@@ -59,9 +59,9 @@ def _initialize_kernel_launch_terminate_on_events(self):
         """
         self.kernel_launch_terminate_on_events = defaultdict(dict)
         for configuration in self.kernel_manager.parent.kernel_launch_terminate_on_events:
-            self.kernel_launch_terminate_on_events[
-                configuration["type"]
-            ][configuration["reason"]] = configuration["timeout_in_seconds"]
+            self.kernel_launch_terminate_on_events[configuration["type"]][
+                configuration["reason"]
+            ] = configuration["timeout_in_seconds"]
 
     def _determine_kernel_images(self, **kwargs: dict[str, Any] | None) -> None:
         """
@@ -237,18 +237,26 @@ def _handle_pending_kernel(self):
         self.log.debug("Sampling kernel container events")
         kernel_pod_events = self.get_container_events()
         for event in kernel_pod_events:
-            if event.type in self.kernel_launch_terminate_on_events and event.reason in self.kernel_launch_terminate_on_events[event.type]:
+            if (
+                event.type in self.kernel_launch_terminate_on_events
+                and event.reason in self.kernel_launch_terminate_on_events[event.type]
+            ):
                 event_key = f"{event.type}{event.reason}"
                 if event_key not in self.kernel_events_to_occurrence_time:
-                    self.kernel_events_to_occurrence_time[event_key] = RemoteProcessProxy.get_current_time()
-                if RemoteProcessProxy.get_time_diff(
-                    RemoteProcessProxy.get_current_time(),
-                    self.kernel_events_to_occurrence_time[event_key]
-                ) >= self.kernel_launch_terminate_on_events[event.type][event.reason]:
+                    self.kernel_events_to_occurrence_time[event_key] = (
+                        RemoteProcessProxy.get_current_time()
+                    )
+                if (
+                    RemoteProcessProxy.get_time_diff(
+                        RemoteProcessProxy.get_current_time(),
+                        self.kernel_events_to_occurrence_time[event_key],
+                    )
+                    >= self.kernel_launch_terminate_on_events[event.type][event.reason]
+                ):
                     self.kill()
                     self.log_and_raise(
                         http_status_code=409,
-                        reason=f"Error starting kernel container; The container encountered an event which may cause a longer than usual startup: '{event.reason} - {event.message[:64]}'"
+                        reason=f"Error starting kernel container; The container encountered an event which may cause a longer than usual startup: '{event.reason} - {event.message[:64]}'",
                     )
 
     def get_process_info(self) -> dict[str, Any]:
diff --git a/enterprise_gateway/services/processproxies/k8s.py b/enterprise_gateway/services/processproxies/k8s.py
index a0065323..2e70b09e 100644
--- a/enterprise_gateway/services/processproxies/k8s.py
+++ b/enterprise_gateway/services/processproxies/k8s.py
@@ -116,7 +116,8 @@ def get_container_events(self) -> list:
         core_v1_api = client.CoreV1Api()
         if self.container_name:
             ret = core_v1_api.list_namespaced_event(
-                namespace=self.kernel_namespace, field_selector=f"involvedObject.name={self.container_name}"
+                namespace=self.kernel_namespace,
+                field_selector=f"involvedObject.name={self.container_name}",
             )
             if ret and ret.items:
                 pod_events = ret.items

From 8432a1b12515e26cd18586cf6d4b3a3efd534b7a Mon Sep 17 00:00:00 2001
From: Oren Zigdon <oren.zigdon@gmail.com>
Date: Sun, 2 Jun 2024 17:14:23 +0300
Subject: [PATCH 07/12] Add type hint for default of
 `kernel_launch_terminate_on_events`

---
 enterprise_gateway/services/kernels/remotemanager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/enterprise_gateway/services/kernels/remotemanager.py b/enterprise_gateway/services/kernels/remotemanager.py
index 2c164959..dbce7f03 100644
--- a/enterprise_gateway/services/kernels/remotemanager.py
+++ b/enterprise_gateway/services/kernels/remotemanager.py
@@ -164,7 +164,7 @@ class RemoteMappingKernelManager(AsyncMappingKernelManager):
     """
 
     kernel_launch_terminate_on_events_env = "EG_KERNEL_LAUNCH_TERMINATE_ON_EVENTS"
-    kernel_launch_terminate_on_events_default_value = []
+    kernel_launch_terminate_on_events_default_value: ClassVar[list] = []
     kernel_launch_terminate_on_events = ListTrait(
         default_value=kernel_launch_terminate_on_events_default_value,
         config=True,

From 4d9cf319e2573f7eded9ccbbda09289a89575285 Mon Sep 17 00:00:00 2001
From: Oren Zigdon <oren.zigdon@gmail.com>
Date: Tue, 4 Jun 2024 12:12:29 +0300
Subject: [PATCH 08/12] Add docstring to `_handle_pending_kernel` which
 explains about possible events reasons and types.

---
 enterprise_gateway/services/processproxies/container.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/enterprise_gateway/services/processproxies/container.py b/enterprise_gateway/services/processproxies/container.py
index 413e4375..a791e2d2 100644
--- a/enterprise_gateway/services/processproxies/container.py
+++ b/enterprise_gateway/services/processproxies/container.py
@@ -234,6 +234,11 @@ async def confirm_remote_startup(self) -> None:
         self.kernel_events_to_occurrence_time = {}
 
     def _handle_pending_kernel(self):
+        """Sample container events and compare them to configured events which may cause termination.
+        The event type and the event reason should match those sampled in the environment to initiate termination.
+        Possible event types: `Warning`, `Normal`.
+        Possible event reasons (may differ in different container platforms and versions): `FailedMount`, `FailedMountAttach`,
+        `FailedSchedule`, `ImagePullBackoff`, etc."""
         self.log.debug("Sampling kernel container events")
         kernel_pod_events = self.get_container_events()
         for event in kernel_pod_events:

From cc4c65009e314a8c73dddb4317fb200e6e94e086 Mon Sep 17 00:00:00 2001
From: Oren Zigdon <oren.zigdon@gmail.com>
Date: Tue, 4 Jun 2024 12:13:15 +0300
Subject: [PATCH 09/12] Add an explanation about event type and reason and how
 to find them in `kernel_launch_terminate_on_events` help.

---
 enterprise_gateway/services/kernels/remotemanager.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/enterprise_gateway/services/kernels/remotemanager.py b/enterprise_gateway/services/kernels/remotemanager.py
index 5c610a17..4f991fa4 100644
--- a/enterprise_gateway/services/kernels/remotemanager.py
+++ b/enterprise_gateway/services/kernels/remotemanager.py
@@ -171,7 +171,10 @@ class RemoteMappingKernelManager(AsyncMappingKernelManager):
         help="""Comma-separated list of dictionaries, each describing an event by `type`, `reason`,
          and `timeout_in_seconds` (e.g. [{"type": "Warning", "reason": "FailedMount", "timeout_in_seconds": 0}]).
          Kernel pod events will be sampled during startup, and if an event described in this list is detected,
-         the kernel launch will be terminated after the set timeout. Only available for container kernels. """,
+         the kernel launch will be terminated after the set timeout. Only available for container kernels. 
+         Make sure to provide the exact event type and reason as it appears in your container platform, 
+         A guide to finding the event type and reason in kubernetes can be found here:
+        `https://www.bluematador.com/blog/kubernetes-events-explained`""",
     )
 
     @default("kernel_launch_terminate_on_events")

From 8990eae699ca61fc349ae1601e5c86f7e1cdbc18 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 4 Jun 2024 09:17:49 +0000
Subject: [PATCH 10/12] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 enterprise_gateway/services/kernels/remotemanager.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/enterprise_gateway/services/kernels/remotemanager.py b/enterprise_gateway/services/kernels/remotemanager.py
index 4f991fa4..55f3685f 100644
--- a/enterprise_gateway/services/kernels/remotemanager.py
+++ b/enterprise_gateway/services/kernels/remotemanager.py
@@ -171,8 +171,8 @@ class RemoteMappingKernelManager(AsyncMappingKernelManager):
         help="""Comma-separated list of dictionaries, each describing an event by `type`, `reason`,
          and `timeout_in_seconds` (e.g. [{"type": "Warning", "reason": "FailedMount", "timeout_in_seconds": 0}]).
          Kernel pod events will be sampled during startup, and if an event described in this list is detected,
-         the kernel launch will be terminated after the set timeout. Only available for container kernels. 
-         Make sure to provide the exact event type and reason as it appears in your container platform, 
+         the kernel launch will be terminated after the set timeout. Only available for container kernels.
+         Make sure to provide the exact event type and reason as it appears in your container platform,
          A guide to finding the event type and reason in kubernetes can be found here:
         `https://www.bluematador.com/blog/kubernetes-events-explained`""",
     )

From 686b3420ff4b0634c20f11873cd9119c396ce69b Mon Sep 17 00:00:00 2001
From: Oren Zigdon <oren.zigdon@gmail.com>
Date: Tue, 25 Jun 2024 13:16:24 +0300
Subject: [PATCH 11/12] Add the docs about `kernel_launch_terminate_on_events`
 to `RemoteMappingKernelManager` cli options.

---
 docs/source/operators/config-cli.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/source/operators/config-cli.md b/docs/source/operators/config-cli.md
index 39c44ae4..5a815a5f 100644
--- a/docs/source/operators/config-cli.md
+++ b/docs/source/operators/config-cli.md
@@ -366,4 +366,14 @@ RemoteMappingKernelManager(AsyncMappingKernelManager) options
 --RemoteMappingKernelManager.shared_context=<Bool>
     Share a single zmq.Context to talk to all my kernels
     Default: True
+--RemoteMappingKernelManager.kernel_launch_terminate_on_events=<list-item-1>...
+    Comma-separated list of dictionaries, each describing an event by `type`, `reason`, and `timeout_in_seconds`
+    (e.g. [{"type": "Warning", "reason": "FailedMount", "timeout_in_seconds": 0}]).
+    Kernel pod events will be sampled during startup, and if an event described in this list is detected,
+    the kernel launch will be terminated after the set timeout. 
+    Only available for container kernels. 
+    Make sure to provide the exact event type and reason as it appears in your container platform, 
+    A guide to finding the event type and reason in kubernetes can be found here:
+    `https://www.bluematador.com/blog/kubernetes-events-explained`
+    Default: []
 ```

From a212f9740a00681a87cbd06a9ad80e023c38dd27 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 25 Jun 2024 10:17:30 +0000
Subject: [PATCH 12/12] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 docs/source/operators/config-cli.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/source/operators/config-cli.md b/docs/source/operators/config-cli.md
index 5a815a5f..2eb80f92 100644
--- a/docs/source/operators/config-cli.md
+++ b/docs/source/operators/config-cli.md
@@ -370,9 +370,9 @@ RemoteMappingKernelManager(AsyncMappingKernelManager) options
     Comma-separated list of dictionaries, each describing an event by `type`, `reason`, and `timeout_in_seconds`
     (e.g. [{"type": "Warning", "reason": "FailedMount", "timeout_in_seconds": 0}]).
     Kernel pod events will be sampled during startup, and if an event described in this list is detected,
-    the kernel launch will be terminated after the set timeout. 
-    Only available for container kernels. 
-    Make sure to provide the exact event type and reason as it appears in your container platform, 
+    the kernel launch will be terminated after the set timeout.
+    Only available for container kernels.
+    Make sure to provide the exact event type and reason as it appears in your container platform,
     A guide to finding the event type and reason in kubernetes can be found here:
     `https://www.bluematador.com/blog/kubernetes-events-explained`
     Default: []