feat: Support graceful shutdown

CHANGELOG.md

@@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file.
 - Default stackableVersion to operator version ([#381]).
 - Configuration overrides for the JVM security properties, such as DNS caching ([#384]).
 - Support PodDisruptionBudgets ([#394]).
+- Support graceful shutdown ([#407]).
 
 ### Changed
 
@@ -28,6 +29,7 @@ All notable changes to this project will be documented in this file.
 [#402]: https://github.com/stackabletech/hdfs-operator/pull/402
 [#404]: https://github.com/stackabletech/hdfs-operator/pull/404
 [#405]: https://github.com/stackabletech/hdfs-operator/pull/405
+[#407]: https://github.com/stackabletech/hdfs-operator/pull/407
 
 ## [23.7.0] - 2023-07-14
 

Cargo.toml

@@ -23,7 +23,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
 snafu = "0.7"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.52.1" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.54.0" }
 strum = { version = "0.25", features = ["derive"] }
 tokio = { version = "1.29", features = ["full"] }
 tracing = "0.1"

deploy/helm/hdfs-operator/crds/crds.yaml

@@ -576,6 +576,10 @@ spec:
                                   type: array
                               type: object
                           type: object
+                        gracefulShutdownTimeout:
+                          description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
+                          nullable: true
+                          type: string
                         logging:
                           default:
                             enableVectorAgent: null
@@ -4061,6 +4065,10 @@ spec:
                                         type: array
                                     type: object
                                 type: object
+                              gracefulShutdownTimeout:
+                                description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
+                                nullable: true
+                                type: string
                               logging:
                                 default:
                                   enableVectorAgent: null
@@ -7605,6 +7613,10 @@ spec:
                                   type: array
                               type: object
                           type: object
+                        gracefulShutdownTimeout:
+                          description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
+                          nullable: true
+                          type: string
                         logging:
                           default:
                             enableVectorAgent: null
@@ -11081,6 +11093,10 @@ spec:
                                         type: array
                                     type: object
                                 type: object
+                              gracefulShutdownTimeout:
+                                description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
+                                nullable: true
+                                type: string
                               logging:
                                 default:
                                   enableVectorAgent: null
@@ -14574,6 +14590,10 @@ spec:
                                   type: array
                               type: object
                           type: object
+                        gracefulShutdownTimeout:
+                          description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
+                          nullable: true
+                          type: string
                         logging:
                           default:
                             enableVectorAgent: null
@@ -18050,6 +18070,10 @@ spec:
                                         type: array
                                     type: object
                                 type: object
+                              gracefulShutdownTimeout:
+                                description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
+                                nullable: true
+                                type: string
                               logging:
                                 default:
                                   enableVectorAgent: null

docs/modules/hdfs/pages/usage-guide/operations/graceful-shutdown.adoc

@@ -1,6 +1,33 @@
 = Graceful shutdown
 
-Graceful shutdown of HDFS nodes is either not supported by the product itself
-or we have not implemented it yet.
+You can configure the graceful shutdown as described in xref:concepts:operations/pod_placement.adoc[].
 
-Outstanding implementation work for the graceful shutdowns of all products where this functionality is relevant is tracked in https://github.com/stackabletech/issues/issues/357
+== JournalNodes
+
+As a default, JournalNodes have `15` minutes to terminate gracefully.
+
+In case they are requested to shut down, they will receive a `SIGTERM`, log the received signal as follows and shut down gracefully.
+https://github.com/apache/hadoop/blob/a585a73c3e02ac62350c136643a5e7f6095a3dbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java#L2004[This] is the relevant code as of version `3.3.4`.
+
+[source,text]
+----
+2023-10-10 13:37:41,525 ERROR server.JournalNode (LogAdapter.java:error(75)) - RECEIVED SIGNAL 15: SIGTERM
+2023-10-10 13:37:41,526 INFO  server.JournalNode (LogAdapter.java:info(51)) - SHUTDOWN_MSG:
+/************************************************************
+SHUTDOWN_MSG: Shutting down JournalNode at hdfs-journalnode-default-0/10.244.0.38
+************************************************************/
+----
+
+== NameNodes
+
+As a default, NameNodes have `15` minutes to terminate gracefully.
+
+In case they are requested to shut down, they will receive a `SIGTERM`, log the received signal and shut down gracefully.
+https://github.com/apache/hadoop/blob/a585a73c3e02ac62350c136643a5e7f6095a3dbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java#L1080[This] is the relevant code as of version `3.3.4`.
+
+== DataNodes
+
+As a default, DataNodes have `30` minutes to terminate gracefully.
+
+In case they are requested to shut down, they will receive a `SIGTERM`, log the received signal and shut down gracefully.
+https://github.com/apache/hadoop/blob/a585a73c3e02ac62350c136643a5e7f6095a3dbb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java#L272[This] is the relevant code as of version `3.3.4`.

rust/crd/src/lib.rs

@@ -23,6 +23,7 @@ use stackable_operator::{
         fragment::{Fragment, ValidationError},
         merge::Merge,
     },
+    duration::Duration,
     k8s_openapi::{
         api::core::v1::PodTemplateSpec,
         apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::LabelSelector},
@@ -156,6 +157,7 @@ pub trait MergedConfig {
         None
     }
     fn affinity(&self) -> &StackableAffinity;
+    fn graceful_shutdown_timeout(&self) -> Option<&Duration>;
     /// Main container shared by all roles
     fn hdfs_logging(&self) -> ContainerLogConfig;
     /// Vector container shared by all roles
@@ -841,6 +843,11 @@ pub struct NameNodeConfig {
     pub logging: Logging<NameNodeContainer>,
     #[fragment_attrs(serde(default))]
     pub affinity: StackableAffinity,
+    #[fragment_attrs(serde(default))]
+    #[fragment_attrs(schemars(
+        description = "Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details."
+    ))]
+    pub graceful_shutdown_timeout: Option<Duration>,
 }
 
 impl MergedConfig for NameNodeConfig {
@@ -852,6 +859,10 @@ impl MergedConfig for NameNodeConfig {
         &self.affinity
     }
 
+    fn graceful_shutdown_timeout(&self) -> Option<&Duration> {
+        self.graceful_shutdown_timeout.as_ref()
+    }
+
     fn hdfs_logging(&self) -> ContainerLogConfig {
         self.logging
             .containers
@@ -916,6 +927,7 @@ impl NameNodeConfigFragment {
             },
             logging: product_logging::spec::default_logging(),
             affinity: get_affinity(cluster_name, role),
+            graceful_shutdown_timeout: Some(Duration::from_minutes_unchecked(15)),
         }
     }
 }
@@ -1001,6 +1013,11 @@ pub struct DataNodeConfig {
     pub logging: Logging<DataNodeContainer>,
     #[fragment_attrs(serde(default))]
     pub affinity: StackableAffinity,
+    #[fragment_attrs(serde(default))]
+    #[fragment_attrs(schemars(
+        description = "Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details."
+    ))]
+    pub graceful_shutdown_timeout: Option<Duration>,
 }
 
 impl MergedConfig for DataNodeConfig {
@@ -1014,6 +1031,10 @@ impl MergedConfig for DataNodeConfig {
         &self.affinity
     }
 
+    fn graceful_shutdown_timeout(&self) -> Option<&Duration> {
+        self.graceful_shutdown_timeout.as_ref()
+    }
+
     fn hdfs_logging(&self) -> ContainerLogConfig {
         self.logging
             .containers
@@ -1069,6 +1090,7 @@ impl DataNodeConfigFragment {
             },
             logging: product_logging::spec::default_logging(),
             affinity: get_affinity(cluster_name, role),
+            graceful_shutdown_timeout: Some(Duration::from_minutes_unchecked(30)),
         }
     }
 }
@@ -1152,6 +1174,11 @@ pub struct JournalNodeConfig {
     pub logging: Logging<JournalNodeContainer>,
     #[fragment_attrs(serde(default))]
     pub affinity: StackableAffinity,
+    #[fragment_attrs(serde(default))]
+    #[fragment_attrs(schemars(
+        description = "Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details."
+    ))]
+    pub graceful_shutdown_timeout: Option<Duration>,
 }
 
 impl MergedConfig for JournalNodeConfig {
@@ -1163,6 +1190,10 @@ impl MergedConfig for JournalNodeConfig {
         &self.affinity
     }
 
+    fn graceful_shutdown_timeout(&self) -> Option<&Duration> {
+        self.graceful_shutdown_timeout.as_ref()
+    }
+
     fn hdfs_logging(&self) -> ContainerLogConfig {
         self.logging
             .containers
@@ -1206,6 +1237,7 @@ impl JournalNodeConfigFragment {
             },
             logging: product_logging::spec::default_logging(),
             affinity: get_affinity(cluster_name, role),
+            graceful_shutdown_timeout: Some(Duration::from_minutes_unchecked(15)),
         }
     }
 }

rust/operator/src/hdfs_controller.rs

@@ -6,7 +6,7 @@ use crate::{
     discovery::build_discovery_configmap,
     event::{build_invalid_replica_message, publish_event},
     kerberos,
-    operations::pdb::add_pdbs,
+    operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs},
     product_logging::{extend_role_group_config_map, resolve_vector_aggregator_address},
     OPERATOR_NAME,
 };
@@ -667,6 +667,8 @@ fn rolegroup_statefulset(
     )
     .context(FailedToCreateContainerAndVolumeConfigurationSnafu)?;
 
+    add_graceful_shutdown_config(merged_config, &mut pb);
+
     let mut pod_template = pb.build_template();
     if let Some(pod_overrides) = hdfs.pod_overrides_for_role(role) {
         pod_template.merge_from(pod_overrides.clone());

rust/operator/src/operations/graceful_shutdown.rs

@@ -0,0 +1,12 @@
+use stackable_hdfs_crd::MergedConfig;
+use stackable_operator::builder::PodBuilder;
+
+pub fn add_graceful_shutdown_config(
+    merged_config: &(dyn MergedConfig + Send + 'static),
+    pod_builder: &mut PodBuilder,
+) {
+    // This must be always set by the merge mechanism, as we provide a default value.
+    if let Some(graceful_shutdown_timeout) = merged_config.graceful_shutdown_timeout() {
+        pod_builder.termination_grace_period_seconds(graceful_shutdown_timeout.as_secs() as i64);
+    }
+}

rust/operator/src/operations/mod.rs

@@ -1 +1,2 @@
+pub mod graceful_shutdown;
 pub mod pdb;

tests/templates/kuttl/smoke/30-assert.yaml.j2

@@ -23,6 +23,7 @@ spec:
         - name: vector
 {% endif %}
         - name: zkfc
+      terminationGracePeriodSeconds: 900
 status:
   readyReplicas: 2
   replicas: 2
@@ -46,6 +47,7 @@ spec:
 {% if lookup('env', 'VECTOR_AGGREGATOR') %}
         - name: vector
 {% endif %}
+      terminationGracePeriodSeconds: 900
 status:
   readyReplicas: 1
   replicas: 1
@@ -69,6 +71,7 @@ spec:
 {% if lookup('env', 'VECTOR_AGGREGATOR') %}
         - name: vector
 {% endif %}
+      terminationGracePeriodSeconds: 1800
 status:
   readyReplicas: {{ test_scenario['values']['number-of-datanodes'] }}
   replicas: {{ test_scenario['values']['number-of-datanodes'] }}