diff --git a/CHANGELOG.md b/CHANGELOG.md index b2148b8d..67a5f624 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file. - Default stackableVersion to operator version ([#360]). - Configuration overrides for the JVM security properties, such as DNS caching ([#365]). - Support PodDisruptionBudgets ([#376]). +- Support graceful shutdown ([#385]). ### Changed @@ -27,6 +28,7 @@ All notable changes to this project will be documented in this file. [#376]: https://github.com/stackabletech/hive-operator/pull/376 [#377]: https://github.com/stackabletech/hive-operator/pull/377 [#382]: https://github.com/stackabletech/hive-operator/pull/382 +[#385]: https://github.com/stackabletech/hive-operator/pull/385 ## [23.7.0] - 2023-07-14 diff --git a/deploy/helm/hive-operator/crds/crds.yaml b/deploy/helm/hive-operator/crds/crds.yaml index 58b9b77e..46fcce16 100644 --- a/deploy/helm/hive-operator/crds/crds.yaml +++ b/deploy/helm/hive-operator/crds/crds.yaml @@ -724,6 +724,10 @@ spec: type: array type: object type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string logging: default: enableVectorAgent: null @@ -4213,6 +4217,10 @@ spec: type: array type: object type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string logging: default: enableVectorAgent: null diff --git a/docs/modules/hive/pages/usage-guide/operations/graceful-shutdown.adoc b/docs/modules/hive/pages/usage-guide/operations/graceful-shutdown.adoc index bd04fc06..36f5f129 100644 --- a/docs/modules/hive/pages/usage-guide/operations/graceful-shutdown.adoc +++ b/docs/modules/hive/pages/usage-guide/operations/graceful-shutdown.adoc @@ -1,7 +1,12 @@ = Graceful shutdown -Graceful shutdown of Zookeeper nodes is either not supported by the product itself -or we have not implemented it yet. +You can configure the graceful shutdown as described in xref:concepts:operations/graceful_shutdown.adoc[]. -Outstanding implementation work for the graceful shutdowns of all products where this functionality is relevant is tracked in -https://github.com/stackabletech/issues/issues/357 +== Hive metastores + +As a default, Hive metastores have `5 minutes` to shut down gracefully. + +The Hive metastore process will receive a `SIGTERM` signal when Kubernetes wants to terminate the Pod. +After the graceful shutdown timeout runs out, and the process still didn't exit, Kubernetes will issue a `SIGKILL` signal. + +However, there is no acknowledge message in the log indicating a graceful shutdown. diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index be3f961e..4d4121d4 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -18,10 +18,16 @@ use stackable_operator::{ k8s_openapi::apimachinery::pkg::api::resource::Quantity, kube::{runtime::reflector::ObjectRef, CustomResource, ResourceExt}, product_config_utils::{ConfigError, Configuration}, - product_logging::{self, spec::Logging}, + product_logging::{ + self, + framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, + spec::Logging, + }, role_utils::{GenericRoleConfig, Role, RoleGroup, RoleGroupRef}, schemars::{self, JsonSchema}, status::condition::{ClusterCondition, HasStatusCondition}, + time::Duration, + utils::COMMON_BASH_TRAP_FUNCTIONS, }; use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; @@ -62,6 +68,8 @@ pub const HIVE_METASTORE_HADOOP_OPTS: &str = "HIVE_METASTORE_HADOOP_OPTS"; pub const HADOOP_HEAPSIZE: &str = "HADOOP_HEAPSIZE"; pub const JVM_HEAP_FACTOR: f32 = 0.8; +const DEFAULT_METASTORE_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5); + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("no metastore role configuration provided"))] @@ -181,25 +189,19 @@ pub enum HiveRole { impl HiveRole { /// Returns the container start command for the metastore service. - pub fn get_command(&self, auto_init_schema: bool, db_type: &str) -> Vec { - if auto_init_schema { - vec![ - "bin/start-metastore".to_string(), - "--config".to_string(), - STACKABLE_CONFIG_DIR.to_string(), - "--db-type".to_string(), - db_type.to_string(), - "--hive-bin-dir".to_string(), - "bin".to_string(), - ] - } else { - vec![ - "/bin/hive".to_string(), - "--config".to_string(), - STACKABLE_CONFIG_DIR.to_string(), - "--service".to_string(), - "metastore".to_string(), - ] + pub fn get_command(&self, db_type: &str) -> String { + formatdoc! {" + {COMMON_BASH_TRAP_FUNCTIONS} + {remove_vector_shutdown_file_command} + prepare_signal_handlers + bin/start-metastore --config {STACKABLE_CONFIG_DIR} --db-type {db_type} --hive-bin-dir bin & + wait_for_termination $! + {create_vector_shutdown_file_command} + ", + remove_vector_shutdown_file_command = + remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), + create_vector_shutdown_file_command = + create_vector_shutdown_file_command(STACKABLE_LOG_DIR), } } @@ -284,12 +286,19 @@ pub struct MetaStoreConfig { /// The location of default database for the Hive warehouse. /// Maps to the `hive.metastore.warehouse.dir` setting. pub warehouse_dir: Option, + #[fragment_attrs(serde(default))] pub resources: Resources, + #[fragment_attrs(serde(default))] pub logging: Logging, + #[fragment_attrs(serde(default))] pub affinity: StackableAffinity, + + /// Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + #[fragment_attrs(serde(default))] + pub graceful_shutdown_timeout: Option, } impl MetaStoreConfig { @@ -330,6 +339,7 @@ impl MetaStoreConfig { }, logging: product_logging::spec::default_logging(), affinity: get_affinity(cluster_name, role), + graceful_shutdown_timeout: Some(DEFAULT_METASTORE_GRACEFUL_SHUTDOWN_TIMEOUT), } } } diff --git a/rust/operator-binary/src/command.rs b/rust/operator-binary/src/command.rs index 9aa75a20..e2dbf839 100644 --- a/rust/operator-binary/src/command.rs +++ b/rust/operator-binary/src/command.rs @@ -50,8 +50,9 @@ pub fn build_container_command_args( args.push(format!("keytool -importcert -file /stackable/certificates/{secret_class}-tls-certificate/ca.crt -alias stackable-{secret_class} -keystore {STACKABLE_TRUST_STORE} -storepass {STACKABLE_TRUST_STORE_PASSWORD} -noprompt")); } } + // metastore start command args.push(start_command); - vec![args.join(" && ")] + vec![args.join("\n")] } diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 7718c8ce..cebcbc64 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -73,7 +73,7 @@ use tracing::warn; use crate::{ command::{self, build_container_command_args, S3_SECRET_DIR}, discovery, - operations::pdb::add_pdbs, + operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, product_logging::{extend_role_group_config_map, resolve_vector_aggregator_address}, OPERATOR_NAME, }; @@ -254,6 +254,11 @@ pub enum Error { FailedToCreatePdb { source: crate::operations::pdb::Error, }, + + #[snafu(display("failed to configure graceful shutdown"))] + GracefulShutdown { + source: crate::operations::graceful_shutdown::Error, + }, } type Result = std::result::Result; @@ -805,14 +810,13 @@ fn build_metastore_rolegroup_statefulset( .image_from_product_image(resolved_product_image) .command(vec![ "/bin/bash".to_string(), - "-c".to_string(), + "-x".to_string(), "-euo".to_string(), "pipefail".to_string(), + "-c".to_string(), ]) .args(build_container_command_args( - HiveRole::MetaStore - .get_command(true, &db_type.unwrap_or_default().to_string()) - .join(" "), + HiveRole::MetaStore.get_command(&db_type.unwrap_or_default().to_string()), s3_connection, )) .add_volume_mount(STACKABLE_CONFIG_DIR_NAME, STACKABLE_CONFIG_DIR) @@ -940,6 +944,8 @@ fn build_metastore_rolegroup_statefulset( )); } + add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?; + let mut pod_template = pod_builder.build_template(); pod_template.merge_from(role.config.pod_overrides.clone()); pod_template.merge_from(rolegroup.config.pod_overrides.clone()); diff --git a/rust/operator-binary/src/operations/graceful_shutdown.rs b/rust/operator-binary/src/operations/graceful_shutdown.rs new file mode 100644 index 00000000..8c1fc2a6 --- /dev/null +++ b/rust/operator-binary/src/operations/graceful_shutdown.rs @@ -0,0 +1,26 @@ +use snafu::{ResultExt, Snafu}; +use stackable_hive_crd::MetaStoreConfig; +use stackable_operator::builder::PodBuilder; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("Failed to set terminationGracePeriod"))] + SetTerminationGracePeriod { + source: stackable_operator::builder::pod::Error, + }, +} + +pub fn add_graceful_shutdown_config( + merged_config: &MetaStoreConfig, + pod_builder: &mut PodBuilder, +) -> Result<(), Error> { + // This must be always set by the merge mechanism, as we provide a default value, + // users can not disable graceful shutdown. + if let Some(graceful_shutdown_timeout) = merged_config.graceful_shutdown_timeout { + pod_builder + .termination_grace_period(&graceful_shutdown_timeout) + .context(SetTerminationGracePeriodSnafu)?; + } + + Ok(()) +} diff --git a/rust/operator-binary/src/operations/mod.rs b/rust/operator-binary/src/operations/mod.rs index d3cf6e9c..92ca2ec7 100644 --- a/rust/operator-binary/src/operations/mod.rs +++ b/rust/operator-binary/src/operations/mod.rs @@ -1 +1,2 @@ +pub mod graceful_shutdown; pub mod pdb; diff --git a/tests/templates/kuttl/smoke/60-assert.yaml.j2 b/tests/templates/kuttl/smoke/60-assert.yaml.j2 index 3e8337b5..f11f2b43 100644 --- a/tests/templates/kuttl/smoke/60-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/60-assert.yaml.j2 @@ -22,6 +22,7 @@ spec: {% if lookup('env', 'VECTOR_AGGREGATOR') %} - name: vector {% endif %} + terminationGracePeriodSeconds: 300 status: readyReplicas: 1 replicas: 1