From 15105697aa0b612b5d98c2fe37e4a04c05a36fdb Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 9 Nov 2023 15:33:41 +0100 Subject: [PATCH 01/34] WIP --- Cargo.lock | 7 + Cargo.toml | 3 +- deploy/helm/hbase-operator/crds/crds.yaml | 17 ++ rust/crd/src/affinity.rs | 2 + rust/crd/src/lib.rs | 189 ++++++++++-- rust/operator-binary/Cargo.toml | 3 +- rust/operator-binary/src/discovery.rs | 12 +- rust/operator-binary/src/hbase_controller.rs | 280 ++++++++++++------ rust/operator-binary/src/kerberos.rs | 270 +++++++++++++++++ rust/operator-binary/src/main.rs | 1 + .../kuttl/kerberos/00-assert.yaml.j2 | 10 + ...tor-aggregator-discovery-configmap.yaml.j2 | 9 + .../kuttl/kerberos/01-assert.yaml.j2 | 14 + .../kerberos/01-install-krb5-kdc.yaml.j2 | 135 +++++++++ .../02-create-kerberos-secretclass.yaml.j2 | 72 +++++ tests/templates/kuttl/kerberos/10-assert.yaml | 12 + .../kuttl/kerberos/10-install-zk.yaml.j2 | 20 ++ tests/templates/kuttl/kerberos/11-assert.yaml | 28 ++ .../kuttl/kerberos/11-install-hdfs.yaml.j2 | 60 ++++ .../kuttl/kerberos/20-access-hdfs.yaml.j2 | 68 +++++ tests/templates/kuttl/kerberos/20-assert.yaml | 11 + tests/templates/kuttl/kerberos/30-assert.yaml | 29 ++ .../kuttl/kerberos/30-install-hbase.yaml.j2 | 53 ++++ .../kuttl/kerberos/40-access-hbase.j2 | 70 +++++ tests/templates/kuttl/kerberos/40-assert.yaml | 11 + tests/test-definition.yaml | 68 +++-- ycsb.yaml | 138 +++++++++ 27 files changed, 1439 insertions(+), 153 deletions(-) create mode 100644 rust/operator-binary/src/kerberos.rs create mode 100644 tests/templates/kuttl/kerberos/00-assert.yaml.j2 create mode 100644 tests/templates/kuttl/kerberos/00-install-vector-aggregator-discovery-configmap.yaml.j2 create mode 100644 tests/templates/kuttl/kerberos/01-assert.yaml.j2 create mode 100644 tests/templates/kuttl/kerberos/01-install-krb5-kdc.yaml.j2 create mode 100644 tests/templates/kuttl/kerberos/02-create-kerberos-secretclass.yaml.j2 create mode 100644 tests/templates/kuttl/kerberos/10-assert.yaml create mode 100644 tests/templates/kuttl/kerberos/10-install-zk.yaml.j2 create mode 100644 tests/templates/kuttl/kerberos/11-assert.yaml create mode 100644 tests/templates/kuttl/kerberos/11-install-hdfs.yaml.j2 create mode 100644 tests/templates/kuttl/kerberos/20-access-hdfs.yaml.j2 create mode 100644 tests/templates/kuttl/kerberos/20-assert.yaml create mode 100644 tests/templates/kuttl/kerberos/30-assert.yaml create mode 100644 tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2 create mode 100644 tests/templates/kuttl/kerberos/40-access-hbase.j2 create mode 100644 tests/templates/kuttl/kerberos/40-assert.yaml create mode 100644 ycsb.yaml diff --git a/Cargo.lock b/Cargo.lock index e0d636a0..88236cdd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -901,6 +901,12 @@ dependencies = [ "hashbrown 0.14.1", ] +[[package]] +name = "indoc" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" + [[package]] name = "instant" version = "0.1.12" @@ -2001,6 +2007,7 @@ dependencies = [ "clap", "fnv", "futures 0.3.28", + "indoc", "product-config", "serde", "snafu 0.7.5", diff --git a/Cargo.toml b/Cargo.toml index 18f621d8..6a89dcd4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,13 +16,14 @@ built = { version = "0.6", features = ["chrono", "git2"] } clap = "4.3" fnv = "1.0" futures = { version = "0.3", features = ["compat"] } +indoc = "2.0" +product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.6.0" } rstest = "0.18" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.7" stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.56.0" } -product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.6.0" } strum = { version = "0.25", features = ["derive"] } tokio = { version = "1.29", features = ["full"] } tracing = "0.1" diff --git a/deploy/helm/hbase-operator/crds/crds.yaml b/deploy/helm/hbase-operator/crds/crds.yaml index 56746624..4196d57d 100644 --- a/deploy/helm/hbase-operator/crds/crds.yaml +++ b/deploy/helm/hbase-operator/crds/crds.yaml @@ -30,6 +30,23 @@ spec: hdfsConfigMapName: description: HDFS cluster connection details from discovery config map type: string + kerberos: + description: Configuration to set up a cluster secured using Kerberos. + nullable: true + properties: + kerberosSecretClass: + default: kerberos + description: Name of the SecretClass providing the keytab for the HDFS services. + type: string + requestNodePrincipals: + default: false + description: Wether a principal including the Kubernetes node name should be requested. The principal could e.g. be `HTTP/my-k8s-worker-0.mycorp.lan`. This feature is disabled by default, as the resulting principals can already by existent e.g. in Active Directory which can cause problems. + type: boolean + tlsSecretClass: + default: tls + description: Name of the SecretClass providing the tls certificates for the WebUIs. + type: string + type: object listenerClass: default: cluster-internal description: |- diff --git a/rust/crd/src/affinity.rs b/rust/crd/src/affinity.rs index 95fbaff8..5d8590c4 100644 --- a/rust/crd/src/affinity.rs +++ b/rust/crd/src/affinity.rs @@ -126,6 +126,7 @@ mod tests { let hbase: HbaseCluster = serde_yaml::from_str(input).expect("illegal test input"); let merged_config = hbase .merged_config( + "simple-hbase", &role, "default", &hbase.spec.cluster_config.hdfs_config_map_name, @@ -258,6 +259,7 @@ mod tests { let hbase: HbaseCluster = serde_yaml::from_str(input).expect("illegal test input"); let merged_config = hbase .merged_config( + "simple-hbase", &HbaseRole::Master, "default", &hbase.spec.cluster_config.hdfs_config_map_name, diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 683d0873..ea94a8e0 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -14,7 +14,7 @@ use stackable_operator::{ }, config::{fragment, fragment::Fragment, fragment::ValidationError, merge::Merge}, k8s_openapi::apimachinery::pkg::api::resource::Quantity, - kube::{runtime::reflector::ObjectRef, CustomResource, ResourceExt}, + kube::{runtime::reflector::ObjectRef, CustomResource}, product_config_utils::{ConfigError, Configuration}, product_logging::{self, spec::Logging}, role_utils::{GenericRoleConfig, Role, RoleGroup, RoleGroupRef}, @@ -32,10 +32,16 @@ pub const APP_NAME: &str = "hbase"; pub const CONFIG_DIR_NAME: &str = "/stackable/conf"; +pub const TLS_STORE_DIR: &str = "/stackable/tls"; +pub const TLS_STORE_VOLUME_NAME: &str = "tls"; +pub const TLS_STORE_PASSWORD: &str = "changeit"; + pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties"; pub const HBASE_ENV_SH: &str = "hbase-env.sh"; pub const HBASE_SITE_XML: &str = "hbase-site.xml"; +pub const SSL_SERVER_XML: &str = "ssl-server.xml"; +pub const SSL_CLIENT_XML: &str = "ssl-client.xml"; pub const HBASE_MANAGES_ZK: &str = "HBASE_MANAGES_ZK"; pub const HBASE_MASTER_OPTS: &str = "HBASE_MASTER_OPTS"; @@ -47,15 +53,20 @@ pub const HBASE_ROOTDIR: &str = "hbase.rootdir"; pub const HBASE_HEAPSIZE: &str = "HBASE_HEAPSIZE"; pub const HBASE_ROOT_DIR_DEFAULT: &str = "/hbase"; -pub const HBASE_UI_PORT_NAME: &str = "ui"; +pub const HBASE_UI_PORT_NAME_HTTP: &str = "ui"; +pub const HBASE_UI_PORT_NAME_HTTPS: &str = "ui-https"; pub const METRICS_PORT_NAME: &str = "metrics"; -pub const HBASE_MASTER_PORT: i32 = 16000; -pub const HBASE_MASTER_UI_PORT: i32 = 16010; -pub const HBASE_REGIONSERVER_PORT: i32 = 16020; -pub const HBASE_REGIONSERVER_UI_PORT: i32 = 16030; -pub const HBASE_REST_PORT: i32 = 8080; -pub const METRICS_PORT: i32 = 8081; +// TODO: Find sane port numbers for https +pub const HBASE_MASTER_PORT: u16 = 16000; +pub const HBASE_MASTER_UI_PORT_HTTP: u16 = 16010; +pub const HBASE_MASTER_UI_PORT_HTTPS: u16 = 16011; +pub const HBASE_REGIONSERVER_PORT: u16 = 16020; +pub const HBASE_REGIONSERVER_UI_PORT_HTTP: u16 = 16030; +pub const HBASE_REGIONSERVER_UI_PORT_HTTPS: u16 = 16031; +// TODO: Think about https +pub const HBASE_REST_PORT: u16 = 8080; +pub const METRICS_PORT: u16 = 8081; pub const JVM_HEAP_FACTOR: f32 = 0.8; @@ -122,6 +133,8 @@ pub struct HbaseClusterConfig { pub vector_aggregator_config_map_name: Option, /// ZooKeeper cluster connection details from discovery config map pub zookeeper_config_map_name: String, + /// Configuration to set up a cluster secured using Kerberos. + pub kerberos: Option, /// This field controls which type of Service the Operator creates for this HbaseCluster: /// /// * cluster-internal: Use a ClusterIP service @@ -155,6 +168,31 @@ impl CurrentlySupportedListenerClasses { } } +#[derive(Clone, Debug, Deserialize, Eq, Hash, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct KerberosConfig { + /// Name of the SecretClass providing the keytab for the HDFS services. + #[serde(default = "default_kerberos_kerberos_secret_class")] + kerberos_secret_class: String, + /// Name of the SecretClass providing the tls certificates for the WebUIs. + #[serde(default = "default_kerberos_tls_secret_class")] + tls_secret_class: String, + /// Wether a principal including the Kubernetes node name should be requested. + /// The principal could e.g. be `HTTP/my-k8s-worker-0.mycorp.lan`. + /// This feature is disabled by default, as the resulting principals can already by existent + /// e.g. in Active Directory which can cause problems. + #[serde(default)] + request_node_principals: bool, +} + +fn default_kerberos_tls_secret_class() -> String { + "tls".to_string() +} + +fn default_kerberos_kerberos_secret_class() -> String { + "kerberos".to_string() +} + #[derive( Clone, Debug, @@ -181,26 +219,6 @@ pub enum HbaseRole { } impl HbaseRole { - /// Returns a port name, the port number, and the protocol for the given role. - pub fn port_properties(&self) -> Vec<(&'static str, i32, &'static str)> { - match self { - HbaseRole::Master => vec![ - ("master", HBASE_MASTER_PORT, "TCP"), - (HBASE_UI_PORT_NAME, HBASE_MASTER_UI_PORT, "TCP"), - (METRICS_PORT_NAME, METRICS_PORT, "TCP"), - ], - HbaseRole::RegionServer => vec![ - ("regionserver", HBASE_REGIONSERVER_PORT, "TCP"), - (HBASE_UI_PORT_NAME, HBASE_REGIONSERVER_UI_PORT, "TCP"), - (METRICS_PORT_NAME, METRICS_PORT, "TCP"), - ], - HbaseRole::RestServer => vec![ - ("rest", HBASE_REST_PORT, "TCP"), - (METRICS_PORT_NAME, METRICS_PORT, "TCP"), - ], - } - } - pub fn default_config( &self, cluster_name: &str, @@ -257,6 +275,25 @@ impl HbaseRole { graceful_shutdown_timeout: Some(graceful_shutdown_timeout), } } + + pub fn kerberos_service_name(&self) -> &'static str { + // FIXME: Use the names below and fix the following errors + // + // 2023-11-08 13:36:58,530 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16020] security.ShellBasedUnixGroupsMapping: unable to return groups for user hbase-master PartialGroupNameException The user name 'hbase-master' is not found. id: 'hbase-master': no such user + // or + // Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.security.AccessDeniedException): org.apache.hadoop.hbase.security.AccessDeniedException: Insufficient permissions (user=hbase-master/hbase-master-default-1.hbase-master-default.kuttl-test-poetic-sunbeam.svc.cluster.local@CLUSTER.LOCAL, scope=hbase:meta, family=table:state, params=[table=hbase:meta,family=table:state],action=WRITE) + // + // match self { + // HbaseRole::Master => "hbase-master", + // HbaseRole::RegionServer => "hbase-regionserver", + // HbaseRole::RestServer => "hbase-restserver", + // } + + // On the other hand, according to docs: + // > A Kerberos principal has three parts, with the form username/fully.qualified.domain.name@YOUR-REALM.COM. We recommend using hbase as the username portion. + + "hbase" + } } #[allow(clippy::derive_partial_eq_without_eq)] @@ -357,7 +394,25 @@ impl Configuration for HbaseConfigFragment { match file { HBASE_ENV_SH => { result.insert(HBASE_MANAGES_ZK.to_string(), Some("false".to_string())); - let mut all_hbase_opts = format!("-Djava.security.properties={CONFIG_DIR_NAME}/{JVM_SECURITY_PROPERTIES_FILE} -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={METRICS_PORT}:/stackable/jmx/{role_name}.yaml"); + // result.insert( + // "KRB5_CONFIG".to_string(), + // Some("/stackable/kerberos/krb5.conf".to_string()), + // ); + + // As we don't have access to the clusterConfig, we always enable the `-Djava.security.krb5.conf` + // config, besides it is not always being used. + + // -Djavax.net.ssl.trustStore={TLS_STORE_DIR}/truststore.p12 \ + // -Djavax.net.ssl.trustStorePassword={TLS_STORE_PASSWORD} \ + // -Djavax.net.ssl.trustStoreType=pkcs12 \ + // -Djavax.net.ssl.keyStore={TLS_STORE_DIR}/keystore.p12 \ + // -Djavax.net.ssl.keyStorePassword={TLS_STORE_PASSWORD} \ + // -Djavax.net.ssl.keyStoreType=pkcs12 \ + let mut all_hbase_opts = format!( + "-Djava.security.properties={CONFIG_DIR_NAME}/{JVM_SECURITY_PROPERTIES_FILE} \ + -Djava.security.krb5.conf=/stackable/kerberos/krb5.conf \ + -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={METRICS_PORT}:/stackable/jmx/{role_name}.yaml", + ); if let Some(hbase_opts) = &self.hbase_opts { all_hbase_opts += " "; all_hbase_opts += hbase_opts; @@ -468,15 +523,89 @@ impl HbaseCluster { } } + pub fn has_kerberos_enabled(&self) -> bool { + self.kerberos_secret_class().is_some() + } + + pub fn kerberos_request_node_principals(&self) -> Option { + self.spec + .cluster_config + .kerberos + .as_ref() + .map(|k| k.request_node_principals) + } + + pub fn kerberos_secret_class(&self) -> Option<&str> { + self.spec + .cluster_config + .kerberos + .as_ref() + .map(|k| k.kerberos_secret_class.as_str()) + } + + pub fn has_https_enabled(&self) -> bool { + self.https_secret_class().is_some() + } + + pub fn https_secret_class(&self) -> Option<&str> { + self.spec + .cluster_config + .kerberos + .as_ref() + .map(|k| k.tls_secret_class.as_str()) + } + + /// Returns required port name and port number tuples depending on the role. + pub fn ports(&self, role: &HbaseRole) -> Vec<(String, u16)> { + match role { + HbaseRole::Master => vec![ + ("master".to_string(), HBASE_MASTER_PORT), + if self.has_https_enabled() { + ( + HBASE_UI_PORT_NAME_HTTPS.to_string(), + HBASE_MASTER_UI_PORT_HTTPS, + ) + } else { + ( + HBASE_UI_PORT_NAME_HTTP.to_string(), + HBASE_MASTER_UI_PORT_HTTP, + ) + }, + (METRICS_PORT_NAME.to_string(), METRICS_PORT), + ], + HbaseRole::RegionServer => vec![ + ("regionserver".to_string(), HBASE_REGIONSERVER_PORT), + if self.has_https_enabled() { + ( + HBASE_UI_PORT_NAME_HTTPS.to_string(), + HBASE_REGIONSERVER_UI_PORT_HTTPS, + ) + } else { + ( + HBASE_UI_PORT_NAME_HTTP.to_string(), + HBASE_REGIONSERVER_UI_PORT_HTTP, + ) + }, + (METRICS_PORT_NAME.to_string(), METRICS_PORT), + ], + // TODO: Respect HTTPS settings + HbaseRole::RestServer => vec![ + ("rest".to_string(), HBASE_REST_PORT), + (METRICS_PORT_NAME.to_string(), METRICS_PORT), + ], + } + } + /// Retrieve and merge resource configs for role and role groups pub fn merged_config( &self, + hbase_name: &str, role: &HbaseRole, role_group: &str, hdfs_discovery_cm_name: &str, ) -> Result { // Initialize the result with all default values as baseline - let conf_defaults = role.default_config(&self.name_any(), hdfs_discovery_cm_name); + let conf_defaults = role.default_config(hbase_name, hdfs_discovery_cm_name); let role = self.get_role(role).context(MissingHbaseRoleSnafu { role: role.to_string(), diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 300bb22e..c7ddf158 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -15,10 +15,11 @@ anyhow.workspace = true clap.workspace = true fnv.workspace = true futures.workspace = true +indoc.workspace = true +product-config.workspace = true serde.workspace = true snafu.workspace = true stackable-operator.workspace = true -product-config.workspace = true strum.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 358d6953..d93ce4c6 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -10,16 +10,24 @@ use stackable_operator::{ }; use crate::{ - hbase_controller::build_recommended_labels, zookeeper::ZookeeperConnectionInformation, + hbase_controller::build_recommended_labels, kerberos::kerberos_discovery_config_properties, + zookeeper::ZookeeperConnectionInformation, }; /// Creates a discovery config map containing the `hbase-site.xml` for clients. pub fn build_discovery_configmap( hbase: &HbaseCluster, + hbase_name: &str, + hbase_namespace: &str, zookeeper_connection_information: &ZookeeperConnectionInformation, resolved_product_image: &ResolvedProductImage, ) -> OperatorResult { - let hbase_site = zookeeper_connection_information.as_hbase_settings(); + let mut hbase_site = zookeeper_connection_information.as_hbase_settings(); + hbase_site.extend(kerberos_discovery_config_properties( + hbase, + hbase_name, + hbase_namespace, + )); ConfigMapBuilder::new() .metadata( diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 96b08a92..80911a75 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -5,6 +5,7 @@ use std::{ sync::Arc, }; +use indoc::formatdoc; use product_config::{ types::PropertyNameKind, writer::{to_hadoop_xml, to_java_properties_string, PropertiesWriterError}, @@ -13,9 +14,8 @@ use product_config::{ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_hbase_crd::{ Container, HbaseCluster, HbaseClusterStatus, HbaseConfig, HbaseConfigFragment, HbaseRole, - APP_NAME, CONFIG_DIR_NAME, HBASE_ENV_SH, HBASE_HEAPSIZE, HBASE_MASTER_PORT, - HBASE_REGIONSERVER_PORT, HBASE_REST_PORT, HBASE_SITE_XML, JVM_HEAP_FACTOR, - JVM_SECURITY_PROPERTIES_FILE, + APP_NAME, CONFIG_DIR_NAME, HBASE_ENV_SH, HBASE_HEAPSIZE, HBASE_SITE_XML, JVM_HEAP_FACTOR, + JVM_SECURITY_PROPERTIES_FILE, SSL_CLIENT_XML, SSL_SERVER_XML, }; use stackable_operator::{ builder::{ @@ -31,14 +31,14 @@ use stackable_operator::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMap, ConfigMapVolumeSource, ContainerPort, HTTPGetAction, Probe, Service, - ServicePort, ServiceSpec, TCPSocketAction, Volume, + ConfigMap, ConfigMapVolumeSource, ContainerPort, Probe, Service, ServicePort, + ServiceSpec, TCPSocketAction, Volume, }, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, DeepMerge, }, - kube::{runtime::controller::Action, Resource}, + kube::{runtime::controller::Action, Resource, ResourceExt}, labels::{role_group_selector_labels, role_selector_labels, ObjectLabels}, logging::controller::ReconcilerError, memory::{BinaryMultiple, MemoryQuantity}, @@ -63,6 +63,10 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ discovery::build_discovery_configmap, + kerberos::{ + add_kerberos_pod_config, kerberos_config_properties, kerberos_container_start_commands, + kerberos_ssl_client_settings, kerberos_ssl_server_settings, + }, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, product_logging::{ extend_role_group_config_map, resolve_vector_aggregator_address, LOG4J_CONFIG_FILE, @@ -241,6 +245,9 @@ pub async fn reconcile_hbase(hbase: Arc, ctx: Arc) -> Result< let client = &ctx.client; + let hbase_name = hbase.name_any(); + let hbase_namespace = hbase.namespace().context(ObjectHasNoNamespaceSnafu)?; + let resolved_product_image = hbase .spec .image @@ -284,6 +291,8 @@ pub async fn reconcile_hbase(hbase: Arc, ctx: Arc) -> Result< // discovery config map let discovery_cm = build_discovery_configmap( &hbase, + &hbase_name, + &hbase_namespace, &zookeeper_connection_information, &resolved_product_image, ) @@ -319,6 +328,7 @@ pub async fn reconcile_hbase(hbase: Arc, ctx: Arc) -> Result< let merged_config = hbase .merged_config( + &hbase_name, &hbase_role, &rolegroup.role_group, &hbase.spec.cluster_config.hdfs_config_map_name, @@ -329,6 +339,8 @@ pub async fn reconcile_hbase(hbase: Arc, ctx: Arc) -> Result< build_rolegroup_service(&hbase, &hbase_role, &rolegroup, &resolved_product_image)?; let rg_configmap = build_rolegroup_config_map( &hbase, + &hbase_name, + &hbase_namespace, &rolegroup, rolegroup_config, &zookeeper_connection_information, @@ -338,6 +350,7 @@ pub async fn reconcile_hbase(hbase: Arc, ctx: Arc) -> Result< )?; let rg_statefulset = build_rolegroup_statefulset( &hbase, + &hbase_name, &hbase_role, &rolegroup, &merged_config, @@ -409,13 +422,13 @@ pub fn build_region_server_role_service( let role_svc_name = hbase .server_role_service_name() .context(GlobalServiceNameNotFoundSnafu)?; - let ports = role - .port_properties() + let ports = hbase + .ports(&role) .into_iter() - .map(|(port_name, port_number, port_protocol)| ServicePort { - name: Some(port_name.into()), - port: port_number, - protocol: Some(port_protocol.into()), + .map(|(name, value)| ServicePort { + name: Some(name), + port: i32::from(value), + protocol: Some("TCP".to_string()), ..ServicePort::default() }) .collect(); @@ -444,46 +457,101 @@ pub fn build_region_server_role_service( } /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator +#[allow(clippy::too_many_arguments)] fn build_rolegroup_config_map( hbase: &HbaseCluster, + hbase_name: &str, + hbase_namespace: &str, rolegroup: &RoleGroupRef, rolegroup_config: &HashMap>, zookeeper_connection_information: &ZookeeperConnectionInformation, - config: &HbaseConfig, + hbase_config: &HbaseConfig, resolved_product_image: &ResolvedProductImage, vector_aggregator_address: Option<&str>, ) -> Result { - let mut hbase_site_config = rolegroup_config - .get(&PropertyNameKind::File(HBASE_SITE_XML.to_string())) - .cloned() - .unwrap_or_default(); - - hbase_site_config.extend(zookeeper_connection_information.as_hbase_settings()); - - let mut hbase_env_config = rolegroup_config - .get(&PropertyNameKind::File(HBASE_ENV_SH.to_string())) - .cloned() - .unwrap_or_default(); - - let memory_limit = MemoryQuantity::try_from( - config - .resources - .memory - .limit - .as_ref() - .context(InvalidJavaHeapConfigSnafu)?, - ) - .context(FailedToConvertJavaHeapSnafu { - unit: BinaryMultiple::Mebi.to_java_memory_unit(), - })?; - let heap_in_mebi = (memory_limit * JVM_HEAP_FACTOR) - .scale_to(BinaryMultiple::Mebi) - .format_for_java() - .context(FailedToConvertJavaHeapSnafu { - unit: BinaryMultiple::Mebi.to_java_memory_unit(), - })?; - - hbase_env_config.insert(HBASE_HEAPSIZE.to_string(), heap_in_mebi); + let mut hbase_site_xml = String::new(); + let mut hbase_env_sh = String::new(); + let mut ssl_server_xml = String::new(); + let mut ssl_client_xml = String::new(); + + for (property_name_kind, config) in rolegroup_config { + match property_name_kind { + PropertyNameKind::File(file_name) if file_name == HBASE_SITE_XML => { + let mut hbase_site_config = BTreeMap::new(); + hbase_site_config.extend(zookeeper_connection_information.as_hbase_settings()); + hbase_site_config.extend(kerberos_config_properties( + hbase, + hbase_name, + hbase_namespace, + )); + + // configOverride come last + hbase_site_config.extend(config.clone()); + hbase_site_xml = to_hadoop_xml( + hbase_site_config + .into_iter() + .map(|(k, v)| (k, Some(v))) + .collect::>() + .iter(), + ); + } + PropertyNameKind::File(file_name) if file_name == HBASE_ENV_SH => { + let mut hbase_env_config = BTreeMap::new(); + + let memory_limit = MemoryQuantity::try_from( + hbase_config + .resources + .memory + .limit + .as_ref() + .context(InvalidJavaHeapConfigSnafu)?, + ) + .context(FailedToConvertJavaHeapSnafu { + unit: BinaryMultiple::Mebi.to_java_memory_unit(), + })?; + let heap_in_mebi = (memory_limit * JVM_HEAP_FACTOR) + .scale_to(BinaryMultiple::Mebi) + .format_for_java() + .context(FailedToConvertJavaHeapSnafu { + unit: BinaryMultiple::Mebi.to_java_memory_unit(), + })?; + hbase_env_config.insert(HBASE_HEAPSIZE.to_string(), heap_in_mebi); + + // configOverride come last + hbase_env_config.extend(config.clone()); + hbase_env_sh = write_hbase_env_sh(hbase_env_config.iter()); + } + PropertyNameKind::File(file_name) if file_name == SSL_SERVER_XML => { + let mut config_opts = BTreeMap::new(); + config_opts.extend(kerberos_ssl_server_settings(hbase)); + + // configOverride come last + config_opts.extend(config.clone()); + ssl_server_xml = to_hadoop_xml( + config_opts + .into_iter() + .map(|(k, v)| (k, Some(v))) + .collect::>() + .iter(), + ); + } + PropertyNameKind::File(file_name) if file_name == SSL_CLIENT_XML => { + let mut config_opts = BTreeMap::new(); + config_opts.extend(kerberos_ssl_client_settings(hbase)); + + // configOverride come last + config_opts.extend(config.clone()); + ssl_client_xml = to_hadoop_xml( + config_opts + .into_iter() + .map(|(k, v)| (k, Some(v))) + .collect::>() + .iter(), + ); + } + _ => {} + } + } let jvm_sec_props: BTreeMap> = rolegroup_config .get(&PropertyNameKind::File( @@ -512,17 +580,8 @@ fn build_rolegroup_config_map( )) .build(), ) - .add_data( - HBASE_SITE_XML, - to_hadoop_xml( - hbase_site_config - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect::>() - .iter(), - ), - ) - .add_data(HBASE_ENV_SH, write_hbase_env_sh(hbase_env_config.iter())) + .add_data(HBASE_SITE_XML, hbase_site_xml) + .add_data(HBASE_ENV_SH, hbase_env_sh) .add_data( JVM_SECURITY_PROPERTIES_FILE, to_java_properties_string(jvm_sec_props.iter()).with_context(|_| { @@ -532,10 +591,19 @@ fn build_rolegroup_config_map( })?, ); + // HBase does not like empty config files: + // Caused by: com.ctc.wstx.exc.WstxEOFException: Unexpected EOF in prolog at [row,col,system-id]: [1,0,"file:/stackable/conf/ssl-server.xml"] + if !ssl_server_xml.is_empty() { + builder.add_data(SSL_SERVER_XML, ssl_server_xml); + } + if !ssl_client_xml.is_empty() { + builder.add_data(SSL_CLIENT_XML, ssl_client_xml); + } + extend_role_group_config_map( rolegroup, vector_aggregator_address, - &config.logging, + &hbase_config.logging, &mut builder, ) .context(InvalidLoggingConfigSnafu { @@ -557,13 +625,13 @@ fn build_rolegroup_service( rolegroup: &RoleGroupRef, resolved_product_image: &ResolvedProductImage, ) -> Result { - let ports = hbase_role - .port_properties() + let ports = hbase + .ports(hbase_role) .into_iter() - .map(|(port_name, port_number, port_protocol)| ServicePort { - name: Some(port_name.into()), - port: port_number, - protocol: Some(port_protocol.into()), + .map(|(name, value)| ServicePort { + name: Some(name), + port: i32::from(value), + protocol: Some("TCP".to_string()), ..ServicePort::default() }) .collect(); @@ -605,6 +673,7 @@ fn build_rolegroup_service( /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the corresponding [`Service`] (from [`build_rolegroup_service`]). fn build_rolegroup_statefulset( hbase: &HbaseCluster, + hbase_name: &str, hbase_role: &HbaseRole, rolegroup_ref: &RoleGroupRef, config: &HbaseConfig, @@ -616,13 +685,13 @@ fn build_rolegroup_statefulset( let role = hbase.get_role(hbase_role); let role_group = role.and_then(|r| r.role_groups.get(&rolegroup_ref.role_group)); - let ports = hbase_role - .port_properties() + let ports = hbase + .ports(hbase_role) .into_iter() - .map(|(port_name, port_number, port_protocol)| ContainerPort { - name: Some(port_name.into()), - container_port: port_number, - protocol: Some(port_protocol.into()), + .map(|(name, value)| ContainerPort { + name: Some(name), + container_port: i32::from(value), + protocol: Some("TCP".to_string()), ..ContainerPort::default() }) .collect(); @@ -630,22 +699,32 @@ fn build_rolegroup_statefulset( let probe_template = match hbase_role { HbaseRole::Master => Probe { tcp_socket: Some(TCPSocketAction { - port: IntOrString::Int(HBASE_MASTER_PORT), + port: IntOrString::String("master".to_string()), ..TCPSocketAction::default() }), ..Probe::default() }, HbaseRole::RegionServer => Probe { tcp_socket: Some(TCPSocketAction { - port: IntOrString::Int(HBASE_REGIONSERVER_PORT), + port: IntOrString::String("regionserver".to_string()), ..TCPSocketAction::default() }), ..Probe::default() }, HbaseRole::RestServer => Probe { - http_get: Some(HTTPGetAction { - port: IntOrString::Int(HBASE_REST_PORT), - ..HTTPGetAction::default() + // We cant use this, as it returns a 401 in case kerberos is enabled. + // http_get: Some(HTTPGetAction { + // port: IntOrString::String("rest".to_string()), + // scheme: Some(if hbase.has_https_enabled() { + // "HTTPS".to_string() + // } else { + // "HTTP".to_string() + // }), + // ..HTTPGetAction::default() + // }), + tcp_socket: Some(TCPSocketAction { + port: IntOrString::String("rest".to_string()), + ..TCPSocketAction::default() }), ..Probe::default() }, @@ -671,8 +750,9 @@ fn build_rolegroup_statefulset( ..probe_template }; - let container = ContainerBuilder::new("hbase") - .expect("ContainerBuilder not created") + let mut container_builder = + ContainerBuilder::new("hbase").expect("ContainerBuilder not created"); + container_builder .image_from_product_image(resolved_product_image) .command(vec![ "/bin/bash".to_string(), @@ -681,21 +761,22 @@ fn build_rolegroup_statefulset( "pipefail".to_string(), "-c".to_string(), ]) - .args(vec![format!( - "\ -mkdir -p {CONFIG_DIR_NAME} -cp {HDFS_DISCOVERY_TMP_DIR}/hdfs-site.xml {CONFIG_DIR_NAME} -cp {HDFS_DISCOVERY_TMP_DIR}/core-site.xml {CONFIG_DIR_NAME} -cp {HBASE_CONFIG_TMP_DIR}/* {CONFIG_DIR_NAME} -cp {HBASE_LOG_CONFIG_TMP_DIR}/{LOG4J_CONFIG_FILE} {CONFIG_DIR_NAME} - -{COMMON_BASH_TRAP_FUNCTIONS} -{remove_vector_shutdown_file_command} -prepare_signal_handlers -bin/hbase {hbase_role_name_in_command} start & -wait_for_termination $! -{create_vector_shutdown_file_command} -", + .args(vec![formatdoc! {" + mkdir -p {CONFIG_DIR_NAME} + cp {HDFS_DISCOVERY_TMP_DIR}/hdfs-site.xml {CONFIG_DIR_NAME} + cp {HDFS_DISCOVERY_TMP_DIR}/core-site.xml {CONFIG_DIR_NAME} + cp {HBASE_CONFIG_TMP_DIR}/* {CONFIG_DIR_NAME} + cp {HBASE_LOG_CONFIG_TMP_DIR}/{LOG4J_CONFIG_FILE} {CONFIG_DIR_NAME} + + {kerberos_container_start_commands} + + {COMMON_BASH_TRAP_FUNCTIONS} + {remove_vector_shutdown_file_command} + prepare_signal_handlers + bin/hbase {hbase_role_name_in_command} start & + wait_for_termination $! + {create_vector_shutdown_file_command} + ", hbase_role_name_in_command = match hbase_role { HbaseRole::Master => "master", HbaseRole::RegionServer => "regionserver", @@ -703,11 +784,12 @@ wait_for_termination $! // instead of just letting the Display impl do it's thing ;P HbaseRole::RestServer => "rest", }, + kerberos_container_start_commands = kerberos_container_start_commands(hbase), remove_vector_shutdown_file_command = remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), create_vector_shutdown_file_command = create_vector_shutdown_file_command(STACKABLE_LOG_DIR), - )]) + }]) .add_env_var("HBASE_CONF_DIR", CONFIG_DIR_NAME) // required by phoenix (for cases where Kerberos is enabled): see https://issues.apache.org/jira/browse/PHOENIX-2369 .add_env_var("HADOOP_CONF_DIR", CONFIG_DIR_NAME) @@ -719,8 +801,7 @@ wait_for_termination $! .resources(config.resources.clone().into()) .startup_probe(startup_probe) .liveness_probe(liveness_probe) - .readiness_probe(readiness_probe) - .build(); + .readiness_probe(readiness_probe); let mut pod_builder = PodBuilder::new(); pod_builder @@ -734,7 +815,6 @@ wait_for_termination $! }) .image_pull_secrets_from_product_image(resolved_product_image) .affinity(&config.affinity) - .add_container(container) .add_volume(stackable_operator::k8s_openapi::api::core::v1::Volume { name: "hbase-config".to_string(), config_map: Some(ConfigMapVolumeSource { @@ -808,7 +888,17 @@ wait_for_termination $! } add_graceful_shutdown_config(config, &mut pod_builder).context(GracefulShutdownSnafu)?; + if hbase.has_kerberos_enabled() { + add_kerberos_pod_config( + hbase, + hbase_name, + hbase_role, + &mut container_builder, + &mut pod_builder, + ); + } + pod_builder.add_container(container_builder.build()); let mut pod_template = pod_builder.build_template(); if let Some(role) = role { pod_template.merge_from(role.config.pod_overrides.clone()); @@ -858,6 +948,8 @@ fn build_roles( let config_types = vec![ PropertyNameKind::File(HBASE_ENV_SH.to_string()), PropertyNameKind::File(HBASE_SITE_XML.to_string()), + PropertyNameKind::File(SSL_SERVER_XML.to_string()), + PropertyNameKind::File(SSL_CLIENT_XML.to_string()), PropertyNameKind::File(JVM_SECURITY_PROPERTIES_FILE.to_string()), ]; diff --git a/rust/operator-binary/src/kerberos.rs b/rust/operator-binary/src/kerberos.rs new file mode 100644 index 00000000..772f9f31 --- /dev/null +++ b/rust/operator-binary/src/kerberos.rs @@ -0,0 +1,270 @@ +use std::collections::BTreeMap; + +use indoc::formatdoc; +use stackable_hbase_crd::{ + HbaseCluster, HbaseRole, CONFIG_DIR_NAME, HBASE_MASTER_UI_PORT_HTTPS, + HBASE_REGIONSERVER_UI_PORT_HTTPS, TLS_STORE_DIR, TLS_STORE_PASSWORD, TLS_STORE_VOLUME_NAME, +}; +use stackable_operator::builder::{ + ContainerBuilder, PodBuilder, SecretFormat, SecretOperatorVolumeSourceBuilder, VolumeBuilder, +}; + +pub fn kerberos_config_properties( + hbase: &HbaseCluster, + hbase_name: &str, + hbase_namespace: &str, +) -> BTreeMap { + if !hbase.has_kerberos_enabled() { + return BTreeMap::new(); + } + + let principal_host_part = principal_host_part(hbase_name, hbase_namespace); + + BTreeMap::from([ + // Kerberos settings + ( + "hbase.security.authentication".to_string(), + "kerberos".to_string(), + ), + ( + "hbase.security.authorization".to_string(), + "true".to_string(), + ), + ("hbase.rpc.protection".to_string(), "privacy".to_string()), + ( + "dfs.data.transfer.protection".to_string(), + "privacy".to_string(), + ), + ( + "hbase.rpc.engine".to_string(), + "org.apache.hadoop.hbase.ipc.SecureRpcEngine".to_string(), + ), + ( + "hbase.master.kerberos.principal".to_string(), + format!( + "{service_name}/{principal_host_part}", + service_name = HbaseRole::Master.kerberos_service_name() + ), + ), + ( + "hbase.regionserver.kerberos.principal".to_string(), + format!( + "{service_name}/{principal_host_part}", + service_name = HbaseRole::RegionServer.kerberos_service_name() + ), + ), + ( + "hbase.rest.kerberos.principal".to_string(), + format!( + "{service_name}/{principal_host_part}", + service_name = HbaseRole::RestServer.kerberos_service_name() + ), + ), + ( + "hbase.master.keytab.file".to_string(), + "/stackable/kerberos/keytab".to_string(), + ), + ( + "hbase.regionserver.keytab.file".to_string(), + "/stackable/kerberos/keytab".to_string(), + ), + ( + "hbase.rest.keytab.file".to_string(), + "/stackable/kerberos/keytab".to_string(), + ), + ( + "hbase.coprocessor.master.classes".to_string(), + "org.apache.hadoop.hbase.security.access.AccessController".to_string(), + ), + ( + "hbase.coprocessor.region.classes".to_string(), + "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.AccessController".to_string(), + ), + + // Rest + ("hbase.rest.authentication.type".to_string(), "kerberos".to_string()), + ("hbase.rest.authentication.kerberos.principal".to_string(), format!( + "HTTP/{principal_host_part}" + )), + ("hbase.rest.authentication.kerberos.keytab".to_string(), "/stackable/kerberos/keytab".to_string()), + + // Enabled https as well + ("hbase.ssl.enabled".to_string(), "true".to_string()), + ("hbase.http.policy".to_string(), "HTTPS_ONLY".to_string()), + // Recommended by the docs https://hbase.apache.org/book.html#hbase.ui.cache + ("hbase.http.filter.no-store.enable".to_string(), "true".to_string()), + // key and truststore come from ssl-server.xml and ssl-client.xml + + // Rest + ("hbase.rest.ssl.enabled".to_string(), "true".to_string()), + ("hbase.rest.ssl.keystore.store".to_string(), format!("{TLS_STORE_DIR}/keystore.p12")), + ("hbase.rest.ssl.keystore.password".to_string(), TLS_STORE_PASSWORD.to_string()), + ("hbase.rest.ssl.keystore.type".to_string(), "pkcs12".to_string()), + // TODO: Check if needed + // ("hbase.rest.support.proxyuser".to_string(), "true".to_string()), + + // Set non-default ports when https is enabled + ("hbase.master.info.port".to_string(), HBASE_MASTER_UI_PORT_HTTPS.to_string()), + ("hbase.regionserver.info.port".to_string(), HBASE_REGIONSERVER_UI_PORT_HTTPS.to_string()), + ]) +} + +pub fn kerberos_discovery_config_properties( + hbase: &HbaseCluster, + hbase_name: &str, + hbase_namespace: &str, +) -> BTreeMap { + if !hbase.has_kerberos_enabled() { + return BTreeMap::new(); + } + + let principal_host_part = principal_host_part(hbase_name, hbase_namespace); + + BTreeMap::from([ + ( + "hbase.security.authentication".to_string(), + "kerberos".to_string(), + ), + ("hbase.rpc.protection".to_string(), "privacy".to_string()), + ("hbase.ssl.enabled".to_string(), "true".to_string()), + ( + "hbase.master.kerberos.principal".to_string(), + format!( + "{service_name}/{principal_host_part}", + service_name = HbaseRole::Master.kerberos_service_name() + ), + ), + ( + "hbase.regionserver.kerberos.principal".to_string(), + format!( + "{service_name}/{principal_host_part}", + service_name = HbaseRole::RegionServer.kerberos_service_name() + ), + ), + ( + "hbase.rest.kerberos.principal".to_string(), + format!( + "{service_name}/{principal_host_part}", + service_name = HbaseRole::RestServer.kerberos_service_name() + ), + ), + ]) +} + +pub fn kerberos_ssl_server_settings(hbase: &HbaseCluster) -> BTreeMap { + if !hbase.has_https_enabled() { + return BTreeMap::new(); + } + + BTreeMap::from([ + ( + "ssl.server.truststore.location".to_string(), + format!("{TLS_STORE_DIR}/truststore.p12"), + ), + ( + "ssl.server.truststore.type".to_string(), + "pkcs12".to_string(), + ), + ( + "ssl.server.truststore.password".to_string(), + TLS_STORE_PASSWORD.to_string(), + ), + ( + "ssl.server.keystore.location".to_string(), + format!("{TLS_STORE_DIR}/keystore.p12"), + ), + ("ssl.server.keystore.type".to_string(), "pkcs12".to_string()), + ( + "ssl.server.keystore.password".to_string(), + TLS_STORE_PASSWORD.to_string(), + ), + ]) +} + +pub fn kerberos_ssl_client_settings(hbase: &HbaseCluster) -> BTreeMap { + if !hbase.has_https_enabled() { + return BTreeMap::new(); + } + + BTreeMap::from([ + ( + "ssl.client.truststore.location".to_string(), + format!("{TLS_STORE_DIR}/truststore.p12"), + ), + ( + "ssl.client.truststore.type".to_string(), + "pkcs12".to_string(), + ), + ( + "ssl.client.truststore.password".to_string(), + TLS_STORE_PASSWORD.to_string(), + ), + ]) +} + +pub fn add_kerberos_pod_config( + hbase: &HbaseCluster, + hbase_name: &str, + role: &HbaseRole, + cb: &mut ContainerBuilder, + pb: &mut PodBuilder, +) { + if let Some(kerberos_secret_class) = hbase.kerberos_secret_class() { + // Keytab + let mut kerberos_secret_operator_volume_builder = + SecretOperatorVolumeSourceBuilder::new(kerberos_secret_class); + kerberos_secret_operator_volume_builder + .with_service_scope(hbase_name) + .with_kerberos_service_name(role.kerberos_service_name()) + .with_kerberos_service_name("HTTP"); + if let Some(true) = hbase.kerberos_request_node_principals() { + kerberos_secret_operator_volume_builder.with_node_scope(); + } + pb.add_volume( + VolumeBuilder::new("kerberos") + .ephemeral(kerberos_secret_operator_volume_builder.build()) + .build(), + ); + cb.add_volume_mount("kerberos", "/stackable/kerberos"); + + // Needed env vars + cb.add_env_var("KRB5_CONFIG", "/stackable/kerberos/krb5.conf"); + // This env var does not only affect the servers, but also the hbase shell + cb.add_env_var( + "HBASE_OPTS", + "-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf", + ); + } + + if let Some(https_secret_class) = hbase.https_secret_class() { + // TLS certs + pb.add_volume( + VolumeBuilder::new(TLS_STORE_VOLUME_NAME) + .ephemeral( + SecretOperatorVolumeSourceBuilder::new(https_secret_class) + .with_pod_scope() + .with_node_scope() + .with_format(SecretFormat::TlsPkcs12) + .with_tls_pkcs12_password(TLS_STORE_PASSWORD) + .build(), + ) + .build(), + ); + cb.add_volume_mount(TLS_STORE_VOLUME_NAME, TLS_STORE_DIR); + } +} + +pub fn kerberos_container_start_commands(hbase: &HbaseCluster) -> String { + if !hbase.has_kerberos_enabled() { + return String::new(); + } + + formatdoc! {" + export KERBEROS_REALM=$(grep -oP 'default_realm = \\K.*' /stackable/kerberos/krb5.conf) + sed -i -e 's/${{env.KERBEROS_REALM}}/'\"$KERBEROS_REALM/g\" {CONFIG_DIR_NAME}/hbase-site.xml", + } +} + +fn principal_host_part(hbase_name: &str, hbase_namespace: &str) -> String { + format!("{hbase_name}.{hbase_namespace}.svc.cluster.local@${{env.KERBEROS_REALM}}") +} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 095ead8c..49f7a058 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -1,5 +1,6 @@ mod discovery; mod hbase_controller; +mod kerberos; mod operations; mod product_logging; mod zookeeper; diff --git a/tests/templates/kuttl/kerberos/00-assert.yaml.j2 b/tests/templates/kuttl/kerberos/00-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/kerberos/00-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/kerberos/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/kerberos/00-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/kerberos/00-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/kerberos/01-assert.yaml.j2 b/tests/templates/kuttl/kerberos/01-assert.yaml.j2 new file mode 100644 index 00000000..d34c1c63 --- /dev/null +++ b/tests/templates/kuttl/kerberos/01-assert.yaml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +{% if test_scenario['values']['kerberos-backend'] == 'mit' %} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: krb5-kdc +status: + readyReplicas: 1 + replicas: 1 +{% endif %} diff --git a/tests/templates/kuttl/kerberos/01-install-krb5-kdc.yaml.j2 b/tests/templates/kuttl/kerberos/01-install-krb5-kdc.yaml.j2 new file mode 100644 index 00000000..d8070ec6 --- /dev/null +++ b/tests/templates/kuttl/kerberos/01-install-krb5-kdc.yaml.j2 @@ -0,0 +1,135 @@ +{% if test_scenario['values']['kerberos-backend'] == 'mit' %} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: krb5-kdc +spec: + selector: + matchLabels: + app: krb5-kdc + template: + metadata: + labels: + app: krb5-kdc + spec: + initContainers: + - name: init + image: docker.stackable.tech/stackable/krb5:1.18.2-stackable0.0.0-dev + args: + - sh + - -euo + - pipefail + - -c + - | + test -e /var/kerberos/krb5kdc/principal || kdb5_util create -s -P asdf + kadmin.local get_principal -terse root/admin || kadmin.local add_principal -pw asdf root/admin + # stackable-secret-operator principal must match the keytab specified in the SecretClass + kadmin.local get_principal -terse stackable-secret-operator || kadmin.local add_principal -e aes256-cts-hmac-sha384-192:normal -pw asdf stackable-secret-operator + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + - mountPath: /var/kerberos/krb5kdc + name: data + containers: + - name: kdc + image: docker.stackable.tech/stackable/krb5:1.18.2-stackable0.0.0-dev + args: + - krb5kdc + - -n + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + - mountPath: /var/kerberos/krb5kdc + name: data + - name: kadmind + image: docker.stackable.tech/stackable/krb5:1.18.2-stackable0.0.0-dev + args: + - kadmind + - -nofork + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + - mountPath: /var/kerberos/krb5kdc + name: data + - name: client + image: docker.stackable.tech/stackable/krb5:1.18.2-stackable0.0.0-dev + tty: true + stdin: true + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + volumes: + - name: config + configMap: + name: krb5-kdc + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: krb5-kdc +spec: + selector: + app: krb5-kdc + ports: + - name: kadmin + port: 749 + - name: kdc + port: 88 + - name: kdc-udp + port: 88 + protocol: UDP +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: krb5-kdc +data: + krb5.conf: | + [logging] + default = STDERR + kdc = STDERR + admin_server = STDERR + # default = FILE:/var/log/krb5libs.log + # kdc = FILE:/var/log/krb5kdc.log + # admin_server = FILE:/vaggr/log/kadmind.log + [libdefaults] + dns_lookup_realm = false + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + rdns = false + default_realm = {{ test_scenario['values']['kerberos-realm'] }} + spake_preauth_groups = edwards25519 + [realms] + {{ test_scenario['values']['kerberos-realm'] }} = { + acl_file = /stackable/config/kadm5.acl + disable_encrypted_timestamp = false + } + [domain_realm] + .cluster.local = {{ test_scenario['values']['kerberos-realm'] }} + cluster.local = {{ test_scenario['values']['kerberos-realm'] }} + kadm5.acl: | + root/admin *e + stackable-secret-operator *e +{% endif %} diff --git a/tests/templates/kuttl/kerberos/02-create-kerberos-secretclass.yaml.j2 b/tests/templates/kuttl/kerberos/02-create-kerberos-secretclass.yaml.j2 new file mode 100644 index 00000000..04ae9a63 --- /dev/null +++ b/tests/templates/kuttl/kerberos/02-create-kerberos-secretclass.yaml.j2 @@ -0,0 +1,72 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl apply -n $NAMESPACE -f - < 0 }} + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/kerberos/11-assert.yaml b/tests/templates/kuttl/kerberos/11-assert.yaml new file mode 100644 index 00000000..47260144 --- /dev/null +++ b/tests/templates/kuttl/kerberos/11-assert.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hdfs-namenode-default +status: + readyReplicas: 2 + replicas: 2 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hdfs-journalnode-default +status: + readyReplicas: 3 + replicas: 3 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hdfs-datanode-default +status: + readyReplicas: 2 + replicas: 2 diff --git a/tests/templates/kuttl/kerberos/11-install-hdfs.yaml.j2 b/tests/templates/kuttl/kerberos/11-install-hdfs.yaml.j2 new file mode 100644 index 00000000..a7a161bb --- /dev/null +++ b/tests/templates/kuttl/kerberos/11-install-hdfs.yaml.j2 @@ -0,0 +1,60 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl apply -n $NAMESPACE -f - < 0 }} + configOverrides: &configOverrides + core-site.xml: + # hadoop.user.group.static.mapping.overrides: "dr.who=;nn=;nm=;jn=;testuser=supergroup;hbase-master=supergroup;hbase-regionserver=supergroup;hbase-restserver=supergroup;" + hadoop.user.group.static.mapping.overrides: "dr.who=;nn=;nm=;jn=;testuser=supergroup;hbase=supergroup" + roleGroups: + default: + replicas: 2 + dataNodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + configOverrides: *configOverrides + roleGroups: + default: + replicas: 2 + journalNodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + configOverrides: *configOverrides + roleGroups: + default: + replicas: 3 + EOF diff --git a/tests/templates/kuttl/kerberos/20-access-hdfs.yaml.j2 b/tests/templates/kuttl/kerberos/20-access-hdfs.yaml.j2 new file mode 100644 index 00000000..c98103f4 --- /dev/null +++ b/tests/templates/kuttl/kerberos/20-access-hdfs.yaml.j2 @@ -0,0 +1,68 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl apply -n $NAMESPACE -f - < 0 }} + roleGroups: + default: + replicas: 2 + regionServers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 2 + restServers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 + EOF diff --git a/tests/templates/kuttl/kerberos/40-access-hbase.j2 b/tests/templates/kuttl/kerberos/40-access-hbase.j2 new file mode 100644 index 00000000..543f52d0 --- /dev/null +++ b/tests/templates/kuttl/kerberos/40-access-hbase.j2 @@ -0,0 +1,70 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl apply -n $NAMESPACE -f - < 'benchmark', IN_MEMORY => false, TTL => 2147483647 , BLOOMFILTER => 'NONE', COMPRESSION => 'NONE'}, SPLITS=> ['user1000000000000000000', 'user2000000000000000000', 'user3000000000000000000', 'user4000000000000000000', 'user5000000000000000000', 'user6000000000000000000', 'user7000000000000000000', 'user8000000000000000000', 'user9000000000000000000'] +# /stackable/ycsb-0.17.0/bin/ycsb.sh load hbase20 -p table=sbernauer -p columnfamily=benchmark -p recordcount=100000 -p fieldcount=10 -p fieldlength=100 -p workload=site.ycsb.workloads.CoreWorkload -threads 10 -s +# /stackable/ycsb-0.17.0/bin/ycsb.sh run hbase20 -p operationcount=100000 -p table=sbernauer -p columnfamily=benchmark -P /stackable/ycsb-0.17.0//workloads/workloada -s -threads 100 -target 50000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ycsb + labels: + app: ycsb +spec: + replicas: 1 + selector: + matchLabels: + app: ycsb + template: + metadata: + labels: + app: ycsb + spec: + containers: + - name: ycsb + image: docker.stackable.tech/soenkeliebau/ycsb + imagePullPolicy: Always + # resources: + # limits: + # cpu: "8" + # memory: 26Gi + # requests: + # cpu: "7" + # memory: 26Gi + env: + - name: KRB5_CONFIG + value: /stackable/kerberos/krb5.conf + - name: HBASE_OPTS + value: -Djava.security.krb5.conf=/stackable/kerberos/krb5.conf + - name: JAVA_OPTS # As HBASE_OPTS does not get picked up + value: -Djava.security.krb5.conf=/stackable/kerberos/krb5.conf + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + command: + - /bin/bash + - -c + - | + set -ex + microdnf install krb5-workstation openssl + + export KERBEROS_REALM=$(grep -oP 'default_realm = \K.*' /stackable/kerberos/krb5.conf) + mkdir -p /stackable/ycsb-0.17.0/conf/ + cat /tmp/hbase-site.xml | sed -e 's/${{env.KERBEROS_REALM}}/'"$KERBEROS_REALM/g" > /stackable/ycsb-0.17.0/conf/hbase-site.xml + + klist -k /stackable/kerberos/keytab + kinit -kt /stackable/kerberos/keytab hbase/access-hbase.${NAMESPACE}.svc.cluster.local + klist + sleep infinity + volumeMounts: + - mountPath: /tmp/hbase-site.xml + name: config-volume-hbase + subPath: hbase-site.xml + - name: kerberos + mountPath: /stackable/kerberos + tolerations: + - key: "app" + operator: "Equal" + value: "ycsb" + effect: "NoSchedule" + volumes: + - name: config-volume-hbase + configMap: + name: hbase + - name: kerberos + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: kerberos-kuttl-test-capital-liger # TODO adopt + secrets.stackable.tech/scope: service=access-hbase + secrets.stackable.tech/kerberos.service.names: hbase # TOOD use testuser + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + securityContext: + runAsUser: 0 +# --- +# apiVersion: v1 +# kind: ConfigMap +# metadata: +# name: hbase-discovery +# data: +# hbase-site.xml: |- +# +# +# +# hbase.master.kerberos.principal +# hbase/hbase.kuttl-test-daring-mammal.svc.cluster.local@REALM_IS_PICKED_FROM_DEFAULT_REALM +# +# +# hbase.regionserver.kerberos.principal +# hbase/hbase.kuttl-test-daring-mammal.svc.cluster.local@REALM_IS_PICKED_FROM_DEFAULT_REALM +# +# +# hbase.rpc.protection +# privacy +# +# +# hbase.security.authentication +# kerberos +# +# +# hbase.ssl.enabled +# true +# +# +# hbase.zookeeper.property.clientPort +# 2282 +# +# +# hbase.zookeeper.quorum +# zookeeper-server-default-0.zookeeper-server-default.kuttl-test-daring-mammal.svc.cluster.local:2282 +# +# +# zookeeper.znode.parent +# /znode-49b43377-5cda-4b8f-a8e6-66c0a1f4def4/hbase +# +# +# log4j.properties: |+ +# log4j.rootLogger=DEBUG, CONSOLE, FILE + +# log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +# log4j.appender.CONSOLE.Threshold=DEBUG +# log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +# log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %.1000m%n From fcbda97779daef2994285bd27c4bb0a5cd3356fd Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 9 Jan 2024 10:54:01 +0100 Subject: [PATCH 02/34] Use resolver 2 --- Cargo.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6becff71..62a000c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,6 @@ [workspace] -members = [ - "rust/crd", "rust/operator-binary" -] +members = ["rust/crd", "rust/operator-binary"] +resolver = "2" [workspace.package] version = "0.0.0-dev" From a5562145f5db8d9f36b332fe96a1bb44aa62e574 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 9 Jan 2024 10:57:58 +0100 Subject: [PATCH 03/34] fix clippy lint --- rust/operator-binary/src/hbase_controller.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 80911a75..ec2d240a 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -1,6 +1,7 @@ //! Ensures that `Pod`s are configured and running for each [`HbaseCluster`] use std::{ collections::{BTreeMap, HashMap}, + fmt::Write, str::FromStr, sync::Arc, }; @@ -990,9 +991,12 @@ fn write_hbase_env_sh<'a, T>(properties: T) -> String where T: Iterator, { - properties - .map(|(variable, value)| format!("export {variable}=\"{value}\"\n")) - .collect() + let mut env_sh = String::new(); + properties.for_each(|(variable, value)| { + let _ = writeln!(env_sh, "export {variable}=\"{value}\""); + }); + + env_sh } pub fn error_policy(_obj: Arc, _error: &Error, _ctx: Arc) -> Action { From 6e3e89869e88c5d5b13418bd1842b591c6d1e40d Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 9 Jan 2024 15:41:40 +0100 Subject: [PATCH 04/34] Improve test and let it pass --- .../kuttl/kerberos/40-access-hbase.j2 | 70 ----------- .../kerberos/40-grant-testuser-access.yaml | 6 + .../kuttl/kerberos/41-access-hbase.j2 | 113 ++++++++++++++++++ .../{40-assert.yaml => 41-assert.yaml} | 0 4 files changed, 119 insertions(+), 70 deletions(-) delete mode 100644 tests/templates/kuttl/kerberos/40-access-hbase.j2 create mode 100644 tests/templates/kuttl/kerberos/40-grant-testuser-access.yaml create mode 100644 tests/templates/kuttl/kerberos/41-access-hbase.j2 rename tests/templates/kuttl/kerberos/{40-assert.yaml => 41-assert.yaml} (100%) diff --git a/tests/templates/kuttl/kerberos/40-access-hbase.j2 b/tests/templates/kuttl/kerberos/40-access-hbase.j2 deleted file mode 100644 index 543f52d0..00000000 --- a/tests/templates/kuttl/kerberos/40-access-hbase.j2 +++ /dev/null @@ -1,70 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: | - kubectl apply -n $NAMESPACE -f - < /stackable/conf/hbase/hbase-site.xml + cat /stackable/conf/hdfs_mount/core-site.xml | sed -e 's/${env.KERBEROS_REALM}/'"$KERBEROS_REALM/g" > /stackable/conf/hbase/core-site.xml + cat /stackable/conf/hdfs_mount/hdfs-site.xml | sed -e 's/${env.KERBEROS_REALM}/'"$KERBEROS_REALM/g" > /stackable/conf/hbase/hdfs-site.xml + + cat > /tmp/hbase-script << 'EOF' + disable 'test'; + drop 'test'; + + create 'test', 'cf1'; + put 'test', 'row1', 'cf1', 42; + put 'test', 'row2', 'cf1', 43; + put 'test', 'row3', 'cf1', 44; + scan 'test'; + count 'test'; + EOF + + if bin/hbase shell /tmp/hbase-script | grep -q '=> 3'; then + echo "test passed" + exit 0 + else + echo "test failed" + echo "Lets re-run to get command output:" + bin/hbase shell /tmp/hbase-script + exit 1 + fi diff --git a/tests/templates/kuttl/kerberos/40-assert.yaml b/tests/templates/kuttl/kerberos/41-assert.yaml similarity index 100% rename from tests/templates/kuttl/kerberos/40-assert.yaml rename to tests/templates/kuttl/kerberos/41-assert.yaml From 92a9063bd617e9692c9d458af2cecc38ddbd1edb Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 10 Jan 2024 13:16:47 +0100 Subject: [PATCH 05/34] Improve test output --- tests/templates/kuttl/kerberos/41-access-hbase.j2 | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/templates/kuttl/kerberos/41-access-hbase.j2 b/tests/templates/kuttl/kerberos/41-access-hbase.j2 index ea1ee963..6811e17f 100644 --- a/tests/templates/kuttl/kerberos/41-access-hbase.j2 +++ b/tests/templates/kuttl/kerberos/41-access-hbase.j2 @@ -100,14 +100,13 @@ data: put 'test', 'row3', 'cf1', 44; scan 'test'; count 'test'; + exit; EOF - if bin/hbase shell /tmp/hbase-script | grep -q '=> 3'; then - echo "test passed" + if bin/hbase shell /tmp/hbase-script | tee /dev/stderr | grep -q '=> 3'; then + echo "Test passed" exit 0 else - echo "test failed" - echo "Lets re-run to get command output:" - bin/hbase shell /tmp/hbase-script + echo "Test failed" exit 1 fi From 8bea54203a4759f7e0b4afe9bd32b7290cef130c Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 10 Jan 2024 13:33:03 +0100 Subject: [PATCH 06/34] revert tests/test-definition.yaml --- tests/test-definition.yaml | 62 ++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 7f151a33..aa418b97 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -2,14 +2,14 @@ dimensions: - name: hbase values: - # - 2.4.12 + - 2.4.12 - 2.4.17 - name: hbase-latest values: - 2.4.17 - name: hdfs values: - # - 3.2.4 + - 3.2.4 - 3.3.6 - name: hdfs-latest values: @@ -23,12 +23,12 @@ dimensions: # Used for zookeeper, hdfs and hbase - name: listener-class values: - # - "cluster-internal" + - "cluster-internal" - "external-unstable" - name: kerberos-realm values: - "CLUSTER.LOCAL" - # - "PROD.MYCORP" + - "PROD.MYCORP" - name: kerberos-backend values: - mit @@ -36,12 +36,12 @@ dimensions: # This will *not* respect the kerberos-realm test attribute, but instead use a hard-coded realm # - activeDirectory tests: - # - name: smoke - # dimensions: - # - hbase - # - hdfs - # - zookeeper - # - listener-class + - name: smoke + dimensions: + - hbase + - hdfs + - zookeeper + - listener-class - name: kerberos dimensions: - hbase @@ -50,26 +50,26 @@ tests: - listener-class - kerberos-realm - kerberos-backend - # - name: orphaned_resources - # dimensions: - # - hbase-latest - # - hdfs-latest - # - zookeeper-latest - # - name: resources - # dimensions: - # - hbase-latest - # - hdfs-latest - # - zookeeper-latest - # - name: logging - # dimensions: - # - hbase - # - hdfs-latest - # - zookeeper-latest - # - name: cluster-operation - # dimensions: - # - hbase-latest - # - hdfs-latest - # - zookeeper-latest + - name: orphaned_resources + dimensions: + - hbase-latest + - hdfs-latest + - zookeeper-latest + - name: resources + dimensions: + - hbase-latest + - hdfs-latest + - zookeeper-latest + - name: logging + dimensions: + - hbase + - hdfs-latest + - zookeeper-latest + - name: cluster-operation + dimensions: + - hbase-latest + - hdfs-latest + - zookeeper-latest suites: - name: nightly patch: @@ -80,6 +80,8 @@ suites: expr: last - name: hdfs expr: last + - name: kerberos-realm + expr: last - name: smoke-latest select: - smoke From a8d2156b67f164936dd70511c07b6e54d96dc082 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 10 Jan 2024 14:54:27 +0100 Subject: [PATCH 07/34] (hopefully) fix tests --- tests/templates/kuttl/kerberos/40-grant-testuser-access.yaml | 4 ++-- tests/templates/kuttl/smoke/60-assert.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/templates/kuttl/kerberos/40-grant-testuser-access.yaml b/tests/templates/kuttl/kerberos/40-grant-testuser-access.yaml index b9586440..4e8ae913 100644 --- a/tests/templates/kuttl/kerberos/40-grant-testuser-access.yaml +++ b/tests/templates/kuttl/kerberos/40-grant-testuser-access.yaml @@ -2,5 +2,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: kubectl exec -n $NAMESPACE -it hbase-master-default-0 -- bash -c 'klist -kt /stackable/kerberos/keytab && kinit -kt /stackable/kerberos/keytab hbase/hbase.'"$NAMESPACE"'.svc.cluster.local && klist' - - script: kubectl exec -n $NAMESPACE -it hbase-master-default-0 -- bash -c "echo -e \"grant 'testuser', 'C';\nexit;\" | bin/hbase shell" + - script: kubectl exec -n $NAMESPACE -it hbase-master-default-0 -c hbase -- bash -c 'klist -kt /stackable/kerberos/keytab && kinit -kt /stackable/kerberos/keytab hbase/hbase.'"$NAMESPACE"'.svc.cluster.local && klist' + - script: kubectl exec -n $NAMESPACE -it hbase-master-default-0 -c hbase -- bash -c "echo -e \"grant 'testuser', 'C';\nexit;\" | bin/hbase shell" diff --git a/tests/templates/kuttl/smoke/60-assert.yaml b/tests/templates/kuttl/smoke/60-assert.yaml index c490693d..f9176864 100644 --- a/tests/templates/kuttl/smoke/60-assert.yaml +++ b/tests/templates/kuttl/smoke/60-assert.yaml @@ -4,6 +4,6 @@ kind: TestAssert metadata: name: test-phoenix-access commands: - - script: kubectl exec --namespace=$NAMESPACE test-hbase-restserver-default-0 -- chmod +x /tmp/test_phoenix.sh - - script: kubectl exec --namespace=$NAMESPACE test-hbase-restserver-default-0 -- /tmp/test_phoenix.sh + - script: kubectl exec --namespace=$NAMESPACE -c hbase test-hbase-restserver-default-0 -- chmod +x /tmp/test_phoenix.sh + - script: kubectl exec --namespace=$NAMESPACE -c hbase test-hbase-restserver-default-0 -- /tmp/test_phoenix.sh timeout: 480 From 85bdb9c9ea5bf09af09d4e3cf54a18b585e3ce0a Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 11 Jan 2024 08:20:34 +0100 Subject: [PATCH 08/34] fix more tests --- tests/templates/kuttl/smoke/60-test-phoenix.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/smoke/60-test-phoenix.yaml b/tests/templates/kuttl/smoke/60-test-phoenix.yaml index ff4f3be2..744a51c2 100644 --- a/tests/templates/kuttl/smoke/60-test-phoenix.yaml +++ b/tests/templates/kuttl/smoke/60-test-phoenix.yaml @@ -2,4 +2,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: kubectl cp --namespace=$NAMESPACE ./test_phoenix.sh test-hbase-restserver-default-0:/tmp + - script: kubectl cp --namespace=$NAMESPACE -c hbase ./test_phoenix.sh test-hbase-restserver-default-0:/tmp From 5485d0eb6aaf753090c35d47b2bb379f5a47b619 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Fri, 12 Jan 2024 11:28:31 +0100 Subject: [PATCH 09/34] snafufy --- rust/operator-binary/src/discovery.rs | 9 ++- rust/operator-binary/src/hbase_controller.rs | 14 +++-- rust/operator-binary/src/kerberos.rs | 64 ++++++++++++++------ 3 files changed, 63 insertions(+), 24 deletions(-) diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 76ec63b8..7ad25286 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -11,7 +11,8 @@ use stackable_operator::{ }; use crate::{ - hbase_controller::build_recommended_labels, kerberos::kerberos_discovery_config_properties, + hbase_controller::build_recommended_labels, + kerberos::{self, kerberos_discovery_config_properties}, zookeeper::ZookeeperConnectionInformation, }; @@ -32,6 +33,9 @@ pub enum Error { #[snafu(display("failed to build object meta data"))] ObjectMeta { source: ObjectMetaBuilderError }, + + #[snafu(display("failed to add Kerberos discovery"))] + AddKerberosDiscovery { source: kerberos::Error }, } /// Creates a discovery config map containing the `hbase-site.xml` for clients. @@ -41,7 +45,8 @@ pub fn build_discovery_configmap( resolved_product_image: &ResolvedProductImage, ) -> Result { let mut hbase_site = zookeeper_connection_information.as_hbase_settings(); - hbase_site.extend(kerberos_discovery_config_properties(hbase)); + hbase_site + .extend(kerberos_discovery_config_properties(hbase).context(AddKerberosDiscoverySnafu)?); ConfigMapBuilder::new() .metadata( diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 3406297b..33a52dd5 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -65,8 +65,9 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ discovery::build_discovery_configmap, kerberos::{ - add_kerberos_pod_config, kerberos_config_properties, kerberos_container_start_commands, - kerberos_ssl_client_settings, kerberos_ssl_server_settings, + self, add_kerberos_pod_config, kerberos_config_properties, + kerberos_container_start_commands, kerberos_ssl_client_settings, + kerberos_ssl_server_settings, }, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, product_logging::{ @@ -233,6 +234,9 @@ pub enum Error { cm_name: String, }, + #[snafu(display("failed to add kerberos config"))] + AddKerberosConfig { source: kerberos::Error }, + #[snafu(display("failed to update status"))] ApplyStatus { source: stackable_operator::error::Error, @@ -516,7 +520,8 @@ fn build_rolegroup_config_map( PropertyNameKind::File(file_name) if file_name == HBASE_SITE_XML => { let mut hbase_site_config = BTreeMap::new(); hbase_site_config.extend(zookeeper_connection_information.as_hbase_settings()); - hbase_site_config.extend(kerberos_config_properties(hbase)); + hbase_site_config + .extend(kerberos_config_properties(hbase).context(AddKerberosConfigSnafu)?); // configOverride come last hbase_site_config.extend(config.clone()); @@ -931,7 +936,8 @@ fn build_rolegroup_statefulset( add_graceful_shutdown_config(config, &mut pod_builder).context(GracefulShutdownSnafu)?; if hbase.has_kerberos_enabled() { - add_kerberos_pod_config(hbase, hbase_role, &mut container_builder, &mut pod_builder); + add_kerberos_pod_config(hbase, hbase_role, &mut container_builder, &mut pod_builder) + .context(AddKerberosConfigSnafu)?; } pod_builder.add_container(container_builder.build()); diff --git a/rust/operator-binary/src/kerberos.rs b/rust/operator-binary/src/kerberos.rs index b7e0e684..a6921876 100644 --- a/rust/operator-binary/src/kerberos.rs +++ b/rust/operator-binary/src/kerberos.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use indoc::formatdoc; +use snafu::{OptionExt, ResultExt, Snafu}; use stackable_hbase_crd::{ HbaseCluster, HbaseRole, CONFIG_DIR_NAME, HBASE_MASTER_UI_PORT_HTTPS, HBASE_REGIONSERVER_UI_PORT_HTTPS, TLS_STORE_DIR, TLS_STORE_PASSWORD, TLS_STORE_VOLUME_NAME, @@ -8,19 +9,35 @@ use stackable_hbase_crd::{ use stackable_operator::{ builder::{ ContainerBuilder, PodBuilder, SecretFormat, SecretOperatorVolumeSourceBuilder, - VolumeBuilder, + SecretOperatorVolumeSourceBuilderError, VolumeBuilder, }, - kube::ResourceExt, + kube::{runtime::reflector::ObjectRef, ResourceExt}, }; -pub fn kerberos_config_properties(hbase: &HbaseCluster) -> BTreeMap { +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("object {hbase} is missing namespace"))] + ObjectMissingNamespace { hbase: ObjectRef }, + + #[snafu(display("failed to add Kerberos secret volume"))] + AddKerberosSecretVolume { + source: SecretOperatorVolumeSourceBuilderError, + }, + + #[snafu(display("failed to add TLS secret volume"))] + AddTlsSecretVolume { + source: SecretOperatorVolumeSourceBuilderError, + }, +} + +pub fn kerberos_config_properties(hbase: &HbaseCluster) -> Result, Error> { if !hbase.has_kerberos_enabled() { - return BTreeMap::new(); + return Ok(BTreeMap::new()); } - let principal_host_part = principal_host_part(hbase); + let principal_host_part = principal_host_part(hbase)?; - BTreeMap::from([ + Ok(BTreeMap::from([ // Kerberos settings ( "hbase.security.authentication".to_string(), @@ -106,17 +123,19 @@ pub fn kerberos_config_properties(hbase: &HbaseCluster) -> BTreeMap BTreeMap { +pub fn kerberos_discovery_config_properties( + hbase: &HbaseCluster, +) -> Result, Error> { if !hbase.has_kerberos_enabled() { - return BTreeMap::new(); + return Ok(BTreeMap::new()); } - let principal_host_part = principal_host_part(hbase); + let principal_host_part = principal_host_part(hbase)?; - BTreeMap::from([ + Ok(BTreeMap::from([ ( "hbase.security.authentication".to_string(), "kerberos".to_string(), @@ -144,7 +163,7 @@ pub fn kerberos_discovery_config_properties(hbase: &HbaseCluster) -> BTreeMap BTreeMap { @@ -203,7 +222,7 @@ pub fn add_kerberos_pod_config( role: &HbaseRole, cb: &mut ContainerBuilder, pb: &mut PodBuilder, -) { +) -> Result<(), Error> { if let Some(kerberos_secret_class) = hbase.kerberos_secret_class() { // Keytab let mut kerberos_secret_operator_volume_builder = @@ -217,7 +236,11 @@ pub fn add_kerberos_pod_config( } pb.add_volume( VolumeBuilder::new("kerberos") - .ephemeral(kerberos_secret_operator_volume_builder.build().unwrap()) // FIXME unwrap + .ephemeral( + kerberos_secret_operator_volume_builder + .build() + .context(AddKerberosSecretVolumeSnafu)?, + ) .build(), ); cb.add_volume_mount("kerberos", "/stackable/kerberos"); @@ -242,12 +265,13 @@ pub fn add_kerberos_pod_config( .with_format(SecretFormat::TlsPkcs12) .with_tls_pkcs12_password(TLS_STORE_PASSWORD) .build() - .unwrap(), // FIXME unwrap, + .context(AddTlsSecretVolumeSnafu)?, ) .build(), ); cb.add_volume_mount(TLS_STORE_VOLUME_NAME, TLS_STORE_DIR); } + Ok(()) } pub fn kerberos_container_start_commands(hbase: &HbaseCluster) -> String { @@ -261,8 +285,12 @@ pub fn kerberos_container_start_commands(hbase: &HbaseCluster) -> String { } } -fn principal_host_part(hbase: &HbaseCluster) -> String { +fn principal_host_part(hbase: &HbaseCluster) -> Result { let hbase_name = hbase.name_any(); - let hbase_namespace = hbase.namespace().unwrap(); // FIXME unwrap - format!("{hbase_name}.{hbase_namespace}.svc.cluster.local@${{env.KERBEROS_REALM}}") + let hbase_namespace = hbase.namespace().context(ObjectMissingNamespaceSnafu { + hbase: ObjectRef::from_obj(hbase), + })?; + Ok(format!( + "{hbase_name}.{hbase_namespace}.svc.cluster.local@${{env.KERBEROS_REALM}}" + )) } From f922e50732b8dc7ef92aea7400543840f408786e Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Fri, 12 Jan 2024 12:38:57 +0100 Subject: [PATCH 10/34] fix: Set rest server http and ui ports correctly --- rust/crd/src/lib.rs | 33 +++++++++++++++----- rust/operator-binary/src/hbase_controller.rs | 22 ++++++------- rust/operator-binary/src/kerberos.rs | 5 ++- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 72e98f42..54161fc5 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -55,20 +55,25 @@ pub const HBASE_UNSAFE_REGIONSERVER_HOSTNAME_DISABLE_MASTER_REVERSEDNS: &str = pub const HBASE_HEAPSIZE: &str = "HBASE_HEAPSIZE"; pub const HBASE_ROOT_DIR_DEFAULT: &str = "/hbase"; -pub const HBASE_UI_PORT_NAME_HTTP: &str = "ui"; +pub const HBASE_UI_PORT_NAME_HTTP: &str = "ui-http"; pub const HBASE_UI_PORT_NAME_HTTPS: &str = "ui-https"; +pub const HBASE_REST_PORT_NAME_HTTP: &str = "rest-http"; +pub const HBASE_REST_PORT_NAME_HTTPS: &str = "rest-https"; pub const METRICS_PORT_NAME: &str = "metrics"; -// TODO: Find sane port numbers for https pub const HBASE_MASTER_PORT: u16 = 16000; +// HBase always uses 16010, regardless of http or https. As most products use different ports for http and https, we +// stick to that to be consistent within the SDP. pub const HBASE_MASTER_UI_PORT_HTTP: u16 = 16010; pub const HBASE_MASTER_UI_PORT_HTTPS: u16 = 16011; pub const HBASE_REGIONSERVER_PORT: u16 = 16020; pub const HBASE_REGIONSERVER_UI_PORT_HTTP: u16 = 16030; pub const HBASE_REGIONSERVER_UI_PORT_HTTPS: u16 = 16031; -// TODO: Think about https -pub const HBASE_REST_PORT: u16 = 8080; -pub const METRICS_PORT: u16 = 8081; +pub const HBASE_REST_PORT_HTTP: u16 = 8080; +pub const HBASE_REST_PORT_HTTPS: u16 = 8081; +pub const HBASE_REST_UI_PORT_HTTP: u16 = 8085; +pub const HBASE_REST_UI_PORT_HTTPS: u16 = 8086; +pub const METRICS_PORT: u16 = 9100; pub const JVM_HEAP_FACTOR: f32 = 0.8; @@ -623,9 +628,23 @@ impl HbaseCluster { }, (METRICS_PORT_NAME.to_string(), METRICS_PORT), ], - // TODO: Respect HTTPS settings HbaseRole::RestServer => vec![ - ("rest".to_string(), HBASE_REST_PORT), + if self.has_https_enabled() { + ( + HBASE_REST_PORT_NAME_HTTPS.to_string(), + HBASE_REST_PORT_HTTPS, + ) + } else { + (HBASE_REST_PORT_NAME_HTTP.to_string(), HBASE_REST_PORT_HTTP) + }, + if self.has_https_enabled() { + ( + HBASE_UI_PORT_NAME_HTTPS.to_string(), + HBASE_REST_UI_PORT_HTTPS, + ) + } else { + (HBASE_UI_PORT_NAME_HTTP.to_string(), HBASE_REST_UI_PORT_HTTP) + }, (METRICS_PORT_NAME.to_string(), METRICS_PORT), ], } diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 33a52dd5..9f8b90bf 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -755,18 +755,18 @@ fn build_rolegroup_statefulset( ..Probe::default() }, HbaseRole::RestServer => Probe { - // We cant use this, as it returns a 401 in case kerberos is enabled. - // http_get: Some(HTTPGetAction { - // port: IntOrString::String("rest".to_string()), - // scheme: Some(if hbase.has_https_enabled() { - // "HTTPS".to_string() - // } else { - // "HTTP".to_string() - // }), - // ..HTTPGetAction::default() - // }), + // We cant use HTTPGetAction, as it returns a 401 in case kerberos is enabled, and there is currently no way + // to tell Kubernetes an 401 is healthy. As an alternative we run curl ourselves and check the http status + // code there. tcp_socket: Some(TCPSocketAction { - port: IntOrString::String("rest".to_string()), + port: IntOrString::String( + if hbase.has_https_enabled() { + "rest-https" + } else { + "rest" + } + .to_string(), + ), ..TCPSocketAction::default() }), ..Probe::default() diff --git a/rust/operator-binary/src/kerberos.rs b/rust/operator-binary/src/kerberos.rs index a6921876..64d626b2 100644 --- a/rust/operator-binary/src/kerberos.rs +++ b/rust/operator-binary/src/kerberos.rs @@ -4,7 +4,8 @@ use indoc::formatdoc; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_hbase_crd::{ HbaseCluster, HbaseRole, CONFIG_DIR_NAME, HBASE_MASTER_UI_PORT_HTTPS, - HBASE_REGIONSERVER_UI_PORT_HTTPS, TLS_STORE_DIR, TLS_STORE_PASSWORD, TLS_STORE_VOLUME_NAME, + HBASE_REGIONSERVER_UI_PORT_HTTPS, HBASE_REST_PORT_HTTPS, HBASE_REST_UI_PORT_HTTPS, + TLS_STORE_DIR, TLS_STORE_PASSWORD, TLS_STORE_VOLUME_NAME, }; use stackable_operator::{ builder::{ @@ -121,8 +122,10 @@ pub fn kerberos_config_properties(hbase: &HbaseCluster) -> Result Date: Fri, 12 Jan 2024 12:53:47 +0100 Subject: [PATCH 11/34] vector container shall be the last in the list --- rust/operator-binary/src/hbase_controller.rs | 21 ++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 9f8b90bf..361d2896 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -793,9 +793,8 @@ fn build_rolegroup_statefulset( ..probe_template }; - let mut container_builder = - ContainerBuilder::new("hbase").expect("ContainerBuilder not created"); - container_builder + let mut hbase_container = ContainerBuilder::new("hbase").expect("ContainerBuilder not created"); + hbase_container .image_from_product_image(resolved_product_image) .command(vec![ "/bin/bash".to_string(), @@ -919,6 +918,14 @@ fn build_rolegroup_statefulset( }); } + add_graceful_shutdown_config(config, &mut pod_builder).context(GracefulShutdownSnafu)?; + if hbase.has_kerberos_enabled() { + add_kerberos_pod_config(hbase, hbase_role, &mut hbase_container, &mut pod_builder) + .context(AddKerberosConfigSnafu)?; + } + pod_builder.add_container(hbase_container.build()); + + // Vector sidecar shall be the last container in the list if config.logging.enable_vector_agent { pod_builder.add_container(product_logging::framework::vector_container( resolved_product_image, @@ -934,14 +941,8 @@ fn build_rolegroup_statefulset( )); } - add_graceful_shutdown_config(config, &mut pod_builder).context(GracefulShutdownSnafu)?; - if hbase.has_kerberos_enabled() { - add_kerberos_pod_config(hbase, hbase_role, &mut container_builder, &mut pod_builder) - .context(AddKerberosConfigSnafu)?; - } - - pod_builder.add_container(container_builder.build()); let mut pod_template = pod_builder.build_template(); + if let Some(role) = role { pod_template.merge_from(role.config.pod_overrides.clone()); } From 6e12f13066004d84854554f5faab101844cc0b2b Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Fri, 12 Jan 2024 13:02:57 +0100 Subject: [PATCH 12/34] fix probe --- rust/operator-binary/src/hbase_controller.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 361d2896..fa8dbbab 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -15,8 +15,9 @@ use product_config::{ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_hbase_crd::{ Container, HbaseCluster, HbaseClusterStatus, HbaseConfig, HbaseConfigFragment, HbaseRole, - APP_NAME, CONFIG_DIR_NAME, HBASE_ENV_SH, HBASE_HEAPSIZE, HBASE_SITE_XML, JVM_HEAP_FACTOR, - JVM_SECURITY_PROPERTIES_FILE, SSL_CLIENT_XML, SSL_SERVER_XML, + APP_NAME, CONFIG_DIR_NAME, HBASE_ENV_SH, HBASE_HEAPSIZE, HBASE_REST_PORT_NAME_HTTP, + HBASE_REST_PORT_NAME_HTTPS, HBASE_SITE_XML, JVM_HEAP_FACTOR, JVM_SECURITY_PROPERTIES_FILE, + SSL_CLIENT_XML, SSL_SERVER_XML, }; use stackable_operator::{ builder::{ @@ -761,9 +762,9 @@ fn build_rolegroup_statefulset( tcp_socket: Some(TCPSocketAction { port: IntOrString::String( if hbase.has_https_enabled() { - "rest-https" + HBASE_REST_PORT_NAME_HTTPS } else { - "rest" + HBASE_REST_PORT_NAME_HTTP } .to_string(), ), From c31b113b010c9aacd2fad537335303f00ea064cd Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Fri, 12 Jan 2024 13:55:04 +0100 Subject: [PATCH 13/34] Add test to check restserver --- tests/templates/kuttl/kerberos/42-assert.yaml | 10 +++++++ .../kuttl/kerberos/42-test-rest-server.yaml | 28 +++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 tests/templates/kuttl/kerberos/42-assert.yaml create mode 100644 tests/templates/kuttl/kerberos/42-test-rest-server.yaml diff --git a/tests/templates/kuttl/kerberos/42-assert.yaml b/tests/templates/kuttl/kerberos/42-assert.yaml new file mode 100644 index 00000000..540f0798 --- /dev/null +++ b/tests/templates/kuttl/kerberos/42-assert.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-rest-server +status: + succeeded: 1 diff --git a/tests/templates/kuttl/kerberos/42-test-rest-server.yaml b/tests/templates/kuttl/kerberos/42-test-rest-server.yaml new file mode 100644 index 00000000..1037149c --- /dev/null +++ b/tests/templates/kuttl/kerberos/42-test-rest-server.yaml @@ -0,0 +1,28 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: test-rest-server +spec: + template: + spec: + containers: + - name: test-rest-server + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev + command: + - /bin/bash + - -c + - | + status_code=$(curl --write-out '%{http_code}' --silent --insecure --output /dev/null "https://hbase-restserver-default:8081") + + if [[ "$status_code" -eq 401 ]] ; then + echo "[PASS] Successfully got 401 as we did not authenticate" + exit 0 + else + echo "[FAIL] Expected the restserver to return a 401 as we did not authenticate" + exit 1 + fi + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsUser: 1000 + restartPolicy: OnFailure From db94e918504ae1e3c2ab7f7678dfa9b50d8d69d4 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Fri, 12 Jan 2024 14:01:59 +0100 Subject: [PATCH 14/34] increase test timeout --- tests/templates/kuttl/kerberos/11-assert.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/kerberos/11-assert.yaml b/tests/templates/kuttl/kerberos/11-assert.yaml index 47260144..8bd2d0ac 100644 --- a/tests/templates/kuttl/kerberos/11-assert.yaml +++ b/tests/templates/kuttl/kerberos/11-assert.yaml @@ -1,7 +1,7 @@ --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 600 +timeout: 900 --- apiVersion: apps/v1 kind: StatefulSet From e151092ad182d9e917d8a040766945b0eeefef93 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 15 Jan 2024 11:20:17 +0100 Subject: [PATCH 15/34] minor cleanup --- rust/crd/src/affinity.rs | 1 - rust/crd/src/lib.rs | 43 ++++++-------------- rust/operator-binary/src/hbase_controller.rs | 3 +- 3 files changed, 14 insertions(+), 33 deletions(-) diff --git a/rust/crd/src/affinity.rs b/rust/crd/src/affinity.rs index 5edc318a..8825be76 100644 --- a/rust/crd/src/affinity.rs +++ b/rust/crd/src/affinity.rs @@ -125,7 +125,6 @@ mod tests { let hbase: HbaseCluster = serde_yaml::from_str(input).expect("illegal test input"); let merged_config = hbase .merged_config( - "simple-hbase", &role, "default", &hbase.spec.cluster_config.hdfs_config_map_name, diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 54161fc5..6e5415a3 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -14,7 +14,7 @@ use stackable_operator::{ }, config::{fragment, fragment::Fragment, fragment::ValidationError, merge::Merge}, k8s_openapi::apimachinery::pkg::api::resource::Quantity, - kube::{runtime::reflector::ObjectRef, CustomResource}, + kube::{runtime::reflector::ObjectRef, CustomResource, ResourceExt}, product_config_utils::{ConfigError, Configuration}, product_logging::{self, spec::Logging}, role_utils::{GenericRoleConfig, Role, RoleGroup, RoleGroupRef}, @@ -312,22 +312,17 @@ impl HbaseRole { } } + /// We could have different service names depended on the role (e.g. "hbase-master", "hbase-regionserver" and + /// "hbase-restserver"). However this produces error messages such as + /// [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16020] security.ShellBasedUnixGroupsMapping: unable to return groups for user hbase-master PartialGroupNameException The user name 'hbase-master' is not found. id: 'hbase-master': no such user + /// or + /// Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.security.AccessDeniedException): org.apache.hadoop.hbase.security.AccessDeniedException: Insufficient permissions (user=hbase-master/hbase-master-default-1.hbase-master-default.kuttl-test-poetic-sunbeam.svc.cluster.local@CLUSTER.LOCAL, scope=hbase:meta, family=table:state, params=[table=hbase:meta,family=table:state],action=WRITE) + /// + /// Also the documentation states: + /// > A Kerberos principal has three parts, with the form username/fully.qualified.domain.name@YOUR-REALM.COM. We recommend using hbase as the username portion. + /// + /// As a result we use "hbase" everywhere (which e.g. differs from the current hdfs implementation) pub fn kerberos_service_name(&self) -> &'static str { - // FIXME: Use the names below and fix the following errors - // - // 2023-11-08 13:36:58,530 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16020] security.ShellBasedUnixGroupsMapping: unable to return groups for user hbase-master PartialGroupNameException The user name 'hbase-master' is not found. id: 'hbase-master': no such user - // or - // Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.security.AccessDeniedException): org.apache.hadoop.hbase.security.AccessDeniedException: Insufficient permissions (user=hbase-master/hbase-master-default-1.hbase-master-default.kuttl-test-poetic-sunbeam.svc.cluster.local@CLUSTER.LOCAL, scope=hbase:meta, family=table:state, params=[table=hbase:meta,family=table:state],action=WRITE) - // - // match self { - // HbaseRole::Master => "hbase-master", - // HbaseRole::RegionServer => "hbase-regionserver", - // HbaseRole::RestServer => "hbase-restserver", - // } - - // On the other hand, according to docs: - // > A Kerberos principal has three parts, with the form username/fully.qualified.domain.name@YOUR-REALM.COM. We recommend using hbase as the username portion. - "hbase" } } @@ -430,20 +425,9 @@ impl Configuration for HbaseConfigFragment { match file { HBASE_ENV_SH => { result.insert(HBASE_MANAGES_ZK.to_string(), Some("false".to_string())); - // result.insert( - // "KRB5_CONFIG".to_string(), - // Some("/stackable/kerberos/krb5.conf".to_string()), - // ); // As we don't have access to the clusterConfig, we always enable the `-Djava.security.krb5.conf` // config, besides it is not always being used. - - // -Djavax.net.ssl.trustStore={TLS_STORE_DIR}/truststore.p12 \ - // -Djavax.net.ssl.trustStorePassword={TLS_STORE_PASSWORD} \ - // -Djavax.net.ssl.trustStoreType=pkcs12 \ - // -Djavax.net.ssl.keyStore={TLS_STORE_DIR}/keystore.p12 \ - // -Djavax.net.ssl.keyStorePassword={TLS_STORE_PASSWORD} \ - // -Djavax.net.ssl.keyStoreType=pkcs12 \ let mut all_hbase_opts = format!( "-Djava.security.properties={CONFIG_DIR_NAME}/{JVM_SECURITY_PROPERTIES_FILE} \ -Djava.security.krb5.conf=/stackable/kerberos/krb5.conf \ @@ -453,7 +437,7 @@ impl Configuration for HbaseConfigFragment { all_hbase_opts += " "; all_hbase_opts += hbase_opts; } - // set the jmx exporter in HBASE_MASTER_OPTS, HBASE_REGIONSERVER_OPTS and HBASE_REST_OPTS instead of HBASE_OPTS + // Set the jmx exporter in HBASE_MASTER_OPTS, HBASE_REGIONSERVER_OPTS and HBASE_REST_OPTS instead of HBASE_OPTS // to prevent a port-conflict i.e. CLI tools read HBASE_OPTS and may then try to re-start the exporter if role_name == HbaseRole::Master.to_string() { result.insert(HBASE_MASTER_OPTS.to_string(), Some(all_hbase_opts)); @@ -653,13 +637,12 @@ impl HbaseCluster { /// Retrieve and merge resource configs for role and role groups pub fn merged_config( &self, - hbase_name: &str, role: &HbaseRole, role_group: &str, hdfs_discovery_cm_name: &str, ) -> Result { // Initialize the result with all default values as baseline - let conf_defaults = role.default_config(hbase_name, hdfs_discovery_cm_name); + let conf_defaults = role.default_config(&self.name_any(), hdfs_discovery_cm_name); let role = self.get_role(role).context(MissingHbaseRoleSnafu { role: role.to_string(), diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index fa8dbbab..07ecc20c 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -40,7 +40,7 @@ use stackable_operator::{ apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, DeepMerge, }, - kube::{runtime::controller::Action, Resource, ResourceExt}, + kube::{runtime::controller::Action, Resource}, kvp::{Label, LabelError, Labels, ObjectLabels}, logging::controller::ReconcilerError, memory::{BinaryMultiple, MemoryQuantity}, @@ -367,7 +367,6 @@ pub async fn reconcile_hbase(hbase: Arc, ctx: Arc) -> Result< let merged_config = hbase .merged_config( - &hbase.name_any(), &hbase_role, &rolegroup.role_group, &hbase.spec.cluster_config.hdfs_config_map_name, From 1b44f553528968c57e35627842b2b190dbb6cd78 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 15 Jan 2024 11:36:13 +0100 Subject: [PATCH 16/34] fix: Also "sed" core-site and hdfs-site --- rust/operator-binary/src/kerberos.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rust/operator-binary/src/kerberos.rs b/rust/operator-binary/src/kerberos.rs index 64d626b2..0539fc62 100644 --- a/rust/operator-binary/src/kerberos.rs +++ b/rust/operator-binary/src/kerberos.rs @@ -99,7 +99,7 @@ pub fn kerberos_config_properties(hbase: &HbaseCluster) -> Result Result Result<(), Error> { if let Some(kerberos_secret_class) = hbase.kerberos_secret_class() { - // Keytab + // Mount keytab let mut kerberos_secret_operator_volume_builder = SecretOperatorVolumeSourceBuilder::new(kerberos_secret_class); kerberos_secret_operator_volume_builder @@ -258,7 +256,7 @@ pub fn add_kerberos_pod_config( } if let Some(https_secret_class) = hbase.https_secret_class() { - // TLS certs + // Mount TLS keystore pb.add_volume( VolumeBuilder::new(TLS_STORE_VOLUME_NAME) .ephemeral( @@ -284,7 +282,9 @@ pub fn kerberos_container_start_commands(hbase: &HbaseCluster) -> String { formatdoc! {" export KERBEROS_REALM=$(grep -oP 'default_realm = \\K.*' /stackable/kerberos/krb5.conf) - sed -i -e 's/${{env.KERBEROS_REALM}}/'\"$KERBEROS_REALM/g\" {CONFIG_DIR_NAME}/hbase-site.xml", + sed -i -e 's/${{env.KERBEROS_REALM}}/'\"$KERBEROS_REALM/g\" {CONFIG_DIR_NAME}/core-site.xml + sed -i -e 's/${{env.KERBEROS_REALM}}/'\"$KERBEROS_REALM/g\" {CONFIG_DIR_NAME}/hbase-site.xml + sed -i -e 's/${{env.KERBEROS_REALM}}/'\"$KERBEROS_REALM/g\" {CONFIG_DIR_NAME}/hdfs-site.xml", } } From a4070baa5449e5cfd1cd584423a7029e378b4dbf Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 15 Jan 2024 14:38:28 +0100 Subject: [PATCH 17/34] add docs --- .../hbase/pages/usage-guide/security.adoc | 60 +++++++++++++++++++ docs/modules/hbase/partials/nav.adoc | 3 +- 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 docs/modules/hbase/pages/usage-guide/security.adoc diff --git a/docs/modules/hbase/pages/usage-guide/security.adoc b/docs/modules/hbase/pages/usage-guide/security.adoc new file mode 100644 index 00000000..8168e940 --- /dev/null +++ b/docs/modules/hbase/pages/usage-guide/security.adoc @@ -0,0 +1,60 @@ += Security + +== Authentication +Currently the only supported authentication mechanism is Kerberos, which is disabled by default. +For Kerberos to work a Kerberos KDC is needed, which the users needs to provide. +The xref:home:secret-operator:secretclass.adoc#backend-kerberoskeytab[secret-operator documentation] states which kind of Kerberos servers are supported and how they can be configured. + +=== 1. Prepare Kerberos server +To configure HDFS to use Kerberos you first need to collect information about your Kerberos server, e.g. hostname and port. +Additionally you need a service-user, which the secret-operator uses to create create principals for the HDFS services. + +=== 2. Create Kerberos SecretClass +Afterwards you need to enter all the needed information into a SecretClass, as described in xref:home:secret-operator:secretclass.adoc#backend-kerberoskeytab[secret-operator documentation]. +The following guide assumes you have named your SecretClass `kerberos`. + +=== 3. Configure HDFS to use SecretClass +The next step is to configure your HdfsCluster to use the newly created SecretClass. +Please follow the xref:home:hdfs-operator:usage-guide/security.adoc[HDFS security guide] to set up and test this. +Please watch out to use the SecretClass named `kerberos`. + +=== 4. Configure HBase to use SecretClass +The last step is to configure the same SecretClass for HBase, which is done similar to HDFS. + +IMPORTANT: HDFS and HBase need to use the same SecretClass (or at least use the same underlying Kerberos server). + +[source,yaml] +---- +spec: + clusterConfig: + authentication: + tlsSecretClass: tls # Optional, defaults to "tls" + kerberos: + secretClass: kerberos # Put your SecretClass name in here +---- + +The `kerberos.secretClass` is used to give HBase the possibility to request keytabs from the secret-operator. + +The `tlsSecretClass` is needed to request TLS certificates, used e.g. for the Web UIs. + +=== 4. Verify that Kerberos authentication is required +Shell into the `hbase-master-default-0` Pod and execute the following commands: + +1. `kdestroy` (just in case you run `kinit` in the Pod already in the past) +2. `echo 'list;' | bin/hbase shell` + +The last command should fail with the error message `ERROR: Found no valid authentication method from options`. +You can also check the RestServer by calling `curl -v --insecure https://hbase-restserver-default:8081`, which should return `HTTP ERROR 401 Authentication required`. + +=== 5. Access HBase +In case you want to access your HBase it is recommended to start up a client Pod that connects to HBase, rather than shelling into the master. +We have an https://github.com/stackabletech/hbase-operator/blob/main/tests/templates/kuttl/kerberos/41-access-hbase.j2[integration test] for this exact purpose, where you can see how to connect and get a valid keytab. + +== Authorization +Together with Kerberos authorization is enabled. +You need to explicitly grant table permissions to all users. +E.g. the integration tests run `grant 'testuser', 'C';` to grant the testuser the permission to create tables. + +== Wire encryption +In case Kerberos is enabled, `Privacy` mode is used for best security. +Wire encryption without Kerberos as well as other wire encryption modes are *not* supported. diff --git a/docs/modules/hbase/partials/nav.adoc b/docs/modules/hbase/partials/nav.adoc index e19700d3..940087ec 100644 --- a/docs/modules/hbase/partials/nav.adoc +++ b/docs/modules/hbase/partials/nav.adoc @@ -3,6 +3,7 @@ ** xref:hbase:getting_started/first_steps.adoc[] * xref:hbase:usage-guide/index.adoc[] ** xref:hbase:usage-guide/listenerclass.adoc[] +** xref:hbase:usage-guide/security.adoc[] ** xref:hbase:usage-guide/resource-requests.adoc[] ** xref:hbase:usage-guide/phoenix.adoc[] ** xref:hbase:usage-guide/compression.adoc[] @@ -19,4 +20,4 @@ *** {crd-docs}/hbase.stackable.tech/hbasecluster/v1alpha1/[HbaseCluster {external-link-icon}^] ** xref:hbase:reference/discovery.adoc[] ** xref:hbase:reference/commandline-parameters.adoc[] -** xref:hbase:reference/environment-variables.adoc[] \ No newline at end of file +** xref:hbase:reference/environment-variables.adoc[] From 90ed2ee6d68ba375435222c3ecbbb4ceffeabaa3 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 15 Jan 2024 14:40:14 +0100 Subject: [PATCH 18/34] remove ycsb.yaml --- ycsb.yaml | 138 ------------------------------------------------------ 1 file changed, 138 deletions(-) delete mode 100644 ycsb.yaml diff --git a/ycsb.yaml b/ycsb.yaml deleted file mode 100644 index 80224b14..00000000 --- a/ycsb.yaml +++ /dev/null @@ -1,138 +0,0 @@ -# create 'sbernauer', {NAME => 'benchmark', IN_MEMORY => false, TTL => 2147483647 , BLOOMFILTER => 'NONE', COMPRESSION => 'NONE'}, SPLITS=> ['user1000000000000000000', 'user2000000000000000000', 'user3000000000000000000', 'user4000000000000000000', 'user5000000000000000000', 'user6000000000000000000', 'user7000000000000000000', 'user8000000000000000000', 'user9000000000000000000'] -# /stackable/ycsb-0.17.0/bin/ycsb.sh load hbase20 -p table=sbernauer -p columnfamily=benchmark -p recordcount=100000 -p fieldcount=10 -p fieldlength=100 -p workload=site.ycsb.workloads.CoreWorkload -threads 10 -s -# /stackable/ycsb-0.17.0/bin/ycsb.sh run hbase20 -p operationcount=100000 -p table=sbernauer -p columnfamily=benchmark -P /stackable/ycsb-0.17.0//workloads/workloada -s -threads 100 -target 50000 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ycsb - labels: - app: ycsb -spec: - replicas: 1 - selector: - matchLabels: - app: ycsb - template: - metadata: - labels: - app: ycsb - spec: - containers: - - name: ycsb - image: docker.stackable.tech/soenkeliebau/ycsb - imagePullPolicy: Always - # resources: - # limits: - # cpu: "8" - # memory: 26Gi - # requests: - # cpu: "7" - # memory: 26Gi - env: - - name: KRB5_CONFIG - value: /stackable/kerberos/krb5.conf - - name: HBASE_OPTS - value: -Djava.security.krb5.conf=/stackable/kerberos/krb5.conf - - name: JAVA_OPTS # As HBASE_OPTS does not get picked up - value: -Djava.security.krb5.conf=/stackable/kerberos/krb5.conf - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - command: - - /bin/bash - - -c - - | - set -ex - microdnf install krb5-workstation openssl - - export KERBEROS_REALM=$(grep -oP 'default_realm = \K.*' /stackable/kerberos/krb5.conf) - mkdir -p /stackable/ycsb-0.17.0/conf/ - cat /tmp/hbase-site.xml | sed -e 's/${{env.KERBEROS_REALM}}/'"$KERBEROS_REALM/g" > /stackable/ycsb-0.17.0/conf/hbase-site.xml - - klist -k /stackable/kerberos/keytab - kinit -kt /stackable/kerberos/keytab hbase/access-hbase.${NAMESPACE}.svc.cluster.local - klist - sleep infinity - volumeMounts: - - mountPath: /tmp/hbase-site.xml - name: config-volume-hbase - subPath: hbase-site.xml - - name: kerberos - mountPath: /stackable/kerberos - tolerations: - - key: "app" - operator: "Equal" - value: "ycsb" - effect: "NoSchedule" - volumes: - - name: config-volume-hbase - configMap: - name: hbase - - name: kerberos - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: kerberos-kuttl-test-capital-liger # TODO adopt - secrets.stackable.tech/scope: service=access-hbase - secrets.stackable.tech/kerberos.service.names: hbase # TOOD use testuser - spec: - storageClassName: secrets.stackable.tech - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "1" - securityContext: - runAsUser: 0 -# --- -# apiVersion: v1 -# kind: ConfigMap -# metadata: -# name: hbase-discovery -# data: -# hbase-site.xml: |- -# -# -# -# hbase.master.kerberos.principal -# hbase/hbase.kuttl-test-daring-mammal.svc.cluster.local@REALM_IS_PICKED_FROM_DEFAULT_REALM -# -# -# hbase.regionserver.kerberos.principal -# hbase/hbase.kuttl-test-daring-mammal.svc.cluster.local@REALM_IS_PICKED_FROM_DEFAULT_REALM -# -# -# hbase.rpc.protection -# privacy -# -# -# hbase.security.authentication -# kerberos -# -# -# hbase.ssl.enabled -# true -# -# -# hbase.zookeeper.property.clientPort -# 2282 -# -# -# hbase.zookeeper.quorum -# zookeeper-server-default-0.zookeeper-server-default.kuttl-test-daring-mammal.svc.cluster.local:2282 -# -# -# zookeeper.znode.parent -# /znode-49b43377-5cda-4b8f-a8e6-66c0a1f4def4/hbase -# -# -# log4j.properties: |+ -# log4j.rootLogger=DEBUG, CONSOLE, FILE - -# log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# log4j.appender.CONSOLE.Threshold=DEBUG -# log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -# log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %.1000m%n From b9343097f7515645c31a6eb591c2efee11729cd9 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 15 Jan 2024 14:45:15 +0100 Subject: [PATCH 19/34] docs link --- docs/modules/hbase/pages/usage-guide/security.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/modules/hbase/pages/usage-guide/security.adoc b/docs/modules/hbase/pages/usage-guide/security.adoc index 8168e940..3d43709e 100644 --- a/docs/modules/hbase/pages/usage-guide/security.adoc +++ b/docs/modules/hbase/pages/usage-guide/security.adoc @@ -10,12 +10,12 @@ To configure HDFS to use Kerberos you first need to collect information about yo Additionally you need a service-user, which the secret-operator uses to create create principals for the HDFS services. === 2. Create Kerberos SecretClass -Afterwards you need to enter all the needed information into a SecretClass, as described in xref:home:secret-operator:secretclass.adoc#backend-kerberoskeytab[secret-operator documentation]. +Afterwards you need to enter all the needed information into a SecretClass, as described in xref:home:secret-operator:secretclass.adoc#backend-kerberoskeytab[secret-operator documentation]. The following guide assumes you have named your SecretClass `kerberos`. === 3. Configure HDFS to use SecretClass The next step is to configure your HdfsCluster to use the newly created SecretClass. -Please follow the xref:home:hdfs-operator:usage-guide/security.adoc[HDFS security guide] to set up and test this. +Please follow the xref:hdfs:usage-guide/security.adoc[HDFS security guide] to set up and test this. Please watch out to use the SecretClass named `kerberos`. === 4. Configure HBase to use SecretClass From f23562f4e5146dd6840ffa5ac498e752475ced9d Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 16 Jan 2024 08:19:19 +0100 Subject: [PATCH 20/34] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6272b94a..dfa22eee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- Support user authentication using Kerberos ([#436]). - More CRD documentation ([#425]). ### Changed @@ -18,6 +19,7 @@ [#425]: https://github.com/stackabletech/hbase-operator/pull/425 +[#436]: https://github.com/stackabletech/hbase-operator/pull/436 [#438]: https://github.com/stackabletech/hbase-operator/pull/438 ## [23.11.0] - 2023-11-24 From d6986d2bac1dadb2e6dfd1cf39f93e7155fc1e5e Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 17 Jan 2024 08:55:30 +0100 Subject: [PATCH 21/34] docs: Add Kerberos discovery --- docs/modules/hbase/pages/reference/discovery.adoc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/modules/hbase/pages/reference/discovery.adoc b/docs/modules/hbase/pages/reference/discovery.adoc index 0b1abf3c..a2268903 100644 --- a/docs/modules/hbase/pages/reference/discovery.adoc +++ b/docs/modules/hbase/pages/reference/discovery.adoc @@ -36,4 +36,11 @@ The resulting discovery ConfigMap is located at `{namespace}/{cluster-name}`. The ConfigMap data values are formatted as Hadoop XML files which allows simple mounting of that ConfigMap into pods that require access to HBase. `hbase-site.xml`:: -Contains the `hbase.zookeeper.quorum` property. +Contains the needed information to connect to Zookeeper and use that to establish a connection to HBase. + +=== Kerberos +In case Kerberos is enabled according to the xref:usage-guide/security.adoc[security documentation], the discovery ConfigMap also includes the information that clients must authenticate themselves using Kerberos. + +If you want to use the discovery ConfigMap outside Stackable services, you need to substitute `${env.KERBEROS_REALM}` with your actual realm (e.g. by using `sed -i -e 's/${{env.KERBEROS_REALM}}/'"$KERBEROS_REALM/g" hbase-site.xml`). + +One example would be the property `hbase.master.kerberos.principal` being set to `hbase/hbase.default.svc.cluster.local@${env.KERBEROS_REALM}`. From c6358382031650f9d4591660a1cb81c26aecfc39 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 17 Jan 2024 08:56:24 +0100 Subject: [PATCH 22/34] Apply suggestions from code review Co-authored-by: Malte Sander --- docs/modules/hbase/pages/usage-guide/security.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/modules/hbase/pages/usage-guide/security.adoc b/docs/modules/hbase/pages/usage-guide/security.adoc index 3d43709e..c5e5d577 100644 --- a/docs/modules/hbase/pages/usage-guide/security.adoc +++ b/docs/modules/hbase/pages/usage-guide/security.adoc @@ -7,7 +7,7 @@ The xref:home:secret-operator:secretclass.adoc#backend-kerberoskeytab[secret-ope === 1. Prepare Kerberos server To configure HDFS to use Kerberos you first need to collect information about your Kerberos server, e.g. hostname and port. -Additionally you need a service-user, which the secret-operator uses to create create principals for the HDFS services. +Additionally, you need a service-user which the secret-operator uses to create principals for the HDFS services. === 2. Create Kerberos SecretClass Afterwards you need to enter all the needed information into a SecretClass, as described in xref:home:secret-operator:secretclass.adoc#backend-kerberoskeytab[secret-operator documentation]. @@ -37,7 +37,7 @@ The `kerberos.secretClass` is used to give HBase the possibility to request keyt The `tlsSecretClass` is needed to request TLS certificates, used e.g. for the Web UIs. -=== 4. Verify that Kerberos authentication is required +=== 5. Verify that Kerberos authentication is required Shell into the `hbase-master-default-0` Pod and execute the following commands: 1. `kdestroy` (just in case you run `kinit` in the Pod already in the past) @@ -46,7 +46,7 @@ Shell into the `hbase-master-default-0` Pod and execute the following commands: The last command should fail with the error message `ERROR: Found no valid authentication method from options`. You can also check the RestServer by calling `curl -v --insecure https://hbase-restserver-default:8081`, which should return `HTTP ERROR 401 Authentication required`. -=== 5. Access HBase +=== 6. Access HBase In case you want to access your HBase it is recommended to start up a client Pod that connects to HBase, rather than shelling into the master. We have an https://github.com/stackabletech/hbase-operator/blob/main/tests/templates/kuttl/kerberos/41-access-hbase.j2[integration test] for this exact purpose, where you can see how to connect and get a valid keytab. From 8411d8943121c6a31e9b4d5eaa307145c1b0b3fe Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 17 Jan 2024 08:56:56 +0100 Subject: [PATCH 23/34] Update docs/modules/hbase/pages/usage-guide/security.adoc Co-authored-by: Malte Sander --- docs/modules/hbase/pages/usage-guide/security.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/hbase/pages/usage-guide/security.adoc b/docs/modules/hbase/pages/usage-guide/security.adoc index c5e5d577..883fa570 100644 --- a/docs/modules/hbase/pages/usage-guide/security.adoc +++ b/docs/modules/hbase/pages/usage-guide/security.adoc @@ -2,7 +2,7 @@ == Authentication Currently the only supported authentication mechanism is Kerberos, which is disabled by default. -For Kerberos to work a Kerberos KDC is needed, which the users needs to provide. +For Kerberos to work a Kerberos KDC is needed, which the users need to provide. The xref:home:secret-operator:secretclass.adoc#backend-kerberoskeytab[secret-operator documentation] states which kind of Kerberos servers are supported and how they can be configured. === 1. Prepare Kerberos server From b911ab14e7c811544eee8d13e1e218d87cc4e553 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 17 Jan 2024 08:57:43 +0100 Subject: [PATCH 24/34] Update rust/crd/src/lib.rs Co-authored-by: Malte Sander --- rust/crd/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 6e5415a3..9bf27c91 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -427,7 +427,7 @@ impl Configuration for HbaseConfigFragment { result.insert(HBASE_MANAGES_ZK.to_string(), Some("false".to_string())); // As we don't have access to the clusterConfig, we always enable the `-Djava.security.krb5.conf` - // config, besides it is not always being used. + // config, besides it not always being used. let mut all_hbase_opts = format!( "-Djava.security.properties={CONFIG_DIR_NAME}/{JVM_SECURITY_PROPERTIES_FILE} \ -Djava.security.krb5.conf=/stackable/kerberos/krb5.conf \ From d3e171921244a74be1d870f0f1d442e930bfa85c Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 17 Jan 2024 12:51:29 +0100 Subject: [PATCH 25/34] refactor cli_role_name out --- rust/crd/src/lib.rs | 10 ++++++++++ rust/operator-binary/src/hbase_controller.rs | 8 +------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 9bf27c91..f2d1fd9a 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -312,6 +312,16 @@ impl HbaseRole { } } + /// Returns the name of the role as it is needed by the `bin/hbase {cli_role_name} start` command. + pub fn cli_role_name(&self) -> String { + match self { + HbaseRole::Master | HbaseRole::RegionServer => self.to_string(), + // Of course it is not called "restserver", so we need to have this match + // instead of just letting the Display impl do it's thing ;P + HbaseRole::RestServer => "rest".to_string(), + } + } + /// We could have different service names depended on the role (e.g. "hbase-master", "hbase-regionserver" and /// "hbase-restserver"). However this produces error messages such as /// [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16020] security.ShellBasedUnixGroupsMapping: unable to return groups for user hbase-master PartialGroupNameException The user name 'hbase-master' is not found. id: 'hbase-master': no such user diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 07ecc20c..cfc734c7 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -819,13 +819,7 @@ fn build_rolegroup_statefulset( wait_for_termination $! {create_vector_shutdown_file_command} ", - hbase_role_name_in_command = match hbase_role { - HbaseRole::Master => "master", - HbaseRole::RegionServer => "regionserver", - // Of course it is not called "restserver", so we need to have this match - // instead of just letting the Display impl do it's thing ;P - HbaseRole::RestServer => "rest", - }, + hbase_role_name_in_command = hbase_role.cli_role_name(), kerberos_container_start_commands = kerberos_container_start_commands(hbase), remove_vector_shutdown_file_command = remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), From afd8a224236233a1d698ee5a7d8f2e87e9492556 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 17 Jan 2024 12:56:54 +0100 Subject: [PATCH 26/34] Update tests/templates/kuttl/kerberos/42-test-rest-server.yaml Co-authored-by: Malte Sander --- tests/templates/kuttl/kerberos/42-test-rest-server.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/templates/kuttl/kerberos/42-test-rest-server.yaml b/tests/templates/kuttl/kerberos/42-test-rest-server.yaml index 1037149c..e27cb477 100644 --- a/tests/templates/kuttl/kerberos/42-test-rest-server.yaml +++ b/tests/templates/kuttl/kerberos/42-test-rest-server.yaml @@ -1,4 +1,5 @@ apiVersion: batch/v1 +--- kind: Job metadata: name: test-rest-server From 4d2bec430795e105af04f380aa5267d2f680ff7b Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 17 Jan 2024 13:01:26 +0100 Subject: [PATCH 27/34] Change port numbers according to Arch meeting decision --- rust/crd/src/lib.rs | 13 +++++++------ .../kuttl/kerberos/42-test-rest-server.yaml | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index f2d1fd9a..f366f993 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -62,17 +62,18 @@ pub const HBASE_REST_PORT_NAME_HTTPS: &str = "rest-https"; pub const METRICS_PORT_NAME: &str = "metrics"; pub const HBASE_MASTER_PORT: u16 = 16000; -// HBase always uses 16010, regardless of http or https. As most products use different ports for http and https, we -// stick to that to be consistent within the SDP. +// HBase always uses 16010, regardless of http or https. On 2024-01-17 we decided in Arch-meeting that we want to stick +// the port numbers to what the product is doing, so we get the least surprise for users - even when this means we have +// inconsistency between Stackable products. pub const HBASE_MASTER_UI_PORT_HTTP: u16 = 16010; -pub const HBASE_MASTER_UI_PORT_HTTPS: u16 = 16011; +pub const HBASE_MASTER_UI_PORT_HTTPS: u16 = HBASE_MASTER_UI_PORT_HTTP; pub const HBASE_REGIONSERVER_PORT: u16 = 16020; pub const HBASE_REGIONSERVER_UI_PORT_HTTP: u16 = 16030; -pub const HBASE_REGIONSERVER_UI_PORT_HTTPS: u16 = 16031; +pub const HBASE_REGIONSERVER_UI_PORT_HTTPS: u16 = HBASE_REGIONSERVER_UI_PORT_HTTP; pub const HBASE_REST_PORT_HTTP: u16 = 8080; -pub const HBASE_REST_PORT_HTTPS: u16 = 8081; +pub const HBASE_REST_PORT_HTTPS: u16 = HBASE_REST_PORT_HTTP; pub const HBASE_REST_UI_PORT_HTTP: u16 = 8085; -pub const HBASE_REST_UI_PORT_HTTPS: u16 = 8086; +pub const HBASE_REST_UI_PORT_HTTPS: u16 = HBASE_REST_UI_PORT_HTTP; pub const METRICS_PORT: u16 = 9100; pub const JVM_HEAP_FACTOR: f32 = 0.8; diff --git a/tests/templates/kuttl/kerberos/42-test-rest-server.yaml b/tests/templates/kuttl/kerberos/42-test-rest-server.yaml index e27cb477..440bc603 100644 --- a/tests/templates/kuttl/kerberos/42-test-rest-server.yaml +++ b/tests/templates/kuttl/kerberos/42-test-rest-server.yaml @@ -13,7 +13,7 @@ spec: - /bin/bash - -c - | - status_code=$(curl --write-out '%{http_code}' --silent --insecure --output /dev/null "https://hbase-restserver-default:8081") + status_code=$(curl --write-out '%{http_code}' --silent --insecure --output /dev/null "https://hbase-restserver-default:8080") if [[ "$status_code" -eq 401 ]] ; then echo "[PASS] Successfully got 401 as we did not authenticate" From cd979e42ffacf286992aa174eec55757dc52d5a3 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 17 Jan 2024 13:42:59 +0100 Subject: [PATCH 28/34] Port unification --- rust/crd/src/lib.rs | 86 +++++++++++++--------------- rust/operator-binary/src/kerberos.rs | 11 +--- 2 files changed, 42 insertions(+), 55 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index f366f993..7e282505 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -65,15 +65,11 @@ pub const HBASE_MASTER_PORT: u16 = 16000; // HBase always uses 16010, regardless of http or https. On 2024-01-17 we decided in Arch-meeting that we want to stick // the port numbers to what the product is doing, so we get the least surprise for users - even when this means we have // inconsistency between Stackable products. -pub const HBASE_MASTER_UI_PORT_HTTP: u16 = 16010; -pub const HBASE_MASTER_UI_PORT_HTTPS: u16 = HBASE_MASTER_UI_PORT_HTTP; +pub const HBASE_MASTER_UI_PORT: u16 = 16010; pub const HBASE_REGIONSERVER_PORT: u16 = 16020; -pub const HBASE_REGIONSERVER_UI_PORT_HTTP: u16 = 16030; -pub const HBASE_REGIONSERVER_UI_PORT_HTTPS: u16 = HBASE_REGIONSERVER_UI_PORT_HTTP; -pub const HBASE_REST_PORT_HTTP: u16 = 8080; -pub const HBASE_REST_PORT_HTTPS: u16 = HBASE_REST_PORT_HTTP; -pub const HBASE_REST_UI_PORT_HTTP: u16 = 8085; -pub const HBASE_REST_UI_PORT_HTTPS: u16 = HBASE_REST_UI_PORT_HTTP; +pub const HBASE_REGIONSERVER_UI_PORT: u16 = 16030; +pub const HBASE_REST_PORT: u16 = 8080; +pub const HBASE_REST_UI_PORT: u16 = 8085; pub const METRICS_PORT: u16 = 9100; pub const JVM_HEAP_FACTOR: f32 = 0.8; @@ -595,51 +591,49 @@ impl HbaseCluster { match role { HbaseRole::Master => vec![ ("master".to_string(), HBASE_MASTER_PORT), - if self.has_https_enabled() { - ( - HBASE_UI_PORT_NAME_HTTPS.to_string(), - HBASE_MASTER_UI_PORT_HTTPS, - ) - } else { - ( - HBASE_UI_PORT_NAME_HTTP.to_string(), - HBASE_MASTER_UI_PORT_HTTP, - ) - }, + ( + if self.has_https_enabled() { + HBASE_UI_PORT_NAME_HTTPS + } else { + HBASE_UI_PORT_NAME_HTTP + } + .to_string(), + HBASE_MASTER_UI_PORT, + ), (METRICS_PORT_NAME.to_string(), METRICS_PORT), ], HbaseRole::RegionServer => vec![ ("regionserver".to_string(), HBASE_REGIONSERVER_PORT), - if self.has_https_enabled() { - ( - HBASE_UI_PORT_NAME_HTTPS.to_string(), - HBASE_REGIONSERVER_UI_PORT_HTTPS, - ) - } else { - ( - HBASE_UI_PORT_NAME_HTTP.to_string(), - HBASE_REGIONSERVER_UI_PORT_HTTP, - ) - }, + ( + if self.has_https_enabled() { + HBASE_UI_PORT_NAME_HTTPS + } else { + HBASE_UI_PORT_NAME_HTTP + } + .to_string(), + HBASE_REGIONSERVER_UI_PORT, + ), (METRICS_PORT_NAME.to_string(), METRICS_PORT), ], HbaseRole::RestServer => vec![ - if self.has_https_enabled() { - ( - HBASE_REST_PORT_NAME_HTTPS.to_string(), - HBASE_REST_PORT_HTTPS, - ) - } else { - (HBASE_REST_PORT_NAME_HTTP.to_string(), HBASE_REST_PORT_HTTP) - }, - if self.has_https_enabled() { - ( - HBASE_UI_PORT_NAME_HTTPS.to_string(), - HBASE_REST_UI_PORT_HTTPS, - ) - } else { - (HBASE_UI_PORT_NAME_HTTP.to_string(), HBASE_REST_UI_PORT_HTTP) - }, + ( + if self.has_https_enabled() { + HBASE_REST_PORT_NAME_HTTPS + } else { + HBASE_REST_PORT_NAME_HTTP + } + .to_string(), + HBASE_REST_PORT, + ), + ( + if self.has_https_enabled() { + HBASE_UI_PORT_NAME_HTTPS + } else { + HBASE_UI_PORT_NAME_HTTP + } + .to_string(), + HBASE_REST_UI_PORT, + ), (METRICS_PORT_NAME.to_string(), METRICS_PORT), ], } diff --git a/rust/operator-binary/src/kerberos.rs b/rust/operator-binary/src/kerberos.rs index 0539fc62..21280d02 100644 --- a/rust/operator-binary/src/kerberos.rs +++ b/rust/operator-binary/src/kerberos.rs @@ -3,9 +3,8 @@ use std::collections::BTreeMap; use indoc::formatdoc; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_hbase_crd::{ - HbaseCluster, HbaseRole, CONFIG_DIR_NAME, HBASE_MASTER_UI_PORT_HTTPS, - HBASE_REGIONSERVER_UI_PORT_HTTPS, HBASE_REST_PORT_HTTPS, HBASE_REST_UI_PORT_HTTPS, - TLS_STORE_DIR, TLS_STORE_PASSWORD, TLS_STORE_VOLUME_NAME, + HbaseCluster, HbaseRole, CONFIG_DIR_NAME, TLS_STORE_DIR, TLS_STORE_PASSWORD, + TLS_STORE_VOLUME_NAME, }; use stackable_operator::{ builder::{ @@ -118,12 +117,6 @@ pub fn kerberos_config_properties(hbase: &HbaseCluster) -> Result Date: Wed, 17 Jan 2024 14:32:37 +0100 Subject: [PATCH 29/34] fix test --- tests/templates/kuttl/kerberos/42-test-rest-server.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/kerberos/42-test-rest-server.yaml b/tests/templates/kuttl/kerberos/42-test-rest-server.yaml index 440bc603..b4fcdb06 100644 --- a/tests/templates/kuttl/kerberos/42-test-rest-server.yaml +++ b/tests/templates/kuttl/kerberos/42-test-rest-server.yaml @@ -1,5 +1,5 @@ -apiVersion: batch/v1 --- +apiVersion: batch/v1 kind: Job metadata: name: test-rest-server From cd864fd99f62387405dce186a833aaeab8e13463 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 18 Jan 2024 09:12:23 +0100 Subject: [PATCH 30/34] Update tests/templates/kuttl/kerberos/30-assert.yaml Co-authored-by: Malte Sander --- tests/templates/kuttl/kerberos/30-assert.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/templates/kuttl/kerberos/30-assert.yaml b/tests/templates/kuttl/kerberos/30-assert.yaml index b20d7a7f..a4fe7035 100644 --- a/tests/templates/kuttl/kerberos/30-assert.yaml +++ b/tests/templates/kuttl/kerberos/30-assert.yaml @@ -1,4 +1,3 @@ - --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert From 9028cee62e5b7713820ece812e054fac554ab19b Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 18 Jan 2024 09:43:12 +0100 Subject: [PATCH 31/34] Fix file name --- docs/modules/hbase/pages/usage-guide/security.adoc | 2 +- .../kerberos/{41-access-hbase.j2 => 41-access-hbase.yaml.j2} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/templates/kuttl/kerberos/{41-access-hbase.j2 => 41-access-hbase.yaml.j2} (100%) diff --git a/docs/modules/hbase/pages/usage-guide/security.adoc b/docs/modules/hbase/pages/usage-guide/security.adoc index 883fa570..ef7b8d45 100644 --- a/docs/modules/hbase/pages/usage-guide/security.adoc +++ b/docs/modules/hbase/pages/usage-guide/security.adoc @@ -48,7 +48,7 @@ You can also check the RestServer by calling `curl -v --insecure https://hbase-r === 6. Access HBase In case you want to access your HBase it is recommended to start up a client Pod that connects to HBase, rather than shelling into the master. -We have an https://github.com/stackabletech/hbase-operator/blob/main/tests/templates/kuttl/kerberos/41-access-hbase.j2[integration test] for this exact purpose, where you can see how to connect and get a valid keytab. +We have an https://github.com/stackabletech/hbase-operator/blob/main/tests/templates/kuttl/kerberos/41-access-hbase.yaml.j2[integration test] for this exact purpose, where you can see how to connect and get a valid keytab. == Authorization Together with Kerberos authorization is enabled. diff --git a/tests/templates/kuttl/kerberos/41-access-hbase.j2 b/tests/templates/kuttl/kerberos/41-access-hbase.yaml.j2 similarity index 100% rename from tests/templates/kuttl/kerberos/41-access-hbase.j2 rename to tests/templates/kuttl/kerberos/41-access-hbase.yaml.j2 From befba730dac58588395b32fce42dd38c09a83cb9 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 18 Jan 2024 09:56:50 +0100 Subject: [PATCH 32/34] refactor!: Align with HDFS CRD --- deploy/helm/hbase-operator/crds/crds.yaml | 29 ++++++++++--------- rust/crd/src/lib.rs | 28 ++++++++---------- rust/crd/src/security.rs | 24 +++++++++++++++ rust/operator-binary/src/kerberos.rs | 22 +++++--------- .../kuttl/kerberos/30-install-hbase.yaml.j2 | 5 ++-- 5 files changed, 63 insertions(+), 45 deletions(-) create mode 100644 rust/crd/src/security.rs diff --git a/deploy/helm/hbase-operator/crds/crds.yaml b/deploy/helm/hbase-operator/crds/crds.yaml index 41f325de..9ff80535 100644 --- a/deploy/helm/hbase-operator/crds/crds.yaml +++ b/deploy/helm/hbase-operator/crds/crds.yaml @@ -31,26 +31,29 @@ spec: clusterConfig: description: Configuration that applies to all roles and role groups. This includes settings for logging, ZooKeeper and HDFS connection, among other things. properties: - hdfsConfigMapName: - description: Name of the [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) for an HDFS cluster. - type: string - kerberos: - description: Configuration to set up a cluster secured using Kerberos. + authentication: + description: Settings related to user [authentication](https://docs.stackable.tech/home/nightly/usage-guide/security). nullable: true properties: - kerberosSecretClass: - default: kerberos - description: Name of the SecretClass providing the keytab for the HDFS services. - type: string - requestNodePrincipals: - default: false - description: Wether a principal including the Kubernetes node name should be requested. The principal could e.g. be `HTTP/my-k8s-worker-0.mycorp.lan`. This feature is disabled by default, as the resulting principals can already by existent e.g. in Active Directory which can cause problems. - type: boolean + kerberos: + description: Kerberos configuration. + properties: + secretClass: + description: Name of the SecretClass providing the keytab for the HBase services. + type: string + required: + - secretClass + type: object tlsSecretClass: default: tls description: Name of the SecretClass providing the tls certificates for the WebUIs. type: string + required: + - kerberos type: object + hdfsConfigMapName: + description: Name of the [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) for an HDFS cluster. + type: string listenerClass: default: cluster-internal description: |- diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 7e282505..bd830531 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeMap, str::FromStr}; +use security::AuthenticationConfig; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ @@ -27,6 +28,7 @@ use strum::{Display, EnumIter, EnumString}; use crate::affinity::get_affinity; pub mod affinity; +pub mod security; pub const APP_NAME: &str = "hbase"; @@ -174,8 +176,8 @@ pub struct HbaseClusterConfig { #[serde(default)] pub listener_class: CurrentlySupportedListenerClasses, - /// Configuration to set up a cluster secured using Kerberos. - pub kerberos: Option, + /// Settings related to user [authentication](DOCS_BASE_URL_PLACEHOLDER/usage-guide/security). + pub authentication: Option, } // TODO: Temporary solution until listener-operator is finished @@ -558,32 +560,26 @@ impl HbaseCluster { self.kerberos_secret_class().is_some() } - pub fn kerberos_request_node_principals(&self) -> Option { + pub fn kerberos_secret_class(&self) -> Option { self.spec .cluster_config - .kerberos + .authentication .as_ref() - .map(|k| k.request_node_principals) - } - - pub fn kerberos_secret_class(&self) -> Option<&str> { - self.spec - .cluster_config - .kerberos - .as_ref() - .map(|k| k.kerberos_secret_class.as_str()) + .map(|a| &a.kerberos) + .map(|k| k.secret_class.clone()) } pub fn has_https_enabled(&self) -> bool { self.https_secret_class().is_some() } - pub fn https_secret_class(&self) -> Option<&str> { + pub fn https_secret_class(&self) -> Option { self.spec .cluster_config - .kerberos + .authentication .as_ref() - .map(|k| k.tls_secret_class.as_str()) + .map(|a| &a.kerberos) + .map(|k| k.secret_class.clone()) } /// Returns required port name and port number tuples depending on the role. diff --git a/rust/crd/src/security.rs b/rust/crd/src/security.rs new file mode 100644 index 00000000..dde0739c --- /dev/null +++ b/rust/crd/src/security.rs @@ -0,0 +1,24 @@ +use serde::{Deserialize, Serialize}; +use stackable_operator::schemars::{self, JsonSchema}; + +#[derive(Clone, Debug, Deserialize, Eq, Hash, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AuthenticationConfig { + /// Name of the SecretClass providing the tls certificates for the WebUIs. + #[serde(default = "default_tls_secret_class")] + pub tls_secret_class: String, + + /// Kerberos configuration. + pub kerberos: KerberosConfig, +} + +fn default_tls_secret_class() -> String { + "tls".to_string() +} + +#[derive(Clone, Debug, Deserialize, Eq, Hash, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct KerberosConfig { + /// Name of the SecretClass providing the keytab for the HBase services. + pub secret_class: String, +} diff --git a/rust/operator-binary/src/kerberos.rs b/rust/operator-binary/src/kerberos.rs index 21280d02..43698361 100644 --- a/rust/operator-binary/src/kerberos.rs +++ b/rust/operator-binary/src/kerberos.rs @@ -219,22 +219,16 @@ pub fn add_kerberos_pod_config( ) -> Result<(), Error> { if let Some(kerberos_secret_class) = hbase.kerberos_secret_class() { // Mount keytab - let mut kerberos_secret_operator_volume_builder = - SecretOperatorVolumeSourceBuilder::new(kerberos_secret_class); - kerberos_secret_operator_volume_builder - .with_service_scope(hbase.name_any()) - .with_kerberos_service_name(role.kerberos_service_name()) - .with_kerberos_service_name("HTTP"); - if let Some(true) = hbase.kerberos_request_node_principals() { - kerberos_secret_operator_volume_builder.with_node_scope(); - } + let kerberos_secret_operator_volume = + SecretOperatorVolumeSourceBuilder::new(kerberos_secret_class) + .with_service_scope(hbase.name_any()) + .with_kerberos_service_name(role.kerberos_service_name()) + .with_kerberos_service_name("HTTP") + .build() + .context(AddKerberosSecretVolumeSnafu)?; pb.add_volume( VolumeBuilder::new("kerberos") - .ephemeral( - kerberos_secret_operator_volume_builder - .build() - .context(AddKerberosSecretVolumeSnafu)?, - ) + .ephemeral(kerberos_secret_operator_volume) .build(), ); cb.add_volume_mount("kerberos", "/stackable/kerberos"); diff --git a/tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2 b/tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2 index f2eee8b0..3d6a70c9 100644 --- a/tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2 +++ b/tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2 @@ -23,9 +23,10 @@ commands: hdfsConfigMapName: hdfs zookeeperConfigMapName: hbase-znode listenerClass: {{ test_scenario['values']['listener-class'] }} - kerberos: + authentication: tlsSecretClass: tls - kerberosSecretClass: kerberos-$NAMESPACE + kerberos: + secretClass: kerberos-$NAMESPACE {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} From 20994934e1a0700e0f563f4e124a5289979fe01a Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 18 Jan 2024 11:05:30 +0100 Subject: [PATCH 33/34] fixup --- rust/crd/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index bd830531..6136d899 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -578,8 +578,7 @@ impl HbaseCluster { .cluster_config .authentication .as_ref() - .map(|a| &a.kerberos) - .map(|k| k.secret_class.clone()) + .map(|a| a.tls_secret_class.clone()) } /// Returns required port name and port number tuples depending on the role. From 8edc644bc00495f279aa512e44a80042bb9e7539 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Fri, 19 Jan 2024 10:50:15 +0100 Subject: [PATCH 34/34] refactor: Move out ui_port_name --- rust/crd/src/lib.rs | 40 +++++++++++++--------------------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 6136d899..29e54132 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -586,28 +586,12 @@ impl HbaseCluster { match role { HbaseRole::Master => vec![ ("master".to_string(), HBASE_MASTER_PORT), - ( - if self.has_https_enabled() { - HBASE_UI_PORT_NAME_HTTPS - } else { - HBASE_UI_PORT_NAME_HTTP - } - .to_string(), - HBASE_MASTER_UI_PORT, - ), + (self.ui_port_name(), HBASE_MASTER_UI_PORT), (METRICS_PORT_NAME.to_string(), METRICS_PORT), ], HbaseRole::RegionServer => vec![ ("regionserver".to_string(), HBASE_REGIONSERVER_PORT), - ( - if self.has_https_enabled() { - HBASE_UI_PORT_NAME_HTTPS - } else { - HBASE_UI_PORT_NAME_HTTP - } - .to_string(), - HBASE_REGIONSERVER_UI_PORT, - ), + (self.ui_port_name(), HBASE_REGIONSERVER_UI_PORT), (METRICS_PORT_NAME.to_string(), METRICS_PORT), ], HbaseRole::RestServer => vec![ @@ -620,20 +604,22 @@ impl HbaseCluster { .to_string(), HBASE_REST_PORT, ), - ( - if self.has_https_enabled() { - HBASE_UI_PORT_NAME_HTTPS - } else { - HBASE_UI_PORT_NAME_HTTP - } - .to_string(), - HBASE_REST_UI_PORT, - ), + (self.ui_port_name(), HBASE_REST_UI_PORT), (METRICS_PORT_NAME.to_string(), METRICS_PORT), ], } } + /// Name of the port used by the Web UI, which depends on HTTPS usage + fn ui_port_name(&self) -> String { + if self.has_https_enabled() { + HBASE_UI_PORT_NAME_HTTPS + } else { + HBASE_UI_PORT_NAME_HTTP + } + .to_string() + } + /// Retrieve and merge resource configs for role and role groups pub fn merged_config( &self,