JVM config overrides (#389)

* Add jvm dns properties. * Update product config property roles. * Update CHANGELOG * Role based defaults for security props. * Update docs * Review feedback. * Review feedback.
stackabletech · Aug 14, 2023 · ffdad72 · ffdad72
1 parent 885d50d
commit ffdad72
Show file tree

Hide file tree

Showing 6 changed files with 311 additions and 6 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,7 @@
 ### Added
 
 - Default stackableVersion to operator version ([#385]).
+- Configuration overrides for the JVM security properties, such as DNS caching ([#389]).
 
 ### Changed
 
@@ -13,6 +14,7 @@
 
 [#382]: https://github.com/stackabletech/hbase-operator/pull/382
 [#385]: https://github.com/stackabletech/hbase-operator/pull/385
+[#389]: https://github.com/stackabletech/hbase-operator/pull/389
 
 ## [23.7.0] - 2023-07-14
 

diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml
@@ -5,6 +5,125 @@ spec:
 
 properties:
 
+  #################################################################################################
+  # security.properties
+  #################################################################################################
+
+  - property:
+      propertyNames:
+        - name: "networkaddress.cache.ttl"
+          kind:
+            type: "file"
+            file: "security.properties"
+      datatype:
+        type: "integer"
+        min: "0"
+      recommendedValues:
+        - fromVersion: "0.0.0"
+          value: "5"
+      roles:
+        - name: "master"
+          required: true
+      asOfVersion: "0.0.0"
+      comment: "master - TTL for successfully resolved domain names."
+      description: "master - TTL for successfully resolved domain names."
+
+  - property:
+      propertyNames:
+        - name: "networkaddress.cache.ttl"
+          kind:
+            type: "file"
+            file: "security.properties"
+      datatype:
+        type: "integer"
+        min: "0"
+      recommendedValues:
+        - fromVersion: "0.0.0"
+          value: "10"
+      roles:
+        - name: "regionserver"
+          required: true
+      asOfVersion: "0.0.0"
+      comment: "regionserver - TTL for successfully resolved domain names."
+      description: "regionserver - TTL for successfully resolved domain names."
+
+  - property:
+      propertyNames:
+        - name: "networkaddress.cache.ttl"
+          kind:
+            type: "file"
+            file: "security.properties"
+      datatype:
+        type: "integer"
+        min: "0"
+      recommendedValues:
+        - fromVersion: "0.0.0"
+          value: "30"
+      roles:
+        - name: "restserver"
+          required: true
+      asOfVersion: "0.0.0"
+      comment: "restserver - TTL for successfully resolved domain names."
+      description: "restserver - TTL for successfully resolved domain names."
+
+  - property:
+      propertyNames:
+        - name: "networkaddress.cache.negative.ttl"
+          kind:
+            type: "file"
+            file: "security.properties"
+      datatype:
+        type: "integer"
+        min: "0"
+      recommendedValues:
+        - fromVersion: "0.0.0"
+          value: "0"
+      roles:
+        - name: "master"
+          required: true
+      asOfVersion: "0.0.0"
+      comment: "master - TTL for domain names that cannot be resolved."
+      description: "master - TTL for domain names that cannot be resolved."
+
+  - property:
+      propertyNames:
+        - name: "networkaddress.cache.negative.ttl"
+          kind:
+            type: "file"
+            file: "security.properties"
+      datatype:
+        type: "integer"
+        min: "0"
+      recommendedValues:
+        - fromVersion: "0.0.0"
+          value: "0"
+      roles:
+        - name: "regionserver"
+          required: true
+      asOfVersion: "0.0.0"
+      comment: "regionserver - TTL for domain names that cannot be resolved."
+      description: "regionserver - TTL for domain names that cannot be resolved."
+
+  - property:
+      propertyNames:
+        - name: "networkaddress.cache.negative.ttl"
+          kind:
+            type: "file"
+            file: "security.properties"
+      datatype:
+        type: "integer"
+        min: "0"
+      recommendedValues:
+        - fromVersion: "0.0.0"
+          value: "0"
+      roles:
+        - name: "restserver"
+          required: true
+      asOfVersion: "0.0.0"
+      comment: "restserver - TTL for domain names that cannot be resolved."
+      description: "restserver - TTL for domain names that cannot be resolved."
+
+
   #################################################################################################
   # hbase-env.sh
   #################################################################################################

diff --git a/deploy/helm/hbase-operator/configs/properties.yaml b/deploy/helm/hbase-operator/configs/properties.yaml
@@ -5,6 +5,125 @@ spec:
 
 properties:
 
+  #################################################################################################
+  # security.properties
+  #################################################################################################
+
+  - property:
+      propertyNames:
+        - name: "networkaddress.cache.ttl"
+          kind:
+            type: "file"
+            file: "security.properties"
+      datatype:
+        type: "integer"
+        min: "0"
+      recommendedValues:
+        - fromVersion: "0.0.0"
+          value: "5"
+      roles:
+        - name: "master"
+          required: true
+      asOfVersion: "0.0.0"
+      comment: "master - TTL for successfully resolved domain names."
+      description: "master - TTL for successfully resolved domain names."
+
+  - property:
+      propertyNames:
+        - name: "networkaddress.cache.ttl"
+          kind:
+            type: "file"
+            file: "security.properties"
+      datatype:
+        type: "integer"
+        min: "0"
+      recommendedValues:
+        - fromVersion: "0.0.0"
+          value: "10"
+      roles:
+        - name: "regionserver"
+          required: true
+      asOfVersion: "0.0.0"
+      comment: "regionserver - TTL for successfully resolved domain names."
+      description: "regionserver - TTL for successfully resolved domain names."
+
+  - property:
+      propertyNames:
+        - name: "networkaddress.cache.ttl"
+          kind:
+            type: "file"
+            file: "security.properties"
+      datatype:
+        type: "integer"
+        min: "0"
+      recommendedValues:
+        - fromVersion: "0.0.0"
+          value: "30"
+      roles:
+        - name: "restserver"
+          required: true
+      asOfVersion: "0.0.0"
+      comment: "restserver - TTL for successfully resolved domain names."
+      description: "restserver - TTL for successfully resolved domain names."
+
+  - property:
+      propertyNames:
+        - name: "networkaddress.cache.negative.ttl"
+          kind:
+            type: "file"
+            file: "security.properties"
+      datatype:
+        type: "integer"
+        min: "0"
+      recommendedValues:
+        - fromVersion: "0.0.0"
+          value: "0"
+      roles:
+        - name: "master"
+          required: true
+      asOfVersion: "0.0.0"
+      comment: "master - TTL for domain names that cannot be resolved."
+      description: "master - TTL for domain names that cannot be resolved."
+
+  - property:
+      propertyNames:
+        - name: "networkaddress.cache.negative.ttl"
+          kind:
+            type: "file"
+            file: "security.properties"
+      datatype:
+        type: "integer"
+        min: "0"
+      recommendedValues:
+        - fromVersion: "0.0.0"
+          value: "0"
+      roles:
+        - name: "regionserver"
+          required: true
+      asOfVersion: "0.0.0"
+      comment: "regionserver - TTL for domain names that cannot be resolved."
+      description: "regionserver - TTL for domain names that cannot be resolved."
+
+  - property:
+      propertyNames:
+        - name: "networkaddress.cache.negative.ttl"
+          kind:
+            type: "file"
+            file: "security.properties"
+      datatype:
+        type: "integer"
+        min: "0"
+      recommendedValues:
+        - fromVersion: "0.0.0"
+          value: "0"
+      roles:
+        - name: "restserver"
+          required: true
+      asOfVersion: "0.0.0"
+      comment: "restserver - TTL for domain names that cannot be resolved."
+      description: "restserver - TTL for domain names that cannot be resolved."
+
+
   #################################################################################################
   # hbase-env.sh
   #################################################################################################

diff --git a/docs/modules/hbase/pages/usage-guide/overrides.adoc b/docs/modules/hbase/pages/usage-guide/overrides.adoc
@@ -11,6 +11,7 @@ For a role or role group, at the same level of `config`, you can specify: `confi
 
 - `hbase-site.xml`
 - `hbase-env.sh`
+- `security.properties`
 
 NOTE: `hdfs-site.xml` is not listed here, the file is always taken from the referenced hdfs cluster. If you want to modify it, have a look at xref:hdfs:usage-guide/configuration-environment-overrides.adoc[HDFS configuration overrides].
 
@@ -50,6 +51,35 @@ All override property values must be strings. The properties will be formatted a
 
 For a full list of configuration options we refer to the HBase https://hbase.apache.org/book.html#config.files[Configuration Documentation].
 
+=== The security.properties file
+
+The `security.properties` file is used to configure JVM security properties. It is very seldom that users need to tweak any of these, but there is one use-case that stands out, and that users need to be aware of: the JVM DNS cache.
+
+The JVM manages it's own cache of successfully resolved host names as well as a cache of host names that cannot be resolved. Some products of the Stackable platform are very sensible to the contents of these caches and their performance is heavily affected by them. As of version 3.4.12, Apache Hbase performs poorly if the positive cache is disabled. To cache resolved host names, and thus speeding up Hbase queries you can configure the TTL of entries in the positive cache like this:
+
+[source,yaml]
+----
+  masters:
+    configOverrides:
+      security.properties:
+        networkaddress.cache.ttl: "5"
+        networkaddress.cache.negative.ttl: "0"
+  regionServers:
+    configOverrides:
+      security.properties:
+        networkaddress.cache.ttl: "10"
+        networkaddress.cache.negative.ttl: "0"
+  restServers:
+    configOverrides:
+      security.properties:
+        networkaddress.cache.ttl: "30"
+        networkaddress.cache.negative.ttl: "0"
+----
+
+NOTE: The operator configures DNS caching by default as shown in the example above.
+
+For details on the JVM security see https://docs.oracle.com/en/java/javase/11/security/java-security-overview1.html
+
 // Environment configuration is not implemented. The environment is managed
 // with the hbase-env.sh configuration file
 

diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs
@@ -27,6 +27,10 @@ use strum::{Display, EnumIter, EnumString};
 
 pub const APP_NAME: &str = "hbase";
 
+pub const CONFIG_DIR_NAME: &str = "/stackable/conf";
+
+pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties";
+
 pub const HBASE_ENV_SH: &str = "hbase-env.sh";
 pub const HBASE_SITE_XML: &str = "hbase-site.xml";
 
@@ -334,7 +338,7 @@ impl Configuration for HbaseConfigFragment {
         match file {
             HBASE_ENV_SH => {
                 result.insert(HBASE_MANAGES_ZK.to_string(), Some("false".to_string()));
-                let mut all_hbase_opts = format!("-javaagent:/stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar={METRICS_PORT}:/stackable/jmx/region-server.yaml");
+                let mut all_hbase_opts = format!("-Djava.security.properties={CONFIG_DIR_NAME}/{JVM_SECURITY_PROPERTIES_FILE} -javaagent:/stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar={METRICS_PORT}:/stackable/jmx/region-server.yaml");
                 if let Some(hbase_opts) = &self.hbase_opts {
                     all_hbase_opts += " ";
                     all_hbase_opts += hbase_opts;