diff --git a/Cargo.lock b/Cargo.lock index ab6169c3..b20f05dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,15 +111,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "async-broadcast" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cd0e2e25ea8e5f7e9df04578dc6cf5c83577fd09b1a46aaf5c85e1c33f2a7e" +checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" dependencies = [ "event-listener", "event-listener-strategy", @@ -146,7 +146,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -157,7 +157,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -270,9 +270,9 @@ checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "cc" -version = "1.2.4" +version = "1.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9157bbaa6b165880c27a4293a474c91cdcf265cc68cc829bf10be0964a391caf" +checksum = "8d6dbb628b8f8555f86d0323c2eb39e3ec81901f4b83e091db8a6a76d316a333" dependencies = [ "jobserver", "libc", @@ -329,7 +329,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -454,7 +454,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -465,7 +465,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -476,7 +476,7 @@ checksum = "bc2323e10c92e1cf4d86e11538512e6dc03ceb586842970b6332af3d4046a046" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -506,7 +506,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -544,7 +544,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -588,7 +588,7 @@ checksum = "0d28318a75d4aead5c4db25382e8ef717932d0346600cacae6357eb5941bc5ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -715,7 +715,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -797,9 +797,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "hashbrown" @@ -949,9 +949,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.3" +version = "0.27.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", "http", @@ -1136,7 +1136,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -1370,7 +1370,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -1409,9 +1409,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.168" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libgit2-sys" @@ -1482,9 +1482,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2ef2593ffb6958c941575cee70c8e257438749971869c4ae5acf6f91a168a61" +checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" dependencies = [ "adler2", ] @@ -1537,9 +1537,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.5" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -1606,7 +1606,7 @@ dependencies = [ "lazy_static", "once_cell", "opentelemetry", - "ordered-float 4.5.0", + "ordered-float 4.6.0", "percent-encoding", "rand", "thiserror 1.0.69", @@ -1625,9 +1625,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.5.0" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c65ee1f9701bf938026630b455d5315f490640234259037edb259798b3bcf85e" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" dependencies = [ "num-traits", ] @@ -1690,7 +1690,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror 2.0.7", + "thiserror 2.0.9", "ucd-trie", ] @@ -1714,7 +1714,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -1745,7 +1745,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -1817,9 +1817,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -1954,7 +1954,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.90", + "syn 2.0.94", "unicode-ident", ] @@ -2010,7 +2010,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.0.1", + "security-framework 3.1.0", ] [[package]] @@ -2041,9 +2041,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "ryu" @@ -2082,7 +2082,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -2115,9 +2115,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.0.1" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1415a607e92bec364ea2cf9264646dcce0f91e6d65281bd6f2819cca3bf39c8" +checksum = "81d3f8c9bfcc3cbb6b0179eb57042d75b1582bdc65c3cb95f3fa999509c03cbc" dependencies = [ "bitflags 2.6.0", "core-foundation 0.10.0", @@ -2128,9 +2128,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +checksum = "1863fd3768cd83c56a7f60faa4dc0d403f1b6df0a38c3c25f44b7894e45370d5" dependencies = [ "core-foundation-sys", "libc", @@ -2144,9 +2144,9 @@ checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" [[package]] name = "serde" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] @@ -2163,13 +2163,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -2180,14 +2180,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.134" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" dependencies = [ "itoa", "memchr", @@ -2308,7 +2308,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -2365,6 +2365,7 @@ dependencies = [ "semver", "serde", "serde_json", + "serde_yaml", "snafu 0.8.5", "stackable-nifi-crd", "stackable-operator", @@ -2377,7 +2378,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.83.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.83.0#3ce7bcbdb58097cde0c0f19488a104c96f69dbc3" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat/jvm-arguments#9a9a99c243b65ef548d72f94796f9677123d6947" dependencies = [ "chrono", "clap", @@ -2415,18 +2416,18 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.83.0#3ce7bcbdb58097cde0c0f19488a104c96f69dbc3" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat/jvm-arguments#9a9a99c243b65ef548d72f94796f9677123d6947" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] name = "stackable-shared" version = "0.0.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.83.0#3ce7bcbdb58097cde0c0f19488a104c96f69dbc3" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat/jvm-arguments#9a9a99c243b65ef548d72f94796f9677123d6947" dependencies = [ "kube", "semver", @@ -2460,7 +2461,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -2482,9 +2483,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.90" +version = "2.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +checksum = "987bc0be1cdea8b10216bd06e2ca407d40b9543468fafd3ddfb02f36e77f71f3" dependencies = [ "proc-macro2", "quote", @@ -2505,7 +2506,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -2519,11 +2520,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.7" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93605438cbd668185516ab499d589afb7ee1859ea3d5fc8f6b0755e1c7443767" +checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" dependencies = [ - "thiserror-impl 2.0.7", + "thiserror-impl 2.0.9", ] [[package]] @@ -2534,18 +2535,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] name = "thiserror-impl" -version = "2.0.7" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d8749b4531af2117677a5fcd12b1348a3fe2b81e36e61ffeac5c4aa3273e36" +checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -2647,7 +2648,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -2781,7 +2782,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -2980,7 +2981,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", "wasm-bindgen-shared", ] @@ -3002,7 +3003,7 @@ checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3138,9 +3139,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.20" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "e6f5bb5257f2407a5425c6e749bfd9692192a73e70a6060516ac04f889087d68" dependencies = [ "memchr", ] @@ -3183,7 +3184,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", "synstructure", ] @@ -3205,7 +3206,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] [[package]] @@ -3225,7 +3226,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", "synstructure", ] @@ -3254,5 +3255,5 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.94", ] diff --git a/Cargo.toml b/Cargo.toml index f07348aa..00da435d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,6 @@ tracing = "0.1" url = { version = "2.5.2" } xml-rs = "0.8" -#[patch."https://github.com/stackabletech/operator-rs.git"] -#stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } +[patch."https://github.com/stackabletech/operator-rs.git"] +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "feat/jvm-arguments" } # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } diff --git a/deploy/helm/nifi-operator/crds/crds.yaml b/deploy/helm/nifi-operator/crds/crds.yaml index b228d736..703d26a3 100644 --- a/deploy/helm/nifi-operator/crds/crds.yaml +++ b/deploy/helm/nifi-operator/crds/crds.yaml @@ -655,6 +655,29 @@ spec: default: {} description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.' type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: Allows overriding JVM arguments. Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) for details on the usage. + properties: + add: + default: [] + items: + type: string + type: array + remove: + default: [] + items: + type: string + type: array + removeRegex: + default: [] + items: + type: string + type: array + type: object podOverrides: default: {} description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information. @@ -1116,6 +1139,29 @@ spec: default: {} description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.' type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: Allows overriding JVM arguments. Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) for details on the usage. + properties: + add: + default: [] + items: + type: string + type: array + remove: + default: [] + items: + type: string + type: array + removeRegex: + default: [] + items: + type: string + type: array + type: object podOverrides: default: {} description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information. diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 6c5eded1..5e294dc1 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -30,7 +30,7 @@ use stackable_operator::{ memory::{BinaryMultiple, MemoryQuantity}, product_config_utils::{self, Configuration}, product_logging::{self, spec::Logging}, - role_utils::{GenericRoleConfig, Role, RoleGroupRef}, + role_utils::{GenericRoleConfig, JavaCommonConfig, Role, RoleGroupRef}, schemars::{self, JsonSchema}, status::condition::{ClusterCondition, HasStatusCondition}, time::Duration, @@ -107,7 +107,7 @@ pub struct NifiSpec { // no doc - docs in Role struct. #[serde(default, skip_serializing_if = "Option::is_none")] - pub nodes: Option>, + pub nodes: Option>, // no doc - docs in ProductImage struct. pub image: ProductImage, diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 53c1782b..afae6eb7 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -31,6 +31,7 @@ url.workspace = true [dev-dependencies] rstest.workspace = true +serde_yaml.workspace = true [build-dependencies] built.workspace = true diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index a5f0c716..6d1fb3d0 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -11,12 +11,13 @@ use stackable_nifi_crd::{ }; use stackable_operator::{ commons::resources::Resources, + config::merge::Merge, memory::{BinaryMultiple, MemoryQuantity}, product_config_utils::{ transform_all_roles_to_config, validate_all_roles_and_groups_config, ValidatedRoleConfigByPropertyKind, }, - role_utils::Role, + role_utils::{GenericRoleConfig, JavaCommonConfig, JvmArgumentOverrides, Role}, }; use strum::{Display, EnumIter}; @@ -97,28 +98,31 @@ pub enum Error { /// Create the NiFi bootstrap.conf pub fn build_bootstrap_conf( - nifi_config: &NifiConfig, + merged_config: &NifiConfig, overrides: BTreeMap, + role_java_common_config: &JavaCommonConfig, + role_group_java_common_config: &JavaCommonConfig, ) -> Result { let mut bootstrap = BTreeMap::new(); // Java command to use when running NiFi bootstrap.insert("java".to_string(), "java".to_string()); // Username to use when running NiFi. This value will be ignored on Windows. bootstrap.insert("run.as".to_string(), "".to_string()); - // Preserve shell environment while runnning as "run.as" user + // Preserve shell environment while ruining as "run.as" user bootstrap.insert("preserve.environment".to_string(), "false".to_string()); // Configure where NiFi's lib and conf directories live bootstrap.insert("lib.dir".to_string(), "./lib".to_string()); bootstrap.insert("conf.dir".to_string(), "./conf".to_string()); - bootstrap.extend(graceful_shutdown_config_properties(nifi_config)); + bootstrap.extend(graceful_shutdown_config_properties(merged_config)); - let mut java_args = Vec::with_capacity(18); - // Disable JSR 199 so that we can use JSP's without running a JDK - java_args.push("-Dorg.apache.jasper.compiler.disablejsr199=true".to_string()); + let mut jvm_args = Vec::new(); // Read memory limits from config - if let Some(heap_size_definition) = &nifi_config.resources.memory.limit { - tracing::debug!("Read {:?} from crd as memory limit", heap_size_definition); + if let Some(heap_size_definition) = &merged_config.resources.memory.limit { + tracing::debug!( + ?heap_size_definition, + "Read heap size definition from CRD as memory limit" + ); let heap_size = MemoryQuantity::try_from(heap_size_definition) .context(InvalidMemoryConfigSnafu)? @@ -130,55 +134,71 @@ pub fn build_bootstrap_conf( .context(InvalidMemoryConfigSnafu)?; tracing::debug!( - "Converted {:?} to {} for java heap config", - &heap_size_definition, - java_heap + ?heap_size_definition, + ?java_heap, + "Converted head size definition to java heap config", ); + // Push heap size config as max and min size to java args - java_args.push(format!("-Xmx{}", java_heap)); - java_args.push(format!("-Xms{}", java_heap)); + jvm_args.push(format!("-Xmx{java_heap}")); + jvm_args.push(format!("-Xms{java_heap}")); } else { - tracing::debug!("No memory limits defined"); + tracing::warn!("No memory limits defined"); } - java_args.push("-Djava.net.preferIPv4Stack=true".to_string()); - - // allowRestrictedHeaders is required for Cluster/Node communications to work properly - java_args.push("-Dsun.net.http.allowRestrictedHeaders=true".to_string()); - java_args.push("-Djava.protocol.handler.pkgs=sun.net.www.protocol".to_string()); - - // The G1GC is known to cause some problems in Java 8 and earlier, but the issues were addressed in Java 9. If using Java 8 or earlier, - // it is recommended that G1GC not be used, especially in conjunction with the Write Ahead Provenance Repository. However, if using a newer - // version of Java, it can result in better performance without significant \"stop-the-world\" delays. - java_args.push("-XX:+UseG1GC".to_string()); - - // Set headless mode by default - java_args.push("-Djava.awt.headless=true".to_string()); - // Root key in hexadecimal format for encrypted sensitive configuration values - //bootstrap.insert("nifi.bootstrap.sensitive.key=".to_string(), "".to_string()); - // Sets the provider of SecureRandom to /dev/urandom to prevent blocking on VMs - java_args.push("-Djava.security.egd=file:/dev/urandom".to_string()); - // Requires JAAS to use only the provided JAAS configuration to authenticate a Subject, without using any "fallback" methods (such as prompting for username/password) - // Please see https://docs.oracle.com/javase/8/docs/technotes/guides/security/jgss/single-signon.html, section "EXCEPTIONS TO THE MODEL" - java_args.push("-Djavax.security.auth.useSubjectCredsOnly=true".to_string()); - - // Zookeeper 3.5 now includes an Admin Server that starts on port 8080, since NiFi is already using that port disable by default. - // Please see https://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_adminserver_config for configuration options. - java_args.push("-Dzookeeper.admin.enableServer=false".to_string()); - - // JVM security properties include especially TTL values for the positive and negative DNS caches. - java_args.push(format!( - "-Djava.security.properties={NIFI_CONFIG_DIRECTORY}/{JVM_SECURITY_PROPERTIES_FILE}" - )); - - // add java args - bootstrap.extend( - java_args - .into_iter() - .enumerate() - .map(|(i, a)| (format!("java.arg.{}", i + 1), a)), - ); - // override with config overrides + jvm_args.extend([ + // The G1GC is known to cause some problems in Java 8 and earlier, but the issues were addressed in Java 9. If using Java 8 or earlier, + // it is recommended that G1GC not be used, especially in conjunction with the Write Ahead Provenance Repository. However, if using a newer + // version of Java, it can result in better performance without significant \"stop-the-world\" delays. + "-XX:+UseG1GC".to_owned(), + // Set headless mode by default + "-Djava.awt.headless=true".to_owned(), + // Disable JSR 199 so that we can use JSP's without running a JDK + "-Dorg.apache.jasper.compiler.disablejsr199=true".to_owned(), + // Note(sbernauer): This has been here since ages, leaving it here for compatibility reasons. + // That being said: IPV6 rocks :rocket:! + "-Djava.net.preferIPv4Stack=true".to_owned(), + // allowRestrictedHeaders is required for Cluster/Node communications to work properly + "-Dsun.net.http.allowRestrictedHeaders=true".to_owned(), + "-Djava.protocol.handler.pkgs=sun.net.www.protocol".to_owned(), + // Sets the provider of SecureRandom to /dev/urandom to prevent blocking on VMs + "-Djava.security.egd=file:/dev/urandom".to_owned(), + // Requires JAAS to use only the provided JAAS configuration to authenticate a Subject, without using any "fallback" methods (such as prompting for username/password) + // Please see https://docs.oracle.com/javase/8/docs/technotes/guides/security/jgss/single-signon.html, section "EXCEPTIONS TO THE MODEL" + "-Djavax.security.auth.useSubjectCredsOnly=true".to_owned(), + // Zookeeper 3.5 now includes an Admin Server that starts on port 8080, since NiFi is already using that port disable by default. + // Please see https://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_adminserver_config for configuration options. + "-Dzookeeper.admin.enableServer=false".to_owned(), + // JVM security properties include especially TTL values for the positive and negative DNS caches. + format!( + "-Djava.security.properties={NIFI_CONFIG_DIRECTORY}/{JVM_SECURITY_PROPERTIES_FILE}" + ), + ]); + + let operator_generated = JavaCommonConfig { + jvm_argument_overrides: JvmArgumentOverrides::new_with_only_additions(jvm_args), + }; + + // Please note that the merge order is different than we normally do! + // This is not trivial, as the merge operation is not purely additive (as it is with e.g. `PodTemplateSpec). + let mut role = role_java_common_config.clone(); + role.merge(&operator_generated); + let mut role_group = role_group_java_common_config.clone(); + role_group.merge(&role); + + // let mut merged_java_config = java_common_config.clone(); + // merged_java_config.merge(&JavaCommonConfig::new(jvm_args)); + + for (index, argument) in role_group + .jvm_argument_overrides + .effective_jvm_config_after_merging() + .iter() + .enumerate() + { + bootstrap.insert(format!("java.arg.{}", index + 1), argument.clone()); + } + + // configOverrides come last bootstrap.extend(overrides); Ok(format_properties(bootstrap)) @@ -669,7 +689,7 @@ pub fn build_state_management_xml() -> String { pub fn validated_product_config( resource: &NifiCluster, version: &str, - role: &Role, + role: &Role, product_config: &ProductConfigManager, ) -> Result { let mut roles = HashMap::new(); @@ -714,3 +734,148 @@ fn storage_quantity_to_nifi(quantity: MemoryQuantity) -> String { .value ) } + +#[cfg(test)] +mod tests { + use super::*; + + use indoc::indoc; + + #[test] + fn test_build_bootstrap_conf_defaults() { + let input = r#" + apiVersion: nifi.stackable.tech/v1alpha1 + kind: NifiCluster + metadata: + name: simple-nifi + spec: + image: + productVersion: 1.27.0 + clusterConfig: + authentication: + - authenticationClass: nifi-admin-credentials-simple + sensitiveProperties: + keySecret: simple-nifi-sensitive-property-key + autoGenerate: true + zookeeperConfigMapName: simple-nifi-znode + nodes: + roleGroups: + default: + replicas: 1 + "#; + let bootstrap_conf = construct_bootstrap_conf(input); + + assert_eq!( + bootstrap_conf, + indoc! {" + conf.dir=./conf + graceful.shutdown.seconds=300 + java=java + java.arg.1=-Xmx3276m + java.arg.10=-Djavax.security.auth.useSubjectCredsOnly=true + java.arg.11=-Dzookeeper.admin.enableServer=false + java.arg.12=-Djava.security.properties=/stackable/nifi/conf/security.properties + java.arg.2=-Xms3276m + java.arg.3=-XX:+UseG1GC + java.arg.4=-Djava.awt.headless=true + java.arg.5=-Dorg.apache.jasper.compiler.disablejsr199=true + java.arg.6=-Djava.net.preferIPv4Stack=true + java.arg.7=-Dsun.net.http.allowRestrictedHeaders=true + java.arg.8=-Djava.protocol.handler.pkgs=sun.net.www.protocol + java.arg.9=-Djava.security.egd=file:/dev/urandom + lib.dir=./lib + preserve.environment=false + run.as= + "} + ); + } + + #[test] + fn test_build_bootstrap_conf_jvm_argument_overrides() { + let input = r#" + apiVersion: nifi.stackable.tech/v1alpha1 + kind: NifiCluster + metadata: + name: simple-nifi + spec: + image: + productVersion: 1.27.0 + clusterConfig: + authentication: + - authenticationClass: nifi-admin-credentials-simple + sensitiveProperties: + keySecret: simple-nifi-sensitive-property-key + autoGenerate: true + zookeeperConfigMapName: simple-nifi-znode + nodes: + config: + resources: + memory: + limit: 42Gi + jvmArgumentOverrides: + remove: + - -XX:+UseG1GC + add: + - -Dhttps.proxyHost=proxy.my.corp + - -Dhttps.proxyPort=8080 + - -Djava.net.preferIPv4Stack=true + roleGroups: + default: + replicas: 1 + jvmArgumentOverrides: + # We need more memory! + removeRegex: + - -Xmx.* + - -Dhttps.proxyPort=.* + add: + - -Xmx40000m + - -Dhttps.proxyPort=1234 + "#; + let bootstrap_conf = construct_bootstrap_conf(input); + + assert_eq!( + bootstrap_conf, + indoc! {" + conf.dir=./conf + graceful.shutdown.seconds=300 + java=java + java.arg.1=-Xms34406m + java.arg.10=-Djava.security.properties=/stackable/nifi/conf/security.properties + java.arg.11=-Dhttps.proxyHost=proxy.my.corp + java.arg.12=-Djava.net.preferIPv4Stack=true + java.arg.13=-Xmx40000m + java.arg.14=-Dhttps.proxyPort=1234 + java.arg.2=-Djava.awt.headless=true + java.arg.3=-Dorg.apache.jasper.compiler.disablejsr199=true + java.arg.4=-Djava.net.preferIPv4Stack=true + java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true + java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + java.arg.7=-Djava.security.egd=file:/dev/urandom + java.arg.8=-Djavax.security.auth.useSubjectCredsOnly=true + java.arg.9=-Dzookeeper.admin.enableServer=false + lib.dir=./lib + preserve.environment=false + run.as= + "} + ); + } + + fn construct_bootstrap_conf(nifi_cluster: &str) -> String { + let nifi: NifiCluster = serde_yaml::from_str(nifi_cluster).expect("illegal test input"); + + let role = NifiRole::Node; + let merged_config = nifi.merged_config(&role, "default").unwrap(); + let nodes = nifi.spec.nodes.unwrap(); + let (role_java_common_config, role_group_java_common_config) = nodes + .get_product_specific_common_configs("default") + .unwrap(); + + build_bootstrap_conf( + &merged_config, + BTreeMap::new(), + role_java_common_config, + role_group_java_common_config, + ) + .unwrap() + } +} diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index ac80b72a..e6fdaa4e 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -66,7 +66,7 @@ use stackable_operator::{ CustomContainerLogConfig, }, }, - role_utils::{GenericRoleConfig, Role, RoleGroupRef}, + role_utils::{GenericRoleConfig, JavaCommonConfig, Role, RoleGroupRef}, status::condition::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, @@ -256,7 +256,7 @@ pub enum Error { source: crate::product_logging::Error, }, - #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] + #[snafu(display("failed to add the logging configuration to the ConfigMap {cm_name:?}"))] InvalidLoggingConfig { source: crate::product_logging::Error, cm_name: String, @@ -278,8 +278,7 @@ pub enum Error { }, #[snafu(display( - "failed to serialize [{JVM_SECURITY_PROPERTIES_FILE}] for {}", - rolegroup + "failed to serialize {JVM_SECURITY_PROPERTIES_FILE:?} for rolegroup {rolegroup:?}", ))] JvmSecurityProperties { source: PropertiesWriterError, @@ -345,6 +344,11 @@ pub enum Error { AddVolumeMount { source: builder::pod::container::Error, }, + + #[snafu(display("failed to get merged jvmArgumentOverrides"))] + GetMergedJvmArgumentOverrides { + source: stackable_operator::role_utils::Error, + }, } type Result = std::result::Result; @@ -554,6 +558,9 @@ pub async fn reconcile_nifi( // For more information see let proxy_hosts = get_proxy_hosts(client, nifi, &updated_role_service).await?; + let (role_java_common_config, role_group_java_common_config) = role + .get_product_specific_common_configs(rolegroup_name) + .context(GetMergedJvmArgumentOverridesSnafu)?; let rg_configmap = build_node_rolegroup_config_map( nifi, &resolved_product_image, @@ -563,6 +570,8 @@ pub async fn reconcile_nifi( &merged_config, vector_aggregator_address.as_deref(), &proxy_hosts, + role_java_common_config, + role_group_java_common_config, ) .await?; @@ -737,6 +746,8 @@ async fn build_node_rolegroup_config_map( merged_config: &NifiConfig, vector_aggregator_address: Option<&str>, proxy_hosts: &str, + role_java_common_config: &JavaCommonConfig, + role_group_java_common_config: &JavaCommonConfig, ) -> Result { tracing::debug!("building rolegroup configmaps"); @@ -782,6 +793,8 @@ async fn build_node_rolegroup_config_map( kind: NIFI_BOOTSTRAP_CONF.to_string(), })? .clone(), + role_java_common_config, + role_group_java_common_config, ) .context(BootstrapConfigSnafu)?, ) @@ -901,7 +914,7 @@ async fn build_node_rolegroup_statefulset( resolved_product_image: &ResolvedProductImage, cluster_info: &KubernetesClusterInfo, rolegroup_ref: &RoleGroupRef, - role: &Role, + role: &Role, rolegroup_config: &HashMap>, merged_config: &NifiConfig, nifi_auth_config: &NifiAuthenticationConfig,