diff --git a/CHANGELOG.md b/CHANGELOG.md index 80944ad8..d193aa04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,13 @@ All notable changes to this project will be documented in this file. - Default stackableVersion to operator version. It is recommended to remove `spec.image.stackableVersion` from your custom resources ([#267], [#268]). - Configuration overrides for the JVM security properties, such as DNS caching ([#272]). - Support PodDisruptionBudgets for HistoryServer ([#288]). +- Support for versions 3.4.1, 3.5.0 ([#291]). +- History server now exports metrics via jmx exporter (port 18081) ([#291]). ### Changed -- `vector` `0.26.0` -> `0.31.0` ([#269]). -- `operator-rs` `0.44.0` -> `0.52.1` ([#267], [#275], [#288]). +- `vector` `0.26.0` -> `0.33.0` ([#269], [#291]). +- `operator-rs` `0.44.0` -> `0.55.0` ([#267], [#275], [#288], [#291]). - Removed usages of SPARK_DAEMON_JAVA_OPTS since it's not a reliable way to pass extra JVM options ([#272]). - [BREAKING] use product image selection instead of version ([#275]). - [BREAKING] refactored application roles to use `CommonConfiguration` structures from the operator framework ([#277]). @@ -23,6 +25,10 @@ All notable changes to this project will be documented in this file. - Dynamic loading of Maven packages ([#281]). +### Removed + +- Removed support for versions 3.2.1, 3.3.0 ([#291]). + [#267]: https://github.com/stackabletech/spark-k8s-operator/pull/267 [#268]: https://github.com/stackabletech/spark-k8s-operator/pull/268 [#269]: https://github.com/stackabletech/spark-k8s-operator/pull/269 @@ -32,6 +38,7 @@ All notable changes to this project will be documented in this file. [#281]: https://github.com/stackabletech/spark-k8s-operator/pull/281 [#286]: https://github.com/stackabletech/spark-k8s-operator/pull/286 [#288]: https://github.com/stackabletech/spark-k8s-operator/pull/288 +[#291]: https://github.com/stackabletech/spark-k8s-operator/pull/291 ## [23.7.0] - 2023-07-14 diff --git a/Cargo.lock b/Cargo.lock index af89e877..8716ff00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1838,8 +1838,8 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "stackable-operator" -version = "0.52.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.52.1#18af9be0473cd6c30d7426e9ade74c90e4abce22" +version = "0.55.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.55.0#bfbc23d3819f815413cb4135e0835acd76aecf97" dependencies = [ "chrono", "clap", @@ -1872,8 +1872,8 @@ dependencies = [ [[package]] name = "stackable-operator-derive" -version = "0.52.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.52.1#18af9be0473cd6c30d7426e9ade74c90e4abce22" +version = "0.55.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.55.0#bfbc23d3819f815413cb4135e0835acd76aecf97" dependencies = [ "darling 0.20.3", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index c0b4dc71..81f73927 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.7" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.52.1" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.55.0" } strum = { version = "0.25", features = ["derive"] } tokio = { version = "1.29", features = ["full"] } tracing = "0.1" diff --git a/deploy/helm/spark-k8s-operator/crds/crds.yaml b/deploy/helm/spark-k8s-operator/crds/crds.yaml index 7088c7b7..581ea19e 100644 --- a/deploy/helm/spark-k8s-operator/crds/crds.yaml +++ b/deploy/helm/spark-k8s-operator/crds/crds.yaml @@ -539,6 +539,7 @@ spec: default: enableVectorAgent: null containers: {} + description: Logging configuration properties: containers: additionalProperties: @@ -546,13 +547,14 @@ spec: - required: - custom - {} - description: Fragment derived from `ContainerLogConfigChoice` + description: Log configuration of the container properties: console: + description: Configuration for the console appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -568,14 +570,16 @@ spec: description: Custom log configuration provided in a ConfigMap properties: configMap: + description: ConfigMap containing the log configuration files nullable: true type: string type: object file: + description: Configuration for the file appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -589,9 +593,10 @@ spec: type: object loggers: additionalProperties: + description: Configuration of a logger properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -604,10 +609,13 @@ spec: type: string type: object default: {} + description: Configuration per logger type: object type: object + description: Log configuration per container type: object enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent nullable: true type: boolean type: object @@ -4049,6 +4057,7 @@ spec: default: enableVectorAgent: null containers: {} + description: Logging configuration properties: containers: additionalProperties: @@ -4056,13 +4065,14 @@ spec: - required: - custom - {} - description: Fragment derived from `ContainerLogConfigChoice` + description: Log configuration of the container properties: console: + description: Configuration for the console appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -4078,14 +4088,16 @@ spec: description: Custom log configuration provided in a ConfigMap properties: configMap: + description: ConfigMap containing the log configuration files nullable: true type: string type: object file: + description: Configuration for the file appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -4099,9 +4111,10 @@ spec: type: object loggers: additionalProperties: + description: Configuration of a logger properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -4114,10 +4127,13 @@ spec: type: string type: object default: {} + description: Configuration per logger type: object type: object + description: Log configuration per container type: object enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent nullable: true type: boolean type: object @@ -7051,6 +7067,7 @@ spec: default: enableVectorAgent: null containers: {} + description: Logging configuration properties: containers: additionalProperties: @@ -7058,13 +7075,14 @@ spec: - required: - custom - {} - description: Fragment derived from `ContainerLogConfigChoice` + description: Log configuration of the container properties: console: + description: Configuration for the console appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -7080,14 +7098,16 @@ spec: description: Custom log configuration provided in a ConfigMap properties: configMap: + description: ConfigMap containing the log configuration files nullable: true type: string type: object file: + description: Configuration for the file appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -7101,9 +7121,10 @@ spec: type: object loggers: additionalProperties: + description: Configuration of a logger properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -7116,10 +7137,13 @@ spec: type: string type: object default: {} + description: Configuration per logger type: object type: object + description: Log configuration per container type: object enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent nullable: true type: boolean type: object @@ -12074,6 +12098,7 @@ spec: default: enableVectorAgent: null containers: {} + description: Logging configuration properties: containers: additionalProperties: @@ -12081,13 +12106,14 @@ spec: - required: - custom - {} - description: Fragment derived from `ContainerLogConfigChoice` + description: Log configuration of the container properties: console: + description: Configuration for the console appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -12103,14 +12129,16 @@ spec: description: Custom log configuration provided in a ConfigMap properties: configMap: + description: ConfigMap containing the log configuration files nullable: true type: string type: object file: + description: Configuration for the file appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -12124,9 +12152,10 @@ spec: type: object loggers: additionalProperties: + description: Configuration of a logger properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -12139,10 +12168,13 @@ spec: type: string type: object default: {} + description: Configuration per logger type: object type: object + description: Log configuration per container type: object enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent nullable: true type: boolean type: object @@ -15507,6 +15539,7 @@ spec: default: enableVectorAgent: null containers: {} + description: Logging configuration properties: containers: additionalProperties: @@ -15514,13 +15547,14 @@ spec: - required: - custom - {} - description: Fragment derived from `ContainerLogConfigChoice` + description: Log configuration of the container properties: console: + description: Configuration for the console appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -15536,14 +15570,16 @@ spec: description: Custom log configuration provided in a ConfigMap properties: configMap: + description: ConfigMap containing the log configuration files nullable: true type: string type: object file: + description: Configuration for the file appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -15557,9 +15593,10 @@ spec: type: object loggers: additionalProperties: + description: Configuration of a logger properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -15572,10 +15609,13 @@ spec: type: string type: object default: {} + description: Configuration per logger type: object type: object + description: Log configuration per container type: object enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent nullable: true type: boolean type: object diff --git a/docs/modules/spark-k8s/examples/example-encapsulated.yaml b/docs/modules/spark-k8s/examples/example-encapsulated.yaml index 52a39f99..bdc6d3f4 100644 --- a/docs/modules/spark-k8s/examples/example-encapsulated.yaml +++ b/docs/modules/spark-k8s/examples/example-encapsulated.yaml @@ -6,7 +6,7 @@ metadata: spec: version: "1.0" sparkImage: - productVersion: 3.3.0 # <1> + productVersion: 3.5.0 # <1> mode: cluster mainClass: org.apache.spark.examples.SparkPi mainApplicationFile: /stackable/spark/examples/jars/spark-examples.jar # <2> diff --git a/docs/modules/spark-k8s/examples/example-history-app.yaml b/docs/modules/spark-k8s/examples/example-history-app.yaml index 825415b7..95ed5350 100644 --- a/docs/modules/spark-k8s/examples/example-history-app.yaml +++ b/docs/modules/spark-k8s/examples/example-history-app.yaml @@ -6,7 +6,7 @@ metadata: spec: version: "1.0" sparkImage: - productVersion: 3.3.0 + productVersion: 3.5.0 pullPolicy: IfNotPresent mode: cluster mainClass: org.apache.spark.examples.SparkPi diff --git a/docs/modules/spark-k8s/examples/example-history-server.yaml b/docs/modules/spark-k8s/examples/example-history-server.yaml index a023b378..64afd14e 100644 --- a/docs/modules/spark-k8s/examples/example-history-server.yaml +++ b/docs/modules/spark-k8s/examples/example-history-server.yaml @@ -5,7 +5,7 @@ metadata: name: spark-history spec: image: - productVersion: 3.3.0 + productVersion: 3.5.0 logFileDirectory: # <1> s3: prefix: eventlogs/ # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml index ee49c0db..a5c90a8f 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml @@ -7,7 +7,7 @@ metadata: spec: version: "1.0" sparkImage: - productVersion: 3.3.0 + productVersion: 3.5.0 mode: cluster mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.1.0.jar # <3> mainClass: tech.stackable.demo.spark.NYTLCReport diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml index 348b8a20..b1c798c5 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml @@ -7,7 +7,7 @@ metadata: spec: version: "1.0" sparkImage: - productVersion: 3.3.0 + productVersion: 3.5.0 mode: cluster mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny_tlc_report.py # <1> args: diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml index 2671d519..13e7fe9c 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml @@ -8,7 +8,7 @@ spec: version: "1.0" image: docker.stackable.tech/stackable/ny-tlc-report:0.1.0 # <1> sparkImage: - productVersion: 3.3.0 + productVersion: 3.5.0 mode: cluster mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py # <2> args: diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml index 1beb37ff..0bce8d0f 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml @@ -7,7 +7,7 @@ metadata: spec: version: "1.0" sparkImage: - productVersion: 3.3.0 + productVersion: 3.5.0 mode: cluster mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.0-SNAPSHOT.jar # <1> mainClass: org.example.App # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml index bf65af2e..b9b06157 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml @@ -6,7 +6,7 @@ metadata: spec: version: "1.0" sparkImage: - productVersion: 3.3.0 + productVersion: 3.5.0 mode: cluster mainApplicationFile: s3a://my-bucket/spark-examples.jar # <1> mainClass: org.apache.spark.examples.SparkPi # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml index 06b4ab90..34213995 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml @@ -7,7 +7,7 @@ metadata: spec: version: "1.0" sparkImage: - productVersion: 3.3.0 + productVersion: 3.5.0 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py args: diff --git a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh index 5a2a37d5..196c0794 100755 --- a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh +++ b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh @@ -59,7 +59,7 @@ metadata: spec: version: "1.0" sparkImage: - productVersion: 3.3.0 + productVersion: 3.5.0 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py driver: diff --git a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 index ac688ab2..90699d20 100755 --- a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 +++ b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 @@ -59,7 +59,7 @@ metadata: spec: version: "1.0" sparkImage: - productVersion: 3.3.0 + productVersion: 3.5.0 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py driver: diff --git a/docs/modules/spark-k8s/pages/crd-reference.adoc b/docs/modules/spark-k8s/pages/crd-reference.adoc index 984cfe65..7a8b1135 100644 --- a/docs/modules/spark-k8s/pages/crd-reference.adoc +++ b/docs/modules/spark-k8s/pages/crd-reference.adoc @@ -24,7 +24,7 @@ Below are listed the CRD fields that can be defined by the user: |User-supplied image containing spark-job dependencies that will be copied to the specified volume mount |`spec.sparkImage` -| Spark image which will be deployed to driver and executor pods, which must contain spark environment needed by the job e.g. `docker.stackable.tech/stackable/spark-k8s:3.3.0-stackable0.3.0` +| Spark image which will be deployed to driver and executor pods, which must contain spark environment needed by the job e.g. `docker.stackable.tech/stackable/spark-k8s:3.5.0-stackable0.0.0-dev` |`spec.sparkImagePullPolicy` | Optional Enum (one of `Always`, `IfNotPresent` or `Never`) that determines the pull policy of the spark job image diff --git a/docs/modules/spark-k8s/pages/usage-guide/history-server.adoc b/docs/modules/spark-k8s/pages/usage-guide/history-server.adoc index 4392d50b..c8d851d0 100644 --- a/docs/modules/spark-k8s/pages/usage-guide/history-server.adoc +++ b/docs/modules/spark-k8s/pages/usage-guide/history-server.adoc @@ -76,7 +76,7 @@ For a role group of the Spark history server, you can specify: `configOverrides` The `security.properties` file is used to configure JVM security properties. It is very seldom that users need to tweak any of these, but there is one use-case that stands out, and that users need to be aware of: the JVM DNS cache. -The JVM manages it's own cache of successfully resolved host names as well as a cache of host names that cannot be resolved. Some products of the Stackable platform are very sensible to the contents of these caches and their performance is heavily affected by them. As of version 3.4.0, Apache Spark may perform poorly if the positive cache is disabled. To cache resolved host names, and thus speeding up queries you can configure the TTL of entries in the positive cache like this: +The JVM manages its own cache of successfully resolved host names as well as a cache of host names that cannot be resolved. Some products of the Stackable platform are very sensible to the contents of these caches and their performance is heavily affected by them. As of version 3.4.0, Apache Spark may perform poorly if the positive cache is disabled. To cache resolved host names, and thus speeding up queries you can configure the TTL of entries in the positive cache like this: [source,yaml] ---- diff --git a/docs/modules/spark-k8s/partials/supported-versions.adoc b/docs/modules/spark-k8s/partials/supported-versions.adoc index 4f54560b..11f8925a 100644 --- a/docs/modules/spark-k8s/partials/supported-versions.adoc +++ b/docs/modules/spark-k8s/partials/supported-versions.adoc @@ -3,5 +3,6 @@ // Stackable Platform documentation. // Please sort the versions in descending order (newest first) -- 3.4.0 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) -- 3.3.0 (Hadoop 3.3.3, Scala 2.12, Python 3.9, Java 11) +- 3.5.0 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) +- 3.4.1 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) +- 3.4.0 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (deprecated) diff --git a/examples/README-examples.md b/examples/README-examples.md index bef8e3b6..f89364d9 100644 --- a/examples/README-examples.md +++ b/examples/README-examples.md @@ -50,10 +50,10 @@ Several resources are needed in this store. These can be loaded like this: ````text kubectl exec minio-mc-0 -- sh -c 'mc alias set test-minio http://test-minio:9000/' -kubectl cp examples/ny-tlc-report-1.1.0-3.3.0.jar minio-mc-0:/tmp +kubectl cp examples/ny-tlc-report-1.1.0-3.5.0.jar minio-mc-0:/tmp kubectl cp apps/ny_tlc_report.py minio-mc-0:/tmp kubectl cp examples/yellow_tripdata_2021-07.csv minio-mc-0:/tmp -kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.3.0.jar test-minio/my-bucket +kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.0.jar test-minio/my-bucket kubectl exec minio-mc-0 -- mc cp /tmp/ny_tlc_report.py test-minio/my-bucket kubectl exec minio-mc-0 -- mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket ```` diff --git a/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.4.0.jar b/examples/ny-tlc-report-1.1.0-3.5.0.jar similarity index 89% rename from tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.4.0.jar rename to examples/ny-tlc-report-1.1.0-3.5.0.jar index 6466711d..c059ccbc 100644 Binary files a/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.4.0.jar and b/examples/ny-tlc-report-1.1.0-3.5.0.jar differ diff --git a/examples/ny-tlc-report-external-dependencies.yaml b/examples/ny-tlc-report-external-dependencies.yaml index c2c9b516..f5d3bc69 100644 --- a/examples/ny-tlc-report-external-dependencies.yaml +++ b/examples/ny-tlc-report-external-dependencies.yaml @@ -7,7 +7,7 @@ metadata: spec: version: "1.0" sparkImage: - productVersion: 3.3.0 + productVersion: 3.5.0 pullPolicy: IfNotPresent mode: cluster mainApplicationFile: s3a://my-bucket/ny_tlc_report.py diff --git a/examples/ny-tlc-report-image.yaml b/examples/ny-tlc-report-image.yaml index a5c16feb..d2a8dfa6 100644 --- a/examples/ny-tlc-report-image.yaml +++ b/examples/ny-tlc-report-image.yaml @@ -8,7 +8,7 @@ spec: version: "1.0" # everything under /jobs will be copied to /stackable/spark/jobs image: docker.stackable.tech/stackable/ny-tlc-report:0.1.0 - sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.3.0-stackable0.0.0-dev + sparkImage: docker.stackable.tech/stackable/spark-k8s:3.5.0-stackable0.0.0-dev sparkImagePullPolicy: IfNotPresent mode: cluster mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py diff --git a/examples/ny-tlc-report.yaml b/examples/ny-tlc-report.yaml index c4fb8e44..dfd14139 100644 --- a/examples/ny-tlc-report.yaml +++ b/examples/ny-tlc-report.yaml @@ -14,9 +14,9 @@ metadata: spec: version: "1.0" sparkImage: - productVersion: 3.3.0 + productVersion: 3.5.0 mode: cluster - mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.3.0.jar + mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.5.0.jar mainClass: tech.stackable.demo.spark.NYTLCReport volumes: - name: cm-job-arguments diff --git a/rust/crd/src/affinity.rs b/rust/crd/src/affinity.rs index 6c60ebe7..6de3ebad 100644 --- a/rust/crd/src/affinity.rs +++ b/rust/crd/src/affinity.rs @@ -47,7 +47,7 @@ mod test { name: spark-history spec: image: - productVersion: 3.3.0 + productVersion: 3.5.0 logFileDirectory: s3: prefix: eventlogs/ diff --git a/rust/operator-binary/src/history/history_controller.rs b/rust/operator-binary/src/history/history_controller.rs index c3c55502..0627fb3c 100644 --- a/rust/operator-binary/src/history/history_controller.rs +++ b/rust/operator-binary/src/history/history_controller.rs @@ -5,7 +5,6 @@ use stackable_operator::{ builder::{ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder, VolumeBuilder}, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::product_image_selection::ResolvedProductImage, - duration::Duration, k8s_openapi::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, @@ -31,6 +30,7 @@ use stackable_operator::{ }, }, role_utils::RoleGroupRef, + time::Duration, }; use stackable_spark_k8s_crd::{ constants::{ @@ -55,6 +55,8 @@ use stackable_operator::k8s_openapi::DeepMerge; use stackable_operator::logging::controller::ReconcilerError; use strum::{EnumDiscriminants, IntoStaticStr}; +const METRICS_PORT: u16 = 18081; + #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] #[allow(clippy::enum_variant_names)] @@ -415,6 +417,7 @@ fn build_stateful_set( .command(vec!["/bin/bash".to_string()]) .args(command_args(s3_log_dir)) .add_container_port("http", 18080) + .add_container_port("metrics", METRICS_PORT.into()) .add_env_vars(env_vars(s3_log_dir)) .add_volume_mounts(s3_log_dir.volume_mounts()) .add_volume_mount(VOLUME_MOUNT_NAME_CONFIG, VOLUME_MOUNT_PATH_CONFIG) @@ -515,15 +518,23 @@ fn build_service( .ownerreference_from_resource(shs, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(labels(shs, app_version_label, &group_name)) + .with_label("prometheus.io/scrape", "true") .build(), spec: Some(ServiceSpec { type_: Some(service_type), cluster_ip: service_cluster_ip, - ports: Some(vec![ServicePort { - name: Some(String::from("http")), - port: 18080, - ..ServicePort::default() - }]), + ports: Some(vec![ + ServicePort { + name: Some(String::from("http")), + port: 18080, + ..ServicePort::default() + }, + ServicePort { + name: Some(String::from("metrics")), + port: METRICS_PORT.into(), + ..ServicePort::default() + }, + ]), selector: Some(selector), ..ServiceSpec::default() }), @@ -634,6 +645,7 @@ fn env_vars(s3logdir: &S3LogDir) -> Vec { format!( "-Djava.security.properties={VOLUME_MOUNT_PATH_CONFIG}/{JVM_SECURITY_PROPERTIES_FILE}" ), + format!("-javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={METRICS_PORT}:/stackable/jmx/config.yaml") ]; if tlscerts::tls_secret_name(&s3logdir.bucket.connection).is_some() { history_opts.extend( diff --git a/rust/operator-binary/src/pod_driver_controller.rs b/rust/operator-binary/src/pod_driver_controller.rs index 478e61a7..272fe1e3 100644 --- a/rust/operator-binary/src/pod_driver_controller.rs +++ b/rust/operator-binary/src/pod_driver_controller.rs @@ -1,6 +1,6 @@ use stackable_operator::{ - client::Client, duration::Duration, k8s_openapi::api::core::v1::Pod, - kube::runtime::controller::Action, + client::Client, k8s_openapi::api::core::v1::Pod, kube::runtime::controller::Action, + time::Duration, }; use stackable_spark_k8s_crd::{ constants::POD_DRIVER_CONTROLLER_NAME, SparkApplication, SparkApplicationStatus, diff --git a/rust/operator-binary/src/spark_k8s_controller.rs b/rust/operator-binary/src/spark_k8s_controller.rs index d29332a9..edb0e09b 100644 --- a/rust/operator-binary/src/spark_k8s_controller.rs +++ b/rust/operator-binary/src/spark_k8s_controller.rs @@ -6,7 +6,7 @@ use std::{ vec, }; -use stackable_operator::{duration::Duration, product_config::writer::to_java_properties_string}; +use stackable_operator::{product_config::writer::to_java_properties_string, time::Duration}; use stackable_spark_k8s_crd::{ constants::*, s3logdir::S3LogDir, tlscerts, RoleConfig, SparkApplication, SparkApplicationRole, SparkContainer, SubmitConfig, diff --git a/tests/README-templating.md b/tests/README-templating.md index 8da9fe91..cd35e45b 100644 --- a/tests/README-templating.md +++ b/tests/README-templating.md @@ -16,13 +16,11 @@ An example of the content for the test definition file is shown here: dimensions: - name: spark values: - - 3.2.1 - - 3.2.2 - - 3.2.3 + - 3.4.1 + - 3.5.0 - name: hadoop values: - - 3.1.0 - - 3.2.0 + - 3.3.4 - name: aws - abc - xyz @@ -39,12 +37,8 @@ In this example the test case uses only two of the three dimensions defined, so ````text └── spark-pi-public-s3 - ├── spark-3.2.1_hadoop-3.1.0 - ├── spark-3.2.1_hadoop-3.2.0 - ├── spark-3.2.2_hadoop-3.1.0 - ├── spark-3.2.2_hadoop-3.2.0 - ├── spark-3.2.3_hadoop-3.1.0 - └── spark-3.2.3_hadoop-3.2.0 + ├── spark-3.4.1_hadoop-3.3.4 + ├── spark-3.5.0_hadoop-3.3.4 ```` The name of a test case defined under `tests` in this file has to refer back to a directory in the `templates/kuttl` directory, which will be used to create the test scenarios. @@ -61,7 +55,7 @@ tests ```` The `kuttl-test.yaml.jinja2` cannot currently be edited, as it comes from the operator templating and any changes would be overwritten again. -This should be fairly easy to solve and we can look at this as soon as it becomes necessary. +This should be fairly easy to solve, and we can look at this as soon as it becomes necessary. ## Using diff --git a/tests/templates/kuttl/iceberg/10-assert.yaml.j2 b/tests/templates/kuttl/iceberg/10-assert.yaml.j2 index 6a97da89..0881ff10 100644 --- a/tests/templates/kuttl/iceberg/10-assert.yaml.j2 +++ b/tests/templates/kuttl/iceberg/10-assert.yaml.j2 @@ -9,9 +9,4 @@ kind: SparkApplication metadata: name: pyspark-iceberg status: -{% if test_scenario['values']['spark'].startswith("3.3") %} - # Spark 3.3 is expected to fail because of this https://issues.apache.org/jira/browse/SPARK-35084 - phase: Failed -{% else %} phase: Succeeded -{% endif %} diff --git a/tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2 index 8dcf96df..943e63e0 100644 --- a/tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2 @@ -45,7 +45,8 @@ spec: mountPath: /stackable/spark/jobs deps: packages: - - org.apache.iceberg:iceberg-spark-runtime-{{ test_scenario['values']['spark'].rstrip('.0') }}_2.12:1.3.1 + # need to extract only the major and minor versions + - org.apache.iceberg:iceberg-spark-runtime-{{ test_scenario['values']['spark'].rsplit('.', maxsplit=1)[0] }}_2.12:1.4.0 volumes: - name: script configMap: diff --git a/tests/templates/kuttl/pod_overrides/spark-examples_3.3.0.jar b/tests/templates/kuttl/pod_overrides/spark-examples_3.3.0.jar deleted file mode 100644 index c5c34c3a..00000000 Binary files a/tests/templates/kuttl/pod_overrides/spark-examples_3.3.0.jar and /dev/null differ diff --git a/tests/templates/kuttl/pod_overrides/spark-examples_3.4.1.jar b/tests/templates/kuttl/pod_overrides/spark-examples_3.4.1.jar new file mode 100644 index 00000000..2dfa1b37 Binary files /dev/null and b/tests/templates/kuttl/pod_overrides/spark-examples_3.4.1.jar differ diff --git a/tests/templates/kuttl/pod_overrides/spark-examples_3.5.0.jar b/tests/templates/kuttl/pod_overrides/spark-examples_3.5.0.jar new file mode 100644 index 00000000..ec899ffe Binary files /dev/null and b/tests/templates/kuttl/pod_overrides/spark-examples_3.5.0.jar differ diff --git a/tests/templates/kuttl/smoke/spark-examples_3.3.0.jar b/tests/templates/kuttl/smoke/spark-examples_3.3.0.jar deleted file mode 100644 index c5c34c3a..00000000 Binary files a/tests/templates/kuttl/smoke/spark-examples_3.3.0.jar and /dev/null differ diff --git a/tests/templates/kuttl/smoke/spark-examples_3.4.1.jar b/tests/templates/kuttl/smoke/spark-examples_3.4.1.jar new file mode 100644 index 00000000..2dfa1b37 Binary files /dev/null and b/tests/templates/kuttl/smoke/spark-examples_3.4.1.jar differ diff --git a/tests/templates/kuttl/smoke/spark-examples_3.5.0.jar b/tests/templates/kuttl/smoke/spark-examples_3.5.0.jar new file mode 100644 index 00000000..ec899ffe Binary files /dev/null and b/tests/templates/kuttl/smoke/spark-examples_3.5.0.jar differ diff --git a/tests/templates/kuttl/spark-history-server/spark-examples_3.3.0.jar b/tests/templates/kuttl/spark-history-server/spark-examples_3.3.0.jar deleted file mode 100644 index c5c34c3a..00000000 Binary files a/tests/templates/kuttl/spark-history-server/spark-examples_3.3.0.jar and /dev/null differ diff --git a/tests/templates/kuttl/spark-history-server/spark-examples_3.4.1.jar b/tests/templates/kuttl/spark-history-server/spark-examples_3.4.1.jar new file mode 100644 index 00000000..2dfa1b37 Binary files /dev/null and b/tests/templates/kuttl/spark-history-server/spark-examples_3.4.1.jar differ diff --git a/tests/templates/kuttl/spark-history-server/spark-examples_3.5.0.jar b/tests/templates/kuttl/spark-history-server/spark-examples_3.5.0.jar new file mode 100644 index 00000000..ec899ffe Binary files /dev/null and b/tests/templates/kuttl/spark-history-server/spark-examples_3.5.0.jar differ diff --git a/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.3.0.jar b/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.4.1.jar similarity index 84% rename from tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.3.0.jar rename to tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.4.1.jar index b6db9c3e..1728484f 100644 Binary files a/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.3.0.jar and b/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.4.1.jar differ diff --git a/examples/ny-tlc-report-1.1.0-3.3.0.jar b/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.0.jar similarity index 84% rename from examples/ny-tlc-report-1.1.0-3.3.0.jar rename to tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.0.jar index b6db9c3e..c059ccbc 100644 Binary files a/examples/ny-tlc-report-1.1.0-3.3.0.jar and b/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.0.jar differ diff --git a/tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.3.0.jar b/tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.3.0.jar deleted file mode 100644 index c5c34c3a..00000000 Binary files a/tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.3.0.jar and /dev/null differ diff --git a/tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.4.1.jar b/tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.4.1.jar new file mode 100644 index 00000000..2dfa1b37 Binary files /dev/null and b/tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.4.1.jar differ diff --git a/tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.5.0.jar b/tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.5.0.jar new file mode 100644 index 00000000..ec899ffe Binary files /dev/null and b/tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.5.0.jar differ diff --git a/tests/templates/kuttl/spark-pi-public-s3/spark-examples_3.3.0.jar b/tests/templates/kuttl/spark-pi-public-s3/spark-examples_3.3.0.jar deleted file mode 100644 index c5c34c3a..00000000 Binary files a/tests/templates/kuttl/spark-pi-public-s3/spark-examples_3.3.0.jar and /dev/null differ diff --git a/tests/templates/kuttl/spark-pi-public-s3/spark-examples_3.4.1.jar b/tests/templates/kuttl/spark-pi-public-s3/spark-examples_3.4.1.jar new file mode 100644 index 00000000..2dfa1b37 Binary files /dev/null and b/tests/templates/kuttl/spark-pi-public-s3/spark-examples_3.4.1.jar differ diff --git a/tests/templates/kuttl/spark-pi-public-s3/spark-examples_3.5.0.jar b/tests/templates/kuttl/spark-pi-public-s3/spark-examples_3.5.0.jar new file mode 100644 index 00000000..ec899ffe Binary files /dev/null and b/tests/templates/kuttl/spark-pi-public-s3/spark-examples_3.5.0.jar differ diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index b399ceb0..d33409c7 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -5,8 +5,8 @@ dimensions: - "false" - name: spark values: - - 3.3.0 - - 3.4.0 + - 3.4.1 + - 3.5.0 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. # - 3.4.0,docker.stackable.tech/sandbox/spark-k8s:3.4.0-stackable0.0.0-dev