From 951db0c820993b4db09d6e45e68e7b8b346bd3a3 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 9 Dec 2024 15:14:53 +0800 Subject: [PATCH 1/4] Spark 3.5.4 --- .github/workflows/master.yml | 1 + bin/docker-image-tool.sh | 4 ++-- docs/deployment/kyuubi_on_kubernetes.md | 4 ++-- docs/extensions/engines/spark/lineage.md | 2 +- .../test/deployment/KyuubiOnKubernetesTestsSuite.scala | 2 +- .../kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala | 2 +- pom.xml | 5 +++-- 7 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 19979080933..312353bccc8 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -444,6 +444,7 @@ jobs: # https://minikube.sigs.k8s.io/docs/handbook/pushing/#7-loading-directly-to-in-cluster-container-runtime minikube image load apache/kyuubi:latest # pre-install spark into minikube + # TODO docker pull apache/spark:3.5.2 minikube image load apache/spark:3.5.2 - name: kubectl pre-check diff --git a/bin/docker-image-tool.sh b/bin/docker-image-tool.sh index 7f6f1d62369..5a495c93489 100755 --- a/bin/docker-image-tool.sh +++ b/bin/docker-image-tool.sh @@ -181,8 +181,8 @@ Examples: $0 -r docker.io/myrepo -t v1.8.1 build $0 -r docker.io/myrepo -t v1.8.1 push - - Build and push with tag "v1.8.1" and Spark-3.5.2 as base image to docker.io/myrepo - $0 -r docker.io/myrepo -t v1.8.1 -b BASE_IMAGE=repo/spark:3.5.2 build + - Build and push with tag "v1.8.1" and Spark-3.5.4 as base image to docker.io/myrepo + $0 -r docker.io/myrepo -t v1.8.1 -b BASE_IMAGE=repo/spark:3.5.4 build $0 -r docker.io/myrepo -t v1.8.1 push - Build and push for multiple archs to docker.io/myrepo diff --git a/docs/deployment/kyuubi_on_kubernetes.md b/docs/deployment/kyuubi_on_kubernetes.md index 1d331b56760..80f3e7509b1 100644 --- a/docs/deployment/kyuubi_on_kubernetes.md +++ b/docs/deployment/kyuubi_on_kubernetes.md @@ -42,8 +42,8 @@ Examples: $0 -r docker.io/myrepo -t v1.8.1 build $0 -r docker.io/myrepo -t v1.8.1 push - - Build and push with tag "v1.8.1" and Spark-3.5.2 as base image to docker.io/myrepo - $0 -r docker.io/myrepo -t v1.8.1 -b BASE_IMAGE=repo/spark:3.5.2 build + - Build and push with tag "v1.8.1" and Spark-3.5.4 as base image to docker.io/myrepo + $0 -r docker.io/myrepo -t v1.8.1 -b BASE_IMAGE=repo/spark:3.5.4 build $0 -r docker.io/myrepo -t v1.8.1 push - Build and push for multiple archs to docker.io/myrepo diff --git a/docs/extensions/engines/spark/lineage.md b/docs/extensions/engines/spark/lineage.md index 26e284df53e..a2414269c7a 100644 --- a/docs/extensions/engines/spark/lineage.md +++ b/docs/extensions/engines/spark/lineage.md @@ -117,7 +117,7 @@ Sometimes, it may be incompatible with other Spark distributions, then you may n For example, ```shell -build/mvn clean package -pl :kyuubi-spark-lineage_2.12 -am -DskipTests -Dspark.version=3.5.2 +build/mvn clean package -pl :kyuubi-spark-lineage_2.12 -am -DskipTests -Dspark.version=3.5.4 ``` The available `spark.version`s are shown in the following table. diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala index ead11b3357d..10565087a61 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala @@ -56,7 +56,7 @@ class KyuubiOnKubernetesWithSparkTestsBase extends WithKyuubiServerOnKubernetes Map( "spark.master" -> s"k8s://$miniKubeApiMaster", // We should update spark docker image in ./github/workflows/master.yml at the same time - "spark.kubernetes.container.image" -> "apache/spark:3.5.2", + "spark.kubernetes.container.image" -> "apache/spark:3.5.4", "spark.kubernetes.container.image.pullPolicy" -> "IfNotPresent", "spark.executor.memory" -> "512M", "spark.driver.memory" -> "1024M", diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala index efbe1e2e969..0aff5ca7981 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala @@ -51,7 +51,7 @@ abstract class SparkOnKubernetesSuiteBase // TODO Support more Spark version // Spark official docker image: https://hub.docker.com/r/apache/spark/tags KyuubiConf().set("spark.master", s"k8s://$apiServerAddress") - .set("spark.kubernetes.container.image", "apache/spark:3.5.2") + .set("spark.kubernetes.container.image", "apache/spark:3.5.4") .set("spark.kubernetes.container.image.pullPolicy", "IfNotPresent") .set("spark.executor.instances", "1") .set("spark.executor.memory", "512M") diff --git a/pom.xml b/pom.xml index 73146363243..82cbb7d695d 100644 --- a/pom.xml +++ b/pom.xml @@ -200,7 +200,7 @@ DO NOT forget to change the following properties when change the minor version of Spark: `delta.version`, `delta.artifact`, `maven.plugin.scalatest.exclude.tags` --> - 3.5.2 + 3.5.4 3.5 spark-${spark.version}-bin-hadoop3${spark.archive.scala.suffix}.tgz @@ -2030,8 +2030,9 @@ extensions/spark/kyuubi-spark-connector-hive - 3.5.2 + 3.5.4 3.5 + https://dist.apache.org/repos/dist/dev/spark/v3.5.4-rc1-bin 3.2.0 delta-spark_${scala.binary.version} org.scalatest.tags.Slow From 49e98a201e87a479cec763b9b59810fdaacb7af5 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 9 Dec 2024 15:28:06 +0800 Subject: [PATCH 2/4] maven repo --- pom.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pom.xml b/pom.xml index 82cbb7d695d..713b656a577 100644 --- a/pom.xml +++ b/pom.xml @@ -1282,6 +1282,18 @@ Maven Repository https://repo.maven.apache.org/maven2 + + + + true + + + false + + spark-3.5.4-rc-staging + Spark 3.5.4 RC Staging + https://repository.apache.org/content/repositories/orgapachespark-1471 + From 0721fa401418b9933ac456bfe1cc92d1635fd089 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 10 Dec 2024 13:49:28 +0800 Subject: [PATCH 3/4] fix --- .../test/deployment/KyuubiOnKubernetesTestsSuite.scala | 2 +- .../kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala | 2 +- pom.xml | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala index 10565087a61..f621f0b68de 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala @@ -56,7 +56,7 @@ class KyuubiOnKubernetesWithSparkTestsBase extends WithKyuubiServerOnKubernetes Map( "spark.master" -> s"k8s://$miniKubeApiMaster", // We should update spark docker image in ./github/workflows/master.yml at the same time - "spark.kubernetes.container.image" -> "apache/spark:3.5.4", + "spark.kubernetes.container.image" -> "apache/spark:3.5.2", // TODO "spark.kubernetes.container.image.pullPolicy" -> "IfNotPresent", "spark.executor.memory" -> "512M", "spark.driver.memory" -> "1024M", diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala index 0aff5ca7981..24872293e6d 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala @@ -51,7 +51,7 @@ abstract class SparkOnKubernetesSuiteBase // TODO Support more Spark version // Spark official docker image: https://hub.docker.com/r/apache/spark/tags KyuubiConf().set("spark.master", s"k8s://$apiServerAddress") - .set("spark.kubernetes.container.image", "apache/spark:3.5.4") + .set("spark.kubernetes.container.image", "apache/spark:3.5.2") // TODO .set("spark.kubernetes.container.image.pullPolicy", "IfNotPresent") .set("spark.executor.instances", "1") .set("spark.executor.memory", "512M") diff --git a/pom.xml b/pom.xml index 713b656a577..c5cfd36b652 100644 --- a/pom.xml +++ b/pom.xml @@ -204,7 +204,7 @@ 3.5 spark-${spark.version}-bin-hadoop3${spark.archive.scala.suffix}.tgz - ${apache.archive.dist}/spark/spark-${spark.version} + https://dist.apache.org/repos/dist/dev/spark/v3.5.4-rc1-bin false 3.46.1.3 2.2.0 @@ -2014,6 +2014,7 @@ 3.3.4 3.3 + ${apache.archive.dist}/spark/spark-${spark.version} 2.3.0 delta-core_${scala.binary.version} org.scalatest.tags.Slow @@ -2029,6 +2030,7 @@ 3.4.3 3.4 + ${apache.archive.dist}/spark/spark-${spark.version} 2.4.0 delta-core_${scala.binary.version} org.scalatest.tags.Slow @@ -2059,6 +2061,7 @@ 4.0.0-preview2 4.0 + ${apache.archive.dist}/spark/spark-${spark.version} 4.13.1 4.0.0rc1 From 8eacc9c97bd4eb55f7a0d3d1a08f2956f91ac04d Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 16 Dec 2024 17:23:00 +0800 Subject: [PATCH 4/4] Spark 3.5.4 RC2 --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index c5cfd36b652..d235d2bcd8a 100644 --- a/pom.xml +++ b/pom.xml @@ -204,7 +204,7 @@ 3.5 spark-${spark.version}-bin-hadoop3${spark.archive.scala.suffix}.tgz - https://dist.apache.org/repos/dist/dev/spark/v3.5.4-rc1-bin + https://dist.apache.org/repos/dist/dev/spark/v3.5.4-rc2-bin false 3.46.1.3 2.2.0 @@ -1292,7 +1292,7 @@ spark-3.5.4-rc-staging Spark 3.5.4 RC Staging - https://repository.apache.org/content/repositories/orgapachespark-1471 + https://repository.apache.org/content/repositories/orgapachespark-1473 @@ -2046,7 +2046,7 @@ 3.5.4 3.5 - https://dist.apache.org/repos/dist/dev/spark/v3.5.4-rc1-bin + https://dist.apache.org/repos/dist/dev/spark/v3.5.4-rc2-bin 3.2.0 delta-spark_${scala.binary.version} org.scalatest.tags.Slow