From 7c234e6bd56f95f711348f6e86e53df2728b3eb9 Mon Sep 17 00:00:00 2001
From: zhengtao <shuaizhentao.szt@alibaba-inc.com>
Date: Fri, 8 Nov 2024 10:28:09 +0800
Subject: [PATCH] add sbt spark version

---
 build/make-distribution.sh                    |  7 ++---
 client-spark/spark-2-shaded/pom.xml           |  4 +--
 client-spark/spark-2/pom.xml                  |  2 +-
 client-spark/spark-3-columnar-common/pom.xml  |  4 +--
 client-spark/spark-3-columnar-shuffle/pom.xml |  6 ++--
 client-spark/spark-3-shaded/pom.xml           | 14 +++++-----
 .../spark-3.5-columnar-shuffle/pom.xml        |  4 +--
 client-spark/spark-3/pom.xml                  |  2 +-
 pom.xml                                       |  8 ------
 project/CelebornBuild.scala                   | 13 +++++++--
 tests/spark-it/pom.xml                        | 28 +++++++++----------
 11 files changed, 44 insertions(+), 48 deletions(-)

diff --git a/build/make-distribution.sh b/build/make-distribution.sh
index 9d42e86614a..e8044ffcb88 100755
--- a/build/make-distribution.sh
+++ b/build/make-distribution.sh
@@ -185,13 +185,10 @@ function build_spark_client {
       | grep -v "WARNING" \
       | tail -n 1)
   SPARK_MAJOR_VERSION=${SPARK_VERSION%%.*}
-  SPARK_MINOR_VERSION=${SPARK_VERSION#*.}
-  SPARK_MINOR_VERSION=${SPARK_MINOR_VERSION%%.*}
-  SPARK_MAJOR_MINOR_VERSION="$SPARK_MAJOR_VERSION.$SPARK_MINOR_VERSION"
   # Store the command as an array because $MVN variable might have spaces in it.
   # Normal quoting tricks don't work.
   # See: http://mywiki.wooledge.org/BashFAQ/050
-  BUILD_COMMAND=("$MVN" clean package $MVN_DIST_OPT -pl :celeborn-client-spark-${SPARK_MAJOR_MINOR_VERSION}-shaded_$SCALA_VERSION -am $@)
+  BUILD_COMMAND=("$MVN" clean package $MVN_DIST_OPT -pl :celeborn-client-spark-${SPARK_VERSION}-shaded_$SCALA_VERSION -am $@)
 
   # Actually build the jar
   echo -e "\nBuilding with..."
@@ -202,7 +199,7 @@ function build_spark_client {
   mkdir -p "$DIST_DIR/spark"
 
   ## Copy spark client jars
-  cp "$PROJECT_DIR"/client-spark/spark-${SPARK_MAJOR_VERSION}-shaded/target/celeborn-client-spark-${SPARK_MAJOR_MINOR_VERSION}-shaded_$SCALA_VERSION-$VERSION.jar "$DIST_DIR/spark/"
+  cp "$PROJECT_DIR"/client-spark/spark-${SPARK_MAJOR_VERSION}-shaded/target/celeborn-client-spark-${SPARK_VERSION}-shaded_$SCALA_VERSION-$VERSION.jar "$DIST_DIR/spark/"
 }
 
 function build_flink_client {
diff --git a/client-spark/spark-2-shaded/pom.xml b/client-spark/spark-2-shaded/pom.xml
index beb3c405553..0d79233d0a8 100644
--- a/client-spark/spark-2-shaded/pom.xml
+++ b/client-spark/spark-2-shaded/pom.xml
@@ -24,14 +24,14 @@
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>celeborn-client-spark-${spark.version.majorminor}-shaded_${scala.binary.version}</artifactId>
+  <artifactId>celeborn-client-spark-${spark.version}-shaded_${scala.binary.version}</artifactId>
   <packaging>jar</packaging>
   <name>Celeborn Shaded Client for Spark 2</name>
 
   <dependencies>
     <dependency>
       <groupId>org.apache.celeborn</groupId>
-      <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+      <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
   </dependencies>
diff --git a/client-spark/spark-2/pom.xml b/client-spark/spark-2/pom.xml
index 1836701fce0..9ed76e34a81 100644
--- a/client-spark/spark-2/pom.xml
+++ b/client-spark/spark-2/pom.xml
@@ -24,7 +24,7 @@
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+  <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
   <packaging>jar</packaging>
   <name>Celeborn Client for Spark 2</name>
 
diff --git a/client-spark/spark-3-columnar-common/pom.xml b/client-spark/spark-3-columnar-common/pom.xml
index 340d6aaaa31..935d01cc307 100644
--- a/client-spark/spark-3-columnar-common/pom.xml
+++ b/client-spark/spark-3-columnar-common/pom.xml
@@ -24,14 +24,14 @@
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-common_${scala.binary.version}</artifactId>
+  <artifactId>celeborn-spark-${spark.version}-columnar-common_${scala.binary.version}</artifactId>
   <packaging>jar</packaging>
   <name>Celeborn Client for Spark 3 Columnar Common</name>
 
   <dependencies>
     <dependency>
       <groupId>org.apache.celeborn</groupId>
-      <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+      <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>
diff --git a/client-spark/spark-3-columnar-shuffle/pom.xml b/client-spark/spark-3-columnar-shuffle/pom.xml
index b26008793ef..8f19934c716 100644
--- a/client-spark/spark-3-columnar-shuffle/pom.xml
+++ b/client-spark/spark-3-columnar-shuffle/pom.xml
@@ -24,14 +24,14 @@
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-shuffle_${scala.binary.version}</artifactId>
+  <artifactId>celeborn-spark-${spark.version}-columnar-shuffle_${scala.binary.version}</artifactId>
   <packaging>jar</packaging>
   <name>Celeborn Client for Spark 3 Columnar Shuffle</name>
 
   <dependencies>
     <dependency>
       <groupId>org.apache.celeborn</groupId>
-      <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-common_${scala.binary.version}</artifactId>
+      <artifactId>celeborn-spark-${spark.version}-columnar-common_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>
@@ -49,7 +49,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.celeborn</groupId>
-      <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+      <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
       <type>test-jar</type>
       <scope>test</scope>
diff --git a/client-spark/spark-3-shaded/pom.xml b/client-spark/spark-3-shaded/pom.xml
index d8a4d9435c8..2e2d0c34373 100644
--- a/client-spark/spark-3-shaded/pom.xml
+++ b/client-spark/spark-3-shaded/pom.xml
@@ -24,14 +24,14 @@
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>celeborn-client-spark-${spark.version.majorminor}-shaded_${scala.binary.version}</artifactId>
+  <artifactId>celeborn-client-spark-${spark.version}-shaded_${scala.binary.version}</artifactId>
   <packaging>jar</packaging>
   <name>Celeborn Shaded Client for Spark 3</name>
 
   <dependencies>
     <dependency>
       <groupId>org.apache.celeborn</groupId>
-      <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+      <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
   </dependencies>
@@ -147,7 +147,7 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-shuffle_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-spark-${spark.version}-columnar-shuffle_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
         </dependency>
       </dependencies>
@@ -157,7 +157,7 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-shuffle_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-spark-${spark.version}-columnar-shuffle_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
         </dependency>
       </dependencies>
@@ -167,7 +167,7 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-shuffle_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-spark-${spark.version}-columnar-shuffle_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
         </dependency>
       </dependencies>
@@ -177,7 +177,7 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-shuffle_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-spark-${spark.version}-columnar-shuffle_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
         </dependency>
       </dependencies>
@@ -187,7 +187,7 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-shuffle_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-spark-${spark.version}-columnar-shuffle_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
         </dependency>
       </dependencies>
diff --git a/client-spark/spark-3.5-columnar-shuffle/pom.xml b/client-spark/spark-3.5-columnar-shuffle/pom.xml
index 37455c4f935..718ba32fc63 100644
--- a/client-spark/spark-3.5-columnar-shuffle/pom.xml
+++ b/client-spark/spark-3.5-columnar-shuffle/pom.xml
@@ -31,7 +31,7 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.celeborn</groupId>
-      <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-common_${scala.binary.version}</artifactId>
+      <artifactId>celeborn-spark-${spark.version}-columnar-common_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>
@@ -49,7 +49,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.celeborn</groupId>
-      <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+      <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
       <type>test-jar</type>
       <scope>test</scope>
diff --git a/client-spark/spark-3/pom.xml b/client-spark/spark-3/pom.xml
index d51c93c486c..ed0b22c62f9 100644
--- a/client-spark/spark-3/pom.xml
+++ b/client-spark/spark-3/pom.xml
@@ -24,7 +24,7 @@
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+  <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
   <packaging>jar</packaging>
   <name>Celeborn Client for Spark 3</name>
 
diff --git a/pom.xml b/pom.xml
index f421ce548ce..d68a665dfff 100644
--- a/pom.xml
+++ b/pom.xml
@@ -68,7 +68,6 @@
 
     <flink.version>1.14.6</flink.version>
     <spark.version>3.4.4</spark.version>
-    <spark.version.majorminor>3.4</spark.version.majorminor>
 
     <!-- use hadoop-3 as default  -->
     <hadoop.version>3.3.6</hadoop.version>
@@ -1374,7 +1373,6 @@
         <scala.version>2.11.12</scala.version>
         <scala.binary.version>2.11</scala.binary.version>
         <spark.version>2.4.8</spark.version>
-        <spark.version.majorminor>2.4</spark.version.majorminor>
         <zstd-jni.version>1.4.4-3</zstd-jni.version>
       </properties>
     </profile>
@@ -1394,7 +1392,6 @@
         <scala.version>2.12.10</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
         <spark.version>3.0.3</spark.version>
-        <spark.version.majorminor>3.0</spark.version.majorminor>
         <zstd-jni.version>1.4.4-3</zstd-jni.version>
         <maven.plugin.silencer.version>1.6.0</maven.plugin.silencer.version>
       </properties>
@@ -1415,7 +1412,6 @@
         <scala.version>2.12.10</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
         <spark.version>3.1.3</spark.version>
-        <spark.version.majorminor>3.1</spark.version.majorminor>
         <zstd-jni.version>1.4.8-1</zstd-jni.version>
         <maven.plugin.silencer.version>1.6.0</maven.plugin.silencer.version>
       </properties>
@@ -1436,7 +1432,6 @@
         <scala.version>2.12.15</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
         <spark.version>3.2.4</spark.version>
-        <spark.version.majorminor>3.2</spark.version.majorminor>
         <zstd-jni.version>1.5.0-4</zstd-jni.version>
       </properties>
     </profile>
@@ -1456,7 +1451,6 @@
         <scala.version>2.12.15</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
         <spark.version>3.3.4</spark.version>
-        <spark.version.majorminor>3.3</spark.version.majorminor>
         <zstd-jni.version>1.5.2-1</zstd-jni.version>
       </properties>
     </profile>
@@ -1476,7 +1470,6 @@
         <scala.version>2.12.17</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
         <spark.version>3.4.4</spark.version>
-        <spark.version.majorminor>3.4</spark.version.majorminor>
         <zstd-jni.version>1.5.2-5</zstd-jni.version>
       </properties>
     </profile>
@@ -1496,7 +1489,6 @@
         <scala.version>2.12.18</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
         <spark.version>3.5.3</spark.version>
-        <spark.version.majorminor>3.5</spark.version.majorminor>
         <zstd-jni.version>1.5.5-4</zstd-jni.version>
       </properties>
     </profile>
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index a28c15d2af8..ad03740f3ee 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -890,6 +890,13 @@ trait SparkClientProjects {
       )
   }
 
+  // the output would be something like: celeborn-client-spark-3.3.4-shaded_2.12-0.6.0-SNAPSHOT.jar
+  def sparkClientShadeJarName(
+                               revision: String,
+                               artifact: Artifact,
+                               scalaBinaryVersionString: String): String =
+    s"celeborn-client-spark-$sparkVersion-shaded_$scalaBinaryVersionString" + "-" + revision + "." + artifact.extension
+
   def sparkClientShade: Project = {
     var p = Project(sparkClientShadedProjectName, file(sparkClientShadedProjectPath))
       .dependsOn(sparkClient)
@@ -903,10 +910,10 @@ trait SparkClientProjects {
         commonSettings,
         releaseSettings,
 
-        // align final shaded jar name with maven.
         (assembly / assemblyJarName) := {
-          val extension = artifact.value.extension
-          s"${moduleName.value}_${scalaBinaryVersion.value}-${version.value}.$extension"
+          val revision: String = version.value
+          val artifactValue: Artifact = artifact.value
+          sparkClientShadeJarName(revision, artifactValue, scalaBinaryVersion.value)
         },
 
         (assembly / test) := { },
diff --git a/tests/spark-it/pom.xml b/tests/spark-it/pom.xml
index bb2aa21798b..6b670f3f077 100644
--- a/tests/spark-it/pom.xml
+++ b/tests/spark-it/pom.xml
@@ -97,13 +97,13 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
         </dependency>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <type>test-jar</type>
           <scope>test</scope>
@@ -115,13 +115,13 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
         </dependency>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <type>test-jar</type>
           <scope>test</scope>
@@ -133,13 +133,13 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
         </dependency>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <type>test-jar</type>
           <scope>test</scope>
@@ -151,13 +151,13 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
         </dependency>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <type>test-jar</type>
           <scope>test</scope>
@@ -169,13 +169,13 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
         </dependency>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <type>test-jar</type>
           <scope>test</scope>
@@ -187,13 +187,13 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
         </dependency>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <type>test-jar</type>
           <scope>test</scope>
@@ -205,13 +205,13 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
         </dependency>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <type>test-jar</type>
           <scope>test</scope>