add sbt spark version

apache · Nov 8, 2024 · 7c234e6 · 7c234e6
1 parent 6760dcf
commit 7c234e6
Show file tree

Hide file tree

Showing 11 changed files with 44 additions and 48 deletions.
diff --git a/build/make-distribution.sh b/build/make-distribution.sh
@@ -185,13 +185,10 @@ function build_spark_client {
       | grep -v "WARNING" \
       | tail -n 1)
   SPARK_MAJOR_VERSION=${SPARK_VERSION%%.*}
-  SPARK_MINOR_VERSION=${SPARK_VERSION#*.}
-  SPARK_MINOR_VERSION=${SPARK_MINOR_VERSION%%.*}
-  SPARK_MAJOR_MINOR_VERSION="$SPARK_MAJOR_VERSION.$SPARK_MINOR_VERSION"
   # Store the command as an array because $MVN variable might have spaces in it.
   # Normal quoting tricks don't work.
   # See: http://mywiki.wooledge.org/BashFAQ/050
-  BUILD_COMMAND=("$MVN" clean package $MVN_DIST_OPT -pl :celeborn-client-spark-${SPARK_MAJOR_MINOR_VERSION}-shaded_$SCALA_VERSION -am $@)
+  BUILD_COMMAND=("$MVN" clean package $MVN_DIST_OPT -pl :celeborn-client-spark-${SPARK_VERSION}-shaded_$SCALA_VERSION -am $@)
 
   # Actually build the jar
   echo -e "\nBuilding with..."
@@ -202,7 +199,7 @@ function build_spark_client {
   mkdir -p "$DIST_DIR/spark"
 
   ## Copy spark client jars
-  cp "$PROJECT_DIR"/client-spark/spark-${SPARK_MAJOR_VERSION}-shaded/target/celeborn-client-spark-${SPARK_MAJOR_MINOR_VERSION}-shaded_$SCALA_VERSION-$VERSION.jar "$DIST_DIR/spark/"
+  cp "$PROJECT_DIR"/client-spark/spark-${SPARK_MAJOR_VERSION}-shaded/target/celeborn-client-spark-${SPARK_VERSION}-shaded_$SCALA_VERSION-$VERSION.jar "$DIST_DIR/spark/"
 }
 
 function build_flink_client {

diff --git a/client-spark/spark-2-shaded/pom.xml b/client-spark/spark-2-shaded/pom.xml
@@ -24,14 +24,14 @@
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>celeborn-client-spark-${spark.version.majorminor}-shaded_${scala.binary.version}</artifactId>
+  <artifactId>celeborn-client-spark-${spark.version}-shaded_${scala.binary.version}</artifactId>
   <packaging>jar</packaging>
   <name>Celeborn Shaded Client for Spark 2</name>
 
   <dependencies>
     <dependency>
       <groupId>org.apache.celeborn</groupId>
-      <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+      <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
   </dependencies>

diff --git a/client-spark/spark-2/pom.xml b/client-spark/spark-2/pom.xml
@@ -24,7 +24,7 @@
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+  <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
   <packaging>jar</packaging>
   <name>Celeborn Client for Spark 2</name>
 

diff --git a/client-spark/spark-3-columnar-common/pom.xml b/client-spark/spark-3-columnar-common/pom.xml
@@ -24,14 +24,14 @@
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-common_${scala.binary.version}</artifactId>
+  <artifactId>celeborn-spark-${spark.version}-columnar-common_${scala.binary.version}</artifactId>
   <packaging>jar</packaging>
   <name>Celeborn Client for Spark 3 Columnar Common</name>
 
   <dependencies>
     <dependency>
       <groupId>org.apache.celeborn</groupId>
-      <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+      <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>

diff --git a/client-spark/spark-3-columnar-shuffle/pom.xml b/client-spark/spark-3-columnar-shuffle/pom.xml
@@ -24,14 +24,14 @@
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-shuffle_${scala.binary.version}</artifactId>
+  <artifactId>celeborn-spark-${spark.version}-columnar-shuffle_${scala.binary.version}</artifactId>
   <packaging>jar</packaging>
   <name>Celeborn Client for Spark 3 Columnar Shuffle</name>
 
   <dependencies>
     <dependency>
       <groupId>org.apache.celeborn</groupId>
-      <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-common_${scala.binary.version}</artifactId>
+      <artifactId>celeborn-spark-${spark.version}-columnar-common_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>
@@ -49,7 +49,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.celeborn</groupId>
-      <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+      <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
       <type>test-jar</type>
       <scope>test</scope>

diff --git a/client-spark/spark-3-shaded/pom.xml b/client-spark/spark-3-shaded/pom.xml
@@ -24,14 +24,14 @@
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>celeborn-client-spark-${spark.version.majorminor}-shaded_${scala.binary.version}</artifactId>
+  <artifactId>celeborn-client-spark-${spark.version}-shaded_${scala.binary.version}</artifactId>
   <packaging>jar</packaging>
   <name>Celeborn Shaded Client for Spark 3</name>
 
   <dependencies>
     <dependency>
       <groupId>org.apache.celeborn</groupId>
-      <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+      <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
   </dependencies>
@@ -147,7 +147,7 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-shuffle_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-spark-${spark.version}-columnar-shuffle_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
         </dependency>
       </dependencies>
@@ -157,7 +157,7 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-shuffle_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-spark-${spark.version}-columnar-shuffle_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
         </dependency>
       </dependencies>
@@ -167,7 +167,7 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-shuffle_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-spark-${spark.version}-columnar-shuffle_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
         </dependency>
       </dependencies>
@@ -177,7 +177,7 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-shuffle_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-spark-${spark.version}-columnar-shuffle_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
         </dependency>
       </dependencies>
@@ -187,7 +187,7 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-shuffle_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-spark-${spark.version}-columnar-shuffle_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
         </dependency>
       </dependencies>

diff --git a/client-spark/spark-3.5-columnar-shuffle/pom.xml b/client-spark/spark-3.5-columnar-shuffle/pom.xml
@@ -31,7 +31,7 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.celeborn</groupId>
-      <artifactId>celeborn-spark-${spark.version.majorminor}-columnar-common_${scala.binary.version}</artifactId>
+      <artifactId>celeborn-spark-${spark.version}-columnar-common_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>
@@ -49,7 +49,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.celeborn</groupId>
-      <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+      <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
       <type>test-jar</type>
       <scope>test</scope>

diff --git a/client-spark/spark-3/pom.xml b/client-spark/spark-3/pom.xml
@@ -24,7 +24,7 @@
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+  <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
   <packaging>jar</packaging>
   <name>Celeborn Client for Spark 3</name>
 

diff --git a/pom.xml b/pom.xml
@@ -68,7 +68,6 @@
 
     <flink.version>1.14.6</flink.version>
     <spark.version>3.4.4</spark.version>
-    <spark.version.majorminor>3.4</spark.version.majorminor>
 
     <!-- use hadoop-3 as default  -->
     <hadoop.version>3.3.6</hadoop.version>
@@ -1374,7 +1373,6 @@
         <scala.version>2.11.12</scala.version>
         <scala.binary.version>2.11</scala.binary.version>
         <spark.version>2.4.8</spark.version>
-        <spark.version.majorminor>2.4</spark.version.majorminor>
         <zstd-jni.version>1.4.4-3</zstd-jni.version>
       </properties>
     </profile>
@@ -1394,7 +1392,6 @@
         <scala.version>2.12.10</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
         <spark.version>3.0.3</spark.version>
-        <spark.version.majorminor>3.0</spark.version.majorminor>
         <zstd-jni.version>1.4.4-3</zstd-jni.version>
         <maven.plugin.silencer.version>1.6.0</maven.plugin.silencer.version>
       </properties>
@@ -1415,7 +1412,6 @@
         <scala.version>2.12.10</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
         <spark.version>3.1.3</spark.version>
-        <spark.version.majorminor>3.1</spark.version.majorminor>
         <zstd-jni.version>1.4.8-1</zstd-jni.version>
         <maven.plugin.silencer.version>1.6.0</maven.plugin.silencer.version>
       </properties>
@@ -1436,7 +1432,6 @@
         <scala.version>2.12.15</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
         <spark.version>3.2.4</spark.version>
-        <spark.version.majorminor>3.2</spark.version.majorminor>
         <zstd-jni.version>1.5.0-4</zstd-jni.version>
       </properties>
     </profile>
@@ -1456,7 +1451,6 @@
         <scala.version>2.12.15</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
         <spark.version>3.3.4</spark.version>
-        <spark.version.majorminor>3.3</spark.version.majorminor>
         <zstd-jni.version>1.5.2-1</zstd-jni.version>
       </properties>
     </profile>
@@ -1476,7 +1470,6 @@
         <scala.version>2.12.17</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
         <spark.version>3.4.4</spark.version>
-        <spark.version.majorminor>3.4</spark.version.majorminor>
         <zstd-jni.version>1.5.2-5</zstd-jni.version>
       </properties>
     </profile>
@@ -1496,7 +1489,6 @@
         <scala.version>2.12.18</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
         <spark.version>3.5.3</spark.version>
-        <spark.version.majorminor>3.5</spark.version.majorminor>
         <zstd-jni.version>1.5.5-4</zstd-jni.version>
       </properties>
     </profile>

diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
@@ -890,6 +890,13 @@ trait SparkClientProjects {
       )
   }
 
+  // the output would be something like: celeborn-client-spark-3.3.4-shaded_2.12-0.6.0-SNAPSHOT.jar
+  def sparkClientShadeJarName(
+                               revision: String,
+                               artifact: Artifact,
+                               scalaBinaryVersionString: String): String =
+    s"celeborn-client-spark-$sparkVersion-shaded_$scalaBinaryVersionString" + "-" + revision + "." + artifact.extension
+
   def sparkClientShade: Project = {
     var p = Project(sparkClientShadedProjectName, file(sparkClientShadedProjectPath))
       .dependsOn(sparkClient)
@@ -903,10 +910,10 @@ trait SparkClientProjects {
         commonSettings,
         releaseSettings,
 
-        // align final shaded jar name with maven.
         (assembly / assemblyJarName) := {
-          val extension = artifact.value.extension
-          s"${moduleName.value}_${scalaBinaryVersion.value}-${version.value}.$extension"
+          val revision: String = version.value
+          val artifactValue: Artifact = artifact.value
+          sparkClientShadeJarName(revision, artifactValue, scalaBinaryVersion.value)
         },
 
         (assembly / test) := { },

diff --git a/tests/spark-it/pom.xml b/tests/spark-it/pom.xml
@@ -97,13 +97,13 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
         </dependency>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <type>test-jar</type>
           <scope>test</scope>
@@ -115,13 +115,13 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
         </dependency>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <type>test-jar</type>
           <scope>test</scope>
@@ -133,13 +133,13 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
         </dependency>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <type>test-jar</type>
           <scope>test</scope>
@@ -151,13 +151,13 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
         </dependency>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <type>test-jar</type>
           <scope>test</scope>
@@ -169,13 +169,13 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
         </dependency>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <type>test-jar</type>
           <scope>test</scope>
@@ -187,13 +187,13 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
         </dependency>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <type>test-jar</type>
           <scope>test</scope>
@@ -205,13 +205,13 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <scope>test</scope>
         </dependency>
         <dependency>
           <groupId>org.apache.celeborn</groupId>
-          <artifactId>celeborn-client-spark-${spark.version.majorminor}_${scala.binary.version}</artifactId>
+          <artifactId>celeborn-client-spark-${spark.version}_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
           <type>test-jar</type>
           <scope>test</scope>