support cross compile for spark 3.2 and spark 3.3

apache · Mar 2, 2024 · ecf8817 · ecf8817
1 parent 23c748b
commit ecf8817
Show file tree

Hide file tree

Showing 3 changed files with 80 additions and 12 deletions.
diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml
@@ -15,6 +15,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
+name: "Java Test"
+description: "Run Java tests"
+inputs:
+  MAVEN_OPTS:
+    description: 'Maven options passed to the mvn command'
+    required: false
+    default: ''
 runs:
   using: "composite"
   steps:
@@ -37,9 +44,9 @@ runs:
     - name: Run Maven compile
       shell: bash
       run: |
-        ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb
+        ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb ${{ inputs.MAVEN_OPTS }}
 
     - name: Run tests
       shell: bash
       run: |
-        SPARK_HOME=`pwd` ./mvnw -B clean install
+        SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.MAVEN_OPTS }}
diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml
@@ -44,13 +44,14 @@ jobs:
         os: [ubuntu-latest]
         java_version: [8, 11, 17]
         test-target: [rust, java]
+        spark-version: ['3.4']
         is_push_event:
           - ${{ github.event_name == 'push' }}
         exclude: # exclude java 11 for pull_request event
           - java_version: 11
             is_push_event: false
       fail-fast: false
-    name: ${{ matrix.test-target }} test on ${{ matrix.os }} with java ${{ matrix.java_version }}
+    name: (${{ matrix.os }}|java ${{ matrix.java_version }}-spark-${{matrix.spark.version}}|${{ matrix.test-target }})
     runs-on: ${{ matrix.os }}
     container:
       image: amd64/rust
@@ -61,24 +62,54 @@ jobs:
         with:
           rust-version: ${{env.RUST_VERSION}}
           jdk-version: ${{ matrix.java_version }}
-
-      - uses: actions/checkout@v4
       - if: matrix.test-target == 'rust'
         name: Rust test steps
         uses: ./.github/actions/rust-test
       - if: matrix.test-target == 'java'
         name: Java test steps
         uses: ./.github/actions/java-test
+        with:
+          MAVEN_OPTS: -Pspark-${{ matrix.spark-version }}
+
+  linux-test-with-old-spark:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        java_version: [8, 11, 17]
+        test-target: [java]
+        spark-version: ['3.2', '3.3']
+        exclude:
+          - java_version: 17
+            spark-version: '3.2'
+          - java_version: 11
+            spark-version: '3.2'
+      fail-fast: false
+    name: (${{ matrix.os }}|java ${{ matrix.java_version }}-spark-${{matrix.spark.version}}|${{ matrix.test-target }})
+    runs-on: ${{ matrix.os }}
+    container:
+      image: amd64/rust
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust & Java toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: ${{env.RUST_VERSION}}
+          jdk-version: ${{ matrix.java_version }}
+      - name: Java test steps
+        uses: ./.github/actions/java-test
+        with:
+          MAVEN_OPTS: -Pspark-${{ matrix.spark-version }}
 
   macos-test:
     strategy:
       matrix:
         os: [macos-13]
         java_version: [8, 11, 17]
         test-target: [rust, java]
+        spark-version: ['3.4']
       fail-fast: false
     if: github.event_name == 'push'
-    name: ${{ matrix.test-target }} test on ${{ matrix.os }} with java ${{ matrix.java_version }}
+    name: (${{ matrix.os }}|java ${{ matrix.java_version }}-spark-${{matrix.spark.version}}|${{ matrix.test-target }})
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
@@ -87,27 +118,28 @@ jobs:
         with:
           rust-version: ${{env.RUST_VERSION}}
           jdk-version: ${{ matrix.java_version }}
-
-      - uses: actions/checkout@v4
       - if: matrix.test-target == 'rust'
         name: Rust test steps
         uses: ./.github/actions/rust-test
       - if: matrix.test-target == 'java'
         name: Java test steps
         uses: ./.github/actions/java-test
+        with:
+          MAVEN_OPTS: -Pspark-${{ inputs.spark-version }}
 
   macos-aarch64-test:
     strategy:
       matrix:
         java_version: [8, 11, 17]
         test-target: [rust, java]
+        spark-version: ['3.4']
         is_push_event:
           - ${{ github.event_name == 'push' }}
         exclude: # exclude java 11 for pull_request event
           - java_version: 11
             is_push_event: false
       fail-fast: false
-    name: ${{ matrix.test-target }} test on macos-aarch64 with java ${{ matrix.java_version }}
+    name: (${{ matrix.os }}|java ${{ matrix.java_version }}-spark-${{matrix.spark.version}}|${{ matrix.test-target }})
     runs-on: macos-14
     steps:
       - uses: actions/checkout@v4
@@ -118,11 +150,40 @@ jobs:
           jdk-version: ${{ matrix.java_version }}
           jdk-architecture: aarch64
           protoc-architecture: aarch_64
-
-      - uses: actions/checkout@v4
       - if: matrix.test-target == 'rust'
         name: Rust test steps
         uses: ./.github/actions/rust-test
       - if: matrix.test-target == 'java'
         name: Java test steps
         uses: ./.github/actions/java-test
+        with:
+          MAVEN_OPTS: -Pspark-${{ inputs.spark-version }}
+
+  macos-aarch64-test-with-old-spark:
+    strategy:
+      matrix:
+        java_version: [8, 17]
+        test-target: [java]
+        spark-version: ['3.2', '3.3']
+        exclude:
+          - java_version: 17
+            spark-version: '3.2'
+          - jav_version: 8
+            spark-version: '3.3'
+      fail-fast: false
+    name: (${{ matrix.os }}|java ${{ matrix.java_version }}-spark-${{matrix.spark.version}}|${{ matrix.test-target }})
+    runs-on: macos-14
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust & Java toolchain
+        uses: ./.github/actions/setup-macos-builder
+        with:
+          rust-version: ${{env.RUST_VERSION}}
+          jdk-version: ${{ matrix.java_version }}
+          jdk-architecture: aarch64
+          protoc-architecture: aarch_64
+      - if: matrix.test-target == 'java'
+        name: Java test steps
+        uses: ./.github/actions/java-test
+        with:
+          MAVEN_OPTS: -Pspark-${{ inputs.spark-version }}
diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala
@@ -183,7 +183,7 @@ abstract class CometTestBase
 
   protected def checkPlanContains(plan: SparkPlan, includePlans: Class[_]*): Unit = {
     includePlans.foreach { case planClass =>
-      if (!plan.exists(op => planClass.isAssignableFrom(op.getClass))) {
+      if (plan.find(op => planClass.isAssignableFrom(op.getClass)).isEmpty) {
         assert(
           false,
           s"Expected plan to contain ${planClass.getSimpleName}, but not.\n" +