diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml index 6c3af7998..4b594e648 100644 --- a/.github/actions/java-test/action.yaml +++ b/.github/actions/java-test/action.yaml @@ -15,6 +15,13 @@ # specific language governing permissions and limitations # under the License. +name: "Java Test" +description: "Run Java tests" +inputs: + MAVEN_OPTS: + description: 'Maven options passed to the mvn command' + required: false + default: '' runs: using: "composite" steps: @@ -37,9 +44,9 @@ runs: - name: Run Maven compile shell: bash run: | - ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb + ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb ${{ inputs.MAVEN_OPTS }} - name: Run tests shell: bash run: | - SPARK_HOME=`pwd` ./mvnw -B clean install + SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.MAVEN_OPTS }} diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index aa69aea31..259b3609d 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -44,13 +44,14 @@ jobs: os: [ubuntu-latest] java_version: [8, 11, 17] test-target: [rust, java] + spark-version: ['3.4'] is_push_event: - ${{ github.event_name == 'push' }} exclude: # exclude java 11 for pull_request event - java_version: 11 is_push_event: false fail-fast: false - name: ${{ matrix.test-target }} test on ${{ matrix.os }} with java ${{ matrix.java_version }} + name: (${{ matrix.os }}|java ${{ matrix.java_version }}-spark-${{matrix.spark.version}}|${{ matrix.test-target }}) runs-on: ${{ matrix.os }} container: image: amd64/rust @@ -61,14 +62,43 @@ jobs: with: rust-version: ${{env.RUST_VERSION}} jdk-version: ${{ matrix.java_version }} - - - uses: actions/checkout@v4 - if: matrix.test-target == 'rust' name: Rust test steps uses: ./.github/actions/rust-test - if: matrix.test-target == 'java' name: Java test steps uses: ./.github/actions/java-test + with: + MAVEN_OPTS: -Pspark-${{ matrix.spark-version }} + + linux-test-with-old-spark: + strategy: + matrix: + os: [ubuntu-latest] + java_version: [8, 11, 17] + test-target: [java] + spark-version: ['3.2', '3.3'] + exclude: + - java_version: 17 + spark-version: '3.2' + - java_version: 11 + spark-version: '3.2' + fail-fast: false + name: (${{ matrix.os }}|java ${{ matrix.java_version }}-spark-${{matrix.spark.version}}|${{ matrix.test-target }}) + runs-on: ${{ matrix.os }} + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust & Java toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: ${{env.RUST_VERSION}} + jdk-version: ${{ matrix.java_version }} + - name: Java test steps + uses: ./.github/actions/java-test + with: + MAVEN_OPTS: -Pspark-${{ matrix.spark-version }} macos-test: strategy: @@ -76,9 +106,10 @@ jobs: os: [macos-13] java_version: [8, 11, 17] test-target: [rust, java] + spark-version: ['3.4'] fail-fast: false if: github.event_name == 'push' - name: ${{ matrix.test-target }} test on ${{ matrix.os }} with java ${{ matrix.java_version }} + name: (${{ matrix.os }}|java ${{ matrix.java_version }}-spark-${{matrix.spark.version}}|${{ matrix.test-target }}) runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -87,27 +118,28 @@ jobs: with: rust-version: ${{env.RUST_VERSION}} jdk-version: ${{ matrix.java_version }} - - - uses: actions/checkout@v4 - if: matrix.test-target == 'rust' name: Rust test steps uses: ./.github/actions/rust-test - if: matrix.test-target == 'java' name: Java test steps uses: ./.github/actions/java-test + with: + MAVEN_OPTS: -Pspark-${{ inputs.spark-version }} macos-aarch64-test: strategy: matrix: java_version: [8, 11, 17] test-target: [rust, java] + spark-version: ['3.4'] is_push_event: - ${{ github.event_name == 'push' }} exclude: # exclude java 11 for pull_request event - java_version: 11 is_push_event: false fail-fast: false - name: ${{ matrix.test-target }} test on macos-aarch64 with java ${{ matrix.java_version }} + name: (${{ matrix.os }}|java ${{ matrix.java_version }}-spark-${{matrix.spark.version}}|${{ matrix.test-target }}) runs-on: macos-14 steps: - uses: actions/checkout@v4 @@ -118,11 +150,40 @@ jobs: jdk-version: ${{ matrix.java_version }} jdk-architecture: aarch64 protoc-architecture: aarch_64 - - - uses: actions/checkout@v4 - if: matrix.test-target == 'rust' name: Rust test steps uses: ./.github/actions/rust-test - if: matrix.test-target == 'java' name: Java test steps uses: ./.github/actions/java-test + with: + MAVEN_OPTS: -Pspark-${{ inputs.spark-version }} + + macos-aarch64-test-with-old-spark: + strategy: + matrix: + java_version: [8, 17] + test-target: [java] + spark-version: ['3.2', '3.3'] + exclude: + - java_version: 17 + spark-version: '3.2' + - jav_version: 8 + spark-version: '3.3' + fail-fast: false + name: (${{ matrix.os }}|java ${{ matrix.java_version }}-spark-${{matrix.spark.version}}|${{ matrix.test-target }}) + runs-on: macos-14 + steps: + - uses: actions/checkout@v4 + - name: Setup Rust & Java toolchain + uses: ./.github/actions/setup-macos-builder + with: + rust-version: ${{env.RUST_VERSION}} + jdk-version: ${{ matrix.java_version }} + jdk-architecture: aarch64 + protoc-architecture: aarch_64 + - if: matrix.test-target == 'java' + name: Java test steps + uses: ./.github/actions/java-test + with: + MAVEN_OPTS: -Pspark-${{ inputs.spark-version }} diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala index 0d7904c01..38a8d7d2f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala @@ -183,7 +183,7 @@ abstract class CometTestBase protected def checkPlanContains(plan: SparkPlan, includePlans: Class[_]*): Unit = { includePlans.foreach { case planClass => - if (!plan.exists(op => planClass.isAssignableFrom(op.getClass))) { + if (plan.find(op => planClass.isAssignableFrom(op.getClass)).isEmpty) { assert( false, s"Expected plan to contain ${planClass.getSimpleName}, but not.\n" +