support built with java 1.8

apache · Feb 21, 2024 · 92c1ef2 · 92c1ef2
1 parent 1c730ec
commit 92c1ef2
Show file tree

Hide file tree

Showing 9 changed files with 81 additions and 73 deletions.
diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml
@@ -37,9 +37,10 @@ runs:
     - name: Run Maven compile
       shell: bash
       run: |
-        ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb
+        echo "JAVA_VERSION=${JAVA_VERSION}"
+        ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb -Djava.version=${JAVA_VERSION}
 
     - name: Run tests
       shell: bash
       run: |
-        SPARK_HOME=`pwd` ./mvnw -B clean install
+        SPARK_HOME=`pwd` ./mvnw -B clean install -Djava.version=${JAVA_VERSION}
diff --git a/.github/actions/rust-test/action.yaml b/.github/actions/rust-test/action.yaml
@@ -51,15 +51,20 @@ runs:
       shell: bash
       run: |
         cd common
-        ../mvnw -B clean compile -DskipTests
+        ../mvnw -B clean compile -DskipTests -Djava.version=${JAVA_VERSION}
 
     - name: Run Cargo test
       shell: bash
       run: |
         cd core
         # This is required to run some JNI related tests on the Rust side
+        JAVA_LD_LIBRARY_PATH=$JAVA_HOME/lib/server:$JAVA_HOME/lib:$JAVA_HOME/lib/jli
+        # special handing for java 1.8 for both linux and mac distributions
+        if [ $JAVA_VERSION == "8" ]; then
+          JAVA_LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:$JAVA_HOME/jre/lib/amd64:$JAVA_HOME/jre/lib/amd64/jli:$JAVA_HOME/jre/lib/server:$JAVA_HOME/jre/lib:$JAVA_HOME/jre/lib/jli
+        fi
         RUST_BACKTRACE=1 \
-        LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_HOME/lib:$JAVA_HOME/lib/server:$JAVA_HOME/lib/jli \
-        DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$JAVA_HOME/lib:$JAVA_HOME/lib/server:$JAVA_HOME/lib/jli \
+        LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_LD_LIBRARY_PATH \
+        DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$JAVA_LD_LIBRARY_PATH \
         cargo test
 
diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml
@@ -38,7 +38,9 @@ runs:
     - name: Install JDK ${{inputs.jdk-version}}
       uses: actions/setup-java@v4
       with:
-        distribution: 'adopt'
+        # distribution is chosen to be zulu as it still offers JDK 8 with Silicon support, which
+        # is not available in the adopt distribution
+        distribution: 'zulu'
         java-version: ${{inputs.jdk-version}}
 
     - name: Set JAVA_HOME

diff --git a/.github/actions/setup-macos-builder/action.yaml b/.github/actions/setup-macos-builder/action.yaml
@@ -59,7 +59,9 @@ runs:
     - name: Install JDK ${{inputs.jdk-version}}
       uses: actions/setup-java@v4
       with:
-        distribution: 'adopt'
+        # distribution is chosen to be zulu as it still offers JDK 8 with Silicon support, which
+        # is not available in the adopt distribution
+        distribution: 'zulu'
         java-version: ${{inputs.jdk-version}}
         architecture: ${{inputs.jdk-architecture}}
 

diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml
@@ -35,105 +35,99 @@ on:
   workflow_dispatch:
 
 env:
-  JAVA_VERSION: 17
   RUST_VERSION: nightly
 
 jobs:
-  linux-rust-test:
-    name: Rust test (amd64)
-    runs-on: ubuntu-latest
+  linux-test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        java_version: [8, 11, 17]
+        test-target: [rust, java]
+      fail-fast: false
+    name: ${{ matrix.test-target }} test on ${{ matrix.os }} with java ${{ matrix.java_version }}
+    runs-on: ${{ matrix.os }}
     container:
       image: amd64/rust
+    env:
+      JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}', matrix.java_version) }}
     steps:
       - uses: actions/checkout@v4
       - name: Setup Rust & Java toolchain
         uses: ./.github/actions/setup-builder
         with:
           rust-version: ${{env.RUST_VERSION}}
-          jdk-version: ${{env.JAVA_VERSION}}
+          jdk-version: ${{ matrix.java_version }}
 
       - uses: actions/checkout@v4
-      - name: Rust test steps
+      - if: matrix.test-target == 'rust'
+        name: Rust test steps
         uses: ./.github/actions/rust-test
-
-  linux-java-test:
-    name: Java test (amd64)
-    runs-on: ubuntu-latest
-    container:
-      image: amd64/rust
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust & Java toolchain
-        uses: ./.github/actions/setup-builder
-        with:
-          rust-version: ${{env.RUST_VERSION}}
-          jdk-version: ${{env.JAVA_VERSION}}
-
-      - uses: actions/checkout@v4
-      - name: Java test steps
+      - if: matrix.test-target == 'java'
+        name: Java test steps
         uses: ./.github/actions/java-test
 
-
-  macos-rust-test:
-    name: Rust test (macos)
-    runs-on: macos-latest
+  macos-test:
+    strategy:
+      matrix:
+        os: [macos-latest]
+        java_version: [8, 11, 17]
+        test-target: [rust, java]
+        is_push_event:
+          - ${{ github.event_name == 'push' }}
+        exclude: # exclude java 11 for pull_request event
+          - java_version: 11
+            is_push_event: false
+      fail-fast: false
+    name: ${{ matrix.test-target }} test on ${{ matrix.os }} with java ${{ matrix.java_version }}
+    runs-on: ${{ matrix.os }}
+    env:
+      JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}', matrix.java_version) }}
     steps:
       - uses: actions/checkout@v4
       - name: Setup Rust & Java toolchain
         uses: ./.github/actions/setup-macos-builder
         with:
           rust-version: ${{env.RUST_VERSION}}
-          jdk-version: ${{env.JAVA_VERSION}}
+          jdk-version: ${{ matrix.java_version }}
 
       - uses: actions/checkout@v4
-      - name: Rust test steps
+      - if: matrix.test-target == 'rust'
+        name: Rust test steps
         uses: ./.github/actions/rust-test
-
-  macos-java-test:
-    name: Java test (macos)
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust & Java toolchain
-        uses: ./.github/actions/setup-macos-builder
-        with:
-          rust-version: ${{env.RUST_VERSION}}
-          jdk-version: ${{env.JAVA_VERSION}}
-
-      - uses: actions/checkout@v4
-      - name: Java test steps
+      - if: matrix.test-target == 'java'
+        name: Java test steps
         uses: ./.github/actions/java-test
 
-  macos-aarch64-rust-test:
-    name: Rust test (macos-aarch64)
+  macos-aarch64-test:
+    strategy:
+      matrix:
+        java_version: [8, 11, 17]
+        test-target: [rust, java]
+        is_push_event:
+          - ${{ github.event_name == 'push' }}
+        exclude: # exclude java 11 for pull_request event
+          - java_version: 11
+            is_push_event: false
+      fail-fast: false
+    name: ${{ matrix.test-target }} (macos-aarch64) test with java ${{ matrix.java_version }}
     runs-on: macos-14
+    env:
+      JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}', matrix.java_version) }}
     steps:
       - uses: actions/checkout@v4
       - name: Setup Rust & Java toolchain
         uses: ./.github/actions/setup-macos-builder
         with:
           rust-version: ${{env.RUST_VERSION}}
-          jdk-version: ${{env.JAVA_VERSION}}
+          jdk-version: ${{ matrix.java_version }}
           jdk-architecture: aarch64
           protoc-architecture: aarch_64
 
       - uses: actions/checkout@v4
-      - name: Rust test steps
+      - if: matrix.test-target == 'rust'
+        name: Rust test steps
         uses: ./.github/actions/rust-test
-
-  macos-aarch64-java-test:
-    name: Java test (macos-aarch64)
-    runs-on: macos-14
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust & Java toolchain
-        uses: ./.github/actions/setup-macos-builder
-        with:
-          rust-version: ${{env.RUST_VERSION}}
-          jdk-version: ${{env.JAVA_VERSION}}
-          jdk-architecture: aarch64
-          protoc-architecture: aarch_64
-
-      - uses: actions/checkout@v4
-      - name: Java test steps
+      - if: matrix.test-target == 'java'
+        name: Java test steps
         uses: ./.github/actions/java-test
diff --git a/common/pom.xml b/common/pom.xml
@@ -80,7 +80,7 @@ under the License.
       <plugin>
         <groupId>io.github.git-commit-id</groupId>
         <artifactId>git-commit-id-maven-plugin</artifactId>
-        <version>5.0.0</version>
+        <version>4.9.9</version>
         <executions>
           <execution>
             <id>get-the-git-infos</id>

diff --git a/common/src/main/java/org/apache/comet/parquet/FileReader.java b/common/src/main/java/org/apache/comet/parquet/FileReader.java
@@ -882,6 +882,8 @@ private class ConsecutivePartList {
     private final SQLMetric readThroughput;
 
     /**
+     * Constructor
+     *
      * @param offset where the first chunk starts
      */
     ConsecutivePartList(long offset) {
@@ -1104,6 +1106,8 @@ private void setReadMetrics(long startNs) {
     }
 
     /**
+     * End position of the last byte of these chunks
+     *
      * @return the position following the last byte of these chunks
      */
     public long endPos() {

diff --git a/pom.xml b/pom.xml
@@ -39,8 +39,8 @@ under the License.
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     <java.version>11</java.version>
-    <maven.compiler.source>11</maven.compiler.source>
-    <maven.compiler.target>11</maven.compiler.target>
+    <maven.compiler.source>${java.version}</maven.compiler.source>
+    <maven.compiler.target>${java.version}</maven.compiler.target>
     <scala.version>2.12.17</scala.version>
     <scala.binary.version>2.12</scala.binary.version>
     <scala.plugin.version>4.7.2</scala.plugin.version>

diff --git a/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala b/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala
@@ -76,7 +76,7 @@ object GenTPCHData {
     // Generate data
     // Since dbgen may uses stdout to output the data, tables.genData needs to run table by table
     val tableNames =
-      if (config.tableFilter.isBlank) tables.tables.map(_.name) else Seq(config.tableFilter)
+      if (config.tableFilter.trim.isEmpty) tables.tables.map(_.name) else Seq(config.tableFilter)
     tableNames.foreach { tableName =>
       tables.genData(
         location = s"${config.location}/tpch/sf${config.scaleFactor}_${config.format}",