From 92c1ef2fad7261c64fc681e9031c73b54c20e9d8 Mon Sep 17 00:00:00 2001 From: Xianjin Date: Sun, 18 Feb 2024 20:39:24 +0800 Subject: [PATCH] support built with java 1.8 --- .github/actions/java-test/action.yaml | 5 +- .github/actions/rust-test/action.yaml | 11 +- .github/actions/setup-builder/action.yaml | 4 +- .../actions/setup-macos-builder/action.yaml | 4 +- .github/workflows/pr_build.yml | 118 +++++++++--------- common/pom.xml | 2 +- .../org/apache/comet/parquet/FileReader.java | 4 + pom.xml | 4 +- .../org/apache/spark/sql/GenTPCHData.scala | 2 +- 9 files changed, 81 insertions(+), 73 deletions(-) diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml index 6c3af7998..9c7de9a82 100644 --- a/.github/actions/java-test/action.yaml +++ b/.github/actions/java-test/action.yaml @@ -37,9 +37,10 @@ runs: - name: Run Maven compile shell: bash run: | - ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb + echo "JAVA_VERSION=${JAVA_VERSION}" + ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb -Djava.version=${JAVA_VERSION} - name: Run tests shell: bash run: | - SPARK_HOME=`pwd` ./mvnw -B clean install + SPARK_HOME=`pwd` ./mvnw -B clean install -Djava.version=${JAVA_VERSION} diff --git a/.github/actions/rust-test/action.yaml b/.github/actions/rust-test/action.yaml index b66b63959..a013a7bae 100644 --- a/.github/actions/rust-test/action.yaml +++ b/.github/actions/rust-test/action.yaml @@ -51,15 +51,20 @@ runs: shell: bash run: | cd common - ../mvnw -B clean compile -DskipTests + ../mvnw -B clean compile -DskipTests -Djava.version=${JAVA_VERSION} - name: Run Cargo test shell: bash run: | cd core # This is required to run some JNI related tests on the Rust side + JAVA_LD_LIBRARY_PATH=$JAVA_HOME/lib/server:$JAVA_HOME/lib:$JAVA_HOME/lib/jli + # special handing for java 1.8 for both linux and mac distributions + if [ $JAVA_VERSION == "8" ]; then + JAVA_LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:$JAVA_HOME/jre/lib/amd64:$JAVA_HOME/jre/lib/amd64/jli:$JAVA_HOME/jre/lib/server:$JAVA_HOME/jre/lib:$JAVA_HOME/jre/lib/jli + fi RUST_BACKTRACE=1 \ - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_HOME/lib:$JAVA_HOME/lib/server:$JAVA_HOME/lib/jli \ - DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$JAVA_HOME/lib:$JAVA_HOME/lib/server:$JAVA_HOME/lib/jli \ + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_LD_LIBRARY_PATH \ + DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$JAVA_LD_LIBRARY_PATH \ cargo test diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml index f1aeb2546..6fa716492 100644 --- a/.github/actions/setup-builder/action.yaml +++ b/.github/actions/setup-builder/action.yaml @@ -38,7 +38,9 @@ runs: - name: Install JDK ${{inputs.jdk-version}} uses: actions/setup-java@v4 with: - distribution: 'adopt' + # distribution is chosen to be zulu as it still offers JDK 8 with Silicon support, which + # is not available in the adopt distribution + distribution: 'zulu' java-version: ${{inputs.jdk-version}} - name: Set JAVA_HOME diff --git a/.github/actions/setup-macos-builder/action.yaml b/.github/actions/setup-macos-builder/action.yaml index 1f680edb3..63010ea8d 100644 --- a/.github/actions/setup-macos-builder/action.yaml +++ b/.github/actions/setup-macos-builder/action.yaml @@ -59,7 +59,9 @@ runs: - name: Install JDK ${{inputs.jdk-version}} uses: actions/setup-java@v4 with: - distribution: 'adopt' + # distribution is chosen to be zulu as it still offers JDK 8 with Silicon support, which + # is not available in the adopt distribution + distribution: 'zulu' java-version: ${{inputs.jdk-version}} architecture: ${{inputs.jdk-architecture}} diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index 51962cd51..b0eace7a2 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -35,105 +35,99 @@ on: workflow_dispatch: env: - JAVA_VERSION: 17 RUST_VERSION: nightly jobs: - linux-rust-test: - name: Rust test (amd64) - runs-on: ubuntu-latest + linux-test: + strategy: + matrix: + os: [ubuntu-latest] + java_version: [8, 11, 17] + test-target: [rust, java] + fail-fast: false + name: ${{ matrix.test-target }} test on ${{ matrix.os }} with java ${{ matrix.java_version }} + runs-on: ${{ matrix.os }} container: image: amd64/rust + env: + JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}', matrix.java_version) }} steps: - uses: actions/checkout@v4 - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{env.JAVA_VERSION}} + jdk-version: ${{ matrix.java_version }} - uses: actions/checkout@v4 - - name: Rust test steps + - if: matrix.test-target == 'rust' + name: Rust test steps uses: ./.github/actions/rust-test - - linux-java-test: - name: Java test (amd64) - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{env.JAVA_VERSION}} - - - uses: actions/checkout@v4 - - name: Java test steps + - if: matrix.test-target == 'java' + name: Java test steps uses: ./.github/actions/java-test - - macos-rust-test: - name: Rust test (macos) - runs-on: macos-latest + macos-test: + strategy: + matrix: + os: [macos-latest] + java_version: [8, 11, 17] + test-target: [rust, java] + is_push_event: + - ${{ github.event_name == 'push' }} + exclude: # exclude java 11 for pull_request event + - java_version: 11 + is_push_event: false + fail-fast: false + name: ${{ matrix.test-target }} test on ${{ matrix.os }} with java ${{ matrix.java_version }} + runs-on: ${{ matrix.os }} + env: + JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}', matrix.java_version) }} steps: - uses: actions/checkout@v4 - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-macos-builder with: rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{env.JAVA_VERSION}} + jdk-version: ${{ matrix.java_version }} - uses: actions/checkout@v4 - - name: Rust test steps + - if: matrix.test-target == 'rust' + name: Rust test steps uses: ./.github/actions/rust-test - - macos-java-test: - name: Java test (macos) - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-macos-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{env.JAVA_VERSION}} - - - uses: actions/checkout@v4 - - name: Java test steps + - if: matrix.test-target == 'java' + name: Java test steps uses: ./.github/actions/java-test - macos-aarch64-rust-test: - name: Rust test (macos-aarch64) + macos-aarch64-test: + strategy: + matrix: + java_version: [8, 11, 17] + test-target: [rust, java] + is_push_event: + - ${{ github.event_name == 'push' }} + exclude: # exclude java 11 for pull_request event + - java_version: 11 + is_push_event: false + fail-fast: false + name: ${{ matrix.test-target }} (macos-aarch64) test with java ${{ matrix.java_version }} runs-on: macos-14 + env: + JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}', matrix.java_version) }} steps: - uses: actions/checkout@v4 - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-macos-builder with: rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{env.JAVA_VERSION}} + jdk-version: ${{ matrix.java_version }} jdk-architecture: aarch64 protoc-architecture: aarch_64 - uses: actions/checkout@v4 - - name: Rust test steps + - if: matrix.test-target == 'rust' + name: Rust test steps uses: ./.github/actions/rust-test - - macos-aarch64-java-test: - name: Java test (macos-aarch64) - runs-on: macos-14 - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-macos-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{env.JAVA_VERSION}} - jdk-architecture: aarch64 - protoc-architecture: aarch_64 - - - uses: actions/checkout@v4 - - name: Java test steps + - if: matrix.test-target == 'java' + name: Java test steps uses: ./.github/actions/java-test diff --git a/common/pom.xml b/common/pom.xml index f885c346d..5757351d9 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -80,7 +80,7 @@ under the License. io.github.git-commit-id git-commit-id-maven-plugin - 5.0.0 + 4.9.9 get-the-git-infos diff --git a/common/src/main/java/org/apache/comet/parquet/FileReader.java b/common/src/main/java/org/apache/comet/parquet/FileReader.java index eddaf3f1a..61cf89bf1 100644 --- a/common/src/main/java/org/apache/comet/parquet/FileReader.java +++ b/common/src/main/java/org/apache/comet/parquet/FileReader.java @@ -882,6 +882,8 @@ private class ConsecutivePartList { private final SQLMetric readThroughput; /** + * Constructor + * * @param offset where the first chunk starts */ ConsecutivePartList(long offset) { @@ -1104,6 +1106,8 @@ private void setReadMetrics(long startNs) { } /** + * End position of the last byte of these chunks + * * @return the position following the last byte of these chunks */ public long endPos() { diff --git a/pom.xml b/pom.xml index 0e72f92c0..b3c354568 100644 --- a/pom.xml +++ b/pom.xml @@ -39,8 +39,8 @@ under the License. UTF-8 UTF-8 11 - 11 - 11 + ${java.version} + ${java.version} 2.12.17 2.12 4.7.2 diff --git a/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala b/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala index 1dcbc743b..e25d4e51e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala +++ b/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala @@ -76,7 +76,7 @@ object GenTPCHData { // Generate data // Since dbgen may uses stdout to output the data, tables.genData needs to run table by table val tableNames = - if (config.tableFilter.isBlank) tables.tables.map(_.name) else Seq(config.tableFilter) + if (config.tableFilter.trim.isEmpty) tables.tables.map(_.name) else Seq(config.tableFilter) tableNames.foreach { tableName => tables.genData( location = s"${config.location}/tpch/sf${config.scaleFactor}_${config.format}",