diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml index ae6d80dee..6c3af7998 100644 --- a/.github/actions/java-test/action.yaml +++ b/.github/actions/java-test/action.yaml @@ -37,18 +37,9 @@ runs: - name: Run Maven compile shell: bash run: | - echo "JAVA_VERSION=${JAVA_VERSION}" - if [ $JAVA_VERSION == "1.8" ]; then - ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb -Djava.version=${JAVA_VERSION} -Dspotless.version=2.29.0 - else - ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb -Djava.version=${JAVA_VERSION} - fi + ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb - name: Run tests shell: bash run: | - if [ $JAVA_VERSION == "1.8" ]; then - SPARK_HOME=`pwd` ./mvnw -B clean install -Djava.version=${JAVA_VERSION} -Dspotless.version=2.29.0 - else - SPARK_HOME=`pwd` ./mvnw -B clean install -Djava.version=${JAVA_VERSION} - fi + SPARK_HOME=`pwd` ./mvnw -B clean install diff --git a/.github/actions/rust-test/action.yaml b/.github/actions/rust-test/action.yaml index cb6c29c82..8f311f77f 100644 --- a/.github/actions/rust-test/action.yaml +++ b/.github/actions/rust-test/action.yaml @@ -51,11 +51,7 @@ runs: shell: bash run: | cd common - if [ $JAVA_VERSION == "1.8" ]; then - ../mvnw -B clean compile -DskipTests -Djava.version=${JAVA_VERSION} -Dspotless.version=2.29.0 - else - ../mvnw -B clean compile -DskipTests -Djava.version=${JAVA_VERSION} - fi + ../mvnw -B clean compile -DskipTests - name: Run Cargo test shell: bash @@ -64,9 +60,6 @@ runs: # This is required to run some JNI related tests on the Rust side JAVA_LD_LIBRARY_PATH=$JAVA_HOME/lib/server:$JAVA_HOME/lib:$JAVA_HOME/lib/jli # special handing for java 1.8 for both linux and mac distributions - if [ $JAVA_VERSION == "8" ]; then - JAVA_LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:$JAVA_HOME/jre/lib/amd64:$JAVA_HOME/jre/lib/amd64/jli:$JAVA_HOME/jre/lib/server:$JAVA_HOME/jre/lib:$JAVA_HOME/jre/lib/jli - fi RUST_BACKTRACE=1 \ LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_LD_LIBRARY_PATH \ DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$JAVA_LD_LIBRARY_PATH \ diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index 316532cee..aa69aea31 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -54,8 +54,6 @@ jobs: runs-on: ${{ matrix.os }} container: image: amd64/rust - env: - JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}', matrix.java_version) }} steps: - uses: actions/checkout@v4 - name: Setup Rust & Java toolchain @@ -82,8 +80,6 @@ jobs: if: github.event_name == 'push' name: ${{ matrix.test-target }} test on ${{ matrix.os }} with java ${{ matrix.java_version }} runs-on: ${{ matrix.os }} - env: - JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}', matrix.java_version) }} steps: - uses: actions/checkout@v4 - name: Setup Rust & Java toolchain @@ -113,8 +109,6 @@ jobs: fail-fast: false name: ${{ matrix.test-target }} test on macos-aarch64 with java ${{ matrix.java_version }} runs-on: macos-14 - env: - JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}', matrix.java_version) }} steps: - uses: actions/checkout@v4 - name: Setup Rust & Java toolchain diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 1793bb9e2..9d10601c0 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -30,7 +30,7 @@ under the License. ## Development Setup -1. Make sure `JAVA_HOME` is set and point to JDK 11 installation. +1. Make sure `JAVA_HOME` is set and point to JDK 8/11/17 installation. 2. Install Rust toolchain. The easiest way is to use [rustup](https://rustup.rs). @@ -39,6 +39,8 @@ under the License. A few common commands are specified in project's `Makefile`: - `make`: compile the entire project, but don't run tests +- `make test-rust`: compile the project and run tests in Rust side +- `make test-java`: compile the project and run tests in Java side - `make test`: compile the project and run tests in both Rust and Java side. - `make release`: compile the project and creates a release build. This @@ -47,6 +49,25 @@ A few common commands are specified in project's `Makefile`: - `make clean`: clean up the workspace - `bin/comet-spark-shell -d . -o spark/target/` run Comet spark shell for V1 datasources - `bin/comet-spark-shell -d . -o spark/target/ --conf spark.sql.sources.useV1SourceList=""` run Comet spark shell for V2 datasources + +## Opening Project in IDEs +Comet is a multi-language project with native code written in Rust and JVM code written in Java and Scala. +For Rust code, the CLion IDE is recommended. For JVM code, IntelliJ IDEA is recommended. + +Before opening the project in an IDE, make sure to run `make` first to generate the necessary files for the IDEs. Currently, it's mostly about +generating protobuf message classes for the JVM side. It's only required to run `make` once after cloning the repo. + +### IntelliJ IDEA +First make sure to install the Scala plugin in IntelliJ IDEA. +After that, you can open the project in IntelliJ IDEA. The IDE should automatically detect the project structure and import as a Maven project. + +### CLion +First make sure to install the Rust plugin in CLion or you can use the dedicated Rust IDE: RustRover. +After that you can open the project in CLion. The IDE should automatically detect the project structure and import as a Cargo project. + +### Running Tests in IDEA +Like other Maven projects, you can run tests in IntelliJ IDEA by right-clicking on the test class or test method and selecting "Run" or "Debug". +However if the tests is related to the native side. Please make sure to run `make core` or `cd core && cargo build` before running the tests in IDEA. ## Benchmark diff --git a/Makefile b/Makefile index fe13fbd9b..d88195b62 100644 --- a/Makefile +++ b/Makefile @@ -21,8 +21,17 @@ all: core jvm core: cd core && cargo build +test-rust: + # We need to compile CometException so that the cargo test can pass + ./mvnw compile -pl common -DskipTests $(PROFILES) + cd core && cargo build && \ + LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${JAVA_HOME}/lib:${JAVA_HOME}/lib/server:${JAVA_HOME}/lib/jli && \ + DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH:+${DYLD_LIBRARY_PATH}:}${JAVA_HOME}/lib:${JAVA_HOME}/lib/server:${JAVA_HOME}/lib/jli \ + RUST_BACKTRACE=1 cargo test jvm: ./mvnw clean package -DskipTests $(PROFILES) +test-jvm: core + SPARK_HOME=`pwd` COMET_CONF_DIR=$(shell pwd)/conf RUST_BACKTRACE=1 ./mvnw verify $(PROFILES) test: ./mvnw clean # We need to compile CometException so that the cargo test can pass diff --git a/pom.xml b/pom.xml index d7cd0764e..aa59d19aa 100644 --- a/pom.xml +++ b/pom.xml @@ -492,9 +492,6 @@ under the License. 3.2.2 3.2 1.12.0 - 1.8 - ${java.version} - ${java.version} @@ -505,9 +502,6 @@ under the License. 3.3.2 3.3 1.12.0 - 11 - ${java.version} - ${java.version} @@ -517,9 +511,6 @@ under the License. 2.12.17 3.4 1.13.1 - 11 - ${java.version} - ${java.version} @@ -531,6 +522,43 @@ under the License. + + jdk1.8 + + 1.8 + + + 1.8 + ${java.version} + ${java.version} + 2.29.0 + + + + + jdk11 + + 11 + + + 11 + ${java.version} + ${java.version} + + + + + jdk17 + + 17 + + + 17 + ${java.version} + ${java.version} + + + semanticdb