diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml index ae6d80dee..6c3af7998 100644 --- a/.github/actions/java-test/action.yaml +++ b/.github/actions/java-test/action.yaml @@ -37,18 +37,9 @@ runs: - name: Run Maven compile shell: bash run: | - echo "JAVA_VERSION=${JAVA_VERSION}" - if [ $JAVA_VERSION == "1.8" ]; then - ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb -Djava.version=${JAVA_VERSION} -Dspotless.version=2.29.0 - else - ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb -Djava.version=${JAVA_VERSION} - fi + ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb - name: Run tests shell: bash run: | - if [ $JAVA_VERSION == "1.8" ]; then - SPARK_HOME=`pwd` ./mvnw -B clean install -Djava.version=${JAVA_VERSION} -Dspotless.version=2.29.0 - else - SPARK_HOME=`pwd` ./mvnw -B clean install -Djava.version=${JAVA_VERSION} - fi + SPARK_HOME=`pwd` ./mvnw -B clean install diff --git a/.github/actions/rust-test/action.yaml b/.github/actions/rust-test/action.yaml index cb6c29c82..bf0d0ba15 100644 --- a/.github/actions/rust-test/action.yaml +++ b/.github/actions/rust-test/action.yaml @@ -51,24 +51,10 @@ runs: shell: bash run: | cd common - if [ $JAVA_VERSION == "1.8" ]; then - ../mvnw -B clean compile -DskipTests -Djava.version=${JAVA_VERSION} -Dspotless.version=2.29.0 - else - ../mvnw -B clean compile -DskipTests -Djava.version=${JAVA_VERSION} - fi + ../mvnw -B clean compile -DskipTests - name: Run Cargo test shell: bash run: | cd core - # This is required to run some JNI related tests on the Rust side - JAVA_LD_LIBRARY_PATH=$JAVA_HOME/lib/server:$JAVA_HOME/lib:$JAVA_HOME/lib/jli - # special handing for java 1.8 for both linux and mac distributions - if [ $JAVA_VERSION == "8" ]; then - JAVA_LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:$JAVA_HOME/jre/lib/amd64:$JAVA_HOME/jre/lib/amd64/jli:$JAVA_HOME/jre/lib/server:$JAVA_HOME/jre/lib:$JAVA_HOME/jre/lib/jli - fi - RUST_BACKTRACE=1 \ - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_LD_LIBRARY_PATH \ - DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$JAVA_LD_LIBRARY_PATH \ - cargo test - + RUST_BACKTRACE=1 cargo test diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index 316532cee..aa69aea31 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -54,8 +54,6 @@ jobs: runs-on: ${{ matrix.os }} container: image: amd64/rust - env: - JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}', matrix.java_version) }} steps: - uses: actions/checkout@v4 - name: Setup Rust & Java toolchain @@ -82,8 +80,6 @@ jobs: if: github.event_name == 'push' name: ${{ matrix.test-target }} test on ${{ matrix.os }} with java ${{ matrix.java_version }} runs-on: ${{ matrix.os }} - env: - JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}', matrix.java_version) }} steps: - uses: actions/checkout@v4 - name: Setup Rust & Java toolchain @@ -113,8 +109,6 @@ jobs: fail-fast: false name: ${{ matrix.test-target }} test on macos-aarch64 with java ${{ matrix.java_version }} runs-on: macos-14 - env: - JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}', matrix.java_version) }} steps: - uses: actions/checkout@v4 - name: Setup Rust & Java toolchain diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 1793bb9e2..25a65999e 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -30,7 +30,7 @@ under the License. ## Development Setup -1. Make sure `JAVA_HOME` is set and point to JDK 11 installation. +1. Make sure `JAVA_HOME` is set and point to JDK 8/11/17 installation. 2. Install Rust toolchain. The easiest way is to use [rustup](https://rustup.rs). @@ -39,6 +39,8 @@ under the License. A few common commands are specified in project's `Makefile`: - `make`: compile the entire project, but don't run tests +- `make test-rust`: compile the project and run tests in Rust side +- `make test-java`: compile the project and run tests in Java side - `make test`: compile the project and run tests in both Rust and Java side. - `make release`: compile the project and creates a release build. This @@ -47,6 +49,25 @@ A few common commands are specified in project's `Makefile`: - `make clean`: clean up the workspace - `bin/comet-spark-shell -d . -o spark/target/` run Comet spark shell for V1 datasources - `bin/comet-spark-shell -d . -o spark/target/ --conf spark.sql.sources.useV1SourceList=""` run Comet spark shell for V2 datasources + +## Development Environment +Comet is a multi-language project with native code written in Rust and JVM code written in Java and Scala. +For Rust code, the CLion IDE is recommended. For JVM code, IntelliJ IDEA is recommended. + +Before opening the project in an IDE, make sure to run `make` first to generate the necessary files for the IDEs. Currently, it's mostly about +generating protobuf message classes for the JVM side. It's only required to run `make` once after cloning the repo. + +### IntelliJ IDEA +First make sure to install the Scala plugin in IntelliJ IDEA. +After that, you can open the project in IntelliJ IDEA. The IDE should automatically detect the project structure and import as a Maven project. + +### CLion +First make sure to install the Rust plugin in CLion or you can use the dedicated Rust IDE: RustRover. +After that you can open the project in CLion. The IDE should automatically detect the project structure and import as a Cargo project. + +### Running Tests in IDEA +Like other Maven projects, you can run tests in IntelliJ IDEA by right-clicking on the test class or test method and selecting "Run" or "Debug". +However if the tests is related to the native side. Please make sure to run `make core` or `cd core && cargo build` before running the tests in IDEA. ## Benchmark diff --git a/Makefile b/Makefile index fe13fbd9b..6f599a081 100644 --- a/Makefile +++ b/Makefile @@ -21,25 +21,22 @@ all: core jvm core: cd core && cargo build -jvm: - ./mvnw clean package -DskipTests $(PROFILES) -test: - ./mvnw clean +test-rust: # We need to compile CometException so that the cargo test can pass ./mvnw compile -pl common -DskipTests $(PROFILES) cd core && cargo build && \ - LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${JAVA_HOME}/lib:${JAVA_HOME}/lib/server:${JAVA_HOME}/lib/jli && \ - DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH:+${DYLD_LIBRARY_PATH}:}${JAVA_HOME}/lib:${JAVA_HOME}/lib/server:${JAVA_HOME}/lib/jli \ RUST_BACKTRACE=1 cargo test +jvm: + ./mvnw clean package -DskipTests $(PROFILES) +test-jvm: core SPARK_HOME=`pwd` COMET_CONF_DIR=$(shell pwd)/conf RUST_BACKTRACE=1 ./mvnw verify $(PROFILES) +test: test-rust test-jvm clean: cd core && cargo clean ./mvnw clean rm -rf .dist bench: - cd core && LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${JAVA_HOME}/lib:${JAVA_HOME}/lib/server:${JAVA_HOME}/lib/jli && \ - DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH:+${DYLD_LIBRARY_PATH}:}${JAVA_HOME}/lib:${JAVA_HOME}/lib/server:${JAVA_HOME}/lib/jli \ - RUSTFLAGS="-Ctarget-cpu=native" cargo bench $(filter-out $@,$(MAKECMDGOALS)) + cd core && RUSTFLAGS="-Ctarget-cpu=native" cargo bench $(filter-out $@,$(MAKECMDGOALS)) format: ./mvnw compile test-compile scalafix:scalafix -Psemanticdb $(PROFILES) ./mvnw spotless:apply $(PROFILES) diff --git a/pom.xml b/pom.xml index d7cd0764e..aa59d19aa 100644 --- a/pom.xml +++ b/pom.xml @@ -492,9 +492,6 @@ under the License. 3.2.2 3.2 1.12.0 - 1.8 - ${java.version} - ${java.version} @@ -505,9 +502,6 @@ under the License. 3.3.2 3.3 1.12.0 - 11 - ${java.version} - ${java.version} @@ -517,9 +511,6 @@ under the License. 2.12.17 3.4 1.13.1 - 11 - ${java.version} - ${java.version} @@ -531,6 +522,43 @@ under the License. + + jdk1.8 + + 1.8 + + + 1.8 + ${java.version} + ${java.version} + 2.29.0 + + + + + jdk11 + + 11 + + + 11 + ${java.version} + ${java.version} + + + + + jdk17 + + 17 + + + 17 + ${java.version} + ${java.version} + + + semanticdb