build: Make the build system work out of box (#136)

.github/actions/java-test/action.yaml

@@ -37,18 +37,9 @@ runs:
     - name: Run Maven compile
       shell: bash
       run: |
-        echo "JAVA_VERSION=${JAVA_VERSION}"
-        if [ $JAVA_VERSION == "1.8" ]; then
-          ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb -Djava.version=${JAVA_VERSION} -Dspotless.version=2.29.0
-        else
-          ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb -Djava.version=${JAVA_VERSION}
-        fi
+        ./mvnw -B compile test-compile scalafix:scalafix -Psemanticdb
 
     - name: Run tests
       shell: bash
       run: |
-        if [ $JAVA_VERSION == "1.8" ]; then
-          SPARK_HOME=`pwd` ./mvnw -B clean install -Djava.version=${JAVA_VERSION} -Dspotless.version=2.29.0
-        else
-          SPARK_HOME=`pwd` ./mvnw -B clean install -Djava.version=${JAVA_VERSION}
-        fi
+        SPARK_HOME=`pwd` ./mvnw -B clean install

.github/actions/rust-test/action.yaml

@@ -51,24 +51,10 @@ runs:
       shell: bash
       run: |
         cd common
-        if [ $JAVA_VERSION == "1.8" ]; then
-          ../mvnw -B clean compile -DskipTests -Djava.version=${JAVA_VERSION} -Dspotless.version=2.29.0
-        else
-          ../mvnw -B clean compile -DskipTests -Djava.version=${JAVA_VERSION}
-        fi
+        ../mvnw -B clean compile -DskipTests
 
     - name: Run Cargo test
       shell: bash
       run: |
         cd core
-        # This is required to run some JNI related tests on the Rust side
-        JAVA_LD_LIBRARY_PATH=$JAVA_HOME/lib/server:$JAVA_HOME/lib:$JAVA_HOME/lib/jli
-        # special handing for java 1.8 for both linux and mac distributions
-        if [ $JAVA_VERSION == "8" ]; then
-          JAVA_LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:$JAVA_HOME/jre/lib/amd64:$JAVA_HOME/jre/lib/amd64/jli:$JAVA_HOME/jre/lib/server:$JAVA_HOME/jre/lib:$JAVA_HOME/jre/lib/jli
-        fi
-        RUST_BACKTRACE=1 \
-        LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_LD_LIBRARY_PATH \
-        DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$JAVA_LD_LIBRARY_PATH \
-        cargo test
-
+        RUST_BACKTRACE=1 cargo test

.github/workflows/pr_build.yml

@@ -54,8 +54,6 @@ jobs:
     runs-on: ${{ matrix.os }}
     container:
       image: amd64/rust
-    env:
-      JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}', matrix.java_version) }}
     steps:
       - uses: actions/checkout@v4
       - name: Setup Rust & Java toolchain
@@ -82,8 +80,6 @@ jobs:
     if: github.event_name == 'push'
     name: ${{ matrix.test-target }} test on ${{ matrix.os }} with java ${{ matrix.java_version }}
     runs-on: ${{ matrix.os }}
-    env:
-      JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}', matrix.java_version) }}
     steps:
       - uses: actions/checkout@v4
       - name: Setup Rust & Java toolchain
@@ -113,8 +109,6 @@ jobs:
       fail-fast: false
     name: ${{ matrix.test-target }} test on macos-aarch64 with java ${{ matrix.java_version }}
     runs-on: macos-14
-    env:
-      JAVA_VERSION: ${{ matrix.java_version == 8 && '1.8' || format('{0}', matrix.java_version) }}
     steps:
       - uses: actions/checkout@v4
       - name: Setup Rust & Java toolchain

DEVELOPMENT.md

@@ -30,7 +30,7 @@ under the License.
 
 ## Development Setup
 
-1. Make sure `JAVA_HOME` is set and point to JDK 11 installation.
+1. Make sure `JAVA_HOME` is set and point to JDK 8/11/17 installation.
 2. Install Rust toolchain. The easiest way is to use
    [rustup](https://rustup.rs).
 
@@ -39,6 +39,8 @@ under the License.
 A few common commands are specified in project's `Makefile`:
 
 - `make`: compile the entire project, but don't run tests
+- `make test-rust`: compile the project and run tests in Rust side
+- `make test-java`: compile the project and run tests in Java side
 - `make test`: compile the project and run tests in both Rust and Java
   side.
 - `make release`: compile the project and creates a release build. This
@@ -47,6 +49,25 @@ A few common commands are specified in project's `Makefile`:
 - `make clean`: clean up the workspace
 - `bin/comet-spark-shell -d . -o spark/target/` run Comet spark shell for V1 datasources
 - `bin/comet-spark-shell -d . -o spark/target/ --conf spark.sql.sources.useV1SourceList=""` run Comet spark shell for V2 datasources
+
+## Development Environment
+Comet is a multi-language project with native code written in Rust and JVM code written in Java and Scala.
+For Rust code, the CLion IDE is recommended. For JVM code, IntelliJ IDEA is recommended. 
+
+Before opening the project in an IDE, make sure to run `make` first to generate the necessary files for the IDEs. Currently, it's mostly about
+generating protobuf message classes for the JVM side. It's only required to run `make` once after cloning the repo.
+
+### IntelliJ IDEA
+First make sure to install the Scala plugin in IntelliJ IDEA. 
+After that, you can open the project in IntelliJ IDEA. The IDE should automatically detect the project structure and import as a Maven project.
+
+### CLion
+First make sure to install the Rust plugin in CLion or you can use the dedicated Rust IDE: RustRover.
+After that you can open the project in CLion. The IDE should automatically detect the project structure and import as a Cargo project.
+
+### Running Tests in IDEA
+Like other Maven projects, you can run tests in IntelliJ IDEA by right-clicking on the test class or test method and selecting "Run" or "Debug".
+However if the tests is related to the native side. Please make sure to run `make core`  or `cd core && cargo build` before running the tests in IDEA.
 
 ## Benchmark
 

Makefile

@@ -21,25 +21,22 @@ all: core jvm
 
 core:
 	cd core && cargo build
-jvm:
-	./mvnw clean package -DskipTests $(PROFILES)
-test:
-	./mvnw clean
+test-rust:
 	# We need to compile CometException so that the cargo test can pass
 	./mvnw compile -pl common -DskipTests $(PROFILES)
 	cd core && cargo build && \
-	LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${JAVA_HOME}/lib:${JAVA_HOME}/lib/server:${JAVA_HOME}/lib/jli && \
-	DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH:+${DYLD_LIBRARY_PATH}:}${JAVA_HOME}/lib:${JAVA_HOME}/lib/server:${JAVA_HOME}/lib/jli \
 	RUST_BACKTRACE=1 cargo test
+jvm:
+	./mvnw clean package -DskipTests $(PROFILES)
+test-jvm: core
 	SPARK_HOME=`pwd` COMET_CONF_DIR=$(shell pwd)/conf RUST_BACKTRACE=1 ./mvnw verify $(PROFILES)
+test: test-rust test-jvm
 clean:
 	cd core && cargo clean
 	./mvnw clean
 	rm -rf .dist
 bench:
-	cd core && LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${JAVA_HOME}/lib:${JAVA_HOME}/lib/server:${JAVA_HOME}/lib/jli && \
-	DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH:+${DYLD_LIBRARY_PATH}:}${JAVA_HOME}/lib:${JAVA_HOME}/lib/server:${JAVA_HOME}/lib/jli \
-	RUSTFLAGS="-Ctarget-cpu=native" cargo bench $(filter-out $@,$(MAKECMDGOALS))
+	cd core && RUSTFLAGS="-Ctarget-cpu=native" cargo bench $(filter-out $@,$(MAKECMDGOALS))
 format:
 	./mvnw compile test-compile scalafix:scalafix -Psemanticdb $(PROFILES)
 	./mvnw spotless:apply $(PROFILES)

pom.xml

@@ -492,9 +492,6 @@ under the License.
         <spark.version>3.2.2</spark.version>
         <spark.version.short>3.2</spark.version.short>
         <parquet.version>1.12.0</parquet.version>
-        <java.version>1.8</java.version>
-        <maven.compiler.source>${java.version}</maven.compiler.source>
-        <maven.compiler.target>${java.version}</maven.compiler.target>
       </properties>
     </profile>
 
@@ -505,9 +502,6 @@ under the License.
         <spark.version>3.3.2</spark.version>
         <spark.version.short>3.3</spark.version.short>
         <parquet.version>1.12.0</parquet.version>
-        <java.version>11</java.version>
-        <maven.compiler.source>${java.version}</maven.compiler.source>
-        <maven.compiler.target>${java.version}</maven.compiler.target>
       </properties>
     </profile>
 
@@ -517,9 +511,6 @@ under the License.
         <scala.version>2.12.17</scala.version>
         <spark.version.short>3.4</spark.version.short>
         <parquet.version>1.13.1</parquet.version>
-        <java.version>11</java.version>
-        <maven.compiler.source>${java.version}</maven.compiler.source>
-        <maven.compiler.target>${java.version}</maven.compiler.target>
       </properties>
     </profile>
 
@@ -531,6 +522,43 @@ under the License.
       </properties>
     </profile>
 
+    <profile>
+      <id>jdk1.8</id>
+      <activation>
+        <jdk>1.8</jdk>
+      </activation>
+      <properties>
+        <java.version>1.8</java.version>
+        <maven.compiler.source>${java.version}</maven.compiler.source>
+        <maven.compiler.target>${java.version}</maven.compiler.target>
+        <spotless.version>2.29.0</spotless.version>
+      </properties>
+    </profile>
+
+    <profile>
+      <id>jdk11</id>
+      <activation>
+        <jdk>11</jdk>
+      </activation>
+      <properties>
+        <java.version>11</java.version>
+        <maven.compiler.source>${java.version}</maven.compiler.source>
+        <maven.compiler.target>${java.version}</maven.compiler.target>
+      </properties>
+    </profile>
+
+    <profile>
+      <id>jdk17</id>
+      <activation>
+        <jdk>17</jdk>
+      </activation>
+      <properties>
+        <java.version>17</java.version>
+        <maven.compiler.source>${java.version}</maven.compiler.source>
+        <maven.compiler.target>${java.version}</maven.compiler.target>
+      </properties>
+    </profile>
+
     <profile>
       <id>semanticdb</id>
       <properties>