improve

.github/workflows/docker-publish.yml

@@ -35,12 +35,24 @@ docker:
     packages: write
   steps:
     - uses: actions/checkout@v3
+    - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+    - name: Extract Maven version
+      id: extract_version
+      run: |
+        COMET_VERSION=$(xmllint --xpath "string(//project/version)" pom.xml)
+        echo "COMET_VERSION=$COMET_VERSION" >> $GITHUB_ENV
+    - name: Echo Maven version
+      run: echo "The current Maven version is ${{ env.COMET_VERSION }}"
     - name: Build and push Docker image
       run: |
-        docker build -t datafusion-comet:spark-3.4-latest -f kube/Dockerfile .
+        docker build -t datafusion-comet:spark-3.4.2-scala-2.12-${{ env.COMET_VERSION }} -f kube/Dockerfile .
         export DOCKER_TAG="$(git describe --exact-match --tags $(git log -n1 --pretty='%h') || echo '')"
         echo "publishing docker tag $DOCKER_TAG"
-        docker tag datafusion-comet:spark-3.4-latest ghcr.io/apache/datafusion-comet:$DOCKER_TAG
+        docker tag datafusion-comet:spark-3.4.2-scala-2.12-${{ env.COMET_VERSION }} ghcr.io/apache/datafusion-comet:$DOCKER_TAG
         docker login ghcr.io -u $DOCKER_USER -p "$DOCKER_PASS"
         docker push ghcr.io/apache/datafusion-comet:$DOCKER_TAG
       env:

kube/Dockerfile

@@ -31,22 +31,23 @@ ENV RUSTFLAGS="-C debuginfo=line-tables-only -C incremental=false"
 ENV SPARK_VERSION=3.4
 ENV SCALA_VERSION=2.12
 
-# Add source files to Docker image
+# copy source files to Docker image
 RUN mkdir /comet
 WORKDIR /comet
 
-ADD .mvn /comet
-ADD common /comet
-ADD fuzz-testing /comet
-ADD native /comet
-ADD spark /comet
-ADD spark-integration /comet
-ADD .scalafix.conf /comet
-ADD Makefile /comet
-ADD pom.xml /comet
+COPY .mvn /comet/.mvn
+COPY common /comet/common
+COPY fuzz-testing /comet/fuzz-testing
+COPY native /comet/native
+COPY spark /comet/spark
+COPY spark-integration /comet/spark-integration
+COPY .scalafix.conf /comet/.scalafix.conf
+COPY Makefile /comet/Makefile
+COPY pom.xml /comet/pom.xml
 
 # Pick the JDK instead of JRE to compile Comet
-RUN JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make release-nogit PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION"
+RUN cd /comet \
+    && JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make release-nogit PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION"
 
 FROM apache/spark:3.4.2
 ENV SPARK_VERSION=3.4

native/Cargo.toml

@@ -39,15 +39,15 @@ arrow-buffer = { version = "52.2.0" }
 arrow-data = { version = "52.2.0" }
 arrow-schema = { version = "52.2.0" }
 parquet = { version = "52.2.0", default-features = false, features = ["experimental"] }
-datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1" }
-datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", features = ["unicode_expressions", "crypto_expressions"] }
-datafusion-functions = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", features = ["crypto_expressions"] }
-datafusion-functions-nested = { git = "https://github.com/apache/datafusion.git",  rev = "41.0.0-rc1", default-features = false }
-datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false }
-datafusion-execution = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false }
-datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false }
-datafusion-physical-expr-common = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false }
-datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false }
+datafusion-common = { version = "41.0.0" }
+datafusion = { default-features = false, version = "41.0.0", features = ["unicode_expressions", "crypto_expressions"] }
+datafusion-functions = { version = "41.0.0", features = ["crypto_expressions"] }
+datafusion-functions-nested = { version = "41.0.0", default-features = false }
+datafusion-expr = { version = "41.0.0", default-features = false }
+datafusion-execution = { version = "41.0.0", default-features = false }
+datafusion-physical-plan = { version = "41.0.0", default-features = false }
+datafusion-physical-expr-common = { version = "41.0.0", default-features = false }
+datafusion-physical-expr = { version = "41.0.0", default-features = false }
 datafusion-comet-spark-expr = { path = "spark-expr", version = "0.2.0" }
 datafusion-comet-proto = { path = "proto", version = "0.2.0" }
 chrono = { version = "0.4", default-features = false, features = ["clock"] }