diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index d68269300..6e69ac98c 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -35,12 +35,24 @@ docker: packages: write steps: - uses: actions/checkout@v3 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + - name: Extract Maven version + id: extract_version + run: | + COMET_VERSION=$(xmllint --xpath "string(//project/version)" pom.xml) + echo "COMET_VERSION=$COMET_VERSION" >> $GITHUB_ENV + - name: Echo Maven version + run: echo "The current Maven version is ${{ env.COMET_VERSION }}" - name: Build and push Docker image run: | - docker build -t datafusion-comet:spark-3.4-latest -f kube/Dockerfile . + docker build -t datafusion-comet:spark-3.4.2-scala-2.12-${{ env.COMET_VERSION }} -f kube/Dockerfile . export DOCKER_TAG="$(git describe --exact-match --tags $(git log -n1 --pretty='%h') || echo '')" echo "publishing docker tag $DOCKER_TAG" - docker tag datafusion-comet:spark-3.4-latest ghcr.io/apache/datafusion-comet:$DOCKER_TAG + docker tag datafusion-comet:spark-3.4.2-scala-2.12-${{ env.COMET_VERSION }} ghcr.io/apache/datafusion-comet:$DOCKER_TAG docker login ghcr.io -u $DOCKER_USER -p "$DOCKER_PASS" docker push ghcr.io/apache/datafusion-comet:$DOCKER_TAG env: diff --git a/kube/Dockerfile b/kube/Dockerfile index 6c9f50caf..18a79e423 100644 --- a/kube/Dockerfile +++ b/kube/Dockerfile @@ -31,22 +31,23 @@ ENV RUSTFLAGS="-C debuginfo=line-tables-only -C incremental=false" ENV SPARK_VERSION=3.4 ENV SCALA_VERSION=2.12 -# Add source files to Docker image +# copy source files to Docker image RUN mkdir /comet WORKDIR /comet -ADD .mvn /comet -ADD common /comet -ADD fuzz-testing /comet -ADD native /comet -ADD spark /comet -ADD spark-integration /comet -ADD .scalafix.conf /comet -ADD Makefile /comet -ADD pom.xml /comet +COPY .mvn /comet/.mvn +COPY common /comet/common +COPY fuzz-testing /comet/fuzz-testing +COPY native /comet/native +COPY spark /comet/spark +COPY spark-integration /comet/spark-integration +COPY .scalafix.conf /comet/.scalafix.conf +COPY Makefile /comet/Makefile +COPY pom.xml /comet/pom.xml # Pick the JDK instead of JRE to compile Comet -RUN JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make release-nogit PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION" +RUN cd /comet \ + && JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make release-nogit PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION" FROM apache/spark:3.4.2 ENV SPARK_VERSION=3.4 diff --git a/native/Cargo.toml b/native/Cargo.toml index 9977ceece..2e73c5445 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -39,15 +39,15 @@ arrow-buffer = { version = "52.2.0" } arrow-data = { version = "52.2.0" } arrow-schema = { version = "52.2.0" } parquet = { version = "52.2.0", default-features = false, features = ["experimental"] } -datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1" } -datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", features = ["unicode_expressions", "crypto_expressions"] } -datafusion-functions = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", features = ["crypto_expressions"] } -datafusion-functions-nested = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false } -datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false } -datafusion-execution = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false } -datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false } -datafusion-physical-expr-common = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false } -datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", rev = "41.0.0-rc1", default-features = false } +datafusion-common = { version = "41.0.0" } +datafusion = { default-features = false, version = "41.0.0", features = ["unicode_expressions", "crypto_expressions"] } +datafusion-functions = { version = "41.0.0", features = ["crypto_expressions"] } +datafusion-functions-nested = { version = "41.0.0", default-features = false } +datafusion-expr = { version = "41.0.0", default-features = false } +datafusion-execution = { version = "41.0.0", default-features = false } +datafusion-physical-plan = { version = "41.0.0", default-features = false } +datafusion-physical-expr-common = { version = "41.0.0", default-features = false } +datafusion-physical-expr = { version = "41.0.0", default-features = false } datafusion-comet-spark-expr = { path = "spark-expr", version = "0.2.0" } datafusion-comet-proto = { path = "proto", version = "0.2.0" } chrono = { version = "0.4", default-features = false, features = ["clock"] }