upmerge

.asf.yaml

@@ -27,7 +27,7 @@ notifications:
   jira_options: link label worklog
 github:
   description: "Apache DataFusion Comet Spark Accelerator"
-  homepage: https://arrow.apache.org/datafusion-comet
+  homepage: https://datafusion.apache.org/comet
   labels:
     - arrow
     - datafusion
@@ -45,7 +45,7 @@ github:
         required_approving_review_count: 1
 
 # publishes the content of the `asf-site` branch to
-# https://arrow.apache.org/datafusion/
+# https://datafusion.apache.org/comet/
 publish:
   whoami: asf-site
-  subdir: datafusion-comet
+  subdir: comet

.github/workflows/benchmark-tpch.yml

@@ -25,10 +25,12 @@ on:
   push:
     paths-ignore:
       - "doc/**"
+      - "docs/**"
       - "**.md"
   pull_request:
     paths-ignore:
       - "doc/**"
+      - "docs/**"
       - "**.md"
   # manual trigger
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow

.github/workflows/benchmark.yml

@@ -25,10 +25,12 @@ on:
   push:
     paths-ignore:
       - "doc/**"
+      - "docs/**"
       - "**.md"
   pull_request:
     paths-ignore:
       - "doc/**"
+      - "docs/**"
       - "**.md"
   # manual trigger
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow

.github/workflows/docs.yaml

@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - .asf.yaml
+      - .github/workflows/docs.yaml
+      - docs/**
+
+name: Deploy DataFusion Comet site
+
+jobs:
+  build-docs:
+    name: Build docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout docs sources
+        uses: actions/checkout@v4
+
+      - name: Checkout asf-site branch
+        uses: actions/checkout@v4
+        with:
+          ref: asf-site
+          path: asf-site
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          set -x
+          python3 -m venv venv
+          source venv/bin/activate
+          pip install -r docs/requirements.txt
+
+      - name: Build docs
+        run: |
+          set -x
+          source venv/bin/activate
+          cd docs
+          ./build.sh
+
+      - name: Copy & push the generated HTML
+        run: |
+          set -x
+          cd asf-site/
+          rsync \
+            -a \
+            --delete \
+            --exclude '/.git/' \
+            ../docs/build/html/ \
+            ./
+          cp ../.asf.yaml .
+          touch .nojekyll
+          git status --porcelain
+          if [ "$(git status --porcelain)" != "" ]; then
+            git config user.name "github-actions[bot]"
+            git config user.email "github-actions[bot]@users.noreply.github.com"
+            git add --all
+            git commit -m 'Publish built docs triggered by ${{ github.sha }}'
+            git push || git push --force
+          fi

.github/workflows/pr_build.yml

@@ -25,10 +25,12 @@ on:
   push:
     paths-ignore:
       - "doc/**"
+      - "docs/**"
       - "**.md"
   pull_request:
     paths-ignore:
       - "doc/**"
+      - "docs/**"
       - "**.md"
   # manual trigger
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow

.github/workflows/spark_sql_test.yml

@@ -25,10 +25,12 @@ on:
   push:
     paths-ignore:
       - "doc/**"
+      - "docs/**"
       - "**.md"
   pull_request:
     paths-ignore:
       - "doc/**"
+      - "docs/**"
       - "**.md"
   # manual trigger
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow

README.md

@@ -17,17 +17,18 @@ specific language governing permissions and limitations
 under the License.
 -->
 
-# Apache Arrow DataFusion Comet
+# Apache DataFusion Comet
 
-Comet is an Apache Spark plugin that uses [Apache Arrow DataFusion](https://arrow.apache.org/datafusion/)
+Apache DataFusion Comet is an Apache Spark plugin that uses [Apache DataFusion](https://datafusion.apache.org/)
 as native runtime to achieve improvement in terms of query efficiency and query runtime.
 
 Comet runs Spark SQL queries using the native DataFusion runtime, which is
 typically faster and more resource efficient than JVM based runtimes.
 
-<a href="doc/comet-overview.png"><img src="doc/comet-system-diagram.png" align="center" width="500" ></a>
+<a href="docs/source/_static/images/comet-overview.png"><img src="docs/source/_static/images/comet-system-diagram.png" align="center" width="500" ></a>
 
 Comet aims to support:
+
 - a native Parquet implementation, including both reader and writer
 - full implementation of Spark operators, including
   Filter/Project/Aggregation/Join/Exchange etc.
@@ -38,7 +39,7 @@ Comet aims to support:
 
 The following diagram illustrates the architecture of Comet:
 
-<a href="doc/comet-overview.png"><img src="doc/comet-overview.png" align="center" height="600" width="750" ></a>
+<a href="docs/source/_static/images/comet-overview.png"><img src="docs/source/_static/images/comet-overview.png" align="center" height="600" width="750" ></a>
 
 ## Current Status
 
@@ -68,74 +69,4 @@ Linux, Apple OSX (Intel and M1)
 
 ## Getting started
 
-Make sure the requirements above are met and software installed on your machine
-
-### Clone repo
-```commandline
-git clone https://github.com/apache/arrow-datafusion-comet.git
-```
-
-### Specify the Spark version and build the Comet
-Spark 3.4 used for the example.
-```
-cd arrow-datafusion-comet
-make release PROFILES="-Pspark-3.4"
-```
-
-### Run Spark with Comet enabled
-Make sure `SPARK_HOME` points to the same Spark version as Comet has built for.
-
-```
-$SPARK_HOME/bin/spark-shell --jars spark/target/comet-spark-spark3.4_2.12-0.1.0-SNAPSHOT.jar \
---conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \
---conf spark.comet.enabled=true \
---conf spark.comet.exec.enabled=true \
---conf spark.comet.exec.all.enabled=true
-```
-
-### Verify Comet enabled for Spark SQL query  
-
-Create a test Parquet source
-```scala
-scala> (0 until 10).toDF("a").write.mode("overwrite").parquet("/tmp/test")
-```
-
-Query the data from the test source and check: 
-- INFO message shows the native Comet library has been initialized.
-- The query plan reflects Comet operators being used for this query instead of Spark ones
-```scala
-scala> spark.read.parquet("/tmp/test").createOrReplaceTempView("t1")
-scala> spark.sql("select * from t1 where a > 5").explain
-INFO src/lib.rs: Comet native library initialized
-== Physical Plan ==
-        *(1) ColumnarToRow
-        +- CometFilter [a#14], (isnotnull(a#14) AND (a#14 > 5))
-          +- CometScan parquet [a#14] Batched: true, DataFilters: [isnotnull(a#14), (a#14 > 5)], 
-             Format: CometParquet, Location: InMemoryFileIndex(1 paths)[file:/tmp/test], PartitionFilters: [], 
-             PushedFilters: [IsNotNull(a), GreaterThan(a,5)], ReadSchema: struct<a:int>
-```
-
-### Enable Comet shuffle
-
-Comet shuffle feature is disabled by default. To enable it, please add related configs:
-
-```
---conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager
---conf spark.comet.exec.shuffle.enabled=true
-```
-
-Above configs enable Comet native shuffle which only supports hash partition and single partition.
-Comet native shuffle doesn't support complex types yet.
-
-Comet doesn't have official release yet so currently the only way to test it is to build jar and include it in your Spark application. Depending on your deployment mode you may also need to set the driver & executor class path(s) to explicitly contain Comet otherwise Spark may use a different class-loader for the Comet components than its internal components which will then fail at runtime. For example:
-
-```
---driver-class-path spark/target/comet-spark-spark3.4_2.12-0.1.0-SNAPSHOT.jar
-```
-
-Some cluster managers may require additional configuration, see https://spark.apache.org/docs/latest/cluster-overview.html
-
-To enable columnar shuffle which supports all partitioning and basic complex types, one more config is required:
-```
---conf spark.comet.columnar.shuffle.enabled=true
-```
+See the [DataFusion Comet User Guide](https://datafusion.apache.org/comet/user-guide/) for installation instructions.

common/pom.xml

@@ -167,6 +167,24 @@ under the License.
         <groupId>net.alchim31.maven</groupId>
         <artifactId>scala-maven-plugin</artifactId>
       </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>build-helper-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>add-shim-source</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>add-source</goal>
+            </goals>
+            <configuration>
+              <sources>
+                <source>src/main/${shims.source}</source>
+              </sources>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
     <resources>
       <resource>