chore: Add documentation publishing infrastructure (#314)

* Add starting point for documentation site

* remove branding guideline

* Remove old logos

* remove logos

* add ASF license headers

* Add ASF links

.github/workflows/docs.yaml

@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - .asf.yaml
+      - .github/workflows/docs.yaml
+      - docs/**
+
+name: Deploy DataFusion Comet site
+
+jobs:
+  build-docs:
+    name: Build docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout docs sources
+        uses: actions/checkout@v4
+
+      - name: Checkout asf-site branch
+        uses: actions/checkout@v4
+        with:
+          ref: asf-site
+          path: asf-site
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          set -x
+          python3 -m venv venv
+          source venv/bin/activate
+          pip install -r docs/requirements.txt
+
+      - name: Build docs
+        run: |
+          set -x
+          source venv/bin/activate
+          cd docs
+          ./build.sh
+
+      - name: Copy & push the generated HTML
+        run: |
+          set -x
+          cd asf-site/
+          rsync \
+            -a \
+            --delete \
+            --exclude '/.git/' \
+            ../docs/build/html/ \
+            ./
+          cp ../.asf.yaml .
+          touch .nojekyll
+          git status --porcelain
+          if [ "$(git status --porcelain)" != "" ]; then
+            git config user.name "github-actions[bot]"
+            git config user.email "github-actions[bot]@users.noreply.github.com"
+            git add --all
+            git commit -m 'Publish built docs triggered by ${{ github.sha }}'
+            git push || git push --force
+          fi

README.md

@@ -19,7 +19,7 @@ under the License.
 
 # Apache DataFusion Comet
 
-Comet is an Apache Spark plugin that uses [Apache DataFusion](https://datafusion.apache.org/datafusion/)
+Apache DataFusion Comet is an Apache Spark plugin that uses [Apache DataFusion](https://datafusion.apache.org/)
 as native runtime to achieve improvement in terms of query efficiency and query runtime.
 
 Comet runs Spark SQL queries using the native DataFusion runtime, which is
@@ -28,6 +28,7 @@ typically faster and more resource efficient than JVM based runtimes.
 <a href="doc/comet-overview.png"><img src="doc/comet-system-diagram.png" align="center" width="500" ></a>
 
 Comet aims to support:
+
 - a native Parquet implementation, including both reader and writer
 - full implementation of Spark operators, including
   Filter/Project/Aggregation/Join/Exchange etc.
@@ -71,18 +72,22 @@ Linux, Apple OSX (Intel and M1)
 Make sure the requirements above are met and software installed on your machine
 
 ### Clone repo
+
 ```commandline
 git clone https://github.com/apache/datafusion-comet.git
 ```
 
 ### Specify the Spark version and build the Comet
+
 Spark 3.4 used for the example.
+
 ```
 cd datafusion-comet
 make release PROFILES="-Pspark-3.4"
 ```
 
 ### Run Spark with Comet enabled
+
 Make sure `SPARK_HOME` points to the same Spark version as Comet has built for.
 
 ```
@@ -93,25 +98,28 @@ $SPARK_HOME/bin/spark-shell --jars spark/target/comet-spark-spark3.4_2.12-0.1.0-
 --conf spark.comet.exec.all.enabled=true
 ```
 
-### Verify Comet enabled for Spark SQL query  
+### Verify Comet enabled for Spark SQL query
 
 Create a test Parquet source
+
 ```scala
 scala> (0 until 10).toDF("a").write.mode("overwrite").parquet("/tmp/test")
 ```
 
-Query the data from the test source and check: 
+Query the data from the test source and check:
+
 - INFO message shows the native Comet library has been initialized.
 - The query plan reflects Comet operators being used for this query instead of Spark ones
+
 ```scala
 scala> spark.read.parquet("/tmp/test").createOrReplaceTempView("t1")
 scala> spark.sql("select * from t1 where a > 5").explain
 INFO src/lib.rs: Comet native library initialized
 == Physical Plan ==
         *(1) ColumnarToRow
         +- CometFilter [a#14], (isnotnull(a#14) AND (a#14 > 5))
-          +- CometScan parquet [a#14] Batched: true, DataFilters: [isnotnull(a#14), (a#14 > 5)], 
-             Format: CometParquet, Location: InMemoryFileIndex(1 paths)[file:/tmp/test], PartitionFilters: [], 
+          +- CometScan parquet [a#14] Batched: true, DataFilters: [isnotnull(a#14), (a#14 > 5)],
+             Format: CometParquet, Location: InMemoryFileIndex(1 paths)[file:/tmp/test], PartitionFilters: [],
              PushedFilters: [IsNotNull(a), GreaterThan(a,5)], ReadSchema: struct<a:int>
 ```
 
@@ -136,6 +144,7 @@ Comet doesn't have official release yet so currently the only way to test it is
 Some cluster managers may require additional configuration, see https://spark.apache.org/docs/latest/cluster-overview.html
 
 To enable columnar shuffle which supports all partitioning and basic complex types, one more config is required:
+
 ```
 --conf spark.comet.columnar.shuffle.enabled=true
 ```

docs/.gitignore

@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+build
+temp
+venv/
+.python-version

docs/Makefile

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/README.md

@@ -0,0 +1,68 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Apache DataFusion Comet Documentation
+
+This folder contains the source content for the Apache DataFusion Comet documentation site. This content is published
+to https://datafusion.apache.org/comet when any changes are merged into the main branch.
+
+## Dependencies
+
+It's recommended to install build dependencies and build the documentation
+inside a Python virtualenv.
+
+- Python
+- `pip install -r requirements.txt`
+
+## Build & Preview
+
+Run the provided script to build the HTML pages.
+
+```bash
+./build.sh
+```
+
+The HTML will be generated into a `build` directory.
+
+Preview the site on Linux by running this command.
+
+```bash
+firefox build/html/index.html
+```
+
+## Making Changes
+
+To make changes to the docs, simply make a Pull Request with your
+proposed changes as normal. When the PR is merged the docs will be
+automatically updated.
+
+## Release Process
+
+This documentation is hosted at https://datafusion.apache.org/comet/
+
+When the PR is merged to the `main` branch of the `datafusion-comet`
+repository, a [github workflow](https://github.com/apache/datafusion-comet/blob/main/.github/workflows/docs.yaml) which:
+
+1. Builds the html content
+2. Pushes the html content to the [`asf-site`](https://github.com/apache/datafusion-comet/tree/asf-site) branch in this repository.
+
+The Apache Software Foundation provides https://datafusion.apache.org/,
+which serves content based on the configuration in
+[.asf.yaml](https://github.com/apache/datafusion-comet/blob/main/.asf.yaml),
+which specifies the target as https://datafusion.apache.org/comet/.

docs/build.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+rm -rf build 2> /dev/null
+rm -rf temp 2> /dev/null
+mkdir temp
+cp -rf source/* temp/
+make SOURCEDIR=`pwd`/temp html

docs/make.bat

@@ -0,0 +1,52 @@
+@rem Licensed to the Apache Software Foundation (ASF) under one
+@rem or more contributor license agreements.  See the NOTICE file
+@rem distributed with this work for additional information
+@rem regarding copyright ownership.  The ASF licenses this file
+@rem to you under the Apache License, Version 2.0 (the
+@rem "License"); you may not use this file except in compliance
+@rem with the License.  You may obtain a copy of the License at
+@rem
+@rem   http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing,
+@rem software distributed under the License is distributed on an
+@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@rem KIND, either express or implied.  See the License for the
+@rem specific language governing permissions and limitations
+@rem under the License.
+
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/requirements.txt

@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+sphinx
+pydata-sphinx-theme==0.8.0
+myst-parser
+maturin
+jinja2