chore: Create initial release process scripts for official ASF source release

core/Cargo.toml

@@ -16,17 +16,21 @@
 # under the License.
 
 [package]
-name = "comet"
+name = "datafusion-comet"
 version = "0.1.0"
+homepage = "https://datafusion.apache.org/comet"
+repository = "https://github.com/apache/datafusion-comet"
+authors = ["Apache DataFusion <[email protected]>"]
+description = "Apache DataFusion Comet: Apache Spark native query execution plugin"
+readme = "README.md"
+license = "Apache-2.0"
 edition = "2021"
 include = [
     "benches/*.rs",
     "src/**/*.rs",
     "Cargo.toml",
 ]
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
 parquet-format = "4.0.0" # This must be kept in sync with that from parquet crate
 arrow = { git = "https://github.com/viirya/arrow-rs.git", rev = "3f1ae0c", features = ["prettyprint", "ffi", "chrono-tz"] }

dev/release/README.md

@@ -0,0 +1,47 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Comet Release Process
+
+This documentation is for creating an official source release of Apache DataFusion Comet. It does not currently
+cover creating binary releases.
+
+The release process is based on the parent Apache DataFusion project, so please refer to the
+[DataFusion Release Process](https://github.com/apache/datafusion/blob/main/dev/release/README.md) for detailed
+instructions if you are not familiar with the release process here.
+
+Here is a brief overview of the steps involved in creating a release:
+
+- Create and merge a PR to update the version & update the changelog
+- Tag the release with a release candidate tag e.g. 0.1.0-rc1
+- Run the create-tarball script to create the source tarball and upload it to the dev site
+- Start an email voting thread
+- Once the vote passes, run the release-tarball script to move the tarball to the release site
+
+## Verifying Release Candidates
+
+The vote email will link to this section of this document, so this is where we will need to provide instructions for
+verifying a release candidate.
+
+The `dev/release/verify-release-candidate.sh` is a script in this repository that can assist in the verification
+process. It checks the hashes and runs the tests.
+
+```shell
+./dev/release/verify-release-candidate.sh 0.1.0 1
+```

dev/release/create-tarball.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Adapted from https://github.com/apache/arrow-rs/tree/master/dev/release/create-tarball.sh
+
+# This script creates a signed tarball in
+# dev/dist/apache-datafusion-comet-<version>-<sha>.tar.gz and uploads it to
+# the "dev" area of the dist.apache.datafusion repository and prepares an
+# email for sending to the [email protected] list for a formal
+# vote.
+#
+# See release/README.md for full release instructions
+#
+# Requirements:
+#
+# 1. gpg setup for signing and have uploaded your public
+# signature to https://pgp.mit.edu/
+#
+# 2. Logged into the apache svn server with the appropriate
+# credentials
+#
+# 3. Install the requests python package
+#
+#
+# Based in part on 02-source.sh from apache/arrow
+#
+
+set -e
+
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)"
+
+if [ "$#" -ne 2 ]; then
+    echo "Usage: $0 <version> <rc>"
+    echo "ex. $0 4.1.0 2"
+    exit
+fi
+
+if [[ -z "${GH_TOKEN}" ]]; then
+    echo "Please set personal github token through GH_TOKEN environment variable"
+    exit
+fi
+
+version=$1
+rc=$2
+tag="${version}-rc${rc}"
+
+echo "Attempting to create ${tarball} from tag ${tag}"
+release_hash=$(cd "${SOURCE_TOP_DIR}" && git rev-list --max-count=1 ${tag})
+
+release=apache-datafusion-comet-${version}
+distdir=${SOURCE_TOP_DIR}/dev/dist/${release}-rc${rc}
+tarname=${release}.tar.gz
+tarball=${distdir}/${tarname}
+url="https://dist.apache.org/repos/dist/dev/datafusion/${release}-rc${rc}"
+
+if [ -z "$release_hash" ]; then
+    echo "Cannot continue: unknown git tag: ${tag}"
+fi
+
+echo "Draft email for [email protected] mailing list"
+echo ""
+echo "---------------------------------------------------------"
+cat <<MAIL
+To: [email protected]
+Subject: [VOTE] Release Apache DataFusion Comet ${version} RC${rc}
+Hi,
+
+I would like to propose a release of Apache DataFusion Comet version ${version}.
+
+This release candidate is based on commit: ${release_hash} [1]
+The proposed release tarball and signatures are hosted at [2].
+The changelog is located at [3].
+
+Please download, verify checksums and signatures, run the unit tests, and vote
+on the release. The vote will be open for at least 72 hours.
+
+Only votes from PMC members are binding, but all members of the community are
+encouraged to test the release and vote with "(non-binding)".
+
+The standard verification procedure is documented at https://github.com/apache/datafusion-comet/blob/main/dev/release/README.md#verifying-release-candidates.
+
+[ ] +1 Release this as Apache DataFusion Comet ${version}
+[ ] +0
+[ ] -1 Do not release this as Apache DataFusion Comet ${version} because...
+
+Here is my vote:
+
++1
+
+[1]: https://github.com/apache/datafusion-comet/tree/${release_hash}
+[2]: ${url}
+[3]: https://github.com/apache/datafusion-comet/blob/${release_hash}/CHANGELOG.md
+MAIL
+echo "---------------------------------------------------------"
+
+
+# create <tarball> containing the files in git at $release_hash
+# the files in the tarball are prefixed with {version} (e.g. 4.0.1)
+mkdir -p ${distdir}
+(cd "${SOURCE_TOP_DIR}" && git archive ${release_hash} --prefix ${release}/ | gzip > ${tarball})
+
+echo "Running rat license checker on ${tarball}"
+${SOURCE_DIR}/run-rat.sh ${tarball}
+
+echo "Signing tarball and creating checksums"
+gpg --armor --output ${tarball}.asc --detach-sig ${tarball}
+# create signing with relative path of tarball
+# so that they can be verified with a command such as
+#  shasum --check apache-datafusion-comet-0.1.0-rc1.tar.gz.sha512
+(cd ${distdir} && shasum -a 256 ${tarname}) > ${tarball}.sha256
+(cd ${distdir} && shasum -a 512 ${tarname}) > ${tarball}.sha512
+
+
+echo "Uploading to datafusion dist/dev to ${url}"
+svn co --depth=empty https://dist.apache.org/repos/dist/dev/datafusion ${SOURCE_TOP_DIR}/dev/dist
+svn add ${distdir}
+svn ci -m "Apache DataFusion Comet ${version} ${rc}" ${distdir}

dev/release/release-tarball.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Adapted from https://github.com/apache/arrow-rs/tree/master/dev/release/release-tarball.sh
+
+# This script copies a tarball from the "dev" area of the
+# dist.apache.datafusion repository to the "release" area
+#
+# This script should only be run after the release has been approved
+# by the Apache DataFusion PMC committee.
+#
+# See release/README.md for full release instructions
+#
+# Based in part on post-01-upload.sh from apache/arrow
+
+
+set -e
+set -u
+
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <version> <rc-num>"
+  echo "ex. $0 4.1.0 2"
+  exit
+fi
+
+version=$1
+rc=$2
+
+tmp_dir=tmp-apache-datafusion-comet-dist
+
+echo "Recreate temporary directory: ${tmp_dir}"
+rm -rf ${tmp_dir}
+mkdir -p ${tmp_dir}
+
+echo "Clone dev dist repository"
+svn \
+  co \
+  https://dist.apache.org/repos/dist/dev/datafusion/apache-datafusion-comet-${version}-rc${rc} \
+  ${tmp_dir}/dev
+
+echo "Clone release dist repository"
+svn co https://dist.apache.org/repos/dist/release/datafusion ${tmp_dir}/release
+
+echo "Copy ${version}-rc${rc} to release working copy"
+release_version=datafusion-comet-${version}
+mkdir -p ${tmp_dir}/release/${release_version}
+cp -r ${tmp_dir}/dev/* ${tmp_dir}/release/${release_version}/
+svn add ${tmp_dir}/release/${release_version}
+
+echo "Commit release"
+svn ci -m "Apache DataFusion Comet ${version}" ${tmp_dir}/release
+
+echo "Clean up"
+rm -rf ${tmp_dir}
+
+echo "Success! The release is available here:"
+echo "  https://dist.apache.org/repos/dist/release/datafusion/${release_version}"