Split binary into vineyard-bdist package (#1236)

Signed-off-by: Tao He <[email protected]>

.github/workflows/build-test-graph.yml

@@ -17,12 +17,12 @@ name: Vineyard Graph CI
 on:
   push:
     paths:
-      modules/graph
+      - modules/graph
   pull_request:
     branches:
       - "*"
     paths:
-      modules/graph
+      - modules/graph
 
 concurrency:
   group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
@@ -227,8 +227,8 @@ jobs:
 
       - name: Get test data
         run: |
-          git clone https://github.com/7br/gstest.git --depth=1
-          git clone https://github.com/7br/gar-test.git --depth=1
+          git clone https://github.com/GraphScope/gstest.git --depth=1
+          git clone https://github.com/GraphScope/gar-test.git --depth=1
 
       - name: Setup tmate session
         if: false

.github/workflows/build-test.yml

@@ -302,7 +302,7 @@ jobs:
           rm -rf build/lib* build/bdist.*
           python3 setup.py bdist_wheel
 
-          for pkg in airflow dask io ml; do
+          for pkg in airflow bdist dask io ml; do
             rm -rf build/lib* build/bdist.*
             python3 setup_${pkg}.py bdist_wheel
           done
@@ -313,7 +313,7 @@ jobs:
 
       - name: Get test data
         run: |
-          git clone https://github.com/7br/gstest.git --depth=1
+          git clone https://github.com/GraphScope/gstest.git --depth=1
 
       - name: Setup tmate session
         if: false

.github/workflows/build-vineyardd-and-wheels-linux.yaml

@@ -128,7 +128,7 @@ jobs:
       - name: Build vineyardctl
         run: |
           go build -a -o vineyardctl k8s/cmd/main.go
-          strip ./vineyardctl
+          strip ./vineyardctl || true
 
       - name: Package vineyardctl artifact on Linux
         run: |
@@ -185,8 +185,8 @@ jobs:
           tar zxvf vineyardd.${{ runner.os }}-generic.${{ github.sha }}.tar.gz
           tar zxvf vineyardctl.${{ runner.os }}-generic.${{ github.sha }}.tar.gz
 
-          mv vineyardd python/vineyard/vineyardd
-          mv vineyardctl python/vineyard/vineyardctl
+          mv vineyardd python/vineyard/bdist/vineyardd
+          mv vineyardctl python/vineyard/bdist/vineyardctl
 
       - name: Login docker registry
         run: |
@@ -208,6 +208,12 @@ jobs:
           sudo docker run --rm -v `pwd`:/target ghcr.io/v6d-io/v6d/vineyard-wheel:${{ matrix.python }}.${{ github.sha }} sh -c "cp -r /work/fixed_wheels/ /target/"
           ls -la ./fixed_wheels/
 
+          if [[ "${{ matrix.python }}" == "cp311-cp311" ]];
+          then
+            python3 setup_bdist.py bdist_wheel
+            cp dist/*.whl fixed_wheels/ || true
+          fi
+
       - name: Extract tag name
         id: tag
         if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}

.github/workflows/build-vineyardd-and-wheels-macos.yaml

@@ -285,8 +285,8 @@ jobs:
           tar zxvf vineyardd.${{ runner.os }}-generic.${{ github.sha }}.tar.gz
           tar zxvf vineyardctl.${{ runner.os }}-generic.${{ github.sha }}.tar.gz
 
-          mv vineyardd python/vineyard/vineyardd
-          mv vineyardctl python/vineyard/vineyardctl
+          mv vineyardd python/vineyard/bdist/vineyardd
+          mv vineyardctl python/vineyard/bdist/vineyardctl
 
       - name: Install python for MacOS
         if: runner.os == 'macOS'
@@ -420,7 +420,15 @@ jobs:
                    -DPYTHON_EXECUTABLE=`which python`
           make vineyard_client_python -j`nproc`
           cd ..
+
+          rm -rf build/lib* build/bdist.*
           python setup.py bdist_wheel
+
+          if [[ "${{ matrix.python }}" == "3.11" ]]; then
+            rm -rf build/lib* build/bdist.*
+            python setup_bdist.py bdist_wheel
+          fi
+
           export LD_LIBRARY_PATH=`pwd`/build/lib:$LD_LIBRARY_PATH
           for wheel in `ls dist/*`; do delocate-wheel -w fixed_wheels -v $wheel; done
           ls -la ./fixed_wheels

python/vineyard/bdist/.gitignore

@@ -0,0 +1,2 @@
+vineyardctl
+vineyardd

python/vineyard/bdist/__init__.py

@@ -0,0 +1,17 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2020-2023 Alibaba Group Holding Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

python/vineyard/deploy/utils.py

@@ -176,6 +176,11 @@ def find_vineyardd_path():
     if vineyardd_path is None:
         vineyardd_path = _check_executable(os.path.join(current_dir, '..', 'vineyardd'))
 
+    if vineyardd_path is None:
+        vineyardd_path = _check_executable(
+            os.path.join(current_dir, '..', 'bdist', 'vineyardd')
+        )
+
     __vineyardd_path = vineyardd_path
     return vineyardd_path
 
@@ -209,6 +214,11 @@ def find_vineyardctl_path():
             os.path.join(current_dir, '..', 'vineyardctl')
         )
 
+    if vineyardctl_path is None:
+        vineyardctl_path = _check_executable(
+            os.path.join(current_dir, '..', 'bdist', 'vineyardctl')
+        )
+
     __vineyardctl_path = vineyardctl_path
     return vineyardctl_path
 

setup.py

@@ -20,7 +20,6 @@
 import subprocess
 import sys
 import textwrap
-import warnings
 from distutils.cmd import Command
 from distutils.util import strtobool
 from typing import List
@@ -186,14 +185,6 @@ def package_data():
         '**/*.yaml.tpl',
         '**/**/*.sh',
     ]
-    if os.path.exists(os.path.join(repo_root, 'python', 'vineyard', 'vineyardd')):
-        artifacts.append('vineyardd')
-    else:
-        warnings.warn('The artifact for `vineyardd` not found')
-    if os.path.exists(os.path.join(repo_root, 'python', 'vineyard', 'vineyardctl')):
-        artifacts.append('vineyardctl')
-    else:
-        warnings.warn('The artifact for `vineyardctl` not found')
     return artifacts
 
 
@@ -249,12 +240,12 @@ def package_data():
         'console_scripts': ['vineyard-codegen=vineyard.core.codegen:main'],
     },
     setup_requires=load_requirements_txt("-setup"),
-    install_requires=load_requirements_txt(),
+    install_requires=load_requirements_txt() + ['vineyard-bdist'],
     extras_require={
         'dev': load_requirements_txt("-dev"),
         'kubernetes': load_requirements_txt("-kubernetes"),
     },
-    platform=["POSIX", "MacOS"],
+    platforms=["POSIX", "MacOS"],
     license="Apache License 2.0",
     classifiers=[
         "Development Status :: 5 - Production/Stable",

setup_airflow.py

@@ -83,7 +83,7 @@ def find_airflow_packages(root):
         'apache-airflow',
         'vineyard',
     ],
-    platform=['POSIX', 'MacOS'],
+    platforms=['POSIX', 'MacOS'],
     license="Apache License 2.0",
     classifiers=[
         "Development Status :: 5 - Production/Stable",

setup_bdist.py

@@ -0,0 +1,122 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright 2020-2023 Alibaba Group Holding Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import textwrap
+import warnings
+
+from setuptools import setup
+from setuptools.command.install import install
+from wheel.bdist_wheel import bdist_wheel
+
+repo_root = os.path.dirname(os.path.abspath(__file__))
+
+
+class bdist_wheel_plat(bdist_wheel):
+    def finalize_options(self):
+        bdist_wheel.finalize_options(self)
+        self.root_is_pure = False
+
+    def get_tag(self):
+        self.root_is_pure = False
+        _, _, plat = bdist_wheel.get_tag(self)
+        return ('py3', 'none', plat)
+
+
+class install_plat(install):
+    def finalize_options(self):
+        self.install_lib = self.install_platlib
+        install.finalize_options(self)
+
+
+def package_data():
+    artifacts = []
+    if os.path.exists(os.path.join(repo_root, 'python', 'vineyard', 'bdist', 'vineyardd')):
+        artifacts.append('vineyardd')
+    else:
+        warnings.warn('The artifact for `vineyardd` not found')
+    if os.path.exists(os.path.join(repo_root, 'python', 'vineyard', 'bdist', 'vineyardctl')):
+        artifacts.append('vineyardctl')
+    else:
+        warnings.warn('The artifact for `vineyardctl` not found')
+    return artifacts
+
+
+with open(
+    os.path.join(repo_root, 'README.rst'),
+    encoding='utf-8',
+    mode='r',
+) as fp:
+    long_description = fp.read()
+
+    # Github doesn't respect "align: center", and pypi disables `.. raw`.
+    replacement = textwrap.dedent(
+        """
+        .. image:: https://v6d.io/_static/vineyard_logo.png
+           :target: https://v6d.io
+           :align: center
+           :alt: vineyard
+           :width: 397px
+
+        vineyard: an in-memory immutable data manager
+        ---------------------------------------------
+        """
+    )
+    long_description = replacement + '\n'.join(long_description.split('\n')[8:])
+
+setup(
+    name='vineyard-bdist',
+    author='The vineyard team',
+    author_email='[email protected]',
+    description='An in-memory immutable data manager',
+    long_description=long_description,
+    long_description_content_type='text/x-rst',
+    url='https://v6d.io',
+    package_dir={'vineyard.bdist': 'python/vineyard/bdist'},
+    packages=['vineyard.bdist'],
+    package_data={
+        'vineyard.bdist': package_data(),
+    },
+    cmdclass={'bdist_wheel': bdist_wheel_plat, 'install': install_plat},
+    zip_safe=False,
+    platforms=["POSIX", "MacOS"],
+    license="Apache License 2.0",
+    classifiers=[
+        "Development Status :: 5 - Production/Stable",
+        "Intended Audience :: Developers",
+        "Intended Audience :: Science/Research",
+        "License :: OSI Approved :: Apache Software License",
+        "Topic :: Software Development :: Libraries",
+        "Topic :: System :: Distributed Computing",
+        "Operating System :: MacOS :: MacOS X",
+        "Operating System :: POSIX",
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+    ],
+    project_urls={
+        "Documentation": "https://v6d.io",
+        "Source": "https://github.com/v6d-io/v6d",
+        "Tracker": "https://github.com/v6d-io/v6d/issues",
+    },
+)

setup_dask.py

@@ -86,7 +86,7 @@ def find_dask_packages(root):
     cmdclass={'bdist_wheel': bdist_wheel_plat, "install": install_plat},
     zip_safe=False,
     install_requires=['vineyard', 'dask[complete]', 'click<8.1'],
-    platform=['POSIX', 'MacOS'],
+    platforms=['POSIX', 'MacOS'],
     license="Apache License 2.0",
     classifiers=[
         "Development Status :: 5 - Production/Stable",

setup_io.py

@@ -123,7 +123,7 @@ def resolve_vineyard_io_package_dir():
             'pytest',
         ],
     },
-    platform=['POSIX', 'MacOS'],
+    platforms=['POSIX', 'MacOS'],
     license="Apache License 2.0",
     classifiers=[
         "Development Status :: 5 - Production/Stable",

setup_ml.py

@@ -86,7 +86,7 @@ def find_ml_packages(root):
     cmdclass={'bdist_wheel': bdist_wheel_plat, 'install': install_plat},
     zip_safe=False,
     install_requires=['vineyard'],
-    platform=['POSIX', 'MacOS'],
+    platforms=['POSIX', 'MacOS'],
     license="Apache License 2.0",
     classifiers=[
         "Development Status :: 5 - Production/Stable",

setup_ray.py

@@ -86,7 +86,7 @@ def find_ray_packages(root):
     cmdclass={'bdist_wheel': bdist_wheel_plat, "install": install_plat},
     zip_safe=False,
     install_requires=['vineyard', 'ray[default]'],
-    platform=['POSIX', 'MacOS'],
+    platforms=['POSIX', 'MacOS'],
     license="Apache License 2.0",
     classifiers=[
         "Development Status :: 5 - Production/Stable",