GraphAr: Support load and write subgraph to graphar with selected vertices/edges and properties #91

Workflow file for this run

.github/workflows/build-test-graph.yml at 8bac0ab

	# Copyright 2020-2023 Alibaba Group Holding Limited.
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	name: Vineyard Graph CI

	on:
	push:
	branches:
	- main
	paths:
	- 'modules/graph/**'
	pull_request:
	branches:
	- main
	paths:
	- 'modules/graph/**'
	workflow_dispatch:

	concurrency:
	group: ${{ github.repository }}-${{ github.event.number \|\| github.head_ref \|\| github.sha }}-${{ github.workflow }}
	cancel-in-progress: true

	env:
	CMAKE_C_COMPILER_LAUNCHER: ccache
	CMAKE_CXX_COMPILER_LAUNCHER: ccache

	jobs:
	ci:
	runs-on: ${{ matrix.os }}
	if: ${{ github.repository == 'v6d-io/v6d' }}
	strategy:
	matrix:
	os: [ubuntu-20.04]
	malloc: [dlmalloc]
	metadata: [etcd]
	env:
	RUNNER_ARGS: "--meta=${{ matrix.metadata }}"
	steps:
	- uses: actions/checkout@v3
	with:
	submodules: recursive

	- name: Generate Summary for Submodules
	run: \|
	git submodule > git-modules.txt
	cat git-modules.txt

	- name: Cache for ccache
	uses: actions/cache@v3
	with:
	path: ~/.ccache
	key: ${{ runner.os }}-${{ matrix.malloc }}-${{ matrix.metadata }}-ccache-graph-${{ hashFiles('**/git-modules.txt') }}
	restore-keys: \|
	${{ runner.os }}-${{ matrix.malloc }}-${{ matrix.metadata }}-ccache-graph-

	- name: Install Dependencies for Linux
	if: runner.os == 'Linux'
	run: \|
	export PATH=/usr/lib/ccache:$PATH

	sudo apt update -y
	sudo apt install -y ca-certificates \
	ccache \
	cmake \
	doxygen \
	fuse3 \
	libboost-all-dev \
	libcurl4-openssl-dev \
	libfuse3-dev \
	libgflags-dev \
	libgoogle-glog-dev \
	libgmock-dev \
	libgrpc-dev \
	libgrpc++-dev \
	libkrb5-dev \
	libmpich-dev \
	libprotobuf-dev \
	librdkafka-dev \
	libgsasl7-dev \
	librdkafka-dev \
	libssl-dev \
	libunwind-dev \
	libuv1-dev \
	libxml2-dev \
	libz-dev \
	lsb-release \
	redis-server \
	pandoc \
	protobuf-compiler-grpc \
	python3-pip \
	uuid-dev \
	wget

	# install apache-arrow
	wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short \| tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
	sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
	sudo apt update
	sudo apt install -y libarrow-dev=14.0.1-1 \
	libarrow-dataset-dev=14.0.1-1 \
	libarrow-acero-dev=14.0.1-1 \
	libarrow-flight-dev=14.0.1-1 \
	libgandiva-dev=14.0.1-1 \
	libparquet-dev=14.0.1-1

	# install clang-format
	sudo curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-1d7ec53d/clang-format-11_linux-amd64 --output /usr/bin/clang-format
	sudo chmod +x /usr/bin/clang-format

	# avoid possible permission errors
	ccache -o cache_dir=~/.ccache
	sudo mkdir -p ~/.ccache
	sudo chmod -R a+wrx ~/.ccache

	# install python packages for codegen, and io adaptors
	sudo pip3 install -U "Pygments>=2.4.1"
	sudo pip3 install -r requirements-setup.txt -r requirements.txt -r requirements-dev.txt

	- name: Setup tmate session
	if: false
	uses: mxschmitt/action-tmate@v3

	- name: CMake
	run: \|
	export PATH=/usr/lib/ccache:$PATH

	mkdir build
	cd build
	cmake .. -DCMAKE_BUILD_TYPE=Debug \
	-DBUILD_SHARED_LIBS=ON \
	-DUSE_ASAN=ON \
	-DWITH_ALLOCATOR=${{ matrix.malloc }} \
	-DBUILD_VINEYARD_COVERAGE=ON \
	-DBUILD_VINEYARD_PYTHON_BINDINGS=OFF \
	-DBUILD_VINEYARD_BASIC=ON \
	-DBUILD_VINEYARD_IO=ON \
	-DBUILD_VINEYARD_IO_KAFKA=ON \
	-DBUILD_VINEYARD_MALLOC=OFF \
	-DBUILD_VINEYARD_HOSSEINMOEIN_DATAFRAME=OFF \
	-DBUILD_VINEYARD_FUSE=OFF \
	-DBUILD_VINEYARD_FUSE_PARQUET=OFF \
	-DBUILD_VINEYARD_GRAPH=ON \
	-DBUILD_VINEYARD_GRAPH_WITH_GAR=ON \
	-DBUILD_VINEYARD_TESTS_ALL=ON

	if [ "${{ matrix.metadata }}" == "redis" ]; then
	cmake .. -DBUILD_VINEYARD_SERVER_REDIS=ON
	fi

	- name: Cpp format and lint
	run: \|
	cd build

	function prepend() { while read line; do echo "${1}${line}"; done; }

	make vineyard_clformat
	GIT_DIFF=$(git diff --ignore-submodules)
	if [[ -n $GIT_DIFF ]]; then
	echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	echo "\| clang-format failures found!"
	echo "\|"
	echo "$GIT_DIFF" \| prepend "\| "
	echo "\|"
	echo "\| Run: "
	echo "\|"
	echo "\| make vineyard_clformat"
	echo "\|"
	echo "\| to fix this error."
	echo "\|"
	echo "\| Ensure you are working with clang-format-11, which can be obtained from"
	echo "\|"
	echo "\| https://github.com/muttleyxd/clang-tools-static-binaries/releases "
	echo "\|"
	echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	exit -1
	fi

	function ec() { [[ "$1" == "-h" ]] && { shift && eval $* > /dev/null 2>&1; ec=$?; echo $ec; } \|\| eval $*; ec=$?; }

	ec make vineyard_cpplint
	if [[ "$ec" != "0" ]]; then
	echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	echo "\| cpplint failures found! Run: "
	echo "\|"
	echo "\| make vineyard_cpplint"
	echo "\|"
	echo "\| to fix this error."
	echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	exit -1
	fi

	- name: Build
	run: \|
	export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64

	pushd build
	make vineyardd vineyard_graph_tests -j`nproc`
	make ccache-stats
	popd

	- name: Get test data
	run: \|
	git clone https://github.com/GraphScope/gstest.git --depth=1
	git clone https://github.com/GraphScope/gar-test.git --depth=1

	- name: Setup tmate session
	if: false
	uses: mxschmitt/action-tmate@v3

	- name: Run graph tests
	run: \|
	export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/local/lib/x86_64-linux-gnu

	export VINEYARD_DATA_DIR=`pwd`/gstest
	export TMPDIR="${TMPDIR:-$(dirname $(mktemp))}"

	rm -rf default.etcd
	rm -rf /dev/shm/etcd*
	python3 test/runner.py $RUNNER_ARGS --with-graph

	- name: Upload Coverage
	if: runner.os == 'Linux'
	uses: codecov/codecov-action@v3
	with:
	verbose: true

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

GraphAr: Support load and write subgraph to graphar with selected vertices/edges and properties #91

Workflow file

GraphAr: Support load and write subgraph to graphar with selected vertices/edges and properties #91

Jobs

Run details

Workflow file for this run