Skip to content

Commit

Permalink
Merge branch 'main' into update-networkx-version
Browse files Browse the repository at this point in the history
  • Loading branch information
haoxins authored Apr 2, 2024
2 parents 5ef1d19 + dbfc0e1 commit 544bef8
Show file tree
Hide file tree
Showing 850 changed files with 44,401 additions and 14,846 deletions.
1 change: 1 addition & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ default.etcd/

# outer build directory
build/
flex/build/

interactive_engine/rust-common/src/proto/*
!interactive_engine/rust-common/src/proto/mod.rs
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/build-graphscope-dev-images.yml
Original file line number Diff line number Diff line change
Expand Up @@ -367,7 +367,7 @@ jobs:
# since the 'vineyard-dev' image isn't actually pushed in 'pull_request'
if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }}

needs: [manifest-push-vineyard-dev-image]
needs: [build-vineyard-dev-image-x86-64]
steps:
- uses: actions/checkout@v3
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/build-graphscope-wheels-macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ jobs:
run: |
. ~/.graphscope_env
python3 -m pip install libclang
git clone --single-branch -b v0.20.3 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
git clone --single-branch -b v0.21.3 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
cd /tmp/v6d
git submodule update --init
cmake . -DCMAKE_INSTALL_PREFIX=/usr/local \
Expand Down
14 changes: 7 additions & 7 deletions .github/workflows/flex.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:

strategy:
matrix:
os: [ubuntu-20.04, ubuntu-22.04]
os: [ubuntu-22.04]
platform: [x86_64]

steps:
Expand All @@ -54,7 +54,7 @@ jobs:
test-cmake-options:
runs-on: ubuntu-20.04
container:
image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.9
image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.10
strategy:
matrix:
BUILD_HQPS: [ON, OFF]
Expand All @@ -75,7 +75,7 @@ jobs:
test-AOCC-compilation:
runs-on: ubuntu-20.04
container:
image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.9
image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.10
steps:
- uses: actions/checkout@v3

Expand Down Expand Up @@ -104,7 +104,7 @@ jobs:
runs-on: ubuntu-20.04
if: ${{ github.repository == 'alibaba/GraphScope' }}
container:
image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.9
image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.10
steps:
- uses: actions/checkout@v3

Expand Down Expand Up @@ -134,10 +134,10 @@ jobs:
mkdir build && cd build
cmake .. && sudo make -j$(nproc)
export FLEX_DATA_DIR=../../../../interactive/examples/modern_graph/
${GITHUB_WORKSPACE}/flex/build/bin/bulk_loader -g ../../../../interactive/examples/modern_graph/modern_graph.yaml -l ../../../../interactive/examples/modern_graph/bulk_load.yaml -d ./data/
${GITHUB_WORKSPACE}/flex/build/bin/bulk_loader -g ../../../../interactive/examples/modern_graph/graph.yaml -l ../../../../interactive/examples/modern_graph/bulk_load.yaml -d ./data/
rm -r ./data/wal
rm -r ./data/runtime/*
./run_grin_test 'flex://schema_file=../../../../interactive/examples/modern_graph/modern_graph.yaml&data_dir=./data/'
./run_grin_test 'flex://schema_file=../../../../interactive/examples/modern_graph/graph.yaml&data_dir=./data/'
- name: Prepare test dataset
env:
Expand Down Expand Up @@ -173,7 +173,7 @@ jobs:
run: |
rm -rf /tmp/csr-data-dir/
cd ${GITHUB_WORKSPACE}/flex/build/
SCHEMA_FILE=../interactive/examples/modern_graph/modern_graph.yaml
SCHEMA_FILE=../interactive/examples/modern_graph/graph.yaml
BULK_LOAD_FILE=../interactive/examples/modern_graph/bulk_load.yaml
GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/gae.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
runs-on: ubuntu-20.04
if: ${{ github.repository == 'alibaba/GraphScope' }}
container:
image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.20.3
image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.21.3
steps:
- uses: actions/checkout@v3

Expand Down
51 changes: 40 additions & 11 deletions .github/workflows/hqps-db-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
runs-on: ubuntu-20.04
if: ${{ github.repository == 'alibaba/GraphScope' }}
container:
image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.9
image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.10
steps:
- uses: actions/checkout@v3

Expand Down Expand Up @@ -101,6 +101,8 @@ jobs:
mkdir -p ${INTERACTIVE_WORKSPACE}/data/graph_algo
cp ${GITHUB_WORKSPACE}/flex/tests/interactive/graph_algo_test.yaml ${INTERACTIVE_WORKSPACE}/data/graph_algo/graph.yaml
cp ${GITHUB_WORKSPACE}/flex/interactive/examples/graph_algo/import.yaml ${INTERACTIVE_WORKSPACE}/data/graph_algo/import.yaml
mkdir -p ${INTERACTIVE_WORKSPACE}/data/modern_graph
cp ${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml ${INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml
# load graph
cd ${GITHUB_WORKSPACE}/flex/build
Expand All @@ -110,14 +112,26 @@ jobs:
GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/movies/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/movies/indices/
export FLEX_DATA_DIR=../interactive/examples/graph_algo
GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/graph_algo/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/graph_algo/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/graph_algo/indices/
export FLEX_DATA_DIR=../interactive/examples/modern_graph
GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml -l ../interactive/examples/modern_graph/bulk_load.yaml -d ${INTERACTIVE_WORKSPACE}/data/modern_graph/indices/
- name: Test HQPS admin http service
env:
GS_TEST_DIR: ${{ github.workspace }}/gstest
FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph
run:
TMP_INTERACTIVE_WORKSPACE: /tmp/temp_workspace
LD_LIBRARY_PATH: /usr/local/lib
run: |
cd ${GITHUB_WORKSPACE}/flex/build/
SCHEMA_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml
BULK_LOAD_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/bulk_load.yaml
mkdir -p ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/
cp ${SCHEMA_FILE} ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml
GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/indices/
cd ${GITHUB_WORKSPACE}/flex/tests/hqps
bash hqps_admin_test.sh /tmp/temp_workspace ./engine_config_test.yaml ${GS_TEST_DIR}
sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ./engine_config_test.yaml
bash hqps_admin_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./engine_config_test.yaml ${GS_TEST_DIR}
sed -i 's/default_graph: modern_graph/default_graph: ldbc/g' ./engine_config_test.yaml
- name: Sample Query test
env:
Expand Down Expand Up @@ -146,7 +160,7 @@ jobs:
cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
echo $cmd
eval ${cmd}
eval ${cmd} || exit 1
done
for i in 1 2 3 4 5 6 7 8 9 10 11 12;
Expand All @@ -155,7 +169,7 @@ jobs:
cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
echo $cmd
eval ${cmd}
eval ${cmd} || exit 1
done
# test movie graph, 8,9,10 are not supported now
Expand All @@ -167,57 +181,72 @@ jobs:
cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml"
echo $cmd
eval ${cmd}
eval ${cmd} || exit 1
done
- name: Test cypher procedure generation
run: |
cd ${GITHUB_WORKSPACE}/flex/bin
./load_plan_and_gen.sh -e=hqps -i=../interactive/examples/modern_graph/get_person_name.cypher -w=/tmp/codegen \
--ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml \
--graph_schema_path=../interactive/examples/modern_graph/modern_graph.yaml
--graph_schema_path=../interactive/examples/modern_graph/graph.yaml
./load_plan_and_gen.sh -e=hqps -i=../interactive/examples/modern_graph/count_vertex_num.cypher -w=/tmp/codegen \
--ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml \
--graph_schema_path=../interactive/examples/modern_graph/modern_graph.yaml
--graph_schema_path=../interactive/examples/modern_graph/graph.yaml
- name: Run End-to-End cypher adhoc ldbc query test
env:
GS_TEST_DIR: ${{ github.workspace }}/gstest
HOME : /home/graphscope/
INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
LD_LIBRARY_PATH: /usr/local/lib
run: |
cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
export ENGINE_TYPE=hiactor
# change the default_graph config in ./engine_config_test.yaml to ldbc
sed -i 's/default_graph: movies/default_graph: ldbc/g' ./engine_config_test.yaml
bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} ldbc \
bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} ldbc \
${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml
- name: Run End-to-End cypher adhoc movie query test
env:
GS_TEST_DIR: ${{ github.workspace }}/gstest
HOME : /home/graphscope/
INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
LD_LIBRARY_PATH: /usr/local/lib
run: |
cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
export ENGINE_TYPE=hiactor
# change the default_graph config in ./engine_config_test.yaml to movies
sed -i 's/default_graph: ldbc/default_graph: movies/g' ./engine_config_test.yaml
bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} movies \
bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} movies \
${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml
- name: Run End-to-End cypher adhoc graph_algo query test
env:
GS_TEST_DIR: ${{ github.workspace }}/gstest
HOME : /home/graphscope/
INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
LD_LIBRARY_PATH: /usr/local/lib
run: |
cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
export ENGINE_TYPE=hiactor
# change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to graph_algo
sed -i 's/default_graph: movies/default_graph: graph_algo/g' ./engine_config_test.yaml
bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} graph_algo \
bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} graph_algo \
${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml
- name: Run Gremlin test on modern graph
env:
GS_TEST_DIR: ${{ github.workspace }}/gstest
INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
LD_LIBRARY_PATH: /usr/local/lib
run: |
cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
export ENGINE_TYPE=hiactor
sed -i 's/default_graph: graph_algo/default_graph: modern_graph/g' ./engine_config_test.yaml
bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} modern_graph \
${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml gremlin
2 changes: 1 addition & 1 deletion .github/workflows/local-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,7 @@ jobs:
# install tensorflow
python3 -m pip install --no-cache-dir pytest pytest-xdist "tensorflow" "pandas" --user
# install pytorch
python3 -m pip install --no-cache-dir "torch" "networkx<=3.0" --index-url https://download.pytorch.org/whl/cpu
python3 -m pip install --no-cache-dir "torch<=2.2.1" "networkx<=3.0" --index-url https://download.pytorch.org/whl/cpu
# install java
sudo apt update -y && sudo apt install openjdk-11-jdk -y
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/networkx-forward-algo-nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ jobs:
run:
shell: bash --noprofile --norc -eo pipefail {0}
container:
image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.20.3
image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.21.3
options:
--shm-size 4096m

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ jobs:
sudo mkdir /opt/graphscope
sudo chown -R $(id -u):$(id -g) /opt/graphscope
python3 -m pip install click
python3 gsctl.py install-deps dev --v6d-version v0.20.3
python3 gsctl.py install-deps dev --v6d-version v0.21.3
- name: Setup tmate session
if: false
Expand Down
89 changes: 89 additions & 0 deletions .github/workflows/stale.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
name: 'Check stale issues and pull requests'

on:
schedule:
- cron: '0 0 12 * *'
workflow_dispatch:

env:
DEFAULT_ASSIGNEE: yecol sighingnow
DAYS_BEFORE_ISSUE_CLOSE: -1
DAYS_BEFORE_PR_CLOSE: -1
DAYS_BEFORE_ISSUE_STALE: 7
DAYS_BEFORE_PR_STALE: 14

jobs:
stale:
permissions:
issues: write
pull-requests: write
runs-on: ubuntu-latest
steps:
- name: Install dependencies
run: |
# install jq
sudo apt-get update
sudo apt-get install -y jq
# install gh cli
type -p curl >/dev/null || (sudo apt update && sudo apt install curl -y)
curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
&& sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
&& sudo apt update \
&& sudo apt install gh -y
- uses: actions/stale@v9
id: stale
with:
days-before-issue-close: ${{ env.DAYS_BEFORE_ISSUE_CLOSE }}
days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}
days-before-issue-stale: ${{ env.DAYS_BEFORE_ISSUE_STALE }}
days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }}
stale-issue-label: stale
stale-pr-label: stale
exempt-pr-labels: work-in-progress,requires-further-info,requires-further-discussion
exempt-issue-labels: work-in-progress,requires-further-info,requires-further-discussion,wontfix,newcomers
labels-to-remove-when-unstale: stale,requires-further-info,requires-further-discussion
exempt-draft-pr: true

- name: Notify assignees
env:
REPO: ${{ github.repository }}
STALED: ${{ steps.stale.outputs.staled-issues-prs }}
GH_TOKEN: ${{ github.token }}
run: |
for N in $(echo "$STALED" | jq -r ".[].number");
do
assignees=$(echo $(gh issue view $N -R $REPO --json assignees) | jq -r ".assignees[].login")
echo "Processing stale issue/pr $N, assignees: $assignees"
message=""
if [ -z "$assignees" ]; then
message="/cc"
for assignee in $DEFAULT_ASSIGNEE; do
message="$message @$assignee"
done
message="$message, this issus/pr has had no activity for a long time, please help to review the status and assign people to work on it."
else
message="/cc"
for assignee in $assignees; do
message="$message @$assignee"
done
message="$message, this issus/pr has had no activity for for a long time, could you folks help to review the status ? \n"
message="$message To suppress further notifications,\n"
message="$message - for issues, \n"
message="$message - if it is waiting for further response from the reporter/author, please help to add the label \`requires-further-info\`, \n"
message="$message - if you have already started working on it, please add the label \`work-in-progress\` to the issue, \n"
message="$message - if this issue requires further designing discussion and not in current plan, or won't be fixed, please add the label \`requires-further-discussion\` or \`wontfix\` to the issue, \n"
message="$message - for pull requests, \n"
message="$message - if you are still working on it and it is not ready for reviewing, please convert this pull request as draft PR, \n"
message="$message - if you have decided to hold this development on, please add the \`requires-further-discussion\` label to the pull request. \n"
message="$message Thanks!"
fi
echo -e "$message" > message-$N
message_content=$(cat message-$N)
echo "Commenting on stale issue/pr $N, assignees: $assignees, message: $message_content"
gh issue comment $N -R $REPO -b "$message_content"
done
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ interactive_engine/executor/ir/Cargo.lock

# java sdk
analytical_engine/java/**/dependency-reduced-pom.xml
analytical_engine/java/**/.ci-friendly-pom.xml

# GraphLearn
coordinator/graphlearn*
Expand All @@ -98,3 +99,5 @@ flex/interactive/logs/*
flex/interactive/examples/sf0.1-raw/
flex/interactive/.running
flex/interactive/.env

**/.cache/
6 changes: 3 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ clean:

client: learning
cd $(CLIENT_DIR) && \
python3 -m pip install ${PIP_ARGS} "torch" "networkx<=3.0" --index-url https://download.pytorch.org/whl/cpu --user && \
python3 -m pip install ${PIP_ARGS} "torch<=2.2.1" "networkx<=3.0" --index-url https://download.pytorch.org/whl/cpu --user && \
python3 -m pip install ${PIP_ARGS} -r requirements.txt -r requirements-dev.txt --user && \
export PATH=$(PATH):$(HOME)/.local/bin && \
python3 setup.py build_ext --inplace --user && \
Expand All @@ -90,7 +90,7 @@ client: learning

coordinator: client
cd $(COORDINATOR_DIR) && \
python3 -m pip install ${PIP_ARGS} "torch" "networkx<=3.0" --index-url https://download.pytorch.org/whl/cpu --user && \
python3 -m pip install ${PIP_ARGS} "torch<=2.2.1" "networkx<=3.0" --index-url https://download.pytorch.org/whl/cpu --user && \
python3 -m pip install ${PIP_ARGS} -r requirements.txt -r requirements-dev.txt --user && \
python3 setup.py build_builtin && \
python3 -m pip install --user --editable $(COORDINATOR_DIR) && \
Expand Down Expand Up @@ -174,7 +174,7 @@ $(LEARNING_DIR)/graphlearn/built/lib/libgraphlearn_shared.$(SUFFIX):

prepare-client:
cd $(CLIENT_DIR) && \
pip3 install ${PIP_ARGS} "torch" "networkx<=3.0" --index-url https://download.pytorch.org/whl/cpu --user && \
pip3 install ${PIP_ARGS} "torch<=2.2.1" "networkx<=3.0" --index-url https://download.pytorch.org/whl/cpu --user && \
pip3 install ${PIP_ARGS} -r requirements.txt --user && \
pip3 install ${PIP_ARGS} -r requirements-dev.txt --user && \
python3 setup.py build_proto
Expand Down
Loading

0 comments on commit 544bef8

Please sign in to comment.