Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(learning): Automate graphlearn-torch docker-build #3753

Merged
merged 10 commits into from
Apr 29, 2024
Merged

fix(learning): Automate graphlearn-torch docker-build #3753

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
4aa1602
automate glt docker build
husimplicity Apr 25, 2024
a3804c2
enable ci
husimplicity Apr 25, 2024
bbbe2bd
minor
husimplicity Apr 25, 2024
853f6b1
minor
husimplicity Apr 25, 2024
b5d858d
minor
husimplicity Apr 26, 2024
fc120f5
Merge branch 'main' into glt-image
husimplicity Apr 26, 2024
30887ba
Merge branch 'main' into glt-image
husimplicity Apr 26, 2024
2527d64
Merge branch 'main' into glt-image
husimplicity Apr 29, 2024
6c79519
minor
husimplicity Apr 29, 2024
0b9f142
Merge branch 'main' into glt-image
husimplicity Apr 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions .github/workflows/build-graphscope-images-linux.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ on:
- 'k8s/dockerfiles/coordinator.Dockerfile'
- 'k8s/dockerfiles/interactive.Dockerfile'
- 'k8s/dockerfiles/learning.Dockerfile'
- 'k8s/dockerfiles/graphlearn-torch.Dockerfile'
- '.github/workflows/build-graphscope-images-linux.yml'

concurrency:
Expand Down Expand Up @@ -61,6 +62,8 @@ jobs:
df -h
make learning CI=false VERSION=${SHORT_SHA}
df -h
make graphlearn-torch CI=false VERSION=${SHORT_SHA}
df -h

- name: Release Nightly Image
if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
Expand All @@ -84,6 +87,7 @@ jobs:
sudo docker tag graphscope/interactive-frontend:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag}
sudo docker tag graphscope/interactive-executor:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag}
sudo docker tag graphscope/learning:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/learning:${tag}
sudo docker tag graphscope/graphlearn-torch:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/graphlearn-torch:${tag}

sudo docker push ${{ env.REGISTRY }}/graphscope/coordinator:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/analytical:${tag}
Expand All @@ -93,6 +97,7 @@ jobs:
sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/learning:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/graphlearn-torch:${tag}

# dataset image
# Note! dataset image are built manually just use the latest one.
Expand Down Expand Up @@ -124,6 +129,7 @@ jobs:
sudo docker tag graphscope/interactive-frontend:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag}
sudo docker tag graphscope/interactive-executor:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag}
sudo docker tag graphscope/learning:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/learning:${tag}
sudo docker tag graphscope/graphlearn-torch:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/graphlearn-torch:${tag}

sudo docker push ${{ env.REGISTRY }}/graphscope/coordinator:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/analytical:${tag}
Expand All @@ -132,6 +138,7 @@ jobs:
sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/learning:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/graphlearn-torch:${tag}

# dataset image
# Note! dataset image are built manually just use the latest one.
Expand All @@ -148,6 +155,7 @@ jobs:
sudo docker tag graphscope/interactive-frontend:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag}
sudo docker tag graphscope/interactive-executor:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag}
sudo docker tag graphscope/learning:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/learning:${tag}
sudo docker tag graphscope/graphlearn-torch:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/graphlearn-torch:${tag}

sudo docker push ${{ env.REGISTRY }}/graphscope/coordinator:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/analytical:${tag}
Expand All @@ -156,6 +164,7 @@ jobs:
sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/learning:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/graphlearn-torch:${tag}


build-image-aarch64:
Expand All @@ -182,6 +191,7 @@ jobs:
make interactive-frontend CI=false VERSION=${SHORT_SHA}
make interactive-executor CI=false VERSION=${SHORT_SHA}
make learning CI=false VERSION=${SHORT_SHA}
# make graphlearn-torch CI=false VERSION=${SHORT_SHA}

- name: Release Nightly Image
if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
Expand All @@ -205,6 +215,7 @@ jobs:
sudo docker tag graphscope/interactive-frontend:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag}
sudo docker tag graphscope/interactive-executor:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag}
sudo docker tag graphscope/learning:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/learning:${tag}
# sudo docker tag graphscope/graphlearn-torch:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/graphlearn-torch:${tag}

sudo docker push ${{ env.REGISTRY }}/graphscope/coordinator:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/analytical:${tag}
Expand All @@ -214,6 +225,7 @@ jobs:
sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/learning:${tag}
# sudo docker push ${{ env.REGISTRY }}/graphscope/graphlearn-torch:${tag}

- name: Extract Tag Name
if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
Expand All @@ -239,6 +251,7 @@ jobs:
sudo docker tag graphscope/interactive-frontend:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag}
sudo docker tag graphscope/interactive-executor:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag}
sudo docker tag graphscope/learning:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/learning:${tag}
# sudo docker tag graphscope/graphlearn-torch:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/graphlearn-torch:${tag}

sudo docker push ${{ env.REGISTRY }}/graphscope/coordinator:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/analytical:${tag}
Expand All @@ -247,6 +260,7 @@ jobs:
sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/learning:${tag}
# sudo docker push ${{ env.REGISTRY }}/graphscope/graphlearn-torch:${tag}

# Release the latest tag
tag=latest-${arch}
Expand All @@ -257,6 +271,7 @@ jobs:
sudo docker tag graphscope/interactive-frontend:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag}
sudo docker tag graphscope/interactive-executor:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag}
sudo docker tag graphscope/learning:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/learning:${tag}
# sudo docker tag graphscope/graphlearn-torch:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/graphlearn-torch:${tag}

sudo docker push ${{ env.REGISTRY }}/graphscope/coordinator:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/analytical:${tag}
Expand All @@ -265,6 +280,7 @@ jobs:
sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope/learning:${tag}
# sudo docker push ${{ env.REGISTRY }}/graphscope/graphlearn-torch:${tag}

- name: Clean Nightly Image
if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
Expand All @@ -282,13 +298,15 @@ jobs:
sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} || true
sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} || true
sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/learning:${tag} || true
# sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/graphlearn-torch:${tag} || true
tag=${SHORT_SHA}
sudo docker rmi -f graphscope/coordinator:${tag} || true
sudo docker rmi -f graphscope/analytical:${tag} || true
sudo docker rmi -f graphscope/analytical-java:${tag} || true
sudo docker rmi -f graphscope/interactive-frontend:${tag} || true
sudo docker rmi -f graphscope/interactive-executor:${tag} || true
sudo docker rmi -f graphscope/learning:${tag} || true
# sudo docker rmi -f graphscope/graphlearn-torch:${tag} || true

- name: Clean Release Image
if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
Expand All @@ -303,20 +321,23 @@ jobs:
sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} || true
sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} || true
sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/learning:${tag} || true
# sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/graphlearn-torch:${tag} || true
tag=latest-${arch}
sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/coordinator:${tag} || true
sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/analytical:${tag} || true
sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/analytical-java:${tag} || true
sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} || true
sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} || true
sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/learning:${tag} || true
# sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/graphlearn-torch:${tag} || true
tag=${SHORT_SHA}
sudo docker rmi -f graphscope/coordinator:${tag} || true
sudo docker rmi -f graphscope/analytical:${tag} || true
sudo docker rmi -f graphscope/analytical-java:${tag} || true
sudo docker rmi -f graphscope/interactive-frontend:${tag} || true
sudo docker rmi -f graphscope/interactive-executor:${tag} || true
sudo docker rmi -f graphscope/learning:${tag} || true
# sudo docker rmi -f graphscope/graphlearn-torch:${tag} || true


manifest-push-image:
Expand Down Expand Up @@ -365,6 +386,10 @@ jobs:
${{ env.REGISTRY }}/graphscope/learning:${version}a${time}-x86_64 \
${{ env.REGISTRY }}/graphscope/learning:${version}a${time}-aarch64

sudo docker manifest create \
${{ env.REGISTRY }}/graphscope/graphlearn-torch:${version}a${time} \
${{ env.REGISTRY }}/graphscope/graphlearn-torch:${version}a${time}-x86_64

sudo docker manifest create \
${{ env.REGISTRY }}/graphscope/dataset:${version}a${time} \
${{ env.REGISTRY }}/graphscope/dataset:${version}a${time}-x86_64
Expand All @@ -375,6 +400,7 @@ jobs:
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/interactive-frontend:${version}a${time}
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/interactive-executor:${version}a${time}
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/learning:${version}a${time}
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/graphlearn-torch:${version}a${time}
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/dataset:${version}a${time}

- name: Extract Tag Name
Expand Down Expand Up @@ -413,6 +439,10 @@ jobs:
${{ env.REGISTRY }}/graphscope/learning:${{ steps.tag.outputs.TAG }}-x86_64 \
${{ env.REGISTRY }}/graphscope/learning:${{ steps.tag.outputs.TAG }}-aarch64

sudo docker manifest create \
${{ env.REGISTRY }}/graphscope/graphlearn-torch:${{ steps.tag.outputs.TAG }} \
${{ env.REGISTRY }}/graphscope/graphlearn-torch:${{ steps.tag.outputs.TAG }}-x86_64

sudo docker manifest create \
${{ env.REGISTRY }}/graphscope/dataset:${{ steps.tag.outputs.TAG }} \
${{ env.REGISTRY }}/graphscope/dataset:${{ steps.tag.outputs.TAG }}-x86_64
Expand Down Expand Up @@ -442,17 +472,23 @@ jobs:
${{ env.REGISTRY }}/graphscope/learning:latest-x86_64 \
${{ env.REGISTRY }}/graphscope/learning:latest-aarch64

sudo docker manifest create \
${{ env.REGISTRY }}/graphscope/graphlearn-torch:latest \
${{ env.REGISTRY }}/graphscope/graphlearn-torch:latest-x86_64

# push
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/coordinator:${{ steps.tag.outputs.TAG }}
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/analytical:${{ steps.tag.outputs.TAG }}
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/interactive-frontend:${{ steps.tag.outputs.TAG }}
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/interactive-executor:${{ steps.tag.outputs.TAG }}
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/learning:${{ steps.tag.outputs.TAG }}
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/graphlearn-torch:${{ steps.tag.outputs.TAG }}
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/dataset:${{ steps.tag.outputs.TAG }}

sudo docker manifest push ${{ env.REGISTRY }}/graphscope/coordinator:latest
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/analytical:latest
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/interactive-frontend:latest
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/interactive-executor:latest
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/learning:latest
sudo docker manifest push ${{ env.REGISTRY }}/graphscope/graphlearn-torch:latest

11 changes: 11 additions & 0 deletions k8s/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,17 @@ learning:
-t graphscope/learning:${VERSION} \
-f $(DOCKERFILES_DIR)/learning.Dockerfile .

graphlearn-torch:
cd $(WORKING_DIR)/.. && \
docker build \
--target graphlearn-torch \
--build-arg REGISTRY=$(REGISTRY) \
--build-arg BUILDER_VERSION=$(BUILDER_VERSION) \
--build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \
--build-arg CI=${CI} \
-t graphscope/graphlearn-torch:${VERSION} \
-f $(DOCKERFILES_DIR)/graphlearn-torch.Dockerfile .

graphscope-store:
cd $(WORKING_DIR)/.. && \
docker build \
Expand Down
43 changes: 43 additions & 0 deletions k8s/dockerfiles/graphlearn-torch.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Graphlearn-torch engine

ARG REGISTRY=registry.cn-hongkong.aliyuncs.com
ARG BUILDER_VERSION=v0.21.3
ARG RUNTIME_VERSION=v0.21.3
FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder

COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope

RUN cd /home/graphscope/GraphScope/; \
mkdir /home/graphscope/install; \
make analytical-install INSTALL_PREFIX=/home/graphscope/install; \
make learning-install INSTALL_PREFIX=/home/graphscope/install; \
WITH_GLTORCH=ON make client INSTALL_PREFIX=/home/graphscope/install; \
cd python; \
python3 -m pip install --user -r requirements.txt; \
python3 setup.py bdist_wheel; \
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/graphscope/GraphScope/learning_engine/graph-learn/graphlearn/built/lib; \
auditwheel repair dist/*.whl --exclude libtorch_cpu.so --exclude libc10.so --exclude libtorch_python.so --exclude libtorch.so; \
python3 -m pip install wheelhouse/*.whl; \
cp wheelhouse/*.whl /home/graphscope/install/; \
cd ../coordinator; \
python3 setup.py bdist_wheel; \
cp dist/*.whl /home/graphscope/install/

############### RUNTIME: GLE #######################
FROM $REGISTRY/graphscope/vineyard-runtime:$RUNTIME_VERSION AS graphlearn-torch

RUN sudo apt-get update -y && \
sudo apt-get install -y python3-pip && \
sudo apt-get clean -y && \
sudo rm -rf /var/lib/apt/lists/*

RUN sudo chmod a+wrx /tmp

RUN pip3 install pip==20.3.4
RUN python3 -m pip install torch==2.2.1 --index-url https://download.pytorch.org/whl/cpu
RUN python3 -m pip install --user ogb torch_geometric pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv -f https://data.pyg.org/whl/torch-2.2.0+cpu.html

COPY --from=builder /home/graphscope/install /opt/graphscope/
RUN python3 -m pip install --no-cache-dir /opt/graphscope/*.whl && sudo rm -rf /opt/graphscope/*.whl

ENV PATH=${PATH}:/home/graphscope/.local/bin
Loading