Skip to content

Commit

Permalink
feat: refactor the coordinator to support Groot under the unified sch…
Browse files Browse the repository at this point in the history
…ema architecture (#4087)

- Refactor the coordinator to support Groot under the unified schema architecture
- Add `flex-api-test` CI test workflow
- Integrate coordinator into Groot's charts
  • Loading branch information
lidongze0629 authored Aug 7, 2024
1 parent f2215cf commit 7f09bb4
Show file tree
Hide file tree
Showing 77 changed files with 6,266 additions and 1,138 deletions.
3 changes: 3 additions & 0 deletions .github/workflows/gss-dummy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ on:
- '!python/graphscope/client/**'
- '!charts/graphscope-store/**'
- '!.github/workflows/gss.yml'
- '!flex/openapi/openapi_coordinator.yaml'
- '!coordinator/gscoordinator/flex/**'
- '!python/graphscope/gsctl/**'
- 'interactive_engine/**.md'
- 'charts/graphscope-store/**.md'

Expand Down
33 changes: 31 additions & 2 deletions .github/workflows/gss.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ on:
- 'interactive_engine/**'
- 'python/graphscope/client/**'
- 'charts/graphscope-store/**'
- 'flex/openapi/openapi_coordinator.yaml'
- 'coordinator/gscoordinator/flex/**'
- 'python/graphscope/gsctl/**'
- '.github/workflows/gss.yml'
- '!interactive_engine/**.md'
- '!charts/graphscope-store/**.md'
Expand All @@ -35,6 +38,7 @@ concurrency:

env:
GSS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-store
COORDINATOR_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/coordinator

jobs:
gremlin-test:
Expand Down Expand Up @@ -131,6 +135,16 @@ jobs:
mv artifacts/data-load-tool/target/data-load-tool-0.0.1-SNAPSHOT.jar artifacts/
docker build -t ${{ env.GSS_IMAGE }}:${SHORT_SHA} \
-f .github/workflows/docker/graphscope-store.Dockerfile .
# build coordinator image
cd ${GITHUB_WORKSPACE}/k8s
make coordinator CI=false VERSION=${SHORT_SHA}
docker tag graphscope/coordinator:${SHORT_SHA} ${{ env.COORDINATOR_IMAGE }}:${SHORT_SHA}
- name: Build gsctl Wheel Package
run: |
cd ${GITHUB_WORKSPACE}/python
python3 setup_flex.py bdist_wheel
python3 setup_gsctl.py bdist_wheel
- name: Setup SSH
run: |
Expand All @@ -149,6 +163,7 @@ jobs:
--cpus='12' --memory='32000mb' --disk-size='40000mb'
minikube image load ${{ env.GSS_IMAGE }}:${SHORT_SHA}
minikube image load ${{ env.COORDINATOR_IMAGE }}:${SHORT_SHA}
- uses: dashanji/kubernetes-log-export-action@v5
env:
Expand All @@ -165,9 +180,24 @@ jobs:
# helm deployment and testing
cd ${GITHUB_WORKSPACE}/charts
helm install ci --set image.tag=${SHORT_SHA},distributed.enabled=true,store.replicaCount=2 ./graphscope-store
helm install ci --set image.tag=${SHORT_SHA},distributed.enabled=true,store.replicaCount=2,portal.enabled=true,portal.coordinatorImage.tag=${SHORT_SHA} ./graphscope-store
helm test ci --timeout 5m0s
- name: Flex api test
run: |
# get coordinator service endpoint
export NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}")
export HTTP_PORT=$(kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services ci-graphscope-store-portal)
export COORDINATOR_SERVICE_ENDPOINT="http://${NODE_IP}:${HTTP_PORT}"
# install gsctl
python3 -m pip install ${GITHUB_WORKSPACE}/python/dist/*.whl
# test
export RUN_ON_MINIKUBE=ON
python3 -m pip install --no-cache-dir pytest pytest-xdist
python3 -m pytest -d --tx popen//python=python3 \
-s -v \
$(dirname $(python3 -c "import graphscope.gsctl as gsctl; print(gsctl.__file__)"))/tests/test_graphscope_insight.py
- name: Test the helm deployment
run: |
# 1. get gss service endpoint
Expand Down Expand Up @@ -236,4 +266,3 @@ jobs:
with:
name: k8s-test-logs
path: ${{ github.workspace }}/helm-installation-logs

33 changes: 32 additions & 1 deletion charts/graphscope-store/templates/NOTES.txt
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,35 @@
export GRPC_PORT=$(kubectl -n {{ .Release.Namespace }} get services {{ $serviceName }} -o jsonpath="{.spec.ports[0].nodePort}")
export GREMLIN_PORT=$(kubectl -n {{ .Release.Namespace }} get services {{ $serviceName }} -o jsonpath="{.spec.ports[1].nodePort}")
echo "GRPC endpoint is: ${NODE_IP}:${GRPC_PORT}"
echo "GREMLIN endpoint is: ${NODE_IP}:${GREMLIN_PORT}"
echo "GREMLIN endpoint is: ${NODE_IP}:${GREMLIN_PORT}"

{{ if .Values.portal.enabled }}
{{- $portalServiceName := include "graphscope-store.portal.fullname" . }}

2. Get flex coordinator endpoint by running these commands:

{{- if contains "NodePort" .Values.portal.service.type }}

export COORDINATOR_NODE_IP=$(kubectl -n {{ .Release.Namespace }} get pod {{ $portalServiceName }}-0 -o jsonpath="{.status.hostIP}")

{{- else if contains "LoadBalancer" .Values.portal.service.type }}

It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status by executing:

kubectl --namespace {{ .Release.Namespace }} get services -o wide -w {{ $portalServiceName }}

After the EXTERNAL_IP is available, executing these commands:

export COORDINATOR_NODE_IP=$(kubectl -n {{ .Release.Namespace }} get svc {{ $portalServiceName }} -o jsonpath="{.status.loadBalancer.ingress[0].ip}")

{{- end }}

export HTTP_PORT=$(kubectl -n {{ .Release.Namespace }} get services {{ $portalServiceName }} -o jsonpath="{.spec.ports[0].nodePort}")
echo "Coordinator service endpoint is: ${COORDINATOR_NODE_IP}:${HTTP_PORT}"

It may take a few minutes for service to be available, you can connect to the coordinator service multiple times by:

gsctl connect --coordinator-endpoint http://${COORDINATOR_NODE_IP}:${HTTP_PORT}

{{- end }}
11 changes: 11 additions & 0 deletions charts/graphscope-store/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ If release name contains chart name it will be used as a full name.
{{- printf "%s-%s" (include "graphscope-store.fullname" .) "store" | trunc 63 | trimSuffix "-" -}}
{{- end -}}

{{- define "graphscope-store.portal.fullname" -}}
{{- printf "%s-%s" (include "graphscope-store.fullname" .) "portal" | trunc 63 | trimSuffix "-" -}}
{{- end -}}

{{/*
Create chart name and version as used by the chart label.
*/}}
Expand All @@ -57,6 +61,13 @@ Return the proper graphscope-store test image name
{{ include "common.images.image" (dict "imageRoot" .Values.test.image "global" .Values.global ) }}
{{- end -}}

{{/*
Return the proper portal image name
*/}}
{{- define "graphscope-store.portal.coordinator.image" -}}
{{ include "common.images.image" (dict "imageRoot" .Values.portal.coordinatorImage "global" .Values.global ) }}
{{- end -}}

{{/*
Return the proper Docker Image Registry Secret Names
*/}}
Expand Down
23 changes: 23 additions & 0 deletions charts/graphscope-store/templates/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -166,3 +166,26 @@ data:
export RUST_BACKTRACE=1
${GRAPHSCOPE_HOME}/groot/bin/store_ctl.sh start ${ROLE} # || sleep infinity
portal_setup.sh: |-
#!/bin/bash
while true
do
export HOST="${INSTANCE_NAME}-graphscope-store-frontend-0.${INSTANCE_NAME}-graphscope-store-frontend-headless"
export GRPC_PORT=${GROOT_GRPC_PORT}
export GREMLIN_PORT=${GROOT_GREMLIN_PORT}
echo "${HOST} ${GRPC_PORT} ${GREMLIN_PORT}"
python3 -c 'import base64;import os;from gremlin_python.driver.client import Client;ip=os.getenv("HOST");gremlin_port=os.getenv("GREMLIN_PORT");graph_url=f"ws://{ip}:{gremlin_port}/gremlin";username=os.getenv("GROOT_USERNAME");password=base64.b64decode(os.getenv("GROOT_PASSWORD")).decode("utf-8");client = Client(graph_url, "g", username=username, password=password); ret = client.submit("g.V().limit(1)").all().result(); client.close();' && break
sleep 3
done
export CONFIG_FILE=/tmp/coordinator_config.yaml
echo "coordinator:" > ${CONFIG_FILE}
echo " http_port: ${SERVICE_PORT}" >> ${CONFIG_FILE}
echo "" >> ${CONFIG_FILE}
echo "launcher_type: hosts" >> ${CONFIG_FILE}
echo "" >> ${CONFIG_FILE}
echo "session:" >> ${CONFIG_FILE}
echo " instance_id: demo" >> ${CONFIG_FILE}
python3 -m gscoordinator --config-file ${CONFIG_FILE}
173 changes: 173 additions & 0 deletions charts/graphscope-store/templates/portal/statefulset.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
{{- if .Values.portal.enabled }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: {{ include "graphscope-store.portal.fullname" . }}
namespace: {{ .Release.Namespace }}
labels: {{- include "common.labels.standard" . | nindent 4 }}
app.kubernetes.io/component: portal
{{- if .Values.commonLabels }}
{{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
{{- end }}
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
spec:
podManagementPolicy: {{ .Values.podManagementPolicy }}
replicas: {{ .Values.portal.replicaCount }}
selector:
matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }}
app.kubernetes.io/component: portal
serviceName: {{ printf "%s-portal-headless" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }}
updateStrategy: {{- include "common.tplvalues.render" (dict "value" .Values.updateStrategy "context" $ ) | nindent 4 }}
template:
metadata:
labels: {{- include "common.labels.standard" . | nindent 8 }}
app.kubernetes.io/component: portal
{{- if .Values.portal.podLabels }}
{{- include "common.tplvalues.render" (dict "value" .Values.portal.podLabels "context" $) | nindent 8 }}
{{- end }}
annotations:
{{- if (include "graphscope-store.createConfigmap" .) }}
checksum/configuration: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
{{- end }}
{{- if .Values.portal.podAnnotations }}
{{- include "common.tplvalues.render" (dict "value" .Values.portal.podAnnotations "context" $) | nindent 8 }}
{{- end }}
spec:
{{- include "graphscope-store.imagePullSecrets" . | nindent 6 }}
{{- if .Values.portal.hostAliases }}
hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.portal.hostAliases "context" $) | nindent 8 }}
{{- end }}
hostNetwork: {{ .Values.portal.hostNetwork }}
hostIPC: {{ .Values.portal.hostIPC }}
{{- if .Values.portal.schedulerName }}
schedulerName: {{ .Values.portal.schedulerName | quote }}
{{- end }}
{{- if .Values.nodeSelector }}
nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }}
{{- end }}
{{- if .Values.dnsPolicy }}
dnsPolicy: {{ .Values.dnsPolicy | quote }}
{{- end }}
{{- if .Values.dnsConfig }}
dnsConfig: {{- include "common.tplvalues.render" (dict "value" .Values.dnsConfig "context" $) | nindent 8 }}
{{- end }}
{{- if .Values.tolerations }}
tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }}
{{- end }}
{{- if .Values.topologySpreadConstraints }}
topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" $) | nindent 8 }}
{{- end }}
{{- if .Values.terminationGracePeriodSeconds }}
terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
{{- end }}
{{- if .Values.priorityClassName }}
priorityClassName: {{ .Values.priorityClassName }}
{{- end }}
{{- if .Values.podSecurityContext.enabled }}
securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "graphscope-store.serviceAccountName" . }}
{{- if .Values.initContainers }}
initContainers:
{{- if .Values.initContainers }}
{{- include "common.tplvalues.render" ( dict "value" .Values.initContainers "context" $ ) | nindent 8 }}
{{- end }}
{{- end }}
containers:
- name: coordinator
image: {{ include "graphscope-store.portal.coordinator.image" . }}
imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
{{- if .Values.containerSecurityContext.enabled }}
securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }}
{{- end }}
command:
- /etc/coordinator/setup.sh
env:
- name: CLUSTER_TYPE
value: "KUBERNETES"
- name: VERSION
value: {{ .Chart.Version | quote }}
- name: GROOT_USERNAME
value: {{ .Values.auth.username | quote }}
- name: GROOT_PASSWORD
value: {{ .Values.auth.password | b64enc | quote }}
- name: GROOT_GRPC_PORT
value: {{ .Values.frontend.service.servicePort | quote }}
- name: GROOT_GREMLIN_PORT
value: {{ .Values.frontend.service.gremlinPort | quote }}
- name: INSTANCE_NAME
value: {{ .Release.Name | quote }}
- name: NAMESPACE
value: {{ .Release.Namespace | quote }}
- name: SERVICE_PORT
value: {{ .Values.portal.service.servicePort | quote }}
- name: WORKSPACE
value: {{ .Values.portal.runtimePath | quote }}
- name: STUDIO_WRAPPER_ENDPOINT
value: {{ .Values.portal.studioWrapperEndpoint | quote }}
- name: SOLUTION
value: "GRAPHSCOPE_INSIGHT"
{{- range $key, $value := .Values.env }}
- name: {{ $key }}
value: {{ $value | quote }}
{{- end }}
ports:
- name: http
containerPort: {{ .Values.portal.service.servicePort }}
{{- if .Values.portal.resources }}
resources: {{- toYaml .Values.portal.resources | nindent 12 }}
{{- end }}
{{- if .Values.livenessProbe.enabled }}
livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.livenessProbe "enabled") "context" $) | nindent 12 }}
tcpSocket:
port: http
{{- end }}
{{- if .Values.readinessProbe.enabled }}
readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.readinessProbe "enabled") "context" $) | nindent 12 }}
tcpSocket:
port: http
{{- end }}
volumeMounts:
- name: runtime
mountPath: {{ .Values.portal.runtimePath | quote }}
- name: config
mountPath: /etc/coordinator/setup.sh
subPath: portal_setup.sh
volumes:
- name: config
configMap:
name: {{ include "graphscope-store.configmapName" . }}
defaultMode: 0755
{{- if and .Values.portal.persistence.enabled .Values.portal.persistence.existingClaim }}
- name: runtime
persistentVolumeClaim:
claimName: {{ printf "%s" (tpl .Values.portal.persistence.existingClaim .) }}
{{- else if not .Values.portal.persistence.enabled }}
- name: runtime
emptyDir: {}
{{- else if and .Values.portal.persistence.enabled (not .Values.portal.persistence.existingClaim) }}
volumeClaimTemplates:
- metadata:
name: runtime
{{- if .Values.persistence.annotations }}
annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }}
{{- end }}
{{- if .Values.persistence.labels }}
labels: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }}
{{- end }}
spec:
accessModes:
{{- range .Values.persistence.accessModes }}
- {{ . | quote }}
{{- end }}
resources:
requests:
storage: {{ .Values.portal.persistence.size | quote }}
{{ include "graphscope-store.storageClass" . | nindent 8 }}
{{- if .Values.portal.persistence.selector }}
selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.portal.persistence.selector "context" $) | nindent 10 }}
{{- end -}}
{{- end }}
{{- end }}
26 changes: 26 additions & 0 deletions charts/graphscope-store/templates/portal/svc-headless.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
{{- if .Values.portal.enabled -}}
apiVersion: v1
kind: Service
metadata:
name: {{ printf "%s-portal-headless" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }}
namespace: {{ .Release.Namespace }}
labels: {{- include "common.labels.standard" . | nindent 4 }}
app.kubernetes.io/component: portal
{{- if .Values.commonLabels }}
{{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
{{- end }}
annotations:
{{- if .Values.commonAnnotations }}
{{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
spec:
type: ClusterIP
clusterIP: None
publishNotReadyAddresses: true
ports:
- name: http
port: {{ .Values.portal.service.servicePort }}
targetPort: http
selector: {{- include "common.labels.matchLabels" . | nindent 4 }}
app.kubernetes.io/component: portal
{{- end -}}
Loading

0 comments on commit 7f09bb4

Please sign in to comment.