diff --git a/.github/workflows/gss.yml b/.github/workflows/gss.yml index ba207bb1a5e6..d55c74cfb3e4 100644 --- a/.github/workflows/gss.yml +++ b/.github/workflows/gss.yml @@ -165,7 +165,7 @@ jobs: # helm deployment and testing cd ${GITHUB_WORKSPACE}/charts - helm install ci --set image.tag=${SHORT_SHA} ./graphscope-store + helm install ci --set image.tag=${SHORT_SHA},distributed.enabled=true,store.replicaCount=2 ./graphscope-store helm test ci --timeout 5m0s - name: Test the helm deployment @@ -206,7 +206,7 @@ jobs: helm uninstall ci sleep 30 cd ${GITHUB_WORKSPACE}/charts - helm install ci --set image.tag=${SHORT_SHA} ./graphscope-store + helm install ci --set image.tag=${SHORT_SHA},distributed.enabled=true,store.replicaCount=2 ./graphscope-store - name: Helm Test with Helm Deployment and PersistentVolume run: | diff --git a/charts/graphscope-store-one-pod/.gitignore b/charts/graphscope-store-one-pod/.gitignore deleted file mode 100644 index ee3892e8794a..000000000000 --- a/charts/graphscope-store-one-pod/.gitignore +++ /dev/null @@ -1 +0,0 @@ -charts/ diff --git a/charts/graphscope-store-one-pod/.helmignore b/charts/graphscope-store-one-pod/.helmignore deleted file mode 100644 index f0c131944441..000000000000 --- a/charts/graphscope-store-one-pod/.helmignore +++ /dev/null @@ -1,21 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj diff --git a/charts/graphscope-store-one-pod/Chart.yaml b/charts/graphscope-store-one-pod/Chart.yaml deleted file mode 100644 index 9b75f1041bcf..000000000000 --- a/charts/graphscope-store-one-pod/Chart.yaml +++ /dev/null @@ -1,31 +0,0 @@ -annotations: - category: Database - licenses: Apache-2.0 -apiVersion: v2 -appVersion: 3.4.0 -description: GraphScope Store is a disk-based row-oriented multi-versioned persistent graph store - and can be used to as a graph database service. GraphScope Store supports the popular - graph query language -- Apache TinkerPop Gremlin. It's also named groot. -home: https://graphscope.io -icon: https://graphscope.io/assets/images/graphscope-logo.svg -keywords: - - GraphScope - - Groot - - database - - cluster -name: graphscope-store -sources: - - https://github.com/alibaba/GraphScope/tree/main/interactive_engine -maintainers: -- name: GraphScope - url: https://github.com/alibaba/graphscope -version: 0.25.0 - -dependencies: -- condition: kafka.enabled - name: kafka - repository: https://charts.bitnami.com/bitnami - version: "20.0.6" -- name: common - repository: https://charts.bitnami.com/bitnami - version: 2.x.x diff --git a/charts/graphscope-store-one-pod/README.md b/charts/graphscope-store-one-pod/README.md deleted file mode 100644 index 3042018d8a23..000000000000 --- a/charts/graphscope-store-one-pod/README.md +++ /dev/null @@ -1 +0,0 @@ -GraphScope store chart's variant that put all components within one pod. \ No newline at end of file diff --git a/charts/graphscope-store-one-pod/templates/NOTES.txt b/charts/graphscope-store-one-pod/templates/NOTES.txt deleted file mode 100644 index 3deb8f8dc7b1..000000000000 --- a/charts/graphscope-store-one-pod/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: - -{{- if contains "NodePort" .Values.frontend.service.type }} - - export NODE_IP=$(kubectl -n {{ .Release.Namespace }} get pod {{ include "graphscope-store.store.fullname" . }}-0 -o jsonpath="{.status.hostIP}") - -{{- else if contains "LoadBalancer" .Values.frontend.service.type }} - - It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status by executing - - kubectl --namespace {{ .Release.Namespace }} get services -o wide -w {{ include "graphscope-store.frontend.fullname" . }} - - After the EXTERNAL_IP is available, executing these commands: - - export NODE_IP=$(kubectl -n {{ .Release.Namespace }} get svc {{ include "graphscope-store.frontend.fullname" . }} -ojsonpath="{.status.loadBalancer.ingress[0].ip}") - -{{- end }} - export GRPC_PORT=$(kubectl -n {{ .Release.Namespace }} get services {{ include "graphscope-store.frontend.fullname" . }} -o jsonpath="{.spec.ports[0].nodePort}") - export GREMLIN_PORT=$(kubectl -n {{ .Release.Namespace }} get services {{ include "graphscope-store.frontend.fullname" . }} -o jsonpath="{.spec.ports[1].nodePort}") - echo "GRPC endpoint is: ${NODE_IP}:${GRPC_PORT}" - echo "GREMLIN endpoint is: ${NODE_IP}:${GREMLIN_PORT}" diff --git a/charts/graphscope-store-one-pod/templates/_helpers.tpl b/charts/graphscope-store-one-pod/templates/_helpers.tpl deleted file mode 100644 index 6a42dc0b2363..000000000000 --- a/charts/graphscope-store-one-pod/templates/_helpers.tpl +++ /dev/null @@ -1,171 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "graphscope-store.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "graphscope-store.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{- define "graphscope-store.coordinator.fullname" -}} -{{- printf "%s-%s" (include "graphscope-store.fullname" .) "coordinator" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{- define "graphscope-store.frontend.fullname" -}} -{{- printf "%s-%s" (include "graphscope-store.fullname" .) "frontend" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{- define "graphscope-store.store.fullname" -}} -{{- printf "%s-%s" (include "graphscope-store.fullname" .) "store" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "graphscope-store.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - - -{{/* -Return the proper graphscope-store image name -*/}} -{{- define "graphscope-store.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global ) }} -{{- end -}} - -{{- define "graphscope-store.otel.collector.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.otel.collector.image "global" .Values.global ) }} -{{- end -}} - -{{/* -Return the proper graphscope-store test image name -*/}} -{{- define "graphscope-store.test.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.test.image "global" .Values.global ) }} -{{- end -}} - -{{/* -Return the proper Docker Image Registry Secret Names -*/}} -{{- define "graphscope-store.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.image) "global" .Values.global) }} -{{- end -}} - -{{/* -Create the name of the service account to use -*/}} -{{- define "graphscope-store.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "graphscope-store.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} - -{{/* -Return the configmap with the graphscope configuration -*/}} -{{- define "graphscope-store.configmapName" -}} -{{- if .Values.existingConfigmap -}} - {{- printf "%s" (tpl .Values.existingConfigmap $) -}} -{{- else -}} - {{- printf "%s" (include "graphscope-store.fullname" .) -}} -{{- end -}} -{{- end -}} - -{{/* -Return true if a configmap object should be created for graphscope-service -*/}} -{{- define "graphscope-store.createConfigmap" -}} -{{- if not .Values.existingConfigmap }} - {{- true -}} -{{- else -}} -{{- end -}} -{{- end -}} - - -{{/* -Return the proper Storage Class -*/}} -{{- define "graphscope-store.storageClass" -}} -{{/* -Helm 2.11 supports the assignment of a value to a variable defined in a different scope, -but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic. -*/}} -{{- if .Values.global -}} - {{- if .Values.global.storageClass -}} - {{- if (eq "-" .Values.global.storageClass) -}} - {{- printf "storageClassName: \"\"" -}} - {{- else }} - {{- printf "storageClassName: %s" .Values.global.storageClass -}} - {{- end -}} - {{- else -}} - {{- if .Values.persistence.storageClass -}} - {{- if (eq "-" .Values.persistence.storageClass) -}} - {{- printf "storageClassName: \"\"" -}} - {{- else }} - {{- printf "storageClassName: %s" .Values.persistence.storageClass -}} - {{- end -}} - {{- end -}} - {{- end -}} -{{- else -}} - {{- if .Values.persistence.storageClass -}} - {{- if (eq "-" .Values.persistence.storageClass) -}} - {{- printf "storageClassName: \"\"" -}} - {{- else }} - {{- printf "storageClassName: %s" .Values.persistence.storageClass -}} - {{- end -}} - {{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create a default fully qualified kafka name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "graphscope-store.kafka.fullname" -}} -{{- printf "%s-%s" .Release.Name "kafka" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Get full broker list. -*/}} -{{- define "graphscope-store.kafka.brokerlist" -}} - -{{- $replicaCount := int .Values.kafka.replicaCount -}} -{{- $releaseNamespace := .Release.Namespace -}} -{{- $clusterDomain := .Values.clusterDomain -}} -{{- $fullname := include "graphscope-store.kafka.fullname" . -}} -{{- $servicePort := int .Values.kafka.service.port -}} - -{{- $brokerList := list }} -{{- range $e, $i := until $replicaCount }} -{{- $brokerList = append $brokerList (printf "%s-%d.%s-headless.%s.svc.%s:%d" $fullname $i $fullname $releaseNamespace $clusterDomain $servicePort) }} -{{- end }} -{{- join "," $brokerList | printf "%s" -}} -{{- end -}} - -{{/* -Create a default fully qualified zookeeper name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "graphscope-store.zookeeper.fullname" -}} -{{- printf "%s-%s" .Release.Name "zookeeper" | trunc 63 | trimSuffix "-" -}} -{{- end -}} diff --git a/charts/graphscope-store-one-pod/templates/configmap.yaml b/charts/graphscope-store-one-pod/templates/configmap.yaml deleted file mode 100644 index fc6a5c88acc1..000000000000 --- a/charts/graphscope-store-one-pod/templates/configmap.yaml +++ /dev/null @@ -1,115 +0,0 @@ -{{- if (include "graphscope-store.createConfigmap" .) }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "graphscope-store.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: configmap - {{- if .Values.commonLabels }} - {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} - {{- end }} - {{- if .Values.commonAnnotations }} - annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} - {{- end }} -data: - groot.config: |- - ## Common Config - role.name= - node.idx= - rpc.port=0 - rpc.max.bytes.mb={{ .Values.rpcMaxBytesMb }} - store.node.count={{ .Values.store.replicaCount }} - frontend.node.count=1 - coordinator.node.count=1 - partition.count={{ .Values.store.replicaCount | mul 16 }} - discovery.mode=zookeeper - - ## Frontend Config - frontend.server.num=1 - enable.hash.generate.eid={{ .Values.enableHashGenerateEid }} - - ## Coordinator Config - snapshot.increase.interval.ms={{ .Values.snapshotIncreaseIntervalMs }} - offsets.persist.interval.ms={{ .Values.offsetsPersistIntervalMs }} - file.meta.store.path={{ .Values.fileMetaStorePath }} - log.recycle.enable={{ .Values.logRecycleEnable }} - - ## Store Config - store.data.path={{ .Values.storeDataPath }} - store.write.thread.count={{ .Values.storeWriteThreadCount }} - store.queue.buffer.size={{ .Values.storeQueueBufferSize }} - - ## Frontend Config - gremlin.server.port=12312 - ## disable neo4j when launching groot server by default - neo4j.bolt.server.disabled=true - - dns.name.prefix.frontend=localhost - dns.name.prefix.coordinator=localhost - dns.name.prefix.store=localhost - - log4rs.config=LOG4RS_CONFIG - - ## Auth config - auth.username={{ .Values.auth.username }} - auth.password={{ .Values.auth.password }} - - # Pegasus config - pegasus.worker.num={{ .Values.pegasus.worker.num }} - pegasus.timeout={{ .Values.pegasus.timeout }} - pegasus.batch.size=1024 - pegasus.output.capacity=16 - - ## Kafka Config - kafka.servers=KAFKA_SERVERS - kafka.topic={{ .Values.kafkaTopic }} - kafka.producer.custom.configs={{ .Values.kafkaProducerCustomConfigs }} - kafka.test.cluster.enable={{ not .Values.kafka.enabled }} - - ## Zk Config - zk.base.path={{ .Values.zkBasePath }} - zk.connect.string=ZK_CONNECT - - ## Secondary config - secondary.instance.enabled={{ .Values.secondary.enabled }} - store.data.secondary.path={{ .Values.secondary.storeDataPath }} - store.gc.interval.ms={{ .Values.storeGcIntervalMs }} - - tracing.enabled={{ .Values.otel.enabled }} - - ## Extra Config - {{- if .Values.extraConfig }} - {{- $config_list := regexSplit ";" .Values.extraConfig -1 }} - {{- range $config_list }} - {{ . }} - {{- end }} - {{- end }} - setup.sh: |- - #!/bin/bash - - sudo chown -R graphscope:graphscope {{ .Values.storeDataPath }} || true - sudo chown -R graphscope:graphscope /etc/groot || true - - until nslookup "${DNS_NAME_SERVICE_KAFKA}"; do - echo "waiting for kafka..."; - sleep 1; - done; - - sudo sed -e "s/KAFKA_SERVERS/${KAFKA_SERVERS}/g" \ - -e "s/ZK_CONNECT/${ZK_CONNECT}/g" \ - -e "s@LOG4RS_CONFIG@${GRAPHSCOPE_HOME}/groot/conf/log4rs.yml@g" \ - /etc/groot/groot.config.tpl | sudo tee -a /etc/groot/groot.config - - export LOG_NAME=graphscope-store - export GROOT_CONF_FILE=/etc/groot/groot.config - - {{- if .Values.otel.enabled }} - export JAVA_TOOL_OPTIONS="-javaagent:/home/graphscope/opentelemetry-javaagent.jar" - export OTEL_SERVICE_NAME="compiler" - export OTEL_TRACES_SAMPLER={{ .Values.otel.traces.sampler.name }} - export OTEL_TRACES_SAMPLER_ARG={{ .Values.otel.traces.sampler.arg }} - {{- end }} - - ${GRAPHSCOPE_HOME}/groot/bin/store_ctl.sh start -{{- end -}} diff --git a/charts/graphscope-store-one-pod/templates/serviceaccount.yaml b/charts/graphscope-store-one-pod/templates/serviceaccount.yaml deleted file mode 100644 index e2fcf8e39bf8..000000000000 --- a/charts/graphscope-store-one-pod/templates/serviceaccount.yaml +++ /dev/null @@ -1,19 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "graphscope-store.serviceAccountName" . }} - namespace: {{ .Release.Namespace | quote }} - labels: {{- include "common.labels.standard" . | nindent 4 }} - {{- if .Values.commonLabels }} - {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} - {{- end }} - annotations: - {{- if .Values.commonAnnotations }} - {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} - {{- end }} - {{- if .Values.serviceAccount.annotations }} - {{- include "common.tplvalues.render" ( dict "value" .Values.serviceAccount.annotations "context" $ ) | nindent 4 }} - {{- end }} -automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} -{{- end }} diff --git a/charts/graphscope-store-one-pod/values.yaml b/charts/graphscope-store-one-pod/values.yaml deleted file mode 100644 index 92dfa0f4ad00..000000000000 --- a/charts/graphscope-store-one-pod/values.yaml +++ /dev/null @@ -1,374 +0,0 @@ -# Default values for graphscope-store. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -## @section Global parameters -## Global Docker image parameters -## Please, note that this will override the image parameters, including dependencies, configured to use the global value -## Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass -## - -## @param global.imageRegistry Global Docker image registry -## @param global.imagePullSecrets Global Docker registry secret names as an array -## @param global.storageClass Global StorageClass for Persistent Volume(s) -## -global: - imageRegistry: "" - ## E.g. - ## imagePullSecrets: - ## - myRegistryKeySecretName - ## - imagePullSecrets: [] - storageClass: "" - -image: - registry: registry.cn-hongkong.aliyuncs.com - repository: graphscope/graphscope-store - # Overrides the image tag whose default is the chart appVersion. - tag: "0.23.0" - digest: "" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) - ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - -test: - image: - registry: registry.cn-hongkong.aliyuncs.com - repository: graphscope/graphscope-store - tag: python-3.9.9-alpine3.14-gremlin - -## GraphScope Store parameters -## -store: - replicaCount: 1 - - ## GraphScope store container's resource requests and limits - ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ - ## - resources: {} - - ## Enable persistence using Persistent Volume Claims - ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ - ## - persistence: - ## If true, use a Persistent Volume Claim, If false, use emptyDir - ## - enabled: true - ## Name of existing PVC to hold GraphScope store data - ## @param persistence.existingClaim A manually managed Persistent Volume and Claim - ## If defined, PVC must be created manually before volume will be bound - ## The value is evaluated as a template - ## - existingClaim: "" - ## Persistent Volume size - ## - size: 1Gi - selector: {} - - - - annotations: {} - -frontend: - service: - ## Service type - ## - type: NodePort - ## Service port - ## - servicePort: 55556 - - gremlinPort: 12312 - - ## Internal port for communication between components. - ## - port: 55555 - - ## Specify the nodePort value for the LoadBalancer and NodePort service types. - ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport - ## - nodePorts: - service: "" - gremlin: "" - ## Service clusterIP - ## - # clusterIP: None - clusterIP: "" - ## Set the LoadBalancer service type to internal only. - ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer - ## - loadBalancerIP: "" - ## Enable client source IP preservation - ## ref http://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip - ## - externalTrafficPolicy: Cluster - ## Load Balancer sources - ## https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service - ## E.g. - ## loadBalancerSourceRanges: - ## - 10.10.10.0/24 - ## - loadBalancerSourceRanges: [] - ## Provide any additional annotations which may be required - ## - annotations: {} - -## @section Common parameters -## - -## @param kubeVersion Override Kubernetes version -## -kubeVersion: "" -## @param nameOverride String to partially override common.names.fullname -## -nameOverride: "" -## @param fullnameOverride String to fully override common.names.fullname -## -fullnameOverride: "" -## @param clusterDomain Default Kubernetes cluster domain -## -clusterDomain: cluster.local -## @param commonLabels Labels to add to all deployed objects -## -commonLabels: {} -## @param commonAnnotations Annotations to add to all deployed objects -## -commonAnnotations: {} -## @param extraDeploy Array of extra objects to deploy with the release -## - -## Configure extra options for containers' liveness, readiness and startup probes -livenessProbe: - enabled: false - initialDelaySeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - periodSeconds: 10 - successThreshold: 1 - -readinessProbe: - enabled: false - initialDelaySeconds: 5 - failureThreshold: 6 - timeoutSeconds: 5 - periodSeconds: 10 - successThreshold: 1 - -startupProbe: - enabled: false - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 1 - failureThreshold: 15 - successThreshold: 1 - -command: -- /etc/groot/setup.sh - -args: [] - -initContainers: [] - -persistence: - ## @param persistence.storageClass PVC Storage Class - ## If defined, storageClassName: - ## If set to "-", storageClassName: "", which disables dynamic provisioning - ## If undefined (the default) or set to null, no storageClassName spec is - ## set, choosing the default provisioner. - ## - storageClass: "" - accessModes: - - ReadWriteOnce - annotations: {} - ## @param persistence.labels Labels for the PVC - ## - labels: {} - -## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod -podSecurityContext: - enabled: true - fsGroup: 1001 - -## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container -containerSecurityContext: - enabled: true - runAsUser: 1001 - runAsNonRoot: true - # allowPrivilegeEscalation: false - -podAffinityPreset: "" -## @param podAntiAffinityPreset Pod anti-affinity preset. Ignored if `affinity` is set. Allowed values: `soft` or `hard` -## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity -## -podAntiAffinityPreset: soft -## Node affinity preset -## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity -## -nodeAffinityPreset: - ## @param nodeAffinityPreset.type Node affinity preset type. Ignored if `affinity` is set. Allowed values: `soft` or `hard` - ## - type: "" - ## @param nodeAffinityPreset.key Node label key to match Ignored if `affinity` is set. - ## E.g. - ## key: "kubernetes.io/e2e-az-name" - ## - key: "" - ## @param nodeAffinityPreset.values Node label values to match. Ignored if `affinity` is set. - ## E.g. - ## values: - ## - e2e-az1 - ## - e2e-az2 - ## - values: [] -## @param affinity Affinity for pod assignment -## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity -## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set -## -affinity: {} -## @param nodeSelector Node labels for pod assignment -## Ref: https://kubernetes.io/docs/user-guide/node-selection/ -## -nodeSelector: {} -## @param tolerations Tolerations for pod assignment -## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ -## -tolerations: [] -## @param topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template -## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods -## -topologySpreadConstraints: [] -## @param terminationGracePeriodSeconds Seconds the pod needs to gracefully terminate -## ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#hook-handler-execution -## -terminationGracePeriodSeconds: "" -## @param podManagementPolicy StatefulSet controller supports relax its ordering guarantees while preserving its uniqueness and identity guarantees. There are two valid pod management policies: OrderedReady and Parallel -## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#pod-management-policy -## -podManagementPolicy: Parallel -## @param priorityClassName Name of the existing priority class to be used by pods -## Ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ -## -priorityClassName: "" - -## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies -## -updateStrategy: - type: RollingUpdate - rollingUpdate: {} - -## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ -pdb: - create: false - minAvailable: "" - maxUnavailable: 1 - -dnsPolicy: "" -dnsConfig: {} - -## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ -## -serviceAccount: - ## @param serviceAccount.create Enable creation of ServiceAccount - ## - create: false - ## @param serviceAccount.name The name of the service account to use. If not set and `create` is `true`, a name is generated - ## If not set and create is true, a name is generated - ## - name: "" - ## @param serviceAccount.automountServiceAccountToken Allows auto mount of ServiceAccountToken on the serviceAccount created - ## Can be set to false if pods using this serviceAccount do not need to use K8s API - ## - automountServiceAccountToken: true - ## @param serviceAccount.annotations Additional custom annotations for the ServiceAccount - ## - annotations: {} - -## -## Kafka chart configuration -## -## https://github.com/bitnami/charts/blob/master/bitnami/kafka/values.yaml -## -kafka: - enabled: true - replicaCount: 1 - service: - ## Kafka port for client connections - ## - port: 9092 - socketRequestMaxBytes: _1048576000 - -## This value is only used when kafka.enabled is set to false -## -externalKafka: - ## Server or list of external kafka servers to use. - ## - servers: [] - - -## Configure GraphScope Components with a custom groot.config file -## -javaOpts: "" - -rpcMaxBytesMb: 20 - -## Coordinator Config -snapshotIncreaseIntervalMs: 1000 -offsetsPersistIntervalMs: 3000 -fileMetaStorePath: "/var/lib/graphscope-store/meta" -logRecycleEnable: true - -## Frontend config -enableHashGenerateEid: false - -## Store Config -storeDataPath: "/var/lib/graphscope-store" -storeWriteThreadCount: 1 -storeQueueBufferSize: 102400 - -storeGcIntervalMs: 5000 - -## Kafka Config -kafkaTopic: "graphscope" -kafkaProducerCustomConfigs: "" - -## Zk Config -zkBasePath: "/graphscope/groot" - -## Key-value pair separated by ; -## For example extraConfig="k1=v1;k2=v2" -extraConfig: "" - -auth: - username: "" - password: "" - -pegasus: - worker: - num: 1 - timeout: 240000 - -secondary: - enabled: false - storeDataPath: "./data_secondary" - -otel: - enabled: false - collector: - image: - registry: docker.io - repository: jaegertracing/all-in-one - tag: "latest" - traces: - sampler: - name: "traceidratio" - arg: "0.1" - diff --git a/charts/graphscope-store/templates/NOTES.txt b/charts/graphscope-store/templates/NOTES.txt index f927c58ff928..874e17dbcf5e 100644 --- a/charts/graphscope-store/templates/NOTES.txt +++ b/charts/graphscope-store/templates/NOTES.txt @@ -1,22 +1,27 @@ 1. Get the application URL by running these commands: +{{- $serviceName := include "graphscope-store.frontend.fullname" . }} +{{ if not .Values.distributed.enabled }} + {{- $serviceName = include "graphscope-store.store.fullname" . }} +{{- end }} + {{- if contains "NodePort" .Values.frontend.service.type }} - export NODE_IP=$(kubectl -n {{ .Release.Namespace }} get pod {{ include "graphscope-store.frontend.fullname" . }}-0 -o jsonpath="{.status.hostIP}") + export NODE_IP=$(kubectl -n {{ .Release.Namespace }} get pod {{ $serviceName }}-0 -o jsonpath="{.status.hostIP}") {{- else if contains "LoadBalancer" .Values.frontend.service.type }} It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status by executing + You can watch the status by executing - kubectl --namespace {{ .Release.Namespace }} get services -o wide -w {{ include "graphscope-store.frontend.fullname" . }} + kubectl --namespace {{ .Release.Namespace }} get services -o wide -w {{ $serviceName }} After the EXTERNAL_IP is available, executing these commands: - export NODE_IP=$(kubectl -n {{ .Release.Namespace }} get svc {{ include "graphscope-store.frontend.fullname" . }} -ojsonpath="{.status.loadBalancer.ingress[0].ip}") + export NODE_IP=$(kubectl -n {{ .Release.Namespace }} get svc {{ $serviceName }} -ojsonpath="{.status.loadBalancer.ingress[0].ip}") {{- end }} - export GRPC_PORT=$(kubectl -n {{ .Release.Namespace }} get services {{ include "graphscope-store.frontend.fullname" . }} -o jsonpath="{.spec.ports[0].nodePort}") - export GREMLIN_PORT=$(kubectl -n {{ .Release.Namespace }} get services {{ include "graphscope-store.frontend.fullname" . }} -o jsonpath="{.spec.ports[1].nodePort}") + export GRPC_PORT=$(kubectl -n {{ .Release.Namespace }} get services {{ $serviceName }} -o jsonpath="{.spec.ports[0].nodePort}") + export GREMLIN_PORT=$(kubectl -n {{ .Release.Namespace }} get services {{ $serviceName }} -o jsonpath="{.spec.ports[1].nodePort}") echo "GRPC endpoint is: ${NODE_IP}:${GRPC_PORT}" echo "GREMLIN endpoint is: ${NODE_IP}:${GREMLIN_PORT}" \ No newline at end of file diff --git a/charts/graphscope-store/templates/_helpers.tpl b/charts/graphscope-store/templates/_helpers.tpl index 6541aee0c5e7..3f1d8f8d0c99 100644 --- a/charts/graphscope-store/templates/_helpers.tpl +++ b/charts/graphscope-store/templates/_helpers.tpl @@ -157,6 +157,8 @@ Get full broker list. {{- end }} {{- join "," $brokerList | printf "%s" -}} {{- end -}} + + {{/* Create a default fully qualified zookeeper name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). diff --git a/charts/graphscope-store/templates/configmap.yaml b/charts/graphscope-store/templates/configmap.yaml index 419f90ccd14c..b27e19047f64 100644 --- a/charts/graphscope-store/templates/configmap.yaml +++ b/charts/graphscope-store/templates/configmap.yaml @@ -1,4 +1,3 @@ -{{- if (include "graphscope-store.createConfigmap" .) }} apiVersion: v1 kind: ConfigMap metadata: @@ -15,58 +14,51 @@ metadata: data: groot.config: |- ## Common Config + +{{- if .Values.distributed.enabled }} + rpc.port=55555 + discovery.mode={{ .Values.discoveryMode }} role.name=ROLE node.idx=INDEX - rpc.port=55555 - rpc.max.bytes.mb={{ .Values.rpcMaxBytesMb }} +{{- else }} + rpc.port=0 + discovery.mode=zookeeper + role.name="" + node.idx=0 +{{- end }} + store.node.count={{ .Values.store.replicaCount }} frontend.node.count={{ .Values.frontend.replicaCount }} coordinator.node.count={{ .Values.coordinator.replicaCount }} partition.count={{ .Values.store.replicaCount | mul 16 }} - discovery.mode={{ .Values.discoveryMode }} ## Frontend Config - frontend.service.port=55556 + frontend.service.port={{ .Values.frontend.service.servicePort }} frontend.server.id=INDEX frontend.server.num={{ .Values.frontend.replicaCount }} - enable.hash.generate.eid={{ .Values.enableHashGenerateEid }} - - ## Coordinator Config - snapshot.increase.interval.ms={{ .Values.snapshotIncreaseIntervalMs }} - offsets.persist.interval.ms={{ .Values.offsetsPersistIntervalMs }} - file.meta.store.path={{ .Values.fileMetaStorePath }} - log.recycle.enable={{ .Values.logRecycleEnable }} ## Store Config store.data.path={{ .Values.storeDataPath }} store.write.thread.count={{ .Values.storeWriteThreadCount }} store.queue.buffer.size={{ .Values.storeQueueBufferSize }} + ## Zk Config + zk.base.path={{ .Values.zkBasePath }} + zk.connect.string={{ printf "%s-headless" (include "graphscope-store.zookeeper.fullname" .) }} + ## Kafka Config - kafka.servers=KAFKA_SERVERS + kafka.servers={{ include "graphscope-store.kafka.brokerlist" . }} kafka.topic={{ .Values.kafkaTopic }} kafka.producer.custom.configs={{ .Values.kafkaProducerCustomConfigs }} kafka.test.cluster.enable=false - ## Zk Config - zk.base.path={{ .Values.zkBasePath }} - zk.connect.string=ZK_CONNECT - ## Frontend Config gremlin.server.port=12312 ## disable neo4j when launching groot server by default neo4j.bolt.server.disabled=true - dns.name.prefix.frontend=FRONTEND - dns.name.prefix.coordinator=COORDINATOR - dns.name.prefix.store=STORE - log4rs.config=LOG4RS_CONFIG - - ## GAIA Config - gaia.rpc.port=60000 - gaia.engine.port=60001 - + release.full.name={{ include "graphscope-store.fullname" . }} ## Auth config auth.username={{ .Values.auth.username }} auth.password={{ .Values.auth.password }} @@ -77,13 +69,23 @@ data: pegasus.batch.size=1024 pegasus.output.capacity=16 + gaia.rpc.port=60000 + gaia.engine.port=60001 + ## Secondary config secondary.instance.enabled={{ .Values.secondary.enabled }} store.data.secondary.path={{ .Values.secondary.storeDataPath }} store.gc.interval.ms={{ .Values.storeGcIntervalMs }} - write.ha.enabled={{ .Values.backup.enabled }} + tracing.enabled={{ .Values.otel.enabled }} + + ## Coordinator Config + rpc.max.bytes.mb={{ .Values.rpcMaxBytesMb }} + snapshot.increase.interval.ms={{ .Values.snapshotIncreaseIntervalMs }} + offsets.persist.interval.ms={{ .Values.offsetsPersistIntervalMs }} + file.meta.store.path={{ .Values.fileMetaStorePath }} + log.recycle.enable={{ .Values.logRecycleEnable }} ## Extra Config {{- if .Values.extraConfig }} @@ -101,20 +103,21 @@ data: [[ `hostname` =~ -([0-9]+)$ ]] || exit 1 ordinal=${BASH_REMATCH[1]} - sudo sed -e "s/GRAPH_NAME/${GRAPH_NAME}/g" \ - -e "s/ROLE/${ROLE}/g" \ + sudo sed -e "s/ROLE/${ROLE}/g" \ -e "s/INDEX/${ordinal}/g" \ - -e "s/ZK_CONNECT/${ZK_CONNECT}/g" \ - -e "s/KAFKA_SERVERS/${KAFKA_SERVERS}/g" \ - -e "s/FRONTEND/${DNS_NAME_PREFIX_FRONTEND}/g" \ - -e "s/COORDINATOR/${DNS_NAME_PREFIX_COORDINATOR}/g" \ - -e "s/STORE/${DNS_NAME_PREFIX_STORE}/g" \ -e "s@LOG4RS_CONFIG@${GRAPHSCOPE_HOME}/groot/conf/log4rs.yml@g" \ /etc/groot/groot.config.tpl | sudo tee -a /etc/groot/groot.config export LOG_NAME=graphscope-store export GROOT_CONF_FILE=/etc/groot/groot.config + {{- if .Values.otel.enabled }} + export JAVA_TOOL_OPTIONS="-javaagent:/home/graphscope/opentelemetry-javaagent.jar" + export OTEL_SERVICE_NAME="compiler" + export OTEL_TRACES_SAMPLER={{ .Values.otel.traces.sampler.name }} + export OTEL_TRACES_SAMPLER_ARG={{ .Values.otel.traces.sampler.arg }} + {{- end }} + # For core and heap profiling # ulimit -c unlimited # sudo mkdir -p /apsara/cloud/data/corefile/ && sudo chown -R graphscope:graphscope /apsara/cloud/data/corefile/ @@ -123,4 +126,3 @@ data: export RUST_BACKTRACE=1 ${GRAPHSCOPE_HOME}/groot/bin/store_ctl.sh start ${ROLE} # || sleep infinity -{{- end -}} diff --git a/charts/graphscope-store/templates/coordinator/statefulset.yaml b/charts/graphscope-store/templates/coordinator/statefulset.yaml index 8910388908de..e8b0f999e475 100644 --- a/charts/graphscope-store/templates/coordinator/statefulset.yaml +++ b/charts/graphscope-store/templates/coordinator/statefulset.yaml @@ -1,10 +1,4 @@ -{{- $frontendFullname := include "graphscope-store.frontend.fullname" . }} -{{- $coordinatorFullname := include "graphscope-store.coordinator.fullname" . }} -{{- $storeFullname := include "graphscope-store.store.fullname" . }} -{{- $kafkaFullname := include "graphscope-store.kafka.fullname" . -}} -{{- $releaseNamespace := .Release.Namespace }} -{{- $clusterDomain := .Values.clusterDomain }} - +{{- if .Values.distributed.enabled }} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -20,7 +14,7 @@ metadata: {{- end }} spec: podManagementPolicy: {{ .Values.podManagementPolicy }} - replicas: {{ .Values.coordinator.replicaCount }} + replicas: 1 selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} app.kubernetes.io/component: coordinator @@ -50,14 +44,6 @@ spec: {{- if .Values.coordinator.schedulerName }} schedulerName: {{ .Values.coordinator.schedulerName | quote }} {{- end }} - {{- if .Values.affinity }} - affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} - {{- else }} - affinity: - podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset "component" "coordinator" "context" $) | nindent 10 }} - podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset "component" "coordinator" "context" $) | nindent 10 }} - nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values) | nindent 10 }} - {{- end }} {{- if .Values.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} {{- end }} @@ -90,7 +76,7 @@ spec: {{- end }} {{- end }} containers: - - name: coordinator + - name: default image: {{ include "graphscope-store.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} {{- if .Values.containerSecurityContext.enabled }} @@ -101,36 +87,29 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} {{- end }} env: - - name: GRAPH_NAME - value: {{ .Values.graphName | quote }} - - name: GROOT_JAVA_OPTS - value: {{ .Values.javaOpts | quote }} - name: ROLE value: "coordinator" - - name: FRONTEND_COUNT - value: {{ .Values.frontend.replicaCount | quote }} - - name: COORDINATOR_COUNT - value: {{ .Values.coordinator.replicaCount | quote }} - - name: STORE_COUNT - value: {{ .Values.store.replicaCount | quote }} - - name: DNS_NAME_PREFIX_FRONTEND - value: {{ $frontendFullname }}-{}.{{ $frontendFullname }}-headless - - name: DNS_NAME_PREFIX_COORDINATOR - value: {{ $coordinatorFullname }}-{}.{{ $coordinatorFullname }}-headless - - name: DNS_NAME_PREFIX_STORE - value: {{ $storeFullname }}-{}.{{ $storeFullname }}-headless - - name: DNS_NAME_SERVICE_KAFKA - value: {{ $kafkaFullname}}-headless.{{ $releaseNamespace }} - - name: KAFKA_SERVERS - value: {{ include "graphscope-store.kafka.brokerlist" . }} - - name: ZK_CONNECT - value: {{ printf "%s-headless" (include "graphscope-store.zookeeper.fullname" .) | quote }} ports: - - name: port + - name: inner-rpc containerPort: 55555 {{- if .Values.coordinator.resources }} resources: {{- toYaml .Values.coordinator.resources | nindent 12 }} {{- end }} + {{- if .Values.livenessProbe.enabled }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.livenessProbe "enabled") "context" $) | nindent 12 }} + tcpSocket: + port: inner-rpc + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.readinessProbe "enabled") "context" $) | nindent 12 }} + tcpSocket: + port: inner-rpc + {{- end }} + {{- if .Values.startupProbe.enabled }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.startupProbe "enabled") "context" $) | nindent 12 }} + tcpSocket: + port: inner-rpc + {{- end }} volumeMounts: - name: meta mountPath: {{ .Values.fileMetaStorePath }} @@ -148,7 +127,7 @@ spec: {{- if and .Values.coordinator.persistence.enabled .Values.coordinator.persistence.existingClaim }} - name: meta persistentVolumeClaim: - claimName: {{ tpl .Values.coordinator.persistence.existingClaim . }} + claimName: {{ printf "%s" (tpl .Values.coordinator.persistence.existingClaim .) }} {{- else if not .Values.coordinator.persistence.enabled }} - name: meta emptyDir: {} @@ -175,3 +154,4 @@ spec: selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.coordinator.persistence.selector "context" $) | nindent 10 }} {{- end -}} {{- end }} +{{- end -}} diff --git a/charts/graphscope-store/templates/coordinator/svc-headless.yaml b/charts/graphscope-store/templates/coordinator/svc-headless.yaml index 681b8f62e78e..ece3c0f2edf6 100644 --- a/charts/graphscope-store/templates/coordinator/svc-headless.yaml +++ b/charts/graphscope-store/templates/coordinator/svc-headless.yaml @@ -1,3 +1,4 @@ +{{- if .Values.distributed.enabled -}} apiVersion: v1 kind: Service metadata: @@ -18,7 +19,8 @@ spec: publishNotReadyAddresses: true ports: - name: port - port: {{ .Values.coordinator.service.port }} + port: 55555 targetPort: port selector: {{- include "common.labels.matchLabels" . | nindent 4 }} app.kubernetes.io/component: coordinator +{{- end -}} diff --git a/charts/graphscope-store/templates/frontend/statefulset.yaml b/charts/graphscope-store/templates/frontend/statefulset.yaml index 56534fd9f3c2..6bd079d117ff 100644 --- a/charts/graphscope-store/templates/frontend/statefulset.yaml +++ b/charts/graphscope-store/templates/frontend/statefulset.yaml @@ -1,10 +1,4 @@ -{{- $frontendFullname := include "graphscope-store.frontend.fullname" . }} -{{- $coordinatorFullname := include "graphscope-store.coordinator.fullname" . }} -{{- $storeFullname := include "graphscope-store.store.fullname" . }} -{{- $kafkaFullname := include "graphscope-store.kafka.fullname" . -}} -{{- $releaseNamespace := .Release.Namespace }} -{{- $clusterDomain := .Values.clusterDomain }} - +{{- if .Values.distributed.enabled }} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -50,14 +44,6 @@ spec: {{- if .Values.frontend.schedulerName }} schedulerName: {{ .Values.frontend.schedulerName | quote }} {{- end }} - {{- if .Values.affinity }} - affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} - {{- else }} - affinity: - podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset "component" "frontend" "context" $) | nindent 10 }} - podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset "component" "frontend" "context" $) | nindent 10 }} - nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values) | nindent 10 }} - {{- end }} {{- if .Values.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} {{- end }} @@ -90,7 +76,7 @@ spec: {{- end }} {{- end }} containers: - - name: frontend + - name: default image: {{ include "graphscope-store.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} {{- if .Values.containerSecurityContext.enabled }} @@ -101,37 +87,18 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} {{- end }} env: - - name: GRAPH_NAME - value: {{ .Values.graphName | quote }} - - name: GROOT_JAVA_OPTS - value: {{ .Values.javaOpts | quote }} - name: ROLE value: "frontend" - - name: FRONTEND_COUNT - value: {{ .Values.frontend.replicaCount | quote }} - - name: COORDINATOR_COUNT - value: {{ .Values.coordinator.replicaCount | quote }} - - name: STORE_COUNT - value: {{ .Values.store.replicaCount | quote }} - - name: DNS_NAME_PREFIX_FRONTEND - value: {{ $frontendFullname }}-{}.{{ $frontendFullname }}-headless - - name: DNS_NAME_PREFIX_COORDINATOR - value: {{ $coordinatorFullname }}-{}.{{ $coordinatorFullname }}-headless - - name: DNS_NAME_PREFIX_STORE - value: {{ $storeFullname }}-{}.{{ $storeFullname }}-headless - - name: DNS_NAME_SERVICE_KAFKA - value: {{ $kafkaFullname}}-headless.{{ $releaseNamespace }} - - name: KAFKA_SERVERS - value: {{ include "graphscope-store.kafka.brokerlist" . }} - - name: ZK_CONNECT - value: {{ printf "%s-headless" (include "graphscope-store.zookeeper.fullname" .) | quote }} ports: - - name: service-port - containerPort: 55556 + - name: grpc + containerPort: {{ .Values.frontend.service.servicePort }} - name: gremlin - containerPort: 12312 - - name: port - containerPort: 55555 + containerPort: {{ .Values.frontend.service.gremlinPort }} + - name: inner-rpc + containerPort: {{ .Values.frontend.service.port }} + {{- if .Values.frontend.resources }} + resources: {{- toYaml .Values.frontend.resources | nindent 12 }} + {{- end }} {{- if .Values.livenessProbe.enabled }} livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.livenessProbe "enabled") "context" $) | nindent 12 }} tcpSocket: @@ -141,20 +108,12 @@ spec: readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.readinessProbe "enabled") "context" $) | nindent 12 }} tcpSocket: port: gremlin - failureThreshold: 5 - initialDelaySeconds: 30 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 1 {{- end }} {{- if .Values.startupProbe.enabled }} startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.startupProbe "enabled") "context" $) | nindent 12 }} tcpSocket: port: gremlin {{- end }} - {{- if .Values.frontend.resources }} - resources: {{- toYaml .Values.frontend.resources | nindent 12 }} - {{- end }} volumeMounts: - name: config mountPath: /etc/groot/groot.config.tpl @@ -167,3 +126,4 @@ spec: configMap: name: {{ include "graphscope-store.configmapName" . }} defaultMode: 0755 +{{- end }} diff --git a/charts/graphscope-store/templates/frontend/svc-headless.yaml b/charts/graphscope-store/templates/frontend/svc-headless.yaml index a3acf236039c..de425f4aa693 100644 --- a/charts/graphscope-store/templates/frontend/svc-headless.yaml +++ b/charts/graphscope-store/templates/frontend/svc-headless.yaml @@ -1,3 +1,4 @@ +{{- if .Values.distributed.enabled -}} apiVersion: v1 kind: Service metadata: @@ -17,11 +18,9 @@ spec: clusterIP: None publishNotReadyAddresses: true ports: - - name: service-port - port: {{ .Values.frontend.service.servicePort }} - targetPort: service-port - name: port port: {{ .Values.frontend.service.port }} targetPort: port selector: {{- include "common.labels.matchLabels" . | nindent 4 }} app.kubernetes.io/component: frontend +{{- end -}} diff --git a/charts/graphscope-store/templates/frontend/svc.yaml b/charts/graphscope-store/templates/frontend/svc.yaml index 76236538e11d..8fb7c7397054 100644 --- a/charts/graphscope-store/templates/frontend/svc.yaml +++ b/charts/graphscope-store/templates/frontend/svc.yaml @@ -1,3 +1,4 @@ +{{- if .Values.distributed.enabled -}} apiVersion: v1 kind: Service metadata: @@ -30,7 +31,7 @@ spec: - name: grpc port: {{ .Values.frontend.service.servicePort }} protocol: TCP - targetPort: service-port + targetPort: grpc {{- if and (or (eq .Values.frontend.service.type "NodePort") (eq .Values.frontend.service.type "LoadBalancer")) (not (empty .Values.frontend.service.nodePorts.service)) }} nodePort: {{ .Values.frontend.service.nodePorts.service }} {{- else if eq .Values.frontend.service.type "ClusterIP" }} @@ -49,3 +50,4 @@ spec: {{- end }} selector: {{- include "common.labels.matchLabels" . | nindent 4 }} app.kubernetes.io/component: frontend +{{- end -}} diff --git a/charts/graphscope-store-one-pod/templates/statefulset.yaml b/charts/graphscope-store/templates/onepod/statefulset.yaml similarity index 65% rename from charts/graphscope-store-one-pod/templates/statefulset.yaml rename to charts/graphscope-store/templates/onepod/statefulset.yaml index 7637be060893..0143756f872b 100644 --- a/charts/graphscope-store-one-pod/templates/statefulset.yaml +++ b/charts/graphscope-store/templates/onepod/statefulset.yaml @@ -1,7 +1,4 @@ -{{- $kafkaFullname := include "graphscope-store.kafka.fullname" . -}} -{{- $releaseNamespace := .Release.Namespace }} -{{- $clusterDomain := .Values.clusterDomain }} - +{{- if not .Values.distributed.enabled }} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -47,14 +44,6 @@ spec: {{- if .Values.store.schedulerName }} schedulerName: {{ .Values.store.schedulerName | quote }} {{- end }} - {{- if .Values.affinity }} - affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} - {{- else }} - affinity: - podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset "component" "store" "context" $) | nindent 10 }} - podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset "component" "store" "context" $) | nindent 10 }} - nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values) | nindent 10 }} - {{- end }} {{- if .Values.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} {{- end }} @@ -87,7 +76,7 @@ spec: {{- end }} {{- end }} containers: - - name: store + - name: default image: {{ include "graphscope-store.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} {{- if .Values.containerSecurityContext.enabled }} @@ -98,30 +87,31 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} {{- end }} env: - - name: GROOT_JAVA_OPTS - value: {{ .Values.javaOpts | quote }} - - name: DNS_NAME_SERVICE_KAFKA - value: {{ $kafkaFullname}}-headless.{{ $releaseNamespace }} - - name: KAFKA_SERVERS - value: {{ include "graphscope-store.kafka.brokerlist" . }} - - name: ZK_CONNECT - value: {{ printf "%s-headless" (include "graphscope-store.zookeeper.fullname" .) | quote }} + - name: ROLE + value: "" ports: - name: gremlin - containerPort: 12312 + containerPort: {{ .Values.frontend.service.gremlinPort }} - name: grpc - containerPort: 55556 + containerPort: {{ .Values.frontend.service.servicePort }} {{- if .Values.store.resources }} resources: {{- toYaml .Values.store.resources | nindent 12 }} {{- end }} - readinessProbe: - failureThreshold: 5 + {{- if .Values.livenessProbe.enabled }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.livenessProbe "enabled") "context" $) | nindent 12 }} + tcpSocket: + port: gremlin + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.readinessProbe "enabled") "context" $) | nindent 12 }} + tcpSocket: + port: gremlin + {{- end }} + {{- if .Values.startupProbe.enabled }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.startupProbe "enabled") "context" $) | nindent 12 }} tcpSocket: - port: gaia-rpc - initialDelaySeconds: 30 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 1 + port: gremlin + {{- end }} volumeMounts: - name: data mountPath: {{ .Values.storeDataPath }} @@ -131,67 +121,21 @@ spec: - name: config mountPath: /etc/groot/setup.sh subPath: setup.sh - {{- if .Values.otel.enabled }} - - env: - - name: METRICS_STORAGE_TYPE - value: prometheus - - name: SPAN_STORAGE_TYPE - value: memory - - name: JAEGER_DISABLED - value: "false" - - name: COLLECTOR_OTLP_ENABLED - value: "true" - image: {{ include "graphscope-store.otel.collector.image" . }} - imagePullPolicy: IfNotPresent - name: jaeger - args: - - "--memory.max-traces=8000" - - "--query.base-path=/jaeger/ui" - - "--prometheus.server-url=http://opentelemetry-demo-prometheus-server:9090" - - "--prometheus.query.normalize-calls=true" - - "--prometheus.query.normalize-duration=true" - ports: - - containerPort: 5778 - protocol: TCP - - containerPort: 16686 - protocol: TCP - - containerPort: 4317 - protocol: TCP - - containerPort: 4318 - protocol: TCP - livenessProbe: - failureThreshold: 5 - httpGet: - path: / - port: 14269 - scheme: HTTP - initialDelaySeconds: 5 - periodSeconds: 15 - successThreshold: 1 - timeoutSeconds: 1 - readinessProbe: - failureThreshold: 3 - httpGet: - path: / - port: 14269 - scheme: HTTP - initialDelaySeconds: 1 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - limits: - memory: 300Mi - {{- end }} + {{- if .Values.store.extraVolumeMounts }} + {{- include "common.tplvalues.render" (dict "value" .Values.store.extraVolumeMounts "context" $) | nindent 12 }} + {{- end }} volumes: - name: config configMap: name: {{ include "graphscope-store.configmapName" . }} defaultMode: 0755 + {{- if .Values.store.extraVolumes }} + {{- include "common.tplvalues.render" (dict "value" .Values.store.extraVolumes "context" $) | nindent 8 }} + {{- end }} {{- if and .Values.store.persistence.enabled .Values.store.persistence.existingClaim }} - name: data persistentVolumeClaim: - claimName: {{ tpl .Values.store.persistence.existingClaim . }} + claimName: {{ printf "%s" (tpl .Values.store.persistence.existingClaim .) }} {{- else if not .Values.store.persistence.enabled }} - name: data emptyDir: {} @@ -218,3 +162,4 @@ spec: selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.store.persistence.selector "context" $) | nindent 10 }} {{- end -}} {{- end }} +{{- end }} diff --git a/charts/graphscope-store-one-pod/templates/svc-headless.yaml b/charts/graphscope-store/templates/onepod/svc-headless.yaml similarity index 70% rename from charts/graphscope-store-one-pod/templates/svc-headless.yaml rename to charts/graphscope-store/templates/onepod/svc-headless.yaml index 4a965dac9d64..0143a651b745 100644 --- a/charts/graphscope-store-one-pod/templates/svc-headless.yaml +++ b/charts/graphscope-store/templates/onepod/svc-headless.yaml @@ -1,10 +1,11 @@ +{{- if not .Values.distributed.enabled }} apiVersion: v1 kind: Service metadata: - name: {{ printf "%s-frontend-headless" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} + name: {{ printf "%s-store-headless" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} namespace: {{ .Release.Namespace }} labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: frontend + app.kubernetes.io/component: store {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} {{- end }} @@ -17,8 +18,9 @@ spec: clusterIP: None publishNotReadyAddresses: true ports: - - name: service-port - port: {{ .Values.frontend.service.servicePort }} - targetPort: service-port + - name: port + port: {{ .Values.frontend.service.port }} + targetPort: port selector: {{- include "common.labels.matchLabels" . | nindent 4 }} app.kubernetes.io/component: store +{{- end -}} diff --git a/charts/graphscope-store-one-pod/templates/svc.yaml b/charts/graphscope-store/templates/onepod/svc.yaml similarity index 94% rename from charts/graphscope-store-one-pod/templates/svc.yaml rename to charts/graphscope-store/templates/onepod/svc.yaml index 96fbd6d90088..d922e519f1cf 100644 --- a/charts/graphscope-store-one-pod/templates/svc.yaml +++ b/charts/graphscope-store/templates/onepod/svc.yaml @@ -1,10 +1,11 @@ +{{- if not .Values.distributed.enabled }} apiVersion: v1 kind: Service metadata: - name: {{ include "graphscope-store.frontend.fullname" . }} + name: {{ include "graphscope-store.store.fullname" . }} namespace: {{ .Release.Namespace }} labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: frontend + app.kubernetes.io/component: store {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} {{- end }} @@ -49,3 +50,4 @@ spec: {{- end }} selector: {{- include "common.labels.matchLabels" . | nindent 4 }} app.kubernetes.io/component: store +{{- end -}} diff --git a/charts/graphscope-store/templates/store/statefulset-backup.yaml b/charts/graphscope-store/templates/store/statefulset-backup.yaml index 37147090e9ab..0f6bf76f3112 100644 --- a/charts/graphscope-store/templates/store/statefulset-backup.yaml +++ b/charts/graphscope-store/templates/store/statefulset-backup.yaml @@ -1,11 +1,5 @@ +{{- if .Values.distributed.enabled }} {{- if .Values.backup.enabled }} -{{- $frontendFullname := include "graphscope-store.frontend.fullname" . }} -{{- $coordinatorFullname := include "graphscope-store.coordinator.fullname" . }} -{{- $storeFullname := include "graphscope-store.store.fullname" . }} -{{- $kafkaFullname := include "graphscope-store.kafka.fullname" . -}} -{{- $releaseNamespace := .Release.Namespace }} -{{- $clusterDomain := .Values.clusterDomain }} - apiVersion: apps/v1 kind: StatefulSet metadata: @@ -13,6 +7,7 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{- include "common.labels.standard" . | nindent 4 }} app.kubernetes.io/component: store + app.kubernetes.io/type: backup {{- if .Values.commonLabels }} {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} {{- end }} @@ -25,12 +20,14 @@ spec: selector: matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} app.kubernetes.io/component: store + app.kubernetes.io/type: backup serviceName: {{ printf "%s-store-headless" (include "common.names.fullname" .) | trunc 63 | trimSuffix "-" }} updateStrategy: {{- include "common.tplvalues.render" (dict "value" .Values.updateStrategy "context" $ ) | nindent 4 }} template: metadata: labels: {{- include "common.labels.standard" . | nindent 8 }} app.kubernetes.io/component: store + app.kubernetes.io/type: backup {{- if .Values.store.podLabels }} {{- include "common.tplvalues.render" (dict "value" .Values.store.podLabels "context" $) | nindent 8 }} {{- end }} @@ -51,14 +48,6 @@ spec: {{- if .Values.store.schedulerName }} schedulerName: {{ .Values.store.schedulerName | quote }} {{- end }} - {{- if .Values.affinity }} - affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} - {{- else }} - affinity: - podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset "component" "store" "context" $) | nindent 10 }} - podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset "component" "store" "context" $) | nindent 10 }} - nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values) | nindent 10 }} - {{- end }} {{- if .Values.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} {{- end }} @@ -91,7 +80,7 @@ spec: {{- end }} {{- end }} containers: - - name: store + - name: default image: {{ include "graphscope-store.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} {{- if .Values.containerSecurityContext.enabled }} @@ -102,32 +91,10 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} {{- end }} env: - - name: GRAPH_NAME - value: {{ .Values.graphName | quote }} - - name: GROOT_JAVA_OPTS - value: {{ .Values.javaOpts | quote }} - name: ROLE value: "store" - - name: FRONTEND_COUNT - value: {{ .Values.frontend.replicaCount | quote }} - - name: COORDINATOR_COUNT - value: {{ .Values.coordinator.replicaCount | quote }} - - name: STORE_COUNT - value: {{ .Values.store.replicaCount | quote }} - - name: DNS_NAME_PREFIX_FRONTEND - value: {{ $frontendFullname }}-{}.{{ $frontendFullname }}-headless - - name: DNS_NAME_PREFIX_COORDINATOR - value: {{ $coordinatorFullname }}-{}.{{ $coordinatorFullname }}-headless - - name: DNS_NAME_PREFIX_STORE - value: {{ $storeFullname }}-{}.{{ $storeFullname }}-headless - - name: DNS_NAME_SERVICE_KAFKA - value: {{ $kafkaFullname}}-headless.{{ $releaseNamespace }} - - name: KAFKA_SERVERS - value: {{ include "graphscope-store.kafka.brokerlist" . }} - - name: ZK_CONNECT - value: {{ printf "%s-headless" (include "graphscope-store.zookeeper.fullname" .) | quote }} ports: - - name: port + - name: inner-rpc containerPort: 55555 - name: gaia-rpc containerPort: 60000 @@ -136,14 +103,21 @@ spec: {{- if .Values.store.resources }} resources: {{- toYaml .Values.store.resources | nindent 12 }} {{- end }} - readinessProbe: - failureThreshold: 5 + {{- if .Values.livenessProbe.enabled }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.livenessProbe "enabled") "context" $) | nindent 12 }} tcpSocket: port: gaia-rpc - initialDelaySeconds: 30 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 1 + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.readinessProbe "enabled") "context" $) | nindent 12 }} + tcpSocket: + port: gaia-rpc + {{- end }} + {{- if .Values.startupProbe.enabled }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.startupProbe "enabled") "context" $) | nindent 12 }} + tcpSocket: + port: gaia-rpc + {{- end }} volumeMounts: - name: data mountPath: {{ .Values.storeDataPath }} @@ -153,15 +127,21 @@ spec: - name: config mountPath: /etc/groot/setup.sh subPath: setup.sh + {{- if .Values.store.extraVolumeMounts }} + {{- include "common.tplvalues.render" (dict "value" .Values.store.extraVolumeMounts "context" $) | nindent 12 }} + {{- end }} volumes: - name: config configMap: name: {{ include "graphscope-store.configmapName" . }} defaultMode: 0755 + {{- if .Values.store.extraVolumes }} + {{- include "common.tplvalues.render" (dict "value" .Values.store.extraVolumes "context" $) | nindent 8 }} + {{- end }} {{- if and .Values.store.persistence.enabled .Values.store.persistence.existingClaim }} - name: data persistentVolumeClaim: - claimName: {{ tpl .Values.store.persistence.existingClaim . }} + claimName: {{ printf "%s" (tpl .Values.store.persistence.existingClaim .) }} {{- else if not .Values.store.persistence.enabled }} - name: data emptyDir: {} @@ -188,4 +168,5 @@ spec: selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.store.persistence.selector "context" $) | nindent 10 }} {{- end -}} {{- end }} -{{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/graphscope-store/templates/store/statefulset.yaml b/charts/graphscope-store/templates/store/statefulset.yaml index 28e404a7992a..f75a0c5a8141 100644 --- a/charts/graphscope-store/templates/store/statefulset.yaml +++ b/charts/graphscope-store/templates/store/statefulset.yaml @@ -1,10 +1,4 @@ -{{- $frontendFullname := include "graphscope-store.frontend.fullname" . }} -{{- $coordinatorFullname := include "graphscope-store.coordinator.fullname" . }} -{{- $storeFullname := include "graphscope-store.store.fullname" . }} -{{- $kafkaFullname := include "graphscope-store.kafka.fullname" . -}} -{{- $releaseNamespace := .Release.Namespace }} -{{- $clusterDomain := .Values.clusterDomain }} - +{{- if .Values.distributed.enabled }} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -50,14 +44,6 @@ spec: {{- if .Values.store.schedulerName }} schedulerName: {{ .Values.store.schedulerName | quote }} {{- end }} - {{- if .Values.affinity }} - affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} - {{- else }} - affinity: - podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset "component" "store" "context" $) | nindent 10 }} - podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset "component" "store" "context" $) | nindent 10 }} - nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values) | nindent 10 }} - {{- end }} {{- if .Values.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} {{- end }} @@ -90,7 +76,7 @@ spec: {{- end }} {{- end }} containers: - - name: store + - name: default image: {{ include "graphscope-store.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} {{- if .Values.containerSecurityContext.enabled }} @@ -101,32 +87,10 @@ spec: args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }} {{- end }} env: - - name: GRAPH_NAME - value: {{ .Values.graphName | quote }} - - name: GROOT_JAVA_OPTS - value: {{ .Values.javaOpts | quote }} - name: ROLE value: "store" - - name: FRONTEND_COUNT - value: {{ .Values.frontend.replicaCount | quote }} - - name: COORDINATOR_COUNT - value: {{ .Values.coordinator.replicaCount | quote }} - - name: STORE_COUNT - value: {{ .Values.store.replicaCount | quote }} - - name: DNS_NAME_PREFIX_FRONTEND - value: {{ $frontendFullname }}-{}.{{ $frontendFullname }}-headless - - name: DNS_NAME_PREFIX_COORDINATOR - value: {{ $coordinatorFullname }}-{}.{{ $coordinatorFullname }}-headless - - name: DNS_NAME_PREFIX_STORE - value: {{ $storeFullname }}-{}.{{ $storeFullname }}-headless - - name: DNS_NAME_SERVICE_KAFKA - value: {{ $kafkaFullname}}-headless.{{ $releaseNamespace }} - - name: KAFKA_SERVERS - value: {{ include "graphscope-store.kafka.brokerlist" . }} - - name: ZK_CONNECT - value: {{ printf "%s-headless" (include "graphscope-store.zookeeper.fullname" .) | quote }} ports: - - name: port + - name: inner-rpc containerPort: 55555 - name: gaia-rpc containerPort: 60000 @@ -135,14 +99,21 @@ spec: {{- if .Values.store.resources }} resources: {{- toYaml .Values.store.resources | nindent 12 }} {{- end }} - readinessProbe: - failureThreshold: 5 + {{- if .Values.livenessProbe.enabled }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.livenessProbe "enabled") "context" $) | nindent 12 }} tcpSocket: port: gaia-rpc - initialDelaySeconds: 30 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 1 + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.readinessProbe "enabled") "context" $) | nindent 12 }} + tcpSocket: + port: gaia-rpc + {{- end }} + {{- if .Values.startupProbe.enabled }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.startupProbe "enabled") "context" $) | nindent 12 }} + tcpSocket: + port: gaia-rpc + {{- end }} volumeMounts: - name: data mountPath: {{ .Values.storeDataPath }} @@ -152,15 +123,21 @@ spec: - name: config mountPath: /etc/groot/setup.sh subPath: setup.sh + {{- if .Values.store.extraVolumeMounts }} + {{- include "common.tplvalues.render" (dict "value" .Values.store.extraVolumeMounts "context" $) | nindent 12 }} + {{- end }} volumes: - name: config configMap: name: {{ include "graphscope-store.configmapName" . }} defaultMode: 0755 + {{- if .Values.store.extraVolumes }} + {{- include "common.tplvalues.render" (dict "value" .Values.store.extraVolumes "context" $) | nindent 8 }} + {{- end }} {{- if and .Values.store.persistence.enabled .Values.store.persistence.existingClaim }} - name: data persistentVolumeClaim: - claimName: {{ tpl .Values.store.persistence.existingClaim . }} + claimName: {{ printf "%s" (tpl .Values.store.persistence.existingClaim .) }} {{- else if not .Values.store.persistence.enabled }} - name: data emptyDir: {} @@ -187,3 +164,4 @@ spec: selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.store.persistence.selector "context" $) | nindent 10 }} {{- end -}} {{- end }} +{{- end -}} diff --git a/charts/graphscope-store/templates/store/svc-headless.yaml b/charts/graphscope-store/templates/store/svc-headless.yaml index 8fd18c6c2980..2781dd4e7c53 100644 --- a/charts/graphscope-store/templates/store/svc-headless.yaml +++ b/charts/graphscope-store/templates/store/svc-headless.yaml @@ -1,3 +1,4 @@ +{{- if .Values.distributed.enabled -}} apiVersion: v1 kind: Service metadata: @@ -18,13 +19,8 @@ spec: publishNotReadyAddresses: true ports: - name: port - port: {{ .Values.store.service.port }} + port: 55555 targetPort: port - - name: gaia-rpc - port: {{ .Values.store.service.gaiaRpc }} - targetPort: gaia-rpc - - name: gaia-engine - port: {{ .Values.store.service.gaiaEngine }} - targetPort: gaia-engine selector: {{- include "common.labels.matchLabels" . | nindent 4 }} app.kubernetes.io/component: store +{{- end -}} diff --git a/charts/graphscope-store/templates/test/test-rpc.yaml b/charts/graphscope-store/templates/test/test-rpc.yaml index 7e22d93f7d2c..21f0b280c1d3 100644 --- a/charts/graphscope-store/templates/test/test-rpc.yaml +++ b/charts/graphscope-store/templates/test/test-rpc.yaml @@ -29,7 +29,11 @@ spec: image: {{ include "graphscope-store.test.image" . }} env: - name: frontend + {{- if .Values.distributed.enabled }} value: {{ include "graphscope-store.frontend.fullname" . }} + {{- else }} + value: {{ include "graphscope-store.store.fullname" . }} + {{- end }} command: - /bin/sh - -c diff --git a/charts/graphscope-store/values.yaml b/charts/graphscope-store/values.yaml index a410badca7ec..ac05dc296c73 100644 --- a/charts/graphscope-store/values.yaml +++ b/charts/graphscope-store/values.yaml @@ -25,7 +25,7 @@ image: registry: registry.cn-hongkong.aliyuncs.com repository: graphscope/graphscope-store # Overrides the image tag whose default is the chart appVersion. - tag: "0.23.0" + tag: "0.28.0" digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' @@ -74,20 +74,6 @@ store: size: 1Gi selector: {} - ## GraphScope store Service parameters - ## - service: - ## Service type - ## - type: ClusterIP - ## Service port - ## - port: 55555 - gaiaRpc: 60000 - gaiaEngine: 60001 - - annotations: {} - ## @param hostAliases pods host aliases ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ ## @@ -114,6 +100,22 @@ store: ## schedulerName: "" + ## @param extraVolumes Optionally specify extra list of additional volumes for the Store pod(s) + ## e.g: + ## extraVolumes: + ## - name: kafka-jaas + ## secret: + ## secretName: kafka-jaas + ## + extraVolumes: [] + ## @param extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Store container(s) + ## extraVolumeMounts: + ## - name: kafka-jaas + ## mountPath: /bitnami/kafka/config/kafka_jaas.conf + ## subPath: kafka_jaas.conf + ## + extraVolumeMounts: [] + ## GraphScope Frontend parameters ## frontend: @@ -223,14 +225,6 @@ coordinator: size: 100Mi selector: {} - - ## Coordinator Service parameters - ## - service: - type: ClusterIP - port: 55555 - annotations: {} - ## @param hostAliases pods host aliases ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ ## @@ -284,26 +278,26 @@ commonAnnotations: {} ## Configure extra options for containers' liveness, readiness and startup probes livenessProbe: enabled: false - initialDelaySeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 + initialDelaySeconds: 30 periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 5 successThreshold: 1 readinessProbe: - enabled: false - initialDelaySeconds: 5 - failureThreshold: 6 - timeoutSeconds: 5 + enabled: true + initialDelaySeconds: 30 periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 5 successThreshold: 1 startupProbe: enabled: false initialDelaySeconds: 30 periodSeconds: 10 - timeoutSeconds: 1 - failureThreshold: 15 + timeoutSeconds: 5 + failureThreshold: 5 successThreshold: 1 command: @@ -427,6 +421,32 @@ serviceAccount: ## annotations: {} + +## Configure GraphScope Components with a custom groot.config file +## +javaOpts: "" + +rpcMaxBytesMb: 20 + +## choose between "file" or "zookeeper" +discoveryMode: "file" + +## Coordinator Config +snapshotIncreaseIntervalMs: 1000 +offsetsPersistIntervalMs: 3000 +fileMetaStorePath: "/etc/groot/my.meta" +logRecycleEnable: true + +## Store Config +storeDataPath: "/var/lib/graphscope-store" +storeDataDownloadPath: "/var/lib/graphscope-store/download" +storeDataSecondaryPath: "/home/graphscope/secondary" +storeWriteThreadCount: 1 +storeQueueBufferSize: 102400 + +storeGcIntervalMs: 5000 + +## Kafka Config ## ## Kafka chart configuration ## @@ -451,34 +471,20 @@ externalKafka: ## servers: [] +kafkaTopic: "graphscope" +kafkaProducerCustomConfigs: "" -## Configure GraphScope Components with a custom groot.config file -## -javaOpts: "" - -rpcMaxBytesMb: 20 - -discoveryMode: "file" - -## Coordinator Config -snapshotIncreaseIntervalMs: 1000 -offsetsPersistIntervalMs: 3000 -fileMetaStorePath: "/etc/groot/my.meta" -logRecycleEnable: true -## Store Config -storeDataPath: "/var/lib/graphscope-store" -storeWriteThreadCount: 1 -storeQueueBufferSize: 102400 +## Zk Config -storeGcIntervalMs: 5000 +## If set, the externalZookeeper.servers will be used to connect to the zookeeper +## If not set, the bundled zookeeper in kafka cluster will be used +externalZookeeper: + servers: [] -## Kafka Config -kafkaTopic: "graphscope" -kafkaProducerCustomConfigs: "" +zkBasePath: "/graphscope/groot" ## Frontend Config -enableHashGenerateEid: false # gremlinServerPort: 12312 ## Key-value pair separated by ; @@ -498,5 +504,21 @@ secondary: enabled: false storeDataPath: "./data_secondary" + backup: enabled: false + +otel: + enabled: false + collector: + image: + registry: docker.io + repository: jaegertracing/all-in-one + tag: "latest" + traces: + sampler: + name: "traceidratio" + arg: "0.1" + +distributed: + enabled: false diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/config/DiscoveryConfig.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/config/DiscoveryConfig.java index 484a5c305595..6f93a27a7f48 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/config/DiscoveryConfig.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/config/DiscoveryConfig.java @@ -15,20 +15,7 @@ */ package com.alibaba.graphscope.groot.common.config; -import com.alibaba.graphscope.groot.common.RoleType; - public class DiscoveryConfig { - public static final String DNS_NAME_PREFIX_FORMAT = "dns.name.prefix.%s"; - - public static final Config DNS_NAME_PREFIX_FRONTEND = - Config.stringConfig( - String.format(DNS_NAME_PREFIX_FORMAT, RoleType.FRONTEND.getName()), ""); - - public static final Config DNS_NAME_PREFIX_COORDINATOR = - Config.stringConfig( - String.format(DNS_NAME_PREFIX_FORMAT, RoleType.COORDINATOR.getName()), ""); - - public static final Config DNS_NAME_PREFIX_STORE = - Config.stringConfig( - String.format(DNS_NAME_PREFIX_FORMAT, RoleType.STORE.getName()), ""); + public static final Config RELEASE_FULL_NAME = + Config.stringConfig("release.full.name", ""); } diff --git a/interactive_engine/executor/store/groot/src/db/graph/codec.rs b/interactive_engine/executor/store/groot/src/db/graph/codec.rs index 787e1e790d6f..b8ab06d6a343 100644 --- a/interactive_engine/executor/store/groot/src/db/graph/codec.rs +++ b/interactive_engine/executor/store/groot/src/db/graph/codec.rs @@ -192,7 +192,7 @@ impl Decoder { if idx > self.src.fixed_len_prop_count { start_off = bytes_to_len(reader.read_bytes(self.src.offsets[idx - 1], 3)); } - if end_off <= start_off { + if end_off < start_off { error!("fatal error! This codec cannot decode the bytes: idx {}, end_off: {}, start_off: {}, props: {:?}, ", idx, end_off, start_off, self.src.props); return None; } diff --git a/interactive_engine/executor/store/groot/src/db/graph/store.rs b/interactive_engine/executor/store/groot/src/db/graph/store.rs index 4736f8d1fee8..de10c4afbb18 100644 --- a/interactive_engine/executor/store/groot/src/db/graph/store.rs +++ b/interactive_engine/executor/store/groot/src/db/graph/store.rs @@ -562,13 +562,17 @@ impl MultiVersionGraph for GraphStore { fn gc(&self, si: i64) -> GraphResult<()> { let vertex_tables = self.vertex_manager.gc(si)?; - info!("garbage collect vertex table {:?}", vertex_tables); + if !vertex_tables.is_empty() { + info!("garbage collect vertex table {:?}", vertex_tables); + } for vt in vertex_tables { let table_prefix = vertex_table_prefix(vt); self.delete_table_by_prefix(table_prefix, true)?; } let edge_tables = self.edge_manager.gc(si)?; - info!("garbage collect edge table {:?}", edge_tables); + if !edge_tables.is_empty() { + info!("garbage collect edge table {:?}", edge_tables); + } for et in edge_tables { let out_table_prefix = edge_table_prefix(et, EdgeDirection::Out); self.delete_table_by_prefix(out_table_prefix, false)?; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/Utils.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/Utils.java index 7c3aec9200f4..381fceff51d0 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/Utils.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/Utils.java @@ -40,18 +40,28 @@ public class Utils { .build(); public static String getHostTemplate(Configs configs, RoleType role) { + String releaseName = DiscoveryConfig.RELEASE_FULL_NAME.get(configs); + // template = "{releaseName}-{role}-{}.{releaseName}-{role}-headless"; + // i.e. demo-graphscope-store-frontend-0.demo-graphscope-store-frontend-headless + String svcTemplate = "%s-%s"; + String svcName = ""; switch (role) { case FRONTEND: - return DiscoveryConfig.DNS_NAME_PREFIX_FRONTEND.get(configs); + svcName = String.format(svcTemplate, releaseName, "frontend"); + break; case COORDINATOR: - return DiscoveryConfig.DNS_NAME_PREFIX_COORDINATOR.get(configs); + svcName = String.format(svcTemplate, releaseName, "coordinator"); + break; case STORE: case GAIA_RPC: case GAIA_ENGINE: - return DiscoveryConfig.DNS_NAME_PREFIX_STORE.get(configs); + svcName = String.format(svcTemplate, releaseName, "store"); + break; default: throw new IllegalArgumentException("invalid role [" + role + "]"); } + String dnsTemplate = "%s-{}.%s-headless"; + return String.format(dnsTemplate, svcName, svcName); } public static int getPort(Configs configs) { diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/CoordinatorSnapshotClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/CoordinatorSnapshotClient.java index af9ccccf406f..6d1d291479d0 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/CoordinatorSnapshotClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/CoordinatorSnapshotClient.java @@ -1,27 +1,28 @@ package com.alibaba.graphscope.groot.coordinator; +import com.alibaba.graphscope.groot.rpc.RpcChannel; import com.alibaba.graphscope.groot.rpc.RpcClient; import com.alibaba.graphscope.proto.groot.*; -import io.grpc.ManagedChannel; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; // send rpc to CoordinatorSnapshotService in Store, to report minimum snapshot current used public class CoordinatorSnapshotClient extends RpcClient { private static final Logger logger = LoggerFactory.getLogger(CoordinatorSnapshotClient.class); - private final CoordinatorSnapshotServiceGrpc.CoordinatorSnapshotServiceBlockingStub stub; - public CoordinatorSnapshotClient(ManagedChannel channel) { + public CoordinatorSnapshotClient(RpcChannel channel) { super(channel); - this.stub = CoordinatorSnapshotServiceGrpc.newBlockingStub(channel); + } + + private CoordinatorSnapshotServiceGrpc.CoordinatorSnapshotServiceBlockingStub getStub() { + return CoordinatorSnapshotServiceGrpc.newBlockingStub(rpcChannel.getChannel()); } public void synchronizeSnapshot(long snapshotId) throws RuntimeException { SynchronizeMinQuerySnapshotIdRequest req = SynchronizeMinQuerySnapshotIdRequest.newBuilder().setSnapshotId(snapshotId).build(); - SynchronizeMinQuerySnapshotIdResponse res = stub.synchronizeMinQuerySnapshotId(req); + SynchronizeMinQuerySnapshotIdResponse res = getStub().synchronizeMinQuerySnapshotId(req); if (!res.getSuccess()) { throw new RuntimeException("Synchronize snapshot to store failed: " + res.getErrMsg()); } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/FrontendSnapshotClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/FrontendSnapshotClient.java index 18413aef423a..2524efbbd21a 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/FrontendSnapshotClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/FrontendSnapshotClient.java @@ -15,6 +15,7 @@ import com.alibaba.graphscope.groot.CompletionCallback; import com.alibaba.graphscope.groot.common.schema.wrapper.GraphDef; +import com.alibaba.graphscope.groot.rpc.RpcChannel; import com.alibaba.graphscope.groot.rpc.RpcClient; import com.alibaba.graphscope.proto.groot.AdvanceQuerySnapshotRequest; import com.alibaba.graphscope.proto.groot.AdvanceQuerySnapshotResponse; @@ -29,16 +30,16 @@ public class FrontendSnapshotClient extends RpcClient { private static final Logger logger = LoggerFactory.getLogger(FrontendSnapshotClient.class); - private FrontendSnapshotGrpc.FrontendSnapshotStub stub; - - public FrontendSnapshotClient(ManagedChannel channel) { + public FrontendSnapshotClient(RpcChannel channel) { super(channel); - this.stub = FrontendSnapshotGrpc.newStub(this.channel); } public FrontendSnapshotClient(FrontendSnapshotGrpc.FrontendSnapshotStub stub) { super((ManagedChannel) stub.getChannel()); - this.stub = stub; + } + + private FrontendSnapshotGrpc.FrontendSnapshotStub getStub() { + return FrontendSnapshotGrpc.newStub(rpcChannel.getChannel()); } public void advanceQuerySnapshot( @@ -48,22 +49,23 @@ public void advanceQuerySnapshot( if (graphDef != null) { builder.setGraphDef(graphDef.toProto()); } - stub.advanceQuerySnapshot( - builder.build(), - new StreamObserver<>() { - @Override - public void onNext(AdvanceQuerySnapshotResponse response) { - long previousSnapshotId = response.getPreviousSnapshotId(); - callback.onCompleted(previousSnapshotId); - } + getStub() + .advanceQuerySnapshot( + builder.build(), + new StreamObserver<>() { + @Override + public void onNext(AdvanceQuerySnapshotResponse response) { + long previousSnapshotId = response.getPreviousSnapshotId(); + callback.onCompleted(previousSnapshotId); + } - @Override - public void onError(Throwable throwable) { - callback.onError(throwable); - } + @Override + public void onError(Throwable throwable) { + callback.onError(throwable); + } - @Override - public void onCompleted() {} - }); + @Override + public void onCompleted() {} + }); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/IngestorSnapshotClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/IngestorSnapshotClient.java index 84d6f2919bef..c4fe18b665ea 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/IngestorSnapshotClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/IngestorSnapshotClient.java @@ -14,6 +14,7 @@ package com.alibaba.graphscope.groot.coordinator; import com.alibaba.graphscope.groot.CompletionCallback; +import com.alibaba.graphscope.groot.rpc.RpcChannel; import com.alibaba.graphscope.groot.rpc.RpcClient; import com.alibaba.graphscope.proto.groot.AdvanceIngestSnapshotIdRequest; import com.alibaba.graphscope.proto.groot.AdvanceIngestSnapshotIdResponse; @@ -23,37 +24,39 @@ import io.grpc.stub.StreamObserver; public class IngestorSnapshotClient extends RpcClient { - private final IngestorSnapshotGrpc.IngestorSnapshotStub stub; - public IngestorSnapshotClient(ManagedChannel channel) { + public IngestorSnapshotClient(RpcChannel channel) { super(channel); - this.stub = IngestorSnapshotGrpc.newStub(channel); } public IngestorSnapshotClient(IngestorSnapshotGrpc.IngestorSnapshotStub stub) { super((ManagedChannel) stub.getChannel()); - this.stub = stub; + } + + private IngestorSnapshotGrpc.IngestorSnapshotStub getStub() { + return IngestorSnapshotGrpc.newStub(rpcChannel.getChannel()); } public void advanceIngestSnapshotId(long writeSnapshotId, CompletionCallback callback) { AdvanceIngestSnapshotIdRequest req = AdvanceIngestSnapshotIdRequest.newBuilder().setSnapshotId(writeSnapshotId).build(); - stub.advanceIngestSnapshotId( - req, - new StreamObserver() { - @Override - public void onNext(AdvanceIngestSnapshotIdResponse response) { - long previousSnapshotId = response.getPreviousSnapshotId(); - callback.onCompleted(previousSnapshotId); - } - - @Override - public void onError(Throwable throwable) { - callback.onError(throwable); - } - - @Override - public void onCompleted() {} - }); + getStub() + .advanceIngestSnapshotId( + req, + new StreamObserver() { + @Override + public void onNext(AdvanceIngestSnapshotIdResponse response) { + long previousSnapshotId = response.getPreviousSnapshotId(); + callback.onCompleted(previousSnapshotId); + } + + @Override + public void onError(Throwable throwable) { + callback.onError(throwable); + } + + @Override + public void onCompleted() {} + }); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/IngestorWriteClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/IngestorWriteClient.java index c02a60125314..bd7f2952eea6 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/IngestorWriteClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/IngestorWriteClient.java @@ -15,18 +15,18 @@ import com.alibaba.graphscope.groot.operation.BatchId; import com.alibaba.graphscope.groot.operation.OperationBatch; +import com.alibaba.graphscope.groot.rpc.RpcChannel; import com.alibaba.graphscope.groot.rpc.RpcClient; import com.alibaba.graphscope.proto.groot.*; -import io.grpc.ManagedChannel; - public class IngestorWriteClient extends RpcClient { - private final IngestorWriteGrpc.IngestorWriteBlockingStub stub; - - public IngestorWriteClient(ManagedChannel channel) { + public IngestorWriteClient(RpcChannel channel) { super(channel); - this.stub = IngestorWriteGrpc.newBlockingStub(channel); + } + + private IngestorWriteGrpc.IngestorWriteBlockingStub getStub() { + return IngestorWriteGrpc.newBlockingStub(rpcChannel.getChannel()); } public BatchId writeIngestor(String requestId, int queueId, OperationBatch operationBatch) { @@ -36,7 +36,7 @@ public BatchId writeIngestor(String requestId, int queueId, OperationBatch opera .setQueueId(queueId) .setOperationBatch(operationBatch.toProto()) .build(); - WriteIngestorResponse response = this.stub.writeIngestor(request); + WriteIngestorResponse response = getStub().writeIngestor(request); return new BatchId(response.getSnapshotId()); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/SnapshotManager.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/SnapshotManager.java index d53307c2d56a..5e733d4c6515 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/SnapshotManager.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/SnapshotManager.java @@ -320,7 +320,7 @@ private void maybeUpdateQuerySnapshotId() { } SnapshotInfo receivedSIInfo = Collections.min(this.storeToSnapshotInfo.values()); if (receivedSIInfo.getSnapshotId() <= this.querySnapshotInfo.getSnapshotId()) { - logger.warn("Received SI vs. current: {}, {}", receivedSIInfo, querySnapshotInfo); + logger.debug("Received SI vs. current: {}, {}", receivedSIInfo, querySnapshotInfo); return; } synchronized (this.querySnapshotLock) { diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/StoreSchemaClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/StoreSchemaClient.java index d820cbe34208..f49edef51122 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/StoreSchemaClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/StoreSchemaClient.java @@ -14,6 +14,7 @@ package com.alibaba.graphscope.groot.coordinator; import com.alibaba.graphscope.groot.common.schema.wrapper.GraphDef; +import com.alibaba.graphscope.groot.rpc.RpcChannel; import com.alibaba.graphscope.groot.rpc.RpcClient; import com.alibaba.graphscope.proto.groot.FetchSchemaRequest; import com.alibaba.graphscope.proto.groot.FetchSchemaResponse; @@ -22,22 +23,21 @@ import io.grpc.ManagedChannel; public class StoreSchemaClient extends RpcClient { - - private final StoreSchemaGrpc.StoreSchemaBlockingStub stub; - - public StoreSchemaClient(ManagedChannel channel) { + public StoreSchemaClient(RpcChannel channel) { super(channel); - this.stub = StoreSchemaGrpc.newBlockingStub(channel); } public StoreSchemaClient(StoreSchemaGrpc.StoreSchemaBlockingStub stub) { super((ManagedChannel) stub.getChannel()); - this.stub = stub; + } + + private StoreSchemaGrpc.StoreSchemaBlockingStub getStub() { + return StoreSchemaGrpc.newBlockingStub(rpcChannel.getChannel()); } public GraphDef fetchSchema() { - FetchSchemaResponse response = - this.stub.fetchSchema(FetchSchemaRequest.newBuilder().build()); + StoreSchemaGrpc.StoreSchemaBlockingStub stub = getStub(); + FetchSchemaResponse response = stub.fetchSchema(FetchSchemaRequest.newBuilder().build()); return GraphDef.parseProto(response.getGraphDef()); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/backup/StoreBackupClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/backup/StoreBackupClient.java index bee9e1c9f557..4e69a2a9963b 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/backup/StoreBackupClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/backup/StoreBackupClient.java @@ -16,6 +16,7 @@ package com.alibaba.graphscope.groot.coordinator.backup; import com.alibaba.graphscope.groot.CompletionCallback; +import com.alibaba.graphscope.groot.rpc.RpcChannel; import com.alibaba.graphscope.groot.rpc.RpcClient; import com.alibaba.graphscope.groot.store.backup.StoreBackupId; import com.alibaba.graphscope.proto.groot.*; @@ -28,38 +29,41 @@ import java.util.Map; public class StoreBackupClient extends RpcClient { - private final StoreBackupGrpc.StoreBackupStub stub; - - public StoreBackupClient(ManagedChannel channel) { + public StoreBackupClient(RpcChannel channel) { super(channel); - this.stub = StoreBackupGrpc.newStub(channel); } public StoreBackupClient(StoreBackupGrpc.StoreBackupStub stub) { super((ManagedChannel) stub.getChannel()); - this.stub = stub; + } + + private StoreBackupGrpc.StoreBackupStub getStub() { + return StoreBackupGrpc.newStub(rpcChannel.getChannel()); } public void createStoreBackup(int globalBackupId, CompletionCallback callback) { CreateStoreBackupRequest req = CreateStoreBackupRequest.newBuilder().setGlobalBackupId(globalBackupId).build(); - stub.createStoreBackup( - req, - new StreamObserver() { - @Override - public void onNext(CreateStoreBackupResponse response) { - StoreBackupIdPb finishedStoreBackupIdPb = response.getStoreBackupId(); - callback.onCompleted(StoreBackupId.parseProto(finishedStoreBackupIdPb)); - } - - @Override - public void onError(Throwable throwable) { - callback.onError(throwable); - } - - @Override - public void onCompleted() {} - }); + getStub() + .createStoreBackup( + req, + new StreamObserver() { + @Override + public void onNext(CreateStoreBackupResponse response) { + StoreBackupIdPb finishedStoreBackupIdPb = + response.getStoreBackupId(); + callback.onCompleted( + StoreBackupId.parseProto(finishedStoreBackupIdPb)); + } + + @Override + public void onError(Throwable throwable) { + callback.onError(throwable); + } + + @Override + public void onCompleted() {} + }); } public void clearUnavailableBackups( @@ -78,24 +82,25 @@ public void clearUnavailableBackups( ClearUnavailableStoreBackupsRequest.newBuilder() .putAllPartitionToReadyBackupIds(partitionToBackupIdListPb) .build(); - stub.clearUnavailableStoreBackups( - req, - new StreamObserver() { - @Override - public void onNext( - ClearUnavailableStoreBackupsResponse - clearUnavailableStoreBackupsResponse) { - callback.onCompleted(null); - } - - @Override - public void onError(Throwable throwable) { - callback.onError(throwable); - } - - @Override - public void onCompleted() {} - }); + getStub() + .clearUnavailableStoreBackups( + req, + new StreamObserver() { + @Override + public void onNext( + ClearUnavailableStoreBackupsResponse + clearUnavailableStoreBackupsResponse) { + callback.onCompleted(null); + } + + @Override + public void onError(Throwable throwable) { + callback.onError(throwable); + } + + @Override + public void onCompleted() {} + }); } public void restoreFromStoreBackup( @@ -107,23 +112,24 @@ public void restoreFromStoreBackup( .setStoreBackupId(storeBackupId.toProto()) .setRestoreRootPath(storeRestoreRootPath) .build(); - stub.restoreFromStoreBackup( - req, - new StreamObserver() { - @Override - public void onNext( - RestoreFromStoreBackupResponse restoreFromStoreBackupResponse) { - callback.onCompleted(null); - } - - @Override - public void onError(Throwable throwable) { - callback.onError(throwable); - } - - @Override - public void onCompleted() {} - }); + getStub() + .restoreFromStoreBackup( + req, + new StreamObserver() { + @Override + public void onNext( + RestoreFromStoreBackupResponse restoreFromStoreBackupResponse) { + callback.onCompleted(null); + } + + @Override + public void onError(Throwable throwable) { + callback.onError(throwable); + } + + @Override + public void onCompleted() {} + }); } public void verifyStoreBackup(StoreBackupId storeBackupId, CompletionCallback callback) { @@ -131,21 +137,23 @@ public void verifyStoreBackup(StoreBackupId storeBackupId, CompletionCallback() { - @Override - public void onNext(VerifyStoreBackupResponse verifyStoreBackupResponse) { - callback.onCompleted(null); - } - - @Override - public void onError(Throwable throwable) { - callback.onError(throwable); - } - - @Override - public void onCompleted() {} - }); + getStub() + .verifyStoreBackup( + req, + new StreamObserver() { + @Override + public void onNext( + VerifyStoreBackupResponse verifyStoreBackupResponse) { + callback.onCompleted(null); + } + + @Override + public void onError(Throwable throwable) { + callback.onError(throwable); + } + + @Override + public void onCompleted() {} + }); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/FileDiscovery.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/FileDiscovery.java index c9b01dd3dcbd..fc9798984932 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/FileDiscovery.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/FileDiscovery.java @@ -36,9 +36,9 @@ public class FileDiscovery implements NodeDiscovery { public FileDiscovery(Configs configs) { this.configs = configs; // Store related nodes - String storePrefix = DiscoveryConfig.DNS_NAME_PREFIX_STORE.get(configs); - String frontendPrefix = DiscoveryConfig.DNS_NAME_PREFIX_FRONTEND.get(configs); - String coordinatorPrefix = DiscoveryConfig.DNS_NAME_PREFIX_COORDINATOR.get(configs); + String storePrefix = Utils.getHostTemplate(configs, STORE); + String frontendPrefix = Utils.getHostTemplate(configs, FRONTEND); + String coordinatorPrefix = Utils.getHostTemplate(configs, COORDINATOR); // Frontend nodes int frontendCount = CommonConfig.FRONTEND_NODE_COUNT.get(configs); diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/BackupClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/BackupClient.java index c30456c67792..5802e9134d4a 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/BackupClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/BackupClient.java @@ -15,6 +15,7 @@ */ package com.alibaba.graphscope.groot.frontend; +import com.alibaba.graphscope.groot.rpc.RpcChannel; import com.alibaba.graphscope.groot.rpc.RpcClient; import com.alibaba.graphscope.proto.groot.*; @@ -23,34 +24,34 @@ import java.util.List; public class BackupClient extends RpcClient { - private final BackupGrpc.BackupBlockingStub stub; - - public BackupClient(ManagedChannel channel) { + public BackupClient(RpcChannel channel) { super(channel); - this.stub = BackupGrpc.newBlockingStub(channel); } public BackupClient(BackupGrpc.BackupBlockingStub stub) { - super((ManagedChannel) stub.getChannel()); - this.stub = stub; + super(new RpcChannel((ManagedChannel) stub.getChannel())); + } + + private BackupGrpc.BackupBlockingStub getStub() { + return BackupGrpc.newBlockingStub(rpcChannel.getChannel()); } public int createNewBackup() { CreateNewBackupRequest request = CreateNewBackupRequest.newBuilder().build(); - CreateNewBackupResponse response = this.stub.createNewBackup(request); + CreateNewBackupResponse response = getStub().createNewBackup(request); return response.getGlobalBackupId(); } public void deleteBackup(int globalBackupId) { DeleteBackupRequest request = DeleteBackupRequest.newBuilder().setGlobalBackupId(globalBackupId).build(); - this.stub.deleteBackup(request); + getStub().deleteBackup(request); } public void purgeOldBackups(int keepAliveNum) { PurgeOldBackupsRequest request = PurgeOldBackupsRequest.newBuilder().setKeepAliveNumber(keepAliveNum).build(); - this.stub.purgeOldBackups(request); + getStub().purgeOldBackups(request); } public void restoreFromBackup( @@ -61,18 +62,18 @@ public void restoreFromBackup( .setMetaRestorePath(metaRestorePath) .setStoreRestorePath(storeRestorePath) .build(); - this.stub.restoreFromBackup(request); + getStub().restoreFromBackup(request); } public void verifyBackup(int globalBackupId) { VerifyBackupRequest request = VerifyBackupRequest.newBuilder().setGlobalBackupId(globalBackupId).build(); - this.stub.verifyBackup(request); + getStub().verifyBackup(request); } public List getBackupInfo() { GetBackupInfoRequest request = GetBackupInfoRequest.newBuilder().build(); - GetBackupInfoResponse response = this.stub.getBackupInfo(request); + GetBackupInfoResponse response = getStub().getBackupInfo(request); return response.getBackupInfoListList(); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/FrontendStoreClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/FrontendStoreClient.java index 5a79f4c8cf73..0fa6aad2ca80 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/FrontendStoreClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/FrontendStoreClient.java @@ -14,21 +14,22 @@ package com.alibaba.graphscope.groot.frontend; import com.alibaba.graphscope.groot.CompletionCallback; +import com.alibaba.graphscope.groot.rpc.RpcChannel; import com.alibaba.graphscope.groot.rpc.RpcClient; import com.alibaba.graphscope.proto.groot.*; -import io.grpc.ManagedChannel; import io.grpc.stub.StreamObserver; import java.util.Map; public class FrontendStoreClient extends RpcClient { - private final FrontendStoreServiceGrpc.FrontendStoreServiceStub stub; - - public FrontendStoreClient(ManagedChannel channel) { + public FrontendStoreClient(RpcChannel channel) { super(channel); - this.stub = FrontendStoreServiceGrpc.newStub(channel); + } + + private FrontendStoreServiceGrpc.FrontendStoreServiceStub getStub() { + return FrontendStoreServiceGrpc.newStub(rpcChannel.getChannel()); } public void storeIngest( @@ -36,97 +37,102 @@ public void storeIngest( IngestDataRequest.Builder builder = IngestDataRequest.newBuilder(); builder.setDataPath(dataPath); builder.putAllConfig(config); - this.stub.storeIngest( - builder.build(), - new StreamObserver<>() { - @Override - public void onNext(IngestDataResponse value) { - callback.onCompleted(null); - } - - @Override - public void onError(Throwable t) { - callback.onError(t); - } - - @Override - public void onCompleted() {} - }); + getStub() + .storeIngest( + builder.build(), + new StreamObserver<>() { + @Override + public void onNext(IngestDataResponse value) { + callback.onCompleted(null); + } + + @Override + public void onError(Throwable t) { + callback.onError(t); + } + + @Override + public void onCompleted() {} + }); } public void storeClearIngest(String path, CompletionCallback callback) { - this.stub.storeClearIngest( - ClearIngestRequest.newBuilder().setDataPath(path).build(), - new StreamObserver<>() { - @Override - public void onNext(ClearIngestResponse storeClearIngestResponse) { - callback.onCompleted(null); - } - - @Override - public void onError(Throwable throwable) { - callback.onError(throwable); - } - - @Override - public void onCompleted() {} - }); + getStub() + .storeClearIngest( + ClearIngestRequest.newBuilder().setDataPath(path).build(), + new StreamObserver<>() { + @Override + public void onNext(ClearIngestResponse storeClearIngestResponse) { + callback.onCompleted(null); + } + + @Override + public void onError(Throwable throwable) { + callback.onError(throwable); + } + + @Override + public void onCompleted() {} + }); } public void storeCompact(CompletionCallback callback) { - this.stub.compactDB( - CompactDBRequest.newBuilder().build(), - new StreamObserver<>() { - @Override - public void onNext(CompactDBResponse value) { - callback.onCompleted(null); - } - - @Override - public void onError(Throwable t) { - callback.onError(t); - } - - @Override - public void onCompleted() {} - }); + getStub() + .compactDB( + CompactDBRequest.newBuilder().build(), + new StreamObserver<>() { + @Override + public void onNext(CompactDBResponse value) { + callback.onCompleted(null); + } + + @Override + public void onError(Throwable t) { + callback.onError(t); + } + + @Override + public void onCompleted() {} + }); } public void reopenSecondary(CompletionCallback callback) { - this.stub.reopenSecondary( - ReopenSecondaryRequest.newBuilder().build(), - new StreamObserver<>() { - @Override - public void onNext(ReopenSecondaryResponse value) { - callback.onCompleted(null); - } - - @Override - public void onError(Throwable t) { - callback.onError(t); - } - - @Override - public void onCompleted() {} - }); + getStub() + .reopenSecondary( + ReopenSecondaryRequest.newBuilder().build(), + new StreamObserver<>() { + @Override + public void onNext(ReopenSecondaryResponse value) { + callback.onCompleted(null); + } + + @Override + public void onError(Throwable t) { + callback.onError(t); + } + + @Override + public void onCompleted() {} + }); } public void getStoreState(CompletionCallback callback) { - this.stub.getState( - GetStoreStateRequest.newBuilder().build(), - new StreamObserver<>() { - @Override - public void onNext(GetStoreStateResponse value) { - callback.onCompleted(value); - } - - @Override - public void onError(Throwable t) { - callback.onError(t); - } - - @Override - public void onCompleted() {} - }); + getStub() + .getState( + GetStoreStateRequest.newBuilder().build(), + new StreamObserver<>() { + @Override + public void onNext(GetStoreStateResponse value) { + callback.onCompleted(value); + } + + @Override + public void onError(Throwable t) { + callback.onError(t); + } + + @Override + public void onCompleted() {} + }); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SchemaClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SchemaClient.java index 91d0afe98496..692f11cc981d 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SchemaClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SchemaClient.java @@ -13,6 +13,7 @@ */ package com.alibaba.graphscope.groot.frontend; +import com.alibaba.graphscope.groot.rpc.RpcChannel; import com.alibaba.graphscope.groot.rpc.RpcClient; import com.alibaba.graphscope.groot.schema.request.DdlException; import com.alibaba.graphscope.proto.groot.DdlRequestBatchPb; @@ -23,17 +24,16 @@ import io.grpc.ManagedChannel; public class SchemaClient extends RpcClient { - - private final SchemaGrpc.SchemaBlockingStub stub; - - public SchemaClient(ManagedChannel channel) { + public SchemaClient(RpcChannel channel) { super(channel); - this.stub = SchemaGrpc.newBlockingStub(channel); } public SchemaClient(SchemaGrpc.SchemaBlockingStub stub) { super((ManagedChannel) stub.getChannel()); - this.stub = stub; + } + + private SchemaGrpc.SchemaBlockingStub getStub() { + return SchemaGrpc.newBlockingStub(rpcChannel.getChannel()); } public long submitBatchDdl( @@ -44,7 +44,7 @@ public long submitBatchDdl( .setSessionId(sessionId) .setDdlRequests(ddlRequestBatchPb) .build(); - SubmitBatchDdlResponse submitBatchDdlResponse = stub.submitBatchDdl(request); + SubmitBatchDdlResponse submitBatchDdlResponse = getStub().submitBatchDdl(request); if (submitBatchDdlResponse.getSuccess()) { return submitBatchDdlResponse.getDdlSnapshotId(); } else { diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SnapshotUpdateClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SnapshotUpdateClient.java index 4db297d19ccc..94fa48992339 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SnapshotUpdateClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SnapshotUpdateClient.java @@ -16,13 +16,12 @@ package com.alibaba.graphscope.groot.frontend; +import com.alibaba.graphscope.groot.rpc.RpcChannel; import com.alibaba.graphscope.groot.rpc.RpcClient; import com.alibaba.graphscope.proto.groot.CoordinatorSnapshotServiceGrpc; import com.alibaba.graphscope.proto.groot.UpdateMinQuerySnapshotIdRequest; import com.alibaba.graphscope.proto.groot.UpdateMinQuerySnapshotIdResponse; -import io.grpc.ManagedChannel; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,11 +29,12 @@ public class SnapshotUpdateClient extends RpcClient { private static final Logger logger = LoggerFactory.getLogger(SnapshotUpdateClient.class); - private final CoordinatorSnapshotServiceGrpc.CoordinatorSnapshotServiceBlockingStub stub; - - public SnapshotUpdateClient(ManagedChannel channel) { + public SnapshotUpdateClient(RpcChannel channel) { super(channel); - this.stub = CoordinatorSnapshotServiceGrpc.newBlockingStub(channel); + } + + private CoordinatorSnapshotServiceGrpc.CoordinatorSnapshotServiceBlockingStub getStub() { + return CoordinatorSnapshotServiceGrpc.newBlockingStub(rpcChannel.getChannel()); } public void updateSnapshot(int frontendId, long snapshotId) throws RuntimeException { @@ -43,7 +43,7 @@ public void updateSnapshot(int frontendId, long snapshotId) throws RuntimeExcept .setFrontendId(frontendId) .setSnapshotId(snapshotId) .build(); - UpdateMinQuerySnapshotIdResponse res = stub.updateMinQuerySnapshotId(req); + UpdateMinQuerySnapshotIdResponse res = getStub().updateMinQuerySnapshotId(req); if (!res.getSuccess()) { throw new RuntimeException("update snapshot fail {} " + res.getErrMsg()); } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/IdAllocateClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/IdAllocateClient.java index 430f1a2faf22..481461e62905 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/IdAllocateClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/IdAllocateClient.java @@ -1,25 +1,24 @@ package com.alibaba.graphscope.groot.frontend.write; +import com.alibaba.graphscope.groot.rpc.RpcChannel; import com.alibaba.graphscope.groot.rpc.RpcClient; import com.alibaba.graphscope.proto.groot.AllocateIdRequest; import com.alibaba.graphscope.proto.groot.AllocateIdResponse; import com.alibaba.graphscope.proto.groot.IdAllocateGrpc; -import io.grpc.ManagedChannel; - public class IdAllocateClient extends RpcClient { - - private IdAllocateGrpc.IdAllocateBlockingStub stub; - - public IdAllocateClient(ManagedChannel channel) { + public IdAllocateClient(RpcChannel channel) { super(channel); - this.stub = IdAllocateGrpc.newBlockingStub(channel); + } + + private IdAllocateGrpc.IdAllocateBlockingStub getStub() { + return IdAllocateGrpc.newBlockingStub(rpcChannel.getChannel()); } public long allocateId(int allocateSize) { AllocateIdRequest req = AllocateIdRequest.newBuilder().setAllocateSize(allocateSize).build(); - AllocateIdResponse response = stub.allocateId(req); + AllocateIdResponse response = getStub().allocateId(req); return response.getStartId(); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/metrics/MetricsCollectClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/metrics/MetricsCollectClient.java index 6214f69730cf..90ebe1c32b61 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/metrics/MetricsCollectClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/metrics/MetricsCollectClient.java @@ -14,42 +14,43 @@ package com.alibaba.graphscope.groot.metrics; import com.alibaba.graphscope.groot.CompletionCallback; +import com.alibaba.graphscope.groot.rpc.RpcChannel; import com.alibaba.graphscope.groot.rpc.RpcClient; import com.alibaba.graphscope.proto.groot.CollectMetricsRequest; import com.alibaba.graphscope.proto.groot.CollectMetricsResponse; import com.alibaba.graphscope.proto.groot.MetricsCollectGrpc; -import io.grpc.ManagedChannel; import io.grpc.stub.StreamObserver; import java.util.Map; public class MetricsCollectClient extends RpcClient { - - private final MetricsCollectGrpc.MetricsCollectStub stub; - - public MetricsCollectClient(ManagedChannel channel) { + public MetricsCollectClient(RpcChannel channel) { super(channel); - this.stub = MetricsCollectGrpc.newStub(channel); + } + + private MetricsCollectGrpc.MetricsCollectStub getStub() { + return MetricsCollectGrpc.newStub(rpcChannel.getChannel()); } public void collectMetrics(CompletionCallback> callback) { - this.stub.collectMetrics( - CollectMetricsRequest.newBuilder().build(), - new StreamObserver() { - @Override - public void onNext(CollectMetricsResponse value) { - Map metricsMap = value.getMetricsMap(); - callback.onCompleted(metricsMap); - } - - @Override - public void onError(Throwable t) { - callback.onError(t); - } - - @Override - public void onCompleted() {} - }); + getStub() + .collectMetrics( + CollectMetricsRequest.newBuilder().build(), + new StreamObserver() { + @Override + public void onNext(CollectMetricsResponse value) { + Map metricsMap = value.getMetricsMap(); + callback.onCompleted(metricsMap); + } + + @Override + public void onError(Throwable t) { + callback.onError(t); + } + + @Override + public void onCompleted() {} + }); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/ChannelManager.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/ChannelManager.java index ea210c9dd5c6..dd764c383208 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/ChannelManager.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/ChannelManager.java @@ -96,21 +96,31 @@ public void registerRole(RoleType role) { logger.debug("role [" + role.getName() + "] registered"); } + private ManagedChannel createChannel(RoleType role, int idx) { + String host = Utils.getHostTemplate(configs, role).replace("{}", String.valueOf(idx)); + int port = Utils.getPort(configs, role, idx); + logger.info("Create channel to {}#{}, {}:{}", role.getName(), idx, host, port); + ManagedChannel channel = + ManagedChannelBuilder.forAddress(host, port) + .maxInboundMessageSize(this.rpcMaxBytes) + .usePlaintext() + .build(); + return channel; + } + public ManagedChannel getChannel(RoleType role, int idx) { Map idToChannel = this.roleToChannels.get(role); if (idToChannel == null) { throw new NodeConnectException("invalid role [" + role + "]"); } + // to avoid thread competition if (idToChannel.get(idx) == null) { - String host = Utils.getHostTemplate(configs, role).replace("{}", String.valueOf(idx)); - int port = Utils.getPort(configs, role, idx); - logger.info("Create channel to {}#{}, {}:{}", role.getName(), idx, host, port); - ManagedChannel channel = - ManagedChannelBuilder.forAddress(host, port) - .maxInboundMessageSize(this.rpcMaxBytes) - .usePlaintext() - .build(); - idToChannel.put(idx, channel); + synchronized (this) { + if (idToChannel.get(idx) == null) { + ManagedChannel channel = createChannel(role, idx); + idToChannel.put(idx, channel); + } + } } return idToChannel.get(idx); } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/RoleClients.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/RoleClients.java index 8d6cdcbcd7eb..3d8839c01327 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/RoleClients.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/RoleClients.java @@ -15,8 +15,6 @@ import com.alibaba.graphscope.groot.common.RoleType; -import io.grpc.ManagedChannel; - import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; @@ -32,12 +30,12 @@ public class RoleClients { private final RoleType targetRole; private final Map clients; - private final Function clientBuilder; + private final Function clientBuilder; public RoleClients( ChannelManager channelManager, RoleType targetRole, - Function clientBuilder) { + Function clientBuilder) { this.channelManager = channelManager; this.targetRole = targetRole; this.clientBuilder = clientBuilder; @@ -51,7 +49,9 @@ public T getClient(int clientId) { synchronized (this) { client = this.clients.get(clientId); if (client == null) { - ManagedChannel channel = this.channelManager.getChannel(targetRole, clientId); + RpcChannel channel = new RpcChannel(channelManager, targetRole, clientId); + // ManagedChannel channel = this.channelManager.getChannel(targetRole, + // clientId); client = clientBuilder.apply(channel); this.clients.put(clientId, client); } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/RpcChannel.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/RpcChannel.java new file mode 100644 index 000000000000..8daac38999ed --- /dev/null +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/RpcChannel.java @@ -0,0 +1,42 @@ +package com.alibaba.graphscope.groot.rpc; + +import com.alibaba.graphscope.groot.common.RoleType; + +import io.grpc.ConnectivityState; +import io.grpc.ManagedChannel; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RpcChannel { + public static final Logger logger = LoggerFactory.getLogger(RpcChannel.class); + private final ChannelManager manager; + private final RoleType targetRole; + private final int index; + + private ManagedChannel channel; + + public RpcChannel(ChannelManager manager, RoleType targetRole, int index) { + this.manager = manager; + this.targetRole = targetRole; + this.index = index; + this.channel = manager.getChannel(targetRole, index); + } + + public RpcChannel(ManagedChannel channel) { + this.channel = channel; + manager = null; + targetRole = null; + index = -1; + } + + public ManagedChannel getChannel() { + ConnectivityState state = channel.getState(false); + if (state == ConnectivityState.TRANSIENT_FAILURE || state == ConnectivityState.SHUTDOWN) { + if (manager != null) { + this.channel = manager.getChannel(targetRole, index); + } + } + return channel; + } +} diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/RpcClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/RpcClient.java index 8ba10b9a848e..eeb00fa01a45 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/RpcClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/RpcClient.java @@ -24,23 +24,31 @@ public abstract class RpcClient implements AutoCloseable { private static final Logger logger = LoggerFactory.getLogger(RpcClient.class); - protected ManagedChannel channel; + // protected ManagedChannel channel; + + protected RpcChannel rpcChannel; + + public RpcClient(RpcChannel channel) { + this.rpcChannel = channel; + } public RpcClient(ManagedChannel channel) { - this.channel = channel; + this(new RpcChannel(channel)); } public void checkChannelState() { - if (channel.getState(true) != ConnectivityState.READY) { - logger.warn("Current channel State: " + channel.getState(true)); + ConnectivityState state = rpcChannel.getChannel().getState(true); + if (state != ConnectivityState.READY) { + logger.warn("Current channel State: " + state); } } public void close() { + ManagedChannel channel = rpcChannel.getChannel(); channel.resetConnectBackoff(); - this.channel.shutdown(); + channel.shutdown(); try { - this.channel.awaitTermination(3000, TimeUnit.MILLISECONDS); + channel.awaitTermination(3000, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { // Ignore } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/KafkaProcessor.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/KafkaProcessor.java index aac5a37e9ecd..68cd0f536ddc 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/KafkaProcessor.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/KafkaProcessor.java @@ -250,7 +250,7 @@ public void replayWAL() throws IOException { long replayFrom = queueOffsetsRef.get().get(0) + 1; logger.info("replay WAL of queue#[{}] from offset [{}]", storeId, replayFrom); if (replayFrom == 0) { - logger.error("It's not normal to replay from the 0 offset, skipped"); + logger.warn("It may not useful to replay from the 0 offset, skipped"); return; } int replayCount = 0; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/SnapshotCommitClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/SnapshotCommitClient.java index 0c43361489d5..89a39e65691b 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/SnapshotCommitClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/SnapshotCommitClient.java @@ -13,6 +13,7 @@ */ package com.alibaba.graphscope.groot.store; +import com.alibaba.graphscope.groot.rpc.RpcChannel; import com.alibaba.graphscope.groot.rpc.RpcClient; import com.alibaba.graphscope.proto.groot.CommitSnapshotIdRequest; import com.alibaba.graphscope.proto.groot.SnapshotCommitGrpc; @@ -22,17 +23,16 @@ import java.util.List; public class SnapshotCommitClient extends RpcClient { - - private SnapshotCommitGrpc.SnapshotCommitBlockingStub stub; - - public SnapshotCommitClient(ManagedChannel channel) { + public SnapshotCommitClient(RpcChannel channel) { super(channel); - this.stub = SnapshotCommitGrpc.newBlockingStub(channel); } public SnapshotCommitClient(SnapshotCommitGrpc.SnapshotCommitBlockingStub stub) { super((ManagedChannel) stub.getChannel()); - this.stub = stub; + } + + private SnapshotCommitGrpc.SnapshotCommitBlockingStub getStub() { + return SnapshotCommitGrpc.newBlockingStub(rpcChannel.getChannel()); } public void commitSnapshotId( @@ -44,6 +44,6 @@ public void commitSnapshotId( .setDdlSnapshotId(ddlSnapshotId) .addAllQueueOffsets(queueOffsets) .build(); - stub.commitSnapshotId(req); + getStub().commitSnapshotId(req); } } diff --git a/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/common/client/RoleClientsTest.java b/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/common/client/RoleClientsTest.java index 50a80578cc9b..cd26eabb581e 100644 --- a/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/common/client/RoleClientsTest.java +++ b/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/common/client/RoleClientsTest.java @@ -21,11 +21,10 @@ import com.alibaba.graphscope.groot.common.exception.NodeConnectException; import com.alibaba.graphscope.groot.rpc.ChannelManager; import com.alibaba.graphscope.groot.rpc.RoleClients; +import com.alibaba.graphscope.groot.rpc.RpcChannel; import com.alibaba.graphscope.groot.rpc.RpcClient; import com.alibaba.graphscope.groot.tests.common.rpc.MockFactory; -import io.grpc.ManagedChannel; - import org.junit.jupiter.api.Test; public class RoleClientsTest { @@ -48,7 +47,7 @@ void testRoleClients() { class MockRoleClient extends RpcClient { - public MockRoleClient(ManagedChannel channel) { + public MockRoleClient(RpcChannel channel) { super(channel); } } diff --git a/k8s/actions-runner-controller/manylinux-ci.yaml b/k8s/actions-runner-controller/manylinux-ci.yaml index 9e77dd6f7805..212b76513a75 100644 --- a/k8s/actions-runner-controller/manylinux-ci.yaml +++ b/k8s/actions-runner-controller/manylinux-ci.yaml @@ -1,7 +1,7 @@ apiVersion: actions.summerwind.dev/v1alpha1 kind: RunnerDeployment metadata: - name: graphscope-runners-manylinux2014 + name: graphscope-runners-manylinux2014-ci-test spec: template: spec: @@ -30,7 +30,7 @@ spec: scaleDownDelaySecondsAfterScaleOut: 900 scaleTargetRef: kind: RunnerDeployment - name: graphscope-runners-manylinux2014 + name: graphscope-runners-manylinux2014-ci-test # Uncomment the below in case the target is not RunnerDeployment but RunnerSet #kind: RunnerSet minReplicas: 0 diff --git a/k8s/actions-runner-controller/ubuntu.yaml b/k8s/actions-runner-controller/ubuntu.yaml index a72e30c21353..e6e292c29eff 100644 --- a/k8s/actions-runner-controller/ubuntu.yaml +++ b/k8s/actions-runner-controller/ubuntu.yaml @@ -23,7 +23,7 @@ spec: requests: cpu: "27.0" memory: "60Gi" - ephemeral-storage: 100Gi + ephemeral-storage: 200Gi # - name: docker # resources: # limits: diff --git a/k8s/dockerfiles/graphscope-store.Dockerfile b/k8s/dockerfiles/graphscope-store.Dockerfile index c70fbda92093..b5f1eebc32ef 100644 --- a/k8s/dockerfiles/graphscope-store.Dockerfile +++ b/k8s/dockerfiles/graphscope-store.Dockerfile @@ -41,7 +41,7 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ RUN apt-get update -y && \ apt-get install -y sudo default-jdk dnsutils tzdata lsof \ - libjemalloc-dev libunwind-dev binutils less python3 python3-pip && \ + libjemalloc-dev libunwind-dev binutils less && \ apt-get clean -y && \ rm -rf /var/lib/apt/lists/* @@ -56,8 +56,10 @@ RUN sudo chmod a+wrx /tmp # install coordinator RUN if [ "${ENABLE_COORDINATOR}" = "true" ]; then \ - pip3 install --upgrade pip \ - && pip3 install /usr/local/groot/wheel/*.whl; \ + apt-get update -y && apt-get install -y python3-pip && \ + apt-get clean -y && rm -rf /var/lib/apt/lists/* && \ + pip3 install --upgrade pip && \ + pip3 install /usr/local/groot/wheel/*.whl; \ fi USER graphscope