K8SPS-291: Add prestop hook to MySQL pods

build/Dockerfile

@@ -85,6 +85,7 @@ COPY --from=go_builder /usr/local/bin/sidecar /opt/percona-server-mysql-operator
 COPY --from=go_builder /usr/local/bin/peer-list /opt/percona-server-mysql-operator/peer-list
 COPY --from=go_builder /usr/local/bin/orc-handler /opt/percona-server-mysql-operator/orc-handler
 COPY build/ps-entrypoint.sh /opt/percona-server-mysql-operator/ps-entrypoint.sh
+COPY build/ps-pre-stop.sh /opt/percona-server-mysql-operator/ps-pre-stop.sh
 COPY build/heartbeat-entrypoint.sh /opt/percona-server-mysql-operator/heartbeat-entrypoint.sh
 COPY build/router-entrypoint.sh /opt/percona-server-mysql-operator/router-entrypoint.sh
 COPY build/router_readiness_check.sh /opt/percona-server-mysql-operator/router_readiness_check.sh

build/ps-init-entrypoint.sh

@@ -7,6 +7,8 @@ OPERATORDIR="/opt/percona-server-mysql-operator"
 BINDIR="/opt/percona"
 
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/ps-entrypoint.sh" "${BINDIR}/ps-entrypoint.sh"
+install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/ps-pre-stop.sh" "${BINDIR}/ps-pre-stop.sh"
+
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/heartbeat-entrypoint.sh" "${BINDIR}/heartbeat-entrypoint.sh"
 
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/orc-entrypoint.sh" "${BINDIR}/orc-entrypoint.sh"
@@ -34,4 +36,4 @@ install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/haproxy_readiness
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/haproxy.cfg" "${BINDIR}/haproxy.cfg"
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/haproxy-global.cfg" "${BINDIR}/haproxy-global.cfg"
 
-install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/pmm-prerun.sh" "${BINDIR}/pmm-prerun.sh"
+install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/pmm-prerun.sh" "${BINDIR}/pmm-prerun.sh"

build/ps-pre-stop.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e
+
+if [ "${CLUSTER_TYPE}" == "async" ]; then
+	exit 0
+fi
+
+LOG_FILE=/var/lib/mysql/pre-stop.log
+NAMESPACE=$(</var/run/secrets/kubernetes.io/serviceaccount/namespace)
+OPERATOR_PASSWORD=$(</etc/mysql/mysql-users-secret/operator)
+FQDN="${HOSTNAME}.${SERVICE_NAME}.${NAMESPACE}"
+POD_IP=$(hostname -I | awk '{print $1}')
+
+echo "$(date +%Y-%m-%dT%H:%M:%S%Z): Removing ${FQDN} from cluster" >>${LOG_FILE}
+mysqlsh -i -h "${POD_IP}" -P 33062 -u operator -p"${OPERATOR_PASSWORD}" -e "dba.getCluster().removeInstance('${FQDN}:3306')" >>${LOG_FILE} 2>&1

cmd/manager/main.go

@@ -84,6 +84,8 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	setupLog.Info("Build info", "GitCommit", GitCommit, "BuildTime", BuildTime)
+
 	namespace, err := k8s.GetWatchNamespace()
 	if err != nil {
 		setupLog.Error(err, "unable to get watch namespace")

e2e-tests/tests/gr-scaling/06-assert.yaml

@@ -1,43 +1,10 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
-timeout: 90
+timeout: 30
 ---
-kind: StatefulSet
-apiVersion: apps/v1
+kind: ConfigMap
+apiVersion: v1
 metadata:
-  name: gr-scaling-mysql
-status:
-  observedGeneration: 2
-  replicas: 1
-  readyReplicas: 1
-  currentReplicas: 1
-  updatedReplicas: 1
-  collisionCount: 0
----
-kind: Deployment
-apiVersion: apps/v1
-metadata:
-  name: gr-scaling-router
-status:
-  observedGeneration: 2
-  replicas: 1
-  updatedReplicas: 1
-  readyReplicas: 1
-  availableReplicas: 1
----
-apiVersion: ps.percona.com/v1alpha1
-kind: PerconaServerMySQL
-metadata:
-  name: gr-scaling
-  finalizers:
-    - delete-mysql-pods-in-order
-status:
-  mysql:
-    ready: 1
-    size: 1
-    state: ready
-  router:
-    ready: 1
-    size: 1
-    state: ready
-  state: ready
+  name: 06-check-group-seeds
+data:
+  gr-scaling-mysql-0.gr-scaling-mysql: ""

e2e-tests/tests/gr-scaling/06-check-group-seeds.yaml

@@ -0,0 +1,21 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 30
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      args=''
+      size=$(kubectl -n ${NAMESPACE} get ps $(get_cluster_name) -o jsonpath='{.spec.mysql.size}')
+      for i in $(seq 0 $((size - 1))); do
+      	host=$(get_mysql_headless_fqdn $(get_cluster_name) $i)
+      	seeds=$(run_mysql "SELECT @@group_replication_group_seeds" "-h ${host} -uroot -proot_password")
+        echo ${seeds}
+        seeds=$(echo ${seeds} | sed -e "s/\.${NAMESPACE}//g")
+      	args="${args} --from-literal=${host}=${seeds}"
+      done
+
+      kubectl create configmap -n "${NAMESPACE}" 06-check-group-seeds ${args}

e2e-tests/tests/gr-scaling/07-assert.yaml

@@ -1,10 +1,43 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
-timeout: 30
+timeout: 90
 ---
-kind: ConfigMap
-apiVersion: v1
+kind: StatefulSet
+apiVersion: apps/v1
 metadata:
-  name: 09-read-from-primary
-data:
-  data: "100500"
+  name: gr-scaling-mysql
+status:
+  observedGeneration: 2
+  replicas: 1
+  readyReplicas: 1
+  currentReplicas: 1
+  updatedReplicas: 1
+  collisionCount: 0
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: gr-scaling-router
+status:
+  observedGeneration: 2
+  replicas: 1
+  updatedReplicas: 1
+  readyReplicas: 1
+  availableReplicas: 1
+---
+apiVersion: ps.percona.com/v1alpha1
+kind: PerconaServerMySQL
+metadata:
+  name: gr-scaling
+  finalizers:
+    - delete-mysql-pods-in-order
+status:
+  mysql:
+    ready: 1
+    size: 1
+    state: ready
+  router:
+    ready: 1
+    size: 1
+    state: ready
+  state: ready

e2e-tests/tests/gr-scaling/08-assert.yaml

@@ -1,43 +1,10 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
-timeout: 180
+timeout: 30
 ---
-kind: StatefulSet
-apiVersion: apps/v1
+kind: ConfigMap
+apiVersion: v1
 metadata:
-  name: gr-scaling-mysql
-status:
-  observedGeneration: 3
-  replicas: 3
-  readyReplicas: 3
-  currentReplicas: 3
-  updatedReplicas: 3
-  collisionCount: 0
----
-kind: Deployment
-apiVersion: apps/v1
-metadata:
-  name: gr-scaling-router
-status:
-  observedGeneration: 3
-  replicas: 3
-  updatedReplicas: 3
-  readyReplicas: 3
-  availableReplicas: 3
----
-apiVersion: ps.percona.com/v1alpha1
-kind: PerconaServerMySQL
-metadata:
-  name: gr-scaling
-  finalizers:
-    - delete-mysql-pods-in-order
-status:
-  mysql:
-    ready: 3
-    size: 3
-    state: ready
-  router:
-    ready: 3
-    size: 3
-    state: ready
-  state: ready
+  name: 09-read-from-primary
+data:
+  data: "100500"

e2e-tests/tests/gr-scaling/09-assert.yaml

@@ -1,10 +1,43 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
-timeout: 30
+timeout: 360
 ---
-kind: ConfigMap
-apiVersion: v1
+kind: StatefulSet
+apiVersion: apps/v1
 metadata:
-  name: 11-read-from-primary
-data:
-  data: "100500"
+  name: gr-scaling-mysql
+status:
+  observedGeneration: 3
+  replicas: 3
+  readyReplicas: 3
+  currentReplicas: 3
+  updatedReplicas: 3
+  collisionCount: 0
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: gr-scaling-router
+status:
+  observedGeneration: 3
+  replicas: 3
+  updatedReplicas: 3
+  readyReplicas: 3
+  availableReplicas: 3
+---
+apiVersion: ps.percona.com/v1alpha1
+kind: PerconaServerMySQL
+metadata:
+  name: gr-scaling
+  finalizers:
+    - delete-mysql-pods-in-order
+status:
+  mysql:
+    ready: 3
+    size: 3
+    state: ready
+  router:
+    ready: 3
+    size: 3
+    state: ready
+  state: ready

e2e-tests/tests/gr-scaling/10-assert.yaml

@@ -5,8 +5,8 @@ timeout: 30
 kind: ConfigMap
 apiVersion: v1
 metadata:
-  name: 12-read-from-replicas
+  name: 10-check-group-seeds
 data:
-  gr-scaling-mysql-0.gr-scaling-mysql: "100500"
-  gr-scaling-mysql-1.gr-scaling-mysql: "100500"
-  gr-scaling-mysql-2.gr-scaling-mysql: "100500"
+  gr-scaling-mysql-0.gr-scaling-mysql: "gr-scaling-mysql-1.gr-scaling-mysql:3306,gr-scaling-mysql-2.gr-scaling-mysql:3306"
+  gr-scaling-mysql-1.gr-scaling-mysql: "gr-scaling-mysql-0.gr-scaling-mysql:3306,gr-scaling-mysql-1.gr-scaling-mysql:3306,gr-scaling-mysql-2.gr-scaling-mysql:3306"
+  gr-scaling-mysql-2.gr-scaling-mysql: "gr-scaling-mysql-0.gr-scaling-mysql:3306,gr-scaling-mysql-1.gr-scaling-mysql:3306,gr-scaling-mysql-2.gr-scaling-mysql:3306"

e2e-tests/tests/gr-scaling/10-check-group-seeds.yaml

@@ -0,0 +1,21 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 30
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      args=''
+      size=$(kubectl -n ${NAMESPACE} get ps $(get_cluster_name) -o jsonpath='{.spec.mysql.size}')
+      for i in $(seq 0 $((size - 1))); do
+      	host=$(get_mysql_headless_fqdn $(get_cluster_name) $i)
+      	seeds=$(run_mysql "SELECT @@group_replication_group_seeds" "-h ${host} -uroot -proot_password")
+        echo ${seeds}
+        seeds=$(echo ${seeds} | sed -e "s/\.${NAMESPACE}//g")
+      	args="${args} --from-literal=${host}=${seeds}"
+      done
+
+      kubectl create configmap -n "${NAMESPACE}" 10-check-group-seeds ${args}

e2e-tests/tests/gr-scaling/11-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 30
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: 11-read-from-primary
+data:
+  data: "100500"

e2e-tests/tests/gr-scaling/12-assert.yaml

@@ -0,0 +1,12 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 30
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: 12-read-from-replicas
+data:
+  gr-scaling-mysql-0.gr-scaling-mysql: "100500"
+  gr-scaling-mysql-1.gr-scaling-mysql: "100500"
+  gr-scaling-mysql-2.gr-scaling-mysql: "100500"

pkg/mysql/mysql.go

@@ -541,6 +541,13 @@ func mysqldContainer(cr *apiv1alpha1.PerconaServerMySQL) corev1.Container {
 		LivenessProbe:            k8s.ExecProbe(spec.LivenessProbe, []string{"/opt/percona/healthcheck", "liveness"}),
 		ReadinessProbe:           k8s.ExecProbe(spec.ReadinessProbe, []string{"/opt/percona/healthcheck", "readiness"}),
 		StartupProbe:             k8s.ExecProbe(spec.StartupProbe, []string{"/opt/percona/bootstrap"}),
+		Lifecycle: &corev1.Lifecycle{
+			PreStop: &corev1.LifecycleHandler{
+				Exec: &corev1.ExecAction{
+					Command: []string{"/opt/percona/ps-pre-stop.sh"},
+				},
+			},
+		},
 	}
 
 	return container