Check milvus pod stopped before finalize

Signed-off-by: shaoyue.chen <[email protected]>
zilliztech · Oct 18, 2023 · 31de7ba · 31de7ba
1 parent d56ab8a
commit 31de7ba
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 2 deletions.
diff --git a/pkg/controllers/component_condition.go b/pkg/controllers/component_condition.go
@@ -158,7 +158,7 @@ func GetComponentConditionGetter() ComponentConditionGetter {
 
 var singletonComponentConditionGetter ComponentConditionGetter = ComponentConditionGetterImpl{}
 
-func CheckMilvusStopped(ctx context.Context, cli client.Client, mc v1beta1.Milvus) (bool, error) {
+var CheckMilvusStopped = func(ctx context.Context, cli client.Client, mc v1beta1.Milvus) (bool, error) {
 	podList := &corev1.PodList{}
 	opts := &client.ListOptions{
 		Namespace: mc.Namespace,

diff --git a/pkg/controllers/milvus_controller.go b/pkg/controllers/milvus_controller.go
@@ -117,6 +117,11 @@ func (r *MilvusReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 			}
 		}
 
+		stopped, err := CheckMilvusStopped(ctx, r.Client, *milvus)
+		if !stopped || err != nil {
+			return ctrl.Result{RequeueAfter: unhealthySyncInterval}, err
+		}
+
 		if controllerutil.ContainsFinalizer(milvus, MilvusFinalizerName) {
 			if err := r.Finalize(ctx, *milvus); err != nil {
 				return ctrl.Result{}, err

diff --git a/pkg/controllers/milvus_controller_test.go b/pkg/controllers/milvus_controller_test.go
@@ -9,6 +9,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/milvus-io/milvus-operator/apis/milvus.io/v1beta1"
@@ -17,7 +18,19 @@ import (
 	"github.com/milvus-io/milvus-operator/pkg/util"
 )
 
+var mockCheckMilvusStopRet = false
+var mockCheckMilvusStopErr error = nil
+var mockCheckMilvusStop = func(ctx context.Context, cli client.Client, mc v1beta1.Milvus) (bool, error) {
+	return mockCheckMilvusStopRet, mockCheckMilvusStopErr
+}
+
 func TestClusterReconciler(t *testing.T) {
+	bak := CheckMilvusStopped
+	defer func() {
+		CheckMilvusStopped = bak
+	}()
+	CheckMilvusStopped = mockCheckMilvusStop
+
 	config.Init(util.GetGitRepoRootDir())
 
 	ctrl := gomock.NewController(t)
@@ -65,7 +78,7 @@ func TestClusterReconciler(t *testing.T) {
 
 	t.Run("case delete remove finalizer", func(t *testing.T) {
 		defer ctrl.Finish()
-
+		mockCheckMilvusStopRet = true
 		m.ObjectMeta.DeletionTimestamp = &metav1.Time{Time: time.Now()}
 		mockClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).
 			Do(func(ctx, key, obj interface{}) {
@@ -88,6 +101,27 @@ func TestClusterReconciler(t *testing.T) {
 		_, err := r.Reconcile(ctx, reconcile.Request{})
 		assert.NoError(t, err)
 	})
+
+	t.Run("milvus not stopped or check failed", func(t *testing.T) {
+		defer ctrl.Finish()
+		mockClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).
+			Do(func(ctx, key, obj interface{}) {
+				o := obj.(*v1beta1.Milvus)
+				*o = m
+			}).
+			Return(nil).Times(2)
+
+		mockCheckMilvusStopRet = false
+		m.Status.Status = v1beta1.StatusDeleting
+		m.ObjectMeta.DeletionTimestamp = &metav1.Time{Time: time.Now()}
+		_, err := r.Reconcile(ctx, reconcile.Request{})
+		assert.NoError(t, err)
+
+		mockCheckMilvusStopErr = errMock
+		ret, err := r.Reconcile(ctx, reconcile.Request{})
+		assert.Error(t, err)
+		assert.True(t, ret.RequeueAfter > 0)
+	})
 }
 
 func TestMilvusReconciler_ReconcileLegacyValues(t *testing.T) {