[KO-245] Deleting PVC of local volumes during node maintenance.

api/v1/aerospikecluster_types.go

@@ -73,6 +73,8 @@ type AerospikeClusterSpec struct { //nolint:govet // for readability
 	SeedsFinderServices SeedsFinderServices `json:"seedsFinderServices,omitempty"`
 	// RosterNodeBlockList is a list of blocked nodeIDs from roster in a strong-consistency setup
 	RosterNodeBlockList []string `json:"rosterNodeBlockList,omitempty"`
+	// CleanLocalPVC is a flag which allows operator to delete local PVC in case of kubernetes node crash
+	CleanLocalPVC bool `json:"cleanLocalPVC,omitempty"`
 }
 
 type SeedsFinderServices struct {

config/crd/bases/asdb.aerospike.com_aerospikeclusters.yaml

@@ -278,6 +278,10 @@ spec:
                     - customInterface
                     type: string
                 type: object
+              cleanLocalPVC:
+                description: CleanLocalPVC is a flag which allows operator to delete
+                  local PVC in case of kubernetes node crash
+                type: boolean
               image:
                 description: Aerospike server image
                 type: string

config/rbac/role.yaml

@@ -98,6 +98,7 @@ rules:
   - ""
   resources:
   - secrets
+  - persistentvolumes
   verbs:
   - get
 - apiGroups:

controllers/aerospikecluster_controller.go

@@ -47,6 +47,38 @@ type AerospikeClusterReconciler struct {
 	Log        logr.Logger
 }
 
+func stsPodPendingPredicate(e event.UpdateEvent) bool {
+	if e.ObjectOld == nil || e.ObjectNew == nil {
+		return false
+	}
+
+	oldSTS, ok := e.ObjectOld.(*appsv1.StatefulSet)
+	if !ok {
+		return false
+	}
+
+	newSTS, ok := e.ObjectNew.(*appsv1.StatefulSet)
+	if !ok {
+		return false
+	}
+
+	if oldSTS.Status.ObservedGeneration == newSTS.Status.ObservedGeneration &&
+		newSTS.Status.AvailableReplicas < oldSTS.Status.AvailableReplicas {
+		return true
+	}
+
+	return false
+}
+
+type StatusPendingPredicate struct {
+	predicate.Funcs
+}
+
+// Update implements default UpdateEvent filter for statefulSets.
+func (StatusPendingPredicate) Update(e event.UpdateEvent) bool {
+	return stsPodPendingPredicate(e)
+}
+
 // SetupWithManager sets up the controller with the Manager
 func (r *AerospikeClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
@@ -57,9 +89,7 @@ func (r *AerospikeClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 					CreateFunc: func(e event.CreateEvent) bool {
 						return false
 					},
-					UpdateFunc: func(e event.UpdateEvent) bool {
-						return false
-					},
+					UpdateFunc: stsPodPendingPredicate,
 				},
 			),
 		).
@@ -68,7 +98,8 @@ func (r *AerospikeClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 				MaxConcurrentReconciles: maxConcurrentReconciles,
 			},
 		).
-		WithEventFilter(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{})).
+		WithEventFilter(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{},
+			StatusPendingPredicate{})).
 		Complete(r)
 }
 
@@ -82,6 +113,7 @@ type RackState struct {
 // +kubebuilder:rbac:groups=core,resources=pods/exec,verbs=create
 // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get
 // +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get

controllers/pod.go

@@ -261,6 +261,12 @@ func (r *SingleClusterReconciler) restartPods(
 			}
 		}
 
+		if r.aeroCluster.Spec.CleanLocalPVC {
+			if err := r.deleteLocalPVCs(pod); err != nil {
+				return reconcileError(err)
+			}
+		}
+
 		if err := r.Client.Delete(context.TODO(), pod); err != nil {
 			r.Log.Error(err, "Failed to delete pod")
 			return reconcileError(err)
@@ -399,6 +405,12 @@ func (r *SingleClusterReconciler) deletePodAndEnsureImageUpdated(
 
 	// Delete pods
 	for _, p := range podsToUpdate {
+		if r.aeroCluster.Spec.CleanLocalPVC {
+			if err := r.deleteLocalPVCs(p); err != nil {
+				return reconcileError(err)
+			}
+		}
+
 		if err := r.Client.Delete(context.TODO(), p); err != nil {
 			return reconcileError(err)
 		}

controllers/pvc.go

@@ -3,10 +3,12 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	asdbv1 "github.com/aerospike/aerospike-kubernetes-operator/api/v1"
@@ -103,6 +105,39 @@ func (r *SingleClusterReconciler) removePVCsAsync(
 	return deletedPVCs, nil
 }
 
+func (r *SingleClusterReconciler) deleteLocalPVCs(pod *corev1.Pod) error {
+	if pod.Status.Phase == corev1.PodPending && utils.IsPodNeedsToMigrate(pod) {
+		rackID, err := strconv.Atoi(pod.Labels[asdbv1.AerospikeRackIDLabel])
+		if err != nil {
+			return err
+		}
+
+		pvcItems, err := r.getPodsPVCList([]string{pod.Name}, rackID)
+		if err != nil {
+			return fmt.Errorf("could not find pvc for pod %v: %v", pod.Name, err)
+		}
+
+		for idx := range pvcItems {
+			pv := &corev1.PersistentVolume{}
+			pvName := types.NamespacedName{Name: pvcItems[idx].Spec.VolumeName}
+
+			if err := r.Client.Get(context.TODO(), pvName, pv); err != nil {
+				return err
+			}
+
+			if pv.Spec.Local != nil {
+				if err := r.Client.Delete(context.TODO(), &pvcItems[idx]); err != nil {
+					return fmt.Errorf(
+						"could not delete pvc %s: %v", pvcItems[idx].Name, err,
+					)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
 func (r *SingleClusterReconciler) waitForPVCTermination(deletedPVCs []corev1.PersistentVolumeClaim) error {
 	if len(deletedPVCs) == 0 {
 		return nil

controllers/statefulset.go

@@ -16,6 +16,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
@@ -597,16 +598,21 @@ func (r *SingleClusterReconciler) updateSTS(
 	// TODO: Add validation. device, file, both should not exist in same storage class
 	r.updateSTSStorage(statefulSet, rackState)
 
-	// Save the updated stateful set.
-	// Can we optimize this? Update stateful set only if there is any change
-	// in it.
-	err := r.Client.Update(context.TODO(), statefulSet, updateOption)
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		found, err := r.getSTS(rackState)
+		if err != nil {
+			return err
+		}
+
+		if reflect.DeepEqual(found.Spec, statefulSet.Spec) {
+			return nil
+		}
+		// Save the updated stateful set.
+		found.Spec = statefulSet.Spec
+		return r.Client.Update(context.TODO(), found, updateOption)
+	})
 	if err != nil {
-		return fmt.Errorf(
-			"failed to update StatefulSet %s: %v",
-			statefulSet.Name,
-			err,
-		)
+		return err
 	}
 
 	r.Log.V(1).Info(
@@ -775,7 +781,16 @@ func (r *SingleClusterReconciler) updateSTSNonPVStorage(
 		)
 
 		// Add volume in statefulSet template
+		perm := corev1.SecretVolumeSourceDefaultMode
 		k8sVolume := createVolumeForVolumeAttachment(volume)
+
+		switch {
+		case k8sVolume.Secret != nil:
+			k8sVolume.Secret.DefaultMode = &perm
+		case k8sVolume.ConfigMap != nil:
+			k8sVolume.ConfigMap.DefaultMode = &perm
+		}
+
 		st.Spec.Template.Spec.Volumes = append(
 			st.Spec.Template.Spec.Volumes, k8sVolume,
 		)
@@ -1190,6 +1205,8 @@ func getDefaultAerospikeInitContainerVolumeMounts() []corev1.VolumeMount {
 func getDefaultSTSVolumes(
 	aeroCluster *asdbv1.AerospikeCluster, rackState *RackState,
 ) []corev1.Volume {
+	defaultMode := corev1.SecretVolumeSourceDefaultMode
+
 	return []corev1.Volume{
 		{
 			Name: confDirName,
@@ -1206,6 +1223,7 @@ func getDefaultSTSVolumes(
 							aeroCluster, rackState.Rack.ID,
 						).Name,
 					},
+					DefaultMode: &defaultMode,
 				},
 			},
 		},

helm-charts/aerospike-kubernetes-operator/templates/aerospike-operator-manager-clusterrole.yaml

@@ -102,6 +102,7 @@ rules:
   - ""
   resources:
   - secrets
+  - persistentvolumes
   verbs:
   - get
 - apiGroups:

pkg/utils/pod.go

@@ -32,7 +32,7 @@ func CheckPodFailed(pod *corev1.Pod) error {
 		return fmt.Errorf("pod %s has failed status", pod.Name)
 	}
 
-	if pod.Status.Phase == corev1.PodPending && isPodReasonUnschedulable(pod) {
+	if pod.Status.Phase == corev1.PodPending && IsPodReasonUnschedulable(pod) {
 		return fmt.Errorf("pod %s is in unschedulable state", pod.Name)
 	}
 
@@ -210,9 +210,22 @@ func isPodError(reason string) bool {
 	return strings.HasSuffix(reason, "Error")
 }
 
-func isPodReasonUnschedulable(pod *corev1.Pod) bool {
+func IsPodReasonUnschedulable(pod *corev1.Pod) bool {
 	for _, condition := range pod.Status.Conditions {
-		if condition.Type == corev1.PodScheduled && condition.Reason == corev1.PodReasonUnschedulable {
+		if condition.Type == corev1.PodScheduled && (condition.Reason == corev1.PodReasonUnschedulable ||
+			condition.Reason == corev1.PodReasonSchedulerError) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func IsPodNeedsToMigrate(pod *corev1.Pod) bool {
+	for _, condition := range pod.Status.Conditions {
+		if condition.Type == corev1.PodScheduled && (condition.Reason == corev1.PodReasonUnschedulable ||
+			condition.Reason == corev1.PodReasonSchedulerError &&
+				strings.Contains(condition.Message, "nodeinfo not found for node name")) {
 			return true
 		}
 	}

test/cluster_helper.go

@@ -1373,11 +1373,14 @@ func getStorageVolumeForAerospike(name, path string) asdbv1.VolumeSpec {
 }
 
 func getStorageVolumeForSecret() asdbv1.VolumeSpec {
+	perm := corev1.SecretVolumeSourceDefaultMode
+
 	return asdbv1.VolumeSpec{
 		Name: aerospikeConfigSecret,
 		Source: asdbv1.VolumeSource{
 			Secret: &corev1.SecretVolumeSource{
-				SecretName: tlsSecretName,
+				SecretName:  tlsSecretName,
+				DefaultMode: &perm,
 			},
 		},
 		Aerospike: &asdbv1.AerospikeServerVolumeAttachment{