Use evict instead of delete when updating image

* Make sure failed to evict pods get unquiesced

controllers/pod.go

@@ -437,10 +437,21 @@ func (r *SingleClusterReconciler) deletePodAndEnsureImageUpdated(
 		return reconcileError(err)
 	}
 
+	failedEvictedPods := make([]*corev1.Pod, 0)
+
 	// Delete pods
 	for _, p := range podsToUpdate {
-		if err := r.Client.Delete(context.TODO(), p); err != nil {
-			return reconcileError(err)
+		if err := r.KubeClient.CoreV1().Pods(p.Namespace).Evict(context.TODO(),
+			&policyv1beta1.Eviction{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      p.Name,
+					Namespace: p.Namespace,
+				},
+			}); err != nil {
+			r.Log.Error(err, fmt.Sprintf("Not evictable pod %s in ns %s. Will QuiesceUndo and retry. Error: %s", p.Name, p.Namespace, err.Error()))
+			// in case of error during the eviction, unquiesce the node since it has been quiesced.
+			failedEvictedPods = append(failedEvictedPods, p)
+			continue
 		}
 
 		r.Log.V(1).Info("Pod deleted", "podName", p.Name)
@@ -450,6 +461,12 @@ func (r *SingleClusterReconciler) deletePodAndEnsureImageUpdated(
 		)
 	}
 
+	if len(failedEvictedPods) > 0 {
+		if err := r.quiesceUndoPods(r.getClientPolicy(), failedEvictedPods); err != nil {
+			r.Log.Error(err, "Unexpected error during quiesce-undo command")
+		}
+	}
+
 	return r.ensurePodsImageUpdated(podsToUpdate)
 }