Merge pull request #210 from fluxcd/gc-prune

Refactor garbage collection

config/manager/deployment.yaml

@@ -17,7 +17,7 @@ spec:
         prometheus.io/scrape: "true"
         prometheus.io/port: "8080"
     spec:
-      terminationGracePeriodSeconds: 10
+      terminationGracePeriodSeconds: 60
       # Required for AWS IAM Role bindings
       # https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html
       securityContext:

controllers/kustomization_controller.go

@@ -382,7 +382,7 @@ func (r *KustomizationReconciler) reconcile(
 	}
 
 	// prune
-	err = r.prune(client, kustomization, snapshot)
+	err = r.prune(client, kustomization, checksum)
 	if err != nil {
 		return kustomizev1.KustomizationNotReady(
 			kustomization,
@@ -552,7 +552,7 @@ func (r *KustomizationReconciler) reconcileDelete(ctx context.Context, log logr.
 			log.Error(err, "Unable to prune for finalizer")
 			return ctrl.Result{}, err
 		}
-		if err := r.prune(client, kustomization, kustomization.Status.Snapshot); err != nil {
+		if err := r.prune(client, kustomization, ""); err != nil {
 			r.event(kustomization, kustomization.Status.LastAppliedRevision, events.EventSeverityError, "pruning for deleted resource failed", nil)
 			// Return the error so we retry the failed garbage collection
 			return ctrl.Result{}, err
@@ -777,15 +777,15 @@ func (r *KustomizationReconciler) applyWithRetry(kustomization kustomizev1.Kusto
 	return changeSet, nil
 }
 
-func (r *KustomizationReconciler) prune(client client.Client, kustomization kustomizev1.Kustomization, snapshot *kustomizev1.Snapshot) error {
-	if !kustomization.Spec.Prune || kustomization.Status.Snapshot == nil || snapshot == nil {
+func (r *KustomizationReconciler) prune(client client.Client, kustomization kustomizev1.Kustomization, newChecksum string) error {
+	if !kustomization.Spec.Prune || kustomization.Status.Snapshot == nil {
 		return nil
 	}
-	if kustomization.DeletionTimestamp.IsZero() && kustomization.Status.Snapshot.Checksum == snapshot.Checksum {
+	if kustomization.DeletionTimestamp.IsZero() && kustomization.Status.Snapshot.Checksum == newChecksum {
 		return nil
 	}
 
-	gc := NewGarbageCollector(client, *kustomization.Status.Snapshot, r.Log)
+	gc := NewGarbageCollector(client, *kustomization.Status.Snapshot, newChecksum, r.Log)
 
 	if output, ok := gc.Prune(kustomization.GetTimeout(),
 		kustomization.GetName(),
@@ -798,7 +798,7 @@ func (r *KustomizationReconciler) prune(client client.Client, kustomization kust
 				strings.ToLower(kustomization.Kind),
 				fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),
 			).Info(fmt.Sprintf("garbage collection completed: %s", output))
-			r.event(kustomization, snapshot.Checksum, events.EventSeverityInfo, output, nil)
+			r.event(kustomization, newChecksum, events.EventSeverityInfo, output, nil)
 		}
 	}
 	return nil

controllers/kustomization_gc.go

@@ -30,16 +30,18 @@ import (
 )
 
 type KustomizeGarbageCollector struct {
-	snapshot kustomizev1.Snapshot
-	log      logr.Logger
+	snapshot    kustomizev1.Snapshot
+	newChecksum string
+	log         logr.Logger
 	client.Client
 }
 
-func NewGarbageCollector(kubeClient client.Client, snapshot kustomizev1.Snapshot, log logr.Logger) *KustomizeGarbageCollector {
+func NewGarbageCollector(kubeClient client.Client, snapshot kustomizev1.Snapshot, newChecksum string, log logr.Logger) *KustomizeGarbageCollector {
 	return &KustomizeGarbageCollector{
-		Client:   kubeClient,
-		snapshot: snapshot,
-		log:      log,
+		Client:      kubeClient,
+		snapshot:    snapshot,
+		newChecksum: newChecksum,
+		log:         log,
 	}
 }
 
@@ -65,10 +67,10 @@ func (kgc *KustomizeGarbageCollector) Prune(timeout time.Duration, name string,
 				Version: gvk.Version,
 			})
 
-			err := kgc.List(ctx, ulist, client.InNamespace(ns), kgc.matchingLabels(name, namespace, kgc.snapshot.Checksum))
+			err := kgc.List(ctx, ulist, client.InNamespace(ns), kgc.matchingLabels(name, namespace))
 			if err == nil {
 				for _, item := range ulist.Items {
-					if item.GetDeletionTimestamp().IsZero() {
+					if kgc.isStale(item) && item.GetDeletionTimestamp().IsZero() {
 						name := fmt.Sprintf("%s/%s/%s", item.GetKind(), item.GetNamespace(), item.GetName())
 						err = kgc.Delete(ctx, &item)
 						if err != nil {
@@ -94,10 +96,10 @@ func (kgc *KustomizeGarbageCollector) Prune(timeout time.Duration, name string,
 			Version: gvk.Version,
 		})
 
-		err := kgc.List(ctx, ulist, kgc.matchingLabels(name, namespace, kgc.snapshot.Checksum))
+		err := kgc.List(ctx, ulist, kgc.matchingLabels(name, namespace))
 		if err == nil {
 			for _, item := range ulist.Items {
-				if item.GetDeletionTimestamp().IsZero() {
+				if kgc.isStale(item) && item.GetDeletionTimestamp().IsZero() {
 					name := fmt.Sprintf("%s/%s", item.GetKind(), item.GetName())
 					err = kgc.Delete(ctx, &item)
 					if err != nil {
@@ -120,8 +122,13 @@ func (kgc *KustomizeGarbageCollector) Prune(timeout time.Duration, name string,
 	return changeSet, true
 }
 
-func (kgc *KustomizeGarbageCollector) matchingLabels(name, namespace, checksum string) client.MatchingLabels {
-	return gcLabels(name, namespace, checksum)
+func (kgc *KustomizeGarbageCollector) isStale(obj unstructured.Unstructured) bool {
+	itemChecksum := obj.GetLabels()[fmt.Sprintf("%s/checksum", kustomizev1.GroupVersion.Group)]
+	return kgc.newChecksum == "" || itemChecksum != kgc.newChecksum
+}
+
+func (kgc *KustomizeGarbageCollector) matchingLabels(name, namespace string) client.MatchingLabels {
+	return selectorLabels(name, namespace)
 }
 
 func gcLabels(name, namespace, checksum string) map[string]string {

go.mod

@@ -30,3 +30,7 @@ require (
 	sigs.k8s.io/kustomize/api v0.7.0
 	sigs.k8s.io/yaml v1.2.0
 )
+
+// controller-runtime v0.6.4 bug:
+// v1.ListOptions is not suitable for converting to \"helm.toolkit.fluxcd.io/v2beta1\" in scheme \"pkg/runtime/scheme.go:101
+replace sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.6.3