Correct cluster drift using patches

This changes the cluster drift correction behavior from performing a
Helm upgrade to performing create and patch API requests based on the
JSON Patch data.

Doing this is much lighter than performing a full release cycle, and
deals with the issue of Helm being unable to restore state of Custom
Resources without the `--force` flag being set. Which has unwanted
side-effects like forcing objects through a deletion/creation cycle.

After a drift correction attempt a Kubernetes Event is emitted, which
contains a summary of the created and patched resources, and a
collection of any (potential) errors.

As the goal is to restore state as best as we can, the drift correction
will be re-attempted until all resources have been restored to the
desired state.

Signed-off-by: Hidde Beydals <[email protected]>

go.mod

@@ -15,13 +15,14 @@ replace (
 )
 
 require (
+	github.com/fluxcd/cli-utils v0.36.0-flux.1
 	github.com/fluxcd/helm-controller/api v0.36.2
 	github.com/fluxcd/pkg/apis/acl v0.1.0
 	github.com/fluxcd/pkg/apis/event v0.6.0
 	github.com/fluxcd/pkg/apis/kustomize v1.2.0
 	github.com/fluxcd/pkg/apis/meta v1.2.0
 	github.com/fluxcd/pkg/runtime v0.43.0
-	github.com/fluxcd/pkg/ssa v0.34.0
+	github.com/fluxcd/pkg/ssa v0.34.1-0.20231127131230-ce912554b6d1
 	github.com/fluxcd/pkg/testserver v0.5.0
 	github.com/fluxcd/source-controller/api v1.1.2
 	github.com/go-logr/logr v1.3.0
@@ -77,7 +78,6 @@ require (
 	github.com/evanphx/json-patch/v5 v5.7.0 // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
 	github.com/fatih/color v1.13.0 // indirect
-	github.com/fluxcd/cli-utils v0.36.0-flux.1 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-gorp/gorp/v3 v3.1.0 // indirect

go.sum

@@ -100,8 +100,8 @@ github.com/fluxcd/pkg/apis/meta v1.2.0 h1:O766PzGAdMdQKybSflGL8oV0+GgCNIkdsxfalR
 github.com/fluxcd/pkg/apis/meta v1.2.0/go.mod h1:fU/Az9AoVyIxC0oI4ihG0NVMNnvrcCzdEym3wxjIQsc=
 github.com/fluxcd/pkg/runtime v0.43.0 h1:dU4cWct5VTpddGzJUU80zxNl80jbbVEN5Y5rbt4YUnw=
 github.com/fluxcd/pkg/runtime v0.43.0/go.mod h1:RuqJ9VEXELjzgurK2+UXBBgVN1vS0hZ7CYVG2xBAEVM=
-github.com/fluxcd/pkg/ssa v0.34.0 h1:hpMo0D7G3faieRYH39e9YD8Jl+aC2hTgUep8ojG5+LE=
-github.com/fluxcd/pkg/ssa v0.34.0/go.mod h1:rhVh0EtYVUOznKXlz6E7JOSgdc8xWbIwA4L5HVtJRLA=
+github.com/fluxcd/pkg/ssa v0.34.1-0.20231127131230-ce912554b6d1 h1:8FrXuX1Z3jc+0WGYfwCo5KPas71X22+Hkuj7CSbP5RM=
+github.com/fluxcd/pkg/ssa v0.34.1-0.20231127131230-ce912554b6d1/go.mod h1:rhVh0EtYVUOznKXlz6E7JOSgdc8xWbIwA4L5HVtJRLA=
 github.com/fluxcd/pkg/testserver v0.5.0 h1:n/Iskk0tXNt2AgIgjz9qeFK/VhEXGfqeazABXZmO2Es=
 github.com/fluxcd/pkg/testserver v0.5.0/go.mod h1:/p4st6d0uPLy8wXydeF/kDJgxUYO9u2NqySuXb9S+Fo=
 github.com/fluxcd/source-controller/api v1.1.2 h1:FfKDKVWnopo+Q2pOAxgHEjrtr4MP41L8aapR4mqBhBk=

internal/action/diff.go

@@ -18,20 +18,27 @@ package action
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
+	"sort"
 	"strings"
 
 	helmaction "helm.sh/helm/v3/pkg/action"
 	helmrelease "helm.sh/helm/v3/pkg/release"
-	"k8s.io/apimachinery/pkg/util/errors"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	apierrutil "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 
+	"github.com/fluxcd/cli-utils/pkg/object"
 	"github.com/fluxcd/pkg/ssa"
 	"github.com/fluxcd/pkg/ssa/jsondiff"
 
 	v2 "github.com/fluxcd/helm-controller/api/v2beta2"
+	"github.com/fluxcd/helm-controller/internal/diff"
 )
 
 // Diff returns a jsondiff.DiffSet of the changes between the state of the
@@ -86,7 +93,6 @@ func Diff(ctx context.Context, config *helmaction.Configuration, rls *helmreleas
 	diffOpts := []jsondiff.ListOption{
 		jsondiff.FieldOwner(fieldOwner),
 		jsondiff.ExclusionSelector{v2.DriftDetectionMetadataKey: v2.DriftDetectionDisabledValue},
-		jsondiff.MaskSecrets(true),
 		jsondiff.Rationalize(true),
 		jsondiff.Graceful(true),
 	}
@@ -119,5 +125,120 @@ func Diff(ctx context.Context, config *helmaction.Configuration, rls *helmreleas
 	if err != nil {
 		errs = append(errs, err)
 	}
-	return set, errors.Reduce(errors.Flatten(errors.NewAggregate(errs)))
+	return set, apierrutil.Reduce(apierrutil.Flatten(apierrutil.NewAggregate(errs)))
+}
+
+// ApplyDiff applies the changes described in the provided jsondiff.DiffSet to
+// the Kubernetes cluster.
+func ApplyDiff(ctx context.Context, config *helmaction.Configuration, diffSet jsondiff.DiffSet, fieldOwner string) (*ssa.ChangeSet, error) {
+	cfg, err := config.RESTClientGetter.ToRESTConfig()
+	if err != nil {
+		return nil, err
+	}
+	c, err := client.New(cfg, client.Options{})
+	if err != nil {
+		return nil, err
+	}
+
+	var toCreate, toPatch sortableDiffs
+	for _, d := range diffSet {
+		switch d.Type {
+		case jsondiff.DiffTypeCreate:
+			toCreate = append(toCreate, d)
+		case jsondiff.DiffTypeUpdate:
+			toPatch = append(toPatch, d)
+		}
+	}
+
+	var (
+		changeSet = ssa.NewChangeSet()
+		errs      []error
+	)
+
+	sort.Sort(toCreate)
+	for _, d := range toCreate {
+		obj := d.DesiredObject.DeepCopyObject().(client.Object)
+		if err := c.Create(ctx, obj, client.FieldOwner(fieldOwner)); err != nil {
+			errs = append(errs, fmt.Errorf("%s creation failure: %w", diff.ResourceName(obj), err))
+			continue
+		}
+		changeSet.Add(objectToChangeSetEntry(obj, ssa.CreatedAction))
+	}
+
+	sort.Sort(toPatch)
+	for _, d := range toPatch {
+		data, err := json.Marshal(d.Patch)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("%s patch failure: %w", diff.ResourceName(d.DesiredObject), err))
+			continue
+		}
+
+		obj := d.DesiredObject.DeepCopyObject().(client.Object)
+		patch := client.RawPatch(types.JSONPatchType, data)
+		if err := c.Patch(ctx, obj, patch, client.FieldOwner(fieldOwner)); err != nil {
+			if obj.GetObjectKind().GroupVersionKind().Kind == "Secret" {
+				err = maskSensitiveErrData(err)
+			}
+			errs = append(errs, fmt.Errorf("%s patch failure: %w", diff.ResourceName(obj), err))
+			continue
+		}
+		changeSet.Add(objectToChangeSetEntry(obj, ssa.ConfiguredAction))
+	}
+
+	return changeSet, apierrutil.NewAggregate(errs)
+}
+
+// objectToChangeSetEntry returns a ssa.ChangeSetEntry for the given object and
+// action.
+func objectToChangeSetEntry(obj client.Object, action ssa.Action) ssa.ChangeSetEntry {
+	return ssa.ChangeSetEntry{
+		ObjMetadata: object.ObjMetadata{
+			GroupKind: obj.GetObjectKind().GroupVersionKind().GroupKind(),
+			Name:      obj.GetName(),
+			Namespace: obj.GetNamespace(),
+		},
+		GroupVersion: obj.GetObjectKind().GroupVersionKind().Version,
+		Subject:      diff.ResourceName(obj),
+		Action:       action,
+	}
+}
+
+// maskSensitiveErrData masks potentially sensitive data from the error message
+// returned by the Kubernetes API server.
+// This avoids leaking any sensitive data in logs or other output when a patch
+// operation fails.
+func maskSensitiveErrData(err error) error {
+	if apierrors.IsInvalid(err) {
+		// The last part of the error message is the reason for the error.
+		if i := strings.LastIndex(err.Error(), `:`); i != -1 {
+			err = errors.New(strings.TrimSpace(err.Error()[i+1:]))
+		}
+	}
+	return err
+}
+
+// sortableDiffs is a sortable slice of jsondiff.Diffs.
+type sortableDiffs []*jsondiff.Diff
+
+// Len returns the length of the slice.
+func (s sortableDiffs) Len() int { return len(s) }
+
+// Swap swaps the elements with indexes i and j.
+func (s sortableDiffs) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+
+// Less returns true if the element with index i should sort before the element
+// with index j.
+// The elements are sorted by GroupKind, Namespace and Name.
+func (s sortableDiffs) Less(i, j int) bool {
+	iDiff, jDiff := s[i], s[j]
+
+	if !ssa.Equals(iDiff.GroupVersionKind().GroupKind(), jDiff.GroupVersionKind().GroupKind()) {
+		return ssa.IsLessThan(iDiff.GroupVersionKind().GroupKind(), jDiff.GroupVersionKind().GroupKind())
+	}
+
+	if iDiff.GetNamespace() != jDiff.GetNamespace() {
+		return iDiff.GetNamespace() < jDiff.GetNamespace()
+	}
+
+	return iDiff.GetName() < jDiff.GetName()
 }