Issue alerts for install, upgrade and reconcile errors

Signed-off-by: Stefan Prodan <[email protected]>

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/fluxcd/pkg/apis/kustomize v1.8.0
 	github.com/fluxcd/pkg/apis/meta v1.9.0
 	github.com/fluxcd/pkg/kustomize v1.15.0
-	github.com/fluxcd/pkg/runtime v0.51.0
+	github.com/fluxcd/pkg/runtime v0.51.1
 	github.com/fluxcd/pkg/ssa v0.43.0
 	github.com/fluxcd/pkg/tar v0.10.0
 	github.com/golang-jwt/jwt/v4 v4.5.1
@@ -59,6 +59,7 @@ require (
 	github.com/emicklei/go-restful/v3 v3.12.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
+	github.com/fluxcd/pkg/apis/event v0.12.0 // indirect
 	github.com/fluxcd/pkg/envsubst v1.3.0 // indirect
 	github.com/fluxcd/pkg/sourceignore v0.10.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
@@ -83,6 +84,8 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/josharian/intern v1.0.0 // indirect

go.sum

@@ -71,8 +71,12 @@ github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0
 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
 github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4=
 github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/fluxcd/cli-utils v0.36.0-flux.11 h1:W0y2uvCVkcE8bgV9jgoGSjzWbLFiNq1AjrWtuxllek8=
 github.com/fluxcd/cli-utils v0.36.0-flux.11/go.mod h1:WZ7xUpZbK+O6HBxA5UWqzWTLSSltdmj4wS1LstS5Dqs=
+github.com/fluxcd/pkg/apis/event v0.12.0 h1:+zQVefTG3+THYRS48dtZkoA1rdbZZNx3t6wnbzprFIE=
+github.com/fluxcd/pkg/apis/event v0.12.0/go.mod h1:aRK2AONnjjSNW61B6Iy3SW4YHozACntnJeGm3fFqDqA=
 github.com/fluxcd/pkg/apis/kustomize v1.8.0 h1:HH6YRa3SMS72KK4cUyb9m5sK/dZH+Eti1qhjWDCgwKg=
 github.com/fluxcd/pkg/apis/kustomize v1.8.0/go.mod h1:QCKIFj1ocdndaWSkrLs5JKvdGNYyTzQX1ZB3lYTwma0=
 github.com/fluxcd/pkg/apis/meta v1.9.0 h1:wPgm7bWNJZ/ImS5GqikOxt362IgLPFBG73dZ27uWRiQ=
@@ -81,8 +85,8 @@ github.com/fluxcd/pkg/envsubst v1.3.0 h1:84Ain+8EBvyzu6y0FsKRwNsvaSiKuqhTqeh/4yo
 github.com/fluxcd/pkg/envsubst v1.3.0/go.mod h1:lz6HvqDnxbX0sIqjr1fxw0oTGYACLVFcOE/srKS0VQQ=
 github.com/fluxcd/pkg/kustomize v1.15.0 h1:lII4FW9EJl0rI20dk+Glg5C2JZhP343FBov7HwW+SQo=
 github.com/fluxcd/pkg/kustomize v1.15.0/go.mod h1:e2SGi7cl28c9cnBVZ8YV8HAS4VBgUsiM6HMqv/AHJWQ=
-github.com/fluxcd/pkg/runtime v0.51.0 h1:F4gKLUBUdvUdtg2lBsg72KUPqlOnaf9ChEL8bmP7CvQ=
-github.com/fluxcd/pkg/runtime v0.51.0/go.mod h1:uMJ+s81+TyNGVjcnn+PIXUGGYs9VA3AK8nDmQWXAnis=
+github.com/fluxcd/pkg/runtime v0.51.1 h1:68C6V/P2l/IwivqzvkgcR6Aa7zKds5ihsvoo0NcLarA=
+github.com/fluxcd/pkg/runtime v0.51.1/go.mod h1:uMJ+s81+TyNGVjcnn+PIXUGGYs9VA3AK8nDmQWXAnis=
 github.com/fluxcd/pkg/sourceignore v0.10.0 h1:z5Bhh0G990uLbwjKNj7SzYqbGkicpGcXxF/Z4ZSVB64=
 github.com/fluxcd/pkg/sourceignore v0.10.0/go.mod h1:d1d9hcFxf+grda6JL3k+mC09nVTtBb9kJVzQn6J77B0=
 github.com/fluxcd/pkg/ssa v0.43.0 h1:XmADD3C0erYZayKfGI0WTsMlW9TtS4bp5gy4Axo1dcA=
@@ -139,6 +143,12 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aN
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA=
 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
+github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
+github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
+github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
@@ -161,6 +171,10 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhn
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=

internal/controller/fluxinstance_controller.go

@@ -15,6 +15,7 @@ import (
 	"github.com/fluxcd/cli-utils/pkg/kstatus/polling"
 	"github.com/fluxcd/pkg/apis/meta"
 	"github.com/fluxcd/pkg/runtime/conditions"
+	"github.com/fluxcd/pkg/runtime/events"
 	"github.com/fluxcd/pkg/runtime/patch"
 	"github.com/fluxcd/pkg/ssa"
 	"github.com/fluxcd/pkg/ssa/normalize"
@@ -170,7 +171,9 @@ func (r *FluxInstanceReconciler) reconcile(ctx context.Context,
 			meta.BuildFailedReason,
 			"%s", msg)
 		log.Error(err, msg)
-		r.EventRecorder.Event(obj, corev1.EventTypeWarning, meta.BuildFailedReason, msg)
+		if eventErr := r.notify(obj, meta.BuildFailedReason, corev1.EventTypeWarning, msg); eventErr != nil {
+			log.Error(eventErr, "failed to emit event for build failure")
+		}
 		return ctrl.Result{}, nil
 	}
 
@@ -195,7 +198,9 @@ func (r *FluxInstanceReconciler) reconcile(ctx context.Context,
 			meta.ReadyCondition,
 			meta.ReconciliationFailedReason,
 			"%s", msg)
-		r.EventRecorder.Event(obj, corev1.EventTypeWarning, meta.ReconciliationFailedReason, msg)
+		if eventErr := r.notify(obj, meta.ReconciliationFailedReason, corev1.EventTypeWarning, msg); eventErr != nil {
+			log.Error(eventErr, "failed to emit event for reconciliation failure")
+		}
 
 		return ctrl.Result{}, err
 	}
@@ -213,7 +218,7 @@ func (r *FluxInstanceReconciler) reconcile(ctx context.Context,
 		map[string]string{fluxcdv1.RevisionAnnotation: obj.Status.LastAppliedRevision},
 		corev1.EventTypeNormal,
 		meta.ReconciliationSucceededReason,
-		msg)
+		"%s", msg)
 
 	return requeueAfter(obj), nil
 }
@@ -520,6 +525,20 @@ func (r *FluxInstanceReconciler) apply(ctx context.Context,
 		}
 	}
 
+	// Send event to notification-controller only if the server-side apply resulted in changes.
+	applyLog := strings.TrimSuffix(changeSetLog.String(), "\n")
+	if applyLog != "" {
+		action := "updated"
+		if len(oldInventory.Entries) == 0 {
+			action = "installed"
+		}
+
+		msg := fmt.Sprintf("Flux %s revision %s\n%s", action, buildResult.Revision, applyLog)
+		if err := r.notify(obj, meta.ReconciliationSucceededReason, corev1.EventTypeNormal, msg); err != nil {
+			log.Error(err, "failed to emit event for reconciliation success")
+		}
+	}
+
 	return nil
 }
 
@@ -626,3 +645,28 @@ func (r *FluxInstanceReconciler) recordMetrics(obj *fluxcdv1.FluxInstance) error
 	reporter.RecordMetrics(unstructured.Unstructured{Object: rawMap})
 	return nil
 }
+
+func (r *FluxInstanceReconciler) notify(obj *fluxcdv1.FluxInstance, reason, eventType, msg string) error {
+	notificationAddress := ""
+	if builder.ContainElementString(obj.GetComponents(), builder.MakeDefaultOptions().NotificationController) {
+		notificationAddress = fmt.Sprintf("http://%s.%s.svc.%s/",
+			builder.MakeDefaultOptions().NotificationController,
+			obj.GetNamespace(),
+			obj.GetCluster().Domain,
+		)
+	}
+
+	eventRecorder, err := events.NewRecorderForScheme(
+		r.Scheme,
+		r.EventRecorder,
+		ctrl.Log,
+		notificationAddress,
+		r.StatusManager)
+	if err != nil {
+		return err
+	}
+
+	go eventRecorder.Eventf(obj, eventType, reason, "%s", msg)
+
+	return nil
+}

internal/controller/fluxinstance_controller_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/fluxcd/pkg/runtime/conditions"
 	. "github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -190,16 +191,18 @@ func TestFluxInstanceReconciler_LifeCycle(t *testing.T) {
 
 	// Check if events were recorded for each step.
 	events := getEvents(result.Name)
-	g.Expect(events).To(HaveLen(5))
+	g.Expect(events).To(HaveLen(7))
 	g.Expect(events[0].Reason).To(Equal(fluxcdv1.OutdatedReason))
 	g.Expect(events[1].Reason).To(Equal(meta.ProgressingReason))
 	g.Expect(events[1].Message).To(HavePrefix("Installing"))
 	g.Expect(events[2].Reason).To(Equal(meta.ReconciliationSucceededReason))
 	g.Expect(events[2].Message).To(HavePrefix("Reconciliation finished"))
-	g.Expect(events[3].Reason).To(Equal(meta.ProgressingReason))
-	g.Expect(events[3].Message).To(HavePrefix("Upgrading"))
-	g.Expect(events[4].Reason).To(Equal(meta.ReconciliationSucceededReason))
-	g.Expect(events[4].Annotations).To(HaveKeyWithValue(fluxcdv1.RevisionAnnotation, resultFinal.Status.LastAppliedRevision))
+	g.Expect(events[3].Reason).To(Equal(meta.ReconciliationSucceededReason))
+	g.Expect(events[3].Message).To(HavePrefix("Flux installed"))
+	g.Expect(events[4].Reason).To(Equal(meta.ProgressingReason))
+	g.Expect(events[4].Message).To(HavePrefix("Upgrading"))
+	g.Expect(events).To(ContainElement(WithTransform(func(e corev1.Event) string { return e.Message }, ContainSubstring("Flux updated"))))
+	g.Expect(events).To(ContainElement(WithTransform(func(e corev1.Event) string { return e.Message }, ContainSubstring("Reconciliation finished"))))
 
 	err = testClient.Delete(ctx, obj)
 	g.Expect(err).ToNot(HaveOccurred())
@@ -486,12 +489,9 @@ func TestFluxInstanceReconciler_Disabled(t *testing.T) {
 	err = testClient.Get(ctx, client.ObjectKeyFromObject(obj), resultFinal)
 	g.Expect(err).ToNot(HaveOccurred())
 
-	// Check if events were recorded for each step.
+	// Check if the ReconciliationDisabled event was recorded.
 	events := getEvents(result.Name)
-	g.Expect(events).To(HaveLen(4))
-	g.Expect(events[1].Reason).To(Equal(meta.ProgressingReason))
-	g.Expect(events[2].Reason).To(Equal(meta.ReconciliationSucceededReason))
-	g.Expect(events[3].Reason).To(Equal("ReconciliationDisabled"))
+	g.Expect(events[len(events)-1].Reason).To(Equal("ReconciliationDisabled"))
 
 	// Check that resources were not deleted.
 	kc := &appsv1.Deployment{}
@@ -672,7 +672,7 @@ func TestFluxInstanceReconciler_NewVersion(t *testing.T) {
 
 	// Check if events were recorded for each step.
 	events := getEvents(obj.Name)
-	g.Expect(events).To(HaveLen(3))
+	g.Expect(events).To(HaveLen(4))
 	g.Expect(events[0].Reason).To(Equal("OutdatedVersion"))
 
 	err = testClient.Delete(ctx, obj)