From 66c01f0654d0605c80f770c43ab6c58473f4eba3 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sat, 21 Dec 2024 11:44:52 +0200 Subject: [PATCH] Issue alerts for install, upgrade and reconcile errors Signed-off-by: Stefan Prodan --- go.mod | 5 +- go.sum | 18 ++++++- .../controller/fluxinstance_controller.go | 50 +++++++++++++++++-- .../fluxinstance_controller_test.go | 25 +++++----- 4 files changed, 80 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index eaff7e2..266da59 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/fluxcd/pkg/apis/kustomize v1.8.0 github.com/fluxcd/pkg/apis/meta v1.9.0 github.com/fluxcd/pkg/kustomize v1.15.0 - github.com/fluxcd/pkg/runtime v0.51.0 + github.com/fluxcd/pkg/runtime v0.51.1 github.com/fluxcd/pkg/ssa v0.43.0 github.com/fluxcd/pkg/tar v0.10.0 github.com/golang-jwt/jwt/v4 v4.5.1 @@ -59,6 +59,7 @@ require ( github.com/emicklei/go-restful/v3 v3.12.0 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect + github.com/fluxcd/pkg/apis/event v0.12.0 // indirect github.com/fluxcd/pkg/envsubst v1.3.0 // indirect github.com/fluxcd/pkg/sourceignore v0.10.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect @@ -83,6 +84,8 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/go.sum b/go.sum index c03954d..574bd51 100644 --- a/go.sum +++ b/go.sum @@ -71,8 +71,12 @@ github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fluxcd/cli-utils v0.36.0-flux.11 h1:W0y2uvCVkcE8bgV9jgoGSjzWbLFiNq1AjrWtuxllek8= github.com/fluxcd/cli-utils v0.36.0-flux.11/go.mod h1:WZ7xUpZbK+O6HBxA5UWqzWTLSSltdmj4wS1LstS5Dqs= +github.com/fluxcd/pkg/apis/event v0.12.0 h1:+zQVefTG3+THYRS48dtZkoA1rdbZZNx3t6wnbzprFIE= +github.com/fluxcd/pkg/apis/event v0.12.0/go.mod h1:aRK2AONnjjSNW61B6Iy3SW4YHozACntnJeGm3fFqDqA= github.com/fluxcd/pkg/apis/kustomize v1.8.0 h1:HH6YRa3SMS72KK4cUyb9m5sK/dZH+Eti1qhjWDCgwKg= github.com/fluxcd/pkg/apis/kustomize v1.8.0/go.mod h1:QCKIFj1ocdndaWSkrLs5JKvdGNYyTzQX1ZB3lYTwma0= github.com/fluxcd/pkg/apis/meta v1.9.0 h1:wPgm7bWNJZ/ImS5GqikOxt362IgLPFBG73dZ27uWRiQ= @@ -81,8 +85,8 @@ github.com/fluxcd/pkg/envsubst v1.3.0 h1:84Ain+8EBvyzu6y0FsKRwNsvaSiKuqhTqeh/4yo github.com/fluxcd/pkg/envsubst v1.3.0/go.mod h1:lz6HvqDnxbX0sIqjr1fxw0oTGYACLVFcOE/srKS0VQQ= github.com/fluxcd/pkg/kustomize v1.15.0 h1:lII4FW9EJl0rI20dk+Glg5C2JZhP343FBov7HwW+SQo= github.com/fluxcd/pkg/kustomize v1.15.0/go.mod h1:e2SGi7cl28c9cnBVZ8YV8HAS4VBgUsiM6HMqv/AHJWQ= -github.com/fluxcd/pkg/runtime v0.51.0 h1:F4gKLUBUdvUdtg2lBsg72KUPqlOnaf9ChEL8bmP7CvQ= -github.com/fluxcd/pkg/runtime v0.51.0/go.mod h1:uMJ+s81+TyNGVjcnn+PIXUGGYs9VA3AK8nDmQWXAnis= +github.com/fluxcd/pkg/runtime v0.51.1 h1:68C6V/P2l/IwivqzvkgcR6Aa7zKds5ihsvoo0NcLarA= +github.com/fluxcd/pkg/runtime v0.51.1/go.mod h1:uMJ+s81+TyNGVjcnn+PIXUGGYs9VA3AK8nDmQWXAnis= github.com/fluxcd/pkg/sourceignore v0.10.0 h1:z5Bhh0G990uLbwjKNj7SzYqbGkicpGcXxF/Z4ZSVB64= github.com/fluxcd/pkg/sourceignore v0.10.0/go.mod h1:d1d9hcFxf+grda6JL3k+mC09nVTtBb9kJVzQn6J77B0= github.com/fluxcd/pkg/ssa v0.43.0 h1:XmADD3C0erYZayKfGI0WTsMlW9TtS4bp5gy4Axo1dcA= @@ -139,6 +143,12 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aN github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= @@ -161,6 +171,10 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhn github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= diff --git a/internal/controller/fluxinstance_controller.go b/internal/controller/fluxinstance_controller.go index 4b04bb4..120e5bd 100644 --- a/internal/controller/fluxinstance_controller.go +++ b/internal/controller/fluxinstance_controller.go @@ -15,6 +15,7 @@ import ( "github.com/fluxcd/cli-utils/pkg/kstatus/polling" "github.com/fluxcd/pkg/apis/meta" "github.com/fluxcd/pkg/runtime/conditions" + "github.com/fluxcd/pkg/runtime/events" "github.com/fluxcd/pkg/runtime/patch" "github.com/fluxcd/pkg/ssa" "github.com/fluxcd/pkg/ssa/normalize" @@ -170,7 +171,9 @@ func (r *FluxInstanceReconciler) reconcile(ctx context.Context, meta.BuildFailedReason, "%s", msg) log.Error(err, msg) - r.EventRecorder.Event(obj, corev1.EventTypeWarning, meta.BuildFailedReason, msg) + if eventErr := r.notify(obj, meta.BuildFailedReason, corev1.EventTypeWarning, msg); eventErr != nil { + log.Error(eventErr, "failed to emit event for build failure") + } return ctrl.Result{}, nil } @@ -195,7 +198,9 @@ func (r *FluxInstanceReconciler) reconcile(ctx context.Context, meta.ReadyCondition, meta.ReconciliationFailedReason, "%s", msg) - r.EventRecorder.Event(obj, corev1.EventTypeWarning, meta.ReconciliationFailedReason, msg) + if eventErr := r.notify(obj, meta.ReconciliationFailedReason, corev1.EventTypeWarning, msg); eventErr != nil { + log.Error(eventErr, "failed to emit event for reconciliation failure") + } return ctrl.Result{}, err } @@ -213,7 +218,7 @@ func (r *FluxInstanceReconciler) reconcile(ctx context.Context, map[string]string{fluxcdv1.RevisionAnnotation: obj.Status.LastAppliedRevision}, corev1.EventTypeNormal, meta.ReconciliationSucceededReason, - msg) + "%s", msg) return requeueAfter(obj), nil } @@ -520,6 +525,20 @@ func (r *FluxInstanceReconciler) apply(ctx context.Context, } } + // Send event to notification-controller only if the server-side apply resulted in changes. + applyLog := strings.TrimSuffix(changeSetLog.String(), "\n") + if applyLog != "" { + action := "updated" + if len(oldInventory.Entries) == 0 { + action = "installed" + } + + msg := fmt.Sprintf("Flux %s revision %s\n%s", action, buildResult.Revision, applyLog) + if err := r.notify(obj, meta.ReconciliationSucceededReason, corev1.EventTypeNormal, msg); err != nil { + log.Error(err, "failed to emit event for reconciliation success") + } + } + return nil } @@ -626,3 +645,28 @@ func (r *FluxInstanceReconciler) recordMetrics(obj *fluxcdv1.FluxInstance) error reporter.RecordMetrics(unstructured.Unstructured{Object: rawMap}) return nil } + +func (r *FluxInstanceReconciler) notify(obj *fluxcdv1.FluxInstance, reason, eventType, msg string) error { + notificationAddress := "" + if builder.ContainElementString(obj.GetComponents(), builder.MakeDefaultOptions().NotificationController) { + notificationAddress = fmt.Sprintf("http://%s.%s.svc.%s/", + builder.MakeDefaultOptions().NotificationController, + obj.GetNamespace(), + obj.GetCluster().Domain, + ) + } + + eventRecorder, err := events.NewRecorderForScheme( + r.Scheme, + r.EventRecorder, + ctrl.Log, + notificationAddress, + r.StatusManager) + if err != nil { + return err + } + + go eventRecorder.Eventf(obj, eventType, reason, "%s", msg) + + return nil +} diff --git a/internal/controller/fluxinstance_controller_test.go b/internal/controller/fluxinstance_controller_test.go index 617d274..b116b87 100644 --- a/internal/controller/fluxinstance_controller_test.go +++ b/internal/controller/fluxinstance_controller_test.go @@ -190,17 +190,21 @@ func TestFluxInstanceReconciler_LifeCycle(t *testing.T) { // Check if events were recorded for each step. events := getEvents(result.Name) - g.Expect(events).To(HaveLen(5)) + g.Expect(events).To(HaveLen(7)) g.Expect(events[0].Reason).To(Equal(fluxcdv1.OutdatedReason)) g.Expect(events[1].Reason).To(Equal(meta.ProgressingReason)) g.Expect(events[1].Message).To(HavePrefix("Installing")) g.Expect(events[2].Reason).To(Equal(meta.ReconciliationSucceededReason)) g.Expect(events[2].Message).To(HavePrefix("Reconciliation finished")) - g.Expect(events[3].Reason).To(Equal(meta.ProgressingReason)) - g.Expect(events[3].Message).To(HavePrefix("Upgrading")) - g.Expect(events[4].Reason).To(Equal(meta.ReconciliationSucceededReason)) - g.Expect(events[4].Annotations).To(HaveKeyWithValue(fluxcdv1.RevisionAnnotation, resultFinal.Status.LastAppliedRevision)) - + g.Expect(events[3].Reason).To(Equal(meta.ReconciliationSucceededReason)) + g.Expect(events[3].Message).To(HavePrefix("Flux installed")) + g.Expect(events[4].Reason).To(Equal(meta.ProgressingReason)) + g.Expect(events[4].Message).To(HavePrefix("Upgrading")) + g.Expect(events[5].Reason).To(Equal(meta.ReconciliationSucceededReason)) + g.Expect(events[5].Annotations).To(HaveKeyWithValue(fluxcdv1.RevisionAnnotation, resultFinal.Status.LastAppliedRevision)) + g.Expect(events[6].Reason).To(Equal(meta.ReconciliationSucceededReason)) + g.Expect(events[6].Message).To(HavePrefix("Flux updated")) + err = testClient.Delete(ctx, obj) g.Expect(err).ToNot(HaveOccurred()) @@ -486,12 +490,9 @@ func TestFluxInstanceReconciler_Disabled(t *testing.T) { err = testClient.Get(ctx, client.ObjectKeyFromObject(obj), resultFinal) g.Expect(err).ToNot(HaveOccurred()) - // Check if events were recorded for each step. + // Check if the ReconciliationDisabled event was recorded. events := getEvents(result.Name) - g.Expect(events).To(HaveLen(4)) - g.Expect(events[1].Reason).To(Equal(meta.ProgressingReason)) - g.Expect(events[2].Reason).To(Equal(meta.ReconciliationSucceededReason)) - g.Expect(events[3].Reason).To(Equal("ReconciliationDisabled")) + g.Expect(events[len(events)-1].Reason).To(Equal("ReconciliationDisabled")) // Check that resources were not deleted. kc := &appsv1.Deployment{} @@ -672,7 +673,7 @@ func TestFluxInstanceReconciler_NewVersion(t *testing.T) { // Check if events were recorded for each step. events := getEvents(obj.Name) - g.Expect(events).To(HaveLen(3)) + g.Expect(events).To(HaveLen(4)) g.Expect(events[0].Reason).To(Equal("OutdatedVersion")) err = testClient.Delete(ctx, obj)