feat(controllers): add logging with k8s events (#262)

* feat(controllers): implement simple event logging

* feat(controllers): add more in-depth event logging

* fix: add mock event broadcasters for tests

deploy/charts/burrito/templates/rbac.yaml

@@ -143,6 +143,7 @@ rules:
   verbs:
   - create
   - update
+  - patch
 - apiGroups:
   - ""
   resources:

internal/controllers/manager.go

@@ -92,36 +92,40 @@ func (c *Controllers) Exec() {
 		switch ctrlType {
 		case "layer":
 			if err = (&terraformlayer.Reconciler{
-				Client:  mgr.GetClient(),
-				Scheme:  mgr.GetScheme(),
-				Config:  c.config,
-				Storage: redis.New(c.config.Redis),
+				Client:   mgr.GetClient(),
+				Scheme:   mgr.GetScheme(),
+				Config:   c.config,
+				Recorder: mgr.GetEventRecorderFor("Burrito"),
+				Storage:  redis.New(c.config.Redis),
 			}).SetupWithManager(mgr); err != nil {
 				log.Fatalf("unable to create layer controller: %s", err)
 			}
 			log.Infof("layer controller started successfully")
 		case "repository":
 			if err = (&terraformrepository.Reconciler{
-				Client: mgr.GetClient(),
-				Scheme: mgr.GetScheme(),
+				Client:   mgr.GetClient(),
+				Scheme:   mgr.GetScheme(),
+				Recorder: mgr.GetEventRecorderFor("Burrito"),
 			}).SetupWithManager(mgr); err != nil {
 				log.Fatalf("unable to create repository controller: %s", err)
 			}
 			log.Infof("repository controller started successfully")
 		case "run":
 			if err = (&terraformrun.Reconciler{
-				Client: mgr.GetClient(),
-				Scheme: mgr.GetScheme(),
-				Config: c.config,
+				Client:   mgr.GetClient(),
+				Scheme:   mgr.GetScheme(),
+				Recorder: mgr.GetEventRecorderFor("Burrito"),
+				Config:   c.config,
 			}).SetupWithManager(mgr); err != nil {
 				log.Fatalf("unable to create run controller: %s", err)
 			}
 			log.Infof("run controller started successfully")
 		case "pullrequest":
 			if err = (&terraformpullrequest.Reconciler{
-				Client: mgr.GetClient(),
-				Scheme: mgr.GetScheme(),
-				Config: c.config,
+				Client:   mgr.GetClient(),
+				Scheme:   mgr.GetScheme(),
+				Recorder: mgr.GetEventRecorderFor("Burrito"),
+				Config:   c.config,
 			}).SetupWithManager(mgr); err != nil {
 				log.Fatalf("unable to create pullrequest controller: %s", err)
 			}

internal/controllers/terraformlayer/controller.go

@@ -24,9 +24,11 @@ import (
 	"github.com/padok-team/burrito/internal/burrito/config"
 	"github.com/padok-team/burrito/internal/lock"
 	"github.com/padok-team/burrito/internal/storage"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/event"
@@ -50,9 +52,10 @@ func (c RealClock) Now() time.Time {
 // Reconciler reconciles a TerraformLayer object
 type Reconciler struct {
 	client.Client
-	Scheme  *runtime.Scheme
-	Config  *config.Config
-	Storage storage.Storage
+	Scheme   *runtime.Scheme
+	Config   *config.Config
+	Storage  storage.Storage
+	Recorder record.EventRecorder
 	Clock
 }
 
@@ -112,6 +115,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	state, conditions := r.GetState(ctx, layer)
 	lastResult, err := r.Storage.Get(storage.GenerateKey(storage.LastPlanResult, layer))
 	if err != nil {
+		r.Recorder.Event(layer, corev1.EventTypeNormal, "Reconciliation", "Failed to get last Result")
 		lastResult = []byte("Error getting last Result")
 	}
 	result, run := state.getHandler()(ctx, r, layer, repository)
@@ -122,6 +126,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	layer.Status = configv1alpha1.TerraformLayerStatus{Conditions: conditions, State: getStateString(state), LastResult: string(lastResult), LastRun: runStatus}
 	err = r.Client.Status().Update(ctx, layer)
 	if err != nil {
+		r.Recorder.Event(layer, corev1.EventTypeWarning, "Reconciliation", "Could not update layer status")
 		log.Errorf("could not update layer %s status: %s", layer.Name, err)
 	}
 	log.Infof("finished reconciliation cycle for layer %s/%s", layer.Namespace, layer.Name)

internal/controllers/terraformlayer/controller_test.go

@@ -15,11 +15,13 @@ import (
 	controller "github.com/padok-team/burrito/internal/controllers/terraformlayer"
 	storage "github.com/padok-team/burrito/internal/storage/mock"
 	utils "github.com/padok-team/burrito/internal/testing"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
@@ -73,6 +75,9 @@ var _ = BeforeSuite(func() {
 		Config:  config.TestConfig(),
 		Storage: storage.New(),
 		Clock:   &MockClock{},
+		Recorder: record.NewBroadcasterForTests(1*time.Second).NewRecorder(scheme.Scheme, corev1.EventSource{
+			Component: "burrito",
+		}),
 	}
 	Expect(err).NotTo(HaveOccurred())
 	Expect(k8sClient).NotTo(BeNil())

internal/controllers/terraformlayer/run.go

@@ -8,6 +8,7 @@ import (
 
 	configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1"
 	log "github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/selection"
@@ -144,6 +145,7 @@ func (r *Reconciler) cleanupRuns(ctx context.Context, layer *configv1alpha1.Terr
 		return err
 	}
 	log.Infof("deleted %d runs for layer %s", len(toDelete), layer.Name)
+	r.Recorder.Event(layer, corev1.EventTypeNormal, "Reconciliation", "Cleaned up old runs")
 	return nil
 }
 

internal/controllers/terraformlayer/states.go

@@ -7,6 +7,7 @@ import (
 
 	configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1"
 	log "github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
@@ -55,9 +56,11 @@ func (s *PlanNeeded) getHandler() Handler {
 		run := r.getRun(layer, repository, "plan")
 		err := r.Client.Create(ctx, &run)
 		if err != nil {
+			r.Recorder.Event(layer, corev1.EventTypeWarning, "Reconciliation", "Failed to create TerraformRun for Plan action")
 			log.Errorf("failed to create TerraformRun for Plan action on layer %s: %s", layer.Name, err)
 			return ctrl.Result{RequeueAfter: r.Config.Controller.Timers.OnError}, nil
 		}
+		r.Recorder.Event(layer, corev1.EventTypeNormal, "Reconciliation", "Created TerraformRun for Plan action")
 		return ctrl.Result{RequeueAfter: r.Config.Controller.Timers.WaitAction}, &run
 	}
 }
@@ -75,9 +78,11 @@ func (s *ApplyNeeded) getHandler() Handler {
 		run := r.getRun(layer, repository, "apply")
 		err := r.Client.Create(ctx, &run)
 		if err != nil {
+			r.Recorder.Event(layer, corev1.EventTypeWarning, "Reconciliation", "Failed to create TerraformRun for Apply action")
 			log.Errorf("failed to create TerraformRun for Apply action on layer %s: %s", layer.Name, err)
 			return ctrl.Result{RequeueAfter: r.Config.Controller.Timers.OnError}, nil
 		}
+		r.Recorder.Event(layer, corev1.EventTypeNormal, "Reconciliation", "Created TerraformRun for Apply action")
 		return ctrl.Result{RequeueAfter: r.Config.Controller.Timers.WaitAction}, &run
 	}
 }

internal/controllers/terraformpullrequest/controller.go

@@ -13,9 +13,11 @@ import (
 	"github.com/padok-team/burrito/internal/storage/redis"
 
 	"github.com/padok-team/burrito/internal/controllers/terraformpullrequest/gitlab"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/event"
@@ -40,6 +42,7 @@ type Reconciler struct {
 	Config    *config.Config
 	Providers []Provider
 	Storage   storage.Storage
+	Recorder  record.EventRecorder
 }
 
 //+kubebuilder:rbac:groups=config.terraform.padok.cloud,resources=terraformpullrequests,verbs=get;list;watch;create;update;patch;delete
@@ -86,6 +89,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	pr.Status = state.Status
 	err = r.Client.Status().Update(ctx, pr)
 	if err != nil {
+		r.Recorder.Event(pr, corev1.EventTypeWarning, "Reconciliation", "Could not update pull request status")
 		log.Errorf("could not update pull request %s status: %s", pr.Name, err)
 	}
 	log.Infof("finished reconciliation cycle for pull request %s/%s", pr.Namespace, pr.Name)

internal/controllers/terraformpullrequest/controller_test.go

@@ -20,14 +20,17 @@ import (
 	"context"
 	"path/filepath"
 	"testing"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
@@ -108,6 +111,9 @@ var _ = BeforeSuite(func() {
 		Providers: []controller.Provider{
 			&provider.Mock{},
 		},
+		Recorder: record.NewBroadcasterForTests(1*time.Second).NewRecorder(scheme.Scheme, corev1.EventSource{
+			Component: "burrito",
+		}),
 	}
 	Expect(err).NotTo(HaveOccurred())
 	Expect(k8sClient).NotTo(BeNil())

internal/controllers/terraformpullrequest/layer.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/selection"
@@ -29,6 +30,7 @@ func (r *Reconciler) getAffectedLayers(repository *configv1alpha1.TerraformRepos
 		}
 	}
 	if provider == nil {
+		r.Recorder.Event(pr, corev1.EventTypeWarning, "Provider error", "Could not find provider (gitlab, github...)")
 		return nil, fmt.Errorf("could not find provider for pull request %s", pr.Name)
 	}
 	changes, err := provider.GetChanges(repository, pr)

internal/controllers/terraformpullrequest/states.go

@@ -2,11 +2,13 @@ package terraformpullrequest
 
 import (
 	"context"
+	"fmt"
 
 	configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1"
 	"github.com/padok-team/burrito/internal/annotations"
 	"github.com/padok-team/burrito/internal/controllers/terraformpullrequest/comment"
 	log "github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
@@ -77,10 +79,12 @@ func planningHandler(ctx context.Context, r *Reconciler, repository *configv1alp
 func discoveryNeededHandler(ctx context.Context, r *Reconciler, repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest, state *State) ctrl.Result {
 	err := r.deleteTempLayers(ctx, pr)
 	if err != nil {
+		r.Recorder.Event(pr, corev1.EventTypeWarning, "Reconciliation", "Failed to delete temp layers for pull request")
 		log.Errorf("failed to delete temp layers for pull request %s: %s", pr.Name, err)
 	}
 	layers, err := r.getAffectedLayers(repository, pr)
 	if err != nil {
+		r.Recorder.Event(pr, corev1.EventTypeWarning, "Reconciliation", "Failed to get affected layers for pull request")
 		log.Errorf("failed to get affected layers for pull request %s: %s", pr.Name, err)
 		return ctrl.Result{RequeueAfter: r.Config.Controller.Timers.OnError}
 	}
@@ -89,8 +93,10 @@ func discoveryNeededHandler(ctx context.Context, r *Reconciler, repository *conf
 		err := r.Client.Create(ctx, &layer)
 		if err != nil {
 			log.Errorf("failed to create layer %s: %s", layer.Name, err)
+			r.Recorder.Event(pr, corev1.EventTypeWarning, "Reconciliation", "Failed to create layer for pull request")
 			return ctrl.Result{RequeueAfter: r.Config.Controller.Timers.OnError}
 		}
+		r.Recorder.Event(pr, corev1.EventTypeNormal, "Reconciliation", fmt.Sprintf("Created layer %s", layer.Name))
 	}
 	state.Status.LastDiscoveredCommit = pr.Annotations[annotations.LastBranchCommit]
 	return ctrl.Result{RequeueAfter: r.Config.Controller.Timers.WaitAction}
@@ -99,6 +105,7 @@ func discoveryNeededHandler(ctx context.Context, r *Reconciler, repository *conf
 func commentNeededHandler(ctx context.Context, r *Reconciler, repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest, state *State) ctrl.Result {
 	layers, err := GetLinkedLayers(r.Client, pr)
 	if err != nil {
+		r.Recorder.Event(pr, corev1.EventTypeWarning, "Reconciliation", "Failed to get linked layers for pull request")
 		log.Errorf("failed to get linked layers for pull request %s: %s", pr.Name, err)
 		return ctrl.Result{RequeueAfter: r.Config.Controller.Timers.OnError}
 	}
@@ -119,9 +126,11 @@ func commentNeededHandler(ctx context.Context, r *Reconciler, repository *config
 	comment := comment.NewDefaultComment(layers, r.Storage)
 	err = provider.Comment(repository, pr, comment)
 	if err != nil {
+		r.Recorder.Event(pr, corev1.EventTypeWarning, "Reconciliation", "Failed to comment pull request")
 		log.Errorf("an error occurred while commenting pull request: %s", err)
 		return ctrl.Result{RequeueAfter: r.Config.Controller.Timers.OnError}
 	}
+	r.Recorder.Event(pr, corev1.EventTypeNormal, "Reconciliation", "Commented pull request")
 	state.Status.LastCommentedCommit = pr.Annotations[annotations.LastBranchCommit]
 	return ctrl.Result{RequeueAfter: r.Config.Controller.Timers.WaitAction}
 }

internal/controllers/terraformrepository/controller.go

@@ -21,6 +21,7 @@ import (
 
 	log "github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -30,7 +31,8 @@ import (
 // RepositoryReconciler reconciles a TerraformRepository object
 type Reconciler struct {
 	client.Client
-	Scheme *runtime.Scheme
+	Scheme   *runtime.Scheme
+	Recorder record.EventRecorder
 }
 
 //+kubebuilder:rbac:groups=config.terraform.padok.cloud,resources=terraformrepositories,verbs=get;list;watch;create;update;patch;delete

internal/controllers/terraformrun/controller.go

@@ -23,9 +23,11 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	log "github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/event"
@@ -48,8 +50,9 @@ func (c RealClock) Now() time.Time {
 // RunReconcilier reconciles a TerraformRun object
 type Reconciler struct {
 	client.Client
-	Scheme *runtime.Scheme
-	Config *config.Config
+	Scheme   *runtime.Scheme
+	Config   *config.Config
+	Recorder record.EventRecorder
 	Clock
 }
 
@@ -85,10 +88,12 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	}
 	layer, err := r.getLinkedLayer(run)
 	if err != nil {
+		r.Recorder.Event(run, corev1.EventTypeWarning, "Reconciliation", "Could not get linked layer")
 		return ctrl.Result{RequeueAfter: r.Config.Controller.Timers.OnError}, err
 	}
 	repo, err := r.getLinkedRepo(run, layer)
 	if err != nil {
+		r.Recorder.Event(run, corev1.EventTypeWarning, "Reconciliation", "Could not get linked repository")
 		return ctrl.Result{RequeueAfter: r.Config.Controller.Timers.OnError}, err
 	}
 	state, conditions := r.GetState(ctx, run, layer, repo)
@@ -102,6 +107,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	}
 	err = r.Client.Status().Update(ctx, run)
 	if err != nil {
+		r.Recorder.Event(run, corev1.EventTypeWarning, "Reconciliation", "Could not update run status")
 		log.Errorf("could not update run %s status: %s", run.Name, err)
 	}
 	log.Infof("finished reconciliation cycle for run %s/%s", run.Namespace, run.Name)

internal/controllers/terraformrun/controller_test.go

@@ -19,6 +19,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
@@ -71,6 +72,9 @@ var _ = BeforeSuite(func() {
 		Scheme: scheme.Scheme,
 		Config: config.TestConfig(),
 		Clock:  &MockClock{},
+		Recorder: record.NewBroadcasterForTests(1*time.Second).NewRecorder(scheme.Scheme, corev1.EventSource{
+			Component: "burrito",
+		}),
 	}
 	Expect(err).NotTo(HaveOccurred())
 	Expect(k8sClient).NotTo(BeNil())